From c087854f71960341933a71442583dbc53d9b4e14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Sat, 26 Nov 2022 21:56:51 +0800 Subject: [PATCH] create --- LICENSE | 2 +- README.md | 78 + egs/aishell/conformer/README.md | 17 + .../conf/decode_asr_transformer.yaml | 6 + .../conformer/conf/train_asr_conformer.yaml | 80 + .../conformer/local/aishell_data_prep.sh | 66 + egs/aishell/conformer/local/prepare_data.sh | 53 + egs/aishell/conformer/path.sh | 5 + egs/aishell/conformer/run.sh | 208 ++ egs/aishell/conformer/utils | 1 + egs/aishell/paraformer/README.md | 24 + .../conf/decode_asr_transformer.yaml | 6 + .../decode_asr_transformer_noctc_1best.yaml | 6 + ..._paraformer_conformer_12e_6d_2048_256.yaml | 91 + ..._paraformer_conformer_20e_6d_1280_320.yaml | 92 + ...aformer_sanm_tf_40e_12d_1280_320_lfr6.yaml | 114 ++ ...aformer_sanm_tf_50e_16d_2048_512_lfr6.yaml | 114 ++ ...aformerbert_conformer_12e_6d_2048_256.yaml | 99 + .../paraformer/local/aishell_data_prep.sh | 66 + egs/aishell/paraformer/local/prepare_data.sh | 53 + egs/aishell/paraformer/path.sh | 5 + egs/aishell/paraformer/run.sh | 208 ++ egs/aishell/paraformer/utils | 1 + egs/aishell/paraformer2/README.md | 18 + .../conf/decode_asr_transformer.yaml | 6 + .../decode_asr_transformer_noctc_1best.yaml | 6 + ..._paraformer_conformer_12e_6d_2048_256.yaml | 92 + ..._paraformer_conformer_20e_6d_1280_320.yaml | 94 + ...aformerbert_conformer_12e_6d_2048_256.yaml | 100 + .../paraformer2/local/aishell_data_prep.sh | 65 + .../paraformer2/local/extract_embeds.sh | 67 + egs/aishell/paraformer2/path.sh | 5 + egs/aishell/paraformer2/run.sh | 226 +++ egs/aishell/paraformer2/utils | 1 + .../conf/decode_asr_transformer.yaml | 6 + .../tranformer/conf/train_asr_conformer.yaml | 80 + .../conf/train_asr_transformer.yaml | 70 + .../tranformer/local/aishell_data_prep.sh | 66 + egs/aishell/tranformer/local/prepare_data.sh | 53 + egs/aishell/tranformer/path.sh | 5 + egs/aishell/tranformer/run.sh | 208 ++ egs/aishell/tranformer/utils/__init__.py | 0 egs/aishell/tranformer/utils/apply_cmvn.py | 79 + egs/aishell/tranformer/utils/apply_cmvn.sh | 29 + .../tranformer/utils/apply_lfr_and_cmvn.py | 143 ++ .../tranformer/utils/apply_lfr_and_cmvn.sh | 38 + .../tranformer/utils/combine_cmvn_file.py | 66 + egs/aishell/tranformer/utils/compute_cmvn.py | 67 + egs/aishell/tranformer/utils/compute_cmvn.sh | 24 + egs/aishell/tranformer/utils/compute_fbank.py | 153 ++ egs/aishell/tranformer/utils/compute_fbank.sh | 51 + egs/aishell/tranformer/utils/compute_wer.py | 157 ++ .../tranformer/utils/easy_asr_infer.sh | 407 ++++ egs/aishell/tranformer/utils/error_rate_zh | 370 ++++ .../tranformer/utils/extract_embeds.py | 47 + egs/aishell/tranformer/utils/filter_scp.pl | 87 + egs/aishell/tranformer/utils/fix_data.sh | 35 + egs/aishell/tranformer/utils/fix_data_feat.sh | 52 + egs/aishell/tranformer/utils/gen_ark_list.sh | 20 + egs/aishell/tranformer/utils/parse_options.sh | 97 + egs/aishell/tranformer/utils/print_args.py | 45 + egs/aishell/tranformer/utils/proc_conf_oss.py | 35 + egs/aishell/tranformer/utils/proce_text.py | 31 + egs/aishell/tranformer/utils/run.pl | 356 ++++ egs/aishell/tranformer/utils/shuffle_list.pl | 44 + egs/aishell/tranformer/utils/split_data.py | 60 + egs/aishell/tranformer/utils/split_scp.pl | 246 +++ .../tranformer/utils/subset_data_dir_tr_cv.sh | 30 + egs/aishell/tranformer/utils/text2token.py | 135 ++ egs/aishell/tranformer/utils/text_tokenize.py | 106 + egs/aishell/tranformer/utils/text_tokenize.sh | 35 + egs/aishell/tranformer/utils/textnorm_zh.py | 834 ++++++++ egs_modelscope/aishell/paraformer/README.md | 38 + egs_modelscope/aishell/paraformer/RESULTS.md | 24 + ...ansformer_noctc_10best_lm_weight_0.15.yaml | 6 + .../decode_asr_transformer_noctc_1best.yaml | 6 + ...paraformer_sanm_50e_16d_2048_512_lfr6.yaml | 91 + .../paraformer/local/aishell_data_prep.sh | 66 + .../aishell/paraformer/modelscope_utils | 1 + .../paraformer/paraformer_large_finetune.sh | 224 ++ .../paraformer/paraformer_large_infer.sh | 70 + egs_modelscope/aishell/paraformer/path.sh | 5 + egs_modelscope/aishell/paraformer/utils | 1 + egs_modelscope/aishell2/paraformer/README.md | 39 + egs_modelscope/aishell2/paraformer/RESULTS.md | 26 + ...ansformer_noctc_10best_lm_weight_0.15.yaml | 6 + .../decode_asr_transformer_noctc_1best.yaml | 6 + ...paraformer_sanm_50e_16d_2048_512_lfr6.yaml | 91 + .../paraformer/local/aishell2_data_prep.sh | 53 + .../aishell2/paraformer/modelscope_utils | 1 + .../paraformer/paraformer_large_finetune.sh | 239 +++ .../paraformer/paraformer_large_infer.sh | 56 + egs_modelscope/aishell2/paraformer/path.sh | 5 + egs_modelscope/aishell2/paraformer/utils | 1 + egs_modelscope/common/README.md | 27 + ...ansformer_noctc_10best_lm_weight_0.15.yaml | 6 + .../decode_asr_transformer_noctc_1best.yaml | 6 + ...paraformer_sanm_50e_16d_2048_512_lfr6.yaml | 91 + .../common/modelscope_common_finetune.sh | 230 +++ .../common/modelscope_common_infer.sh | 78 + .../modelscope_common_infer_after_finetune.sh | 66 + .../common/modelscope_utils/download_model.py | 21 + .../modelscope_utils/modelscope_infer.sh | 90 + .../common/modelscope_utils/update_config.py | 41 + egs_modelscope/common/path.sh | 5 + egs_modelscope/common/utils | 1 + egs_modelscope/speechio/paraformer/README.md | 24 + egs_modelscope/speechio/paraformer/RESULTS.md | 42 + .../speechio/paraformer/modelscope_utils | 1 + .../paraformer/paraformer_large_infer.sh | 81 + egs_modelscope/speechio/paraformer/path.sh | 5 + egs_modelscope/speechio/paraformer/utils | 1 + .../wenetspeech/paraformer/README.md | 24 + .../wenetspeech/paraformer/RESULTS.md | 25 + .../wenetspeech/paraformer/modelscope_utils | 1 + .../paraformer/paraformer_large_infer.sh | 56 + egs_modelscope/wenetspeech/paraformer/path.sh | 5 + egs_modelscope/wenetspeech/paraformer/utils | 1 + funasr/__init__.py | 8 + funasr/bin/__init__.py | 0 funasr/bin/aggregate_stats_dirs.py | 108 + funasr/bin/asr_inference.py | 548 +++++ funasr/bin/asr_inference_launch.py | 225 ++ funasr/bin/asr_inference_paraformer.py | 528 +++++ funasr/bin/asr_inference_uniasr.py | 543 +++++ funasr/bin/asr_train.py | 46 + funasr/bin/asr_train_paraformer.py | 46 + funasr/bin/asr_train_uniasr.py | 46 + funasr/bin/lm_calc_perplexity.py | 210 ++ funasr/bin/lm_train.py | 22 + funasr/bin/modelscope_infer.py | 82 + funasr/datasets/__init__.py | 0 funasr/datasets/collate_fn.py | 83 + funasr/datasets/dataset.py | 444 ++++ funasr/datasets/iterable_dataset.py | 237 +++ funasr/datasets/large_datasets/__init__.py | 0 .../large_datasets/build_dataloader.py | 41 + .../large_datasets/datapipes/__init__.py | 0 .../large_datasets/datapipes/batch.py | 148 ++ .../large_datasets/datapipes/filter.py | 24 + .../datasets/large_datasets/datapipes/map.py | 22 + funasr/datasets/large_datasets/dataset.py | 175 ++ .../datasets/large_datasets/utils/__init__.py | 0 .../datasets/large_datasets/utils/filter.py | 15 + .../large_datasets/utils/low_frame_rate.py | 30 + .../datasets/large_datasets/utils/padding.py | 35 + .../datasets/large_datasets/utils/tokenize.py | 17 + funasr/datasets/preprocessor.py | 496 +++++ funasr/fileio/__init__.py | 0 funasr/fileio/datadir_writer.py | 77 + funasr/fileio/npy_scp.py | 97 + funasr/fileio/rand_gen_dataset.py | 86 + funasr/fileio/read_text.py | 81 + funasr/fileio/sound_scp.py | 131 ++ funasr/iterators/__init__.py | 0 funasr/iterators/abs_iter_factory.py | 9 + funasr/iterators/chunk_iter_factory.py | 215 ++ funasr/iterators/multiple_iter_factory.py | 37 + funasr/iterators/sequence_iter_factory.py | 143 ++ funasr/layers/__init__.py | 0 funasr/layers/abs_normalize.py | 14 + funasr/layers/complex_utils.py | 191 ++ funasr/layers/global_mvn.py | 121 ++ funasr/layers/inversible_interface.py | 14 + funasr/layers/label_aggregation.py | 82 + funasr/layers/log_mel.py | 83 + funasr/layers/mask_along_axis.py | 340 ++++ funasr/layers/sinc_conv.py | 273 +++ funasr/layers/stft.py | 229 +++ funasr/layers/time_warp.py | 88 + funasr/layers/utterance_mvn.py | 88 + funasr/lm/__init__.py | 0 funasr/lm/abs_model.py | 29 + funasr/lm/espnet_model.py | 131 ++ funasr/lm/seq_rnn_lm.py | 174 ++ funasr/lm/transformer_lm.py | 131 ++ funasr/losses/label_smoothing_loss.py | 63 + funasr/main_funcs/__init__.py | 0 funasr/main_funcs/average_nbest_models.py | 127 ++ funasr/main_funcs/calculate_all_attentions.py | 160 ++ funasr/main_funcs/collect_stats.py | 126 ++ funasr/main_funcs/pack_funcs.py | 302 +++ funasr/models/ctc.py | 187 ++ funasr/models/decoder/__init__.py | 0 funasr/models/decoder/abs_decoder.py | 19 + funasr/models/decoder/rnn_decoder.py | 334 +++ funasr/models/decoder/sanm_decoder.py | 616 ++++++ funasr/models/decoder/transformer_decoder.py | 766 +++++++ funasr/models/e2e_asr.py | 458 +++++ funasr/models/e2e_asr_common.py | 249 +++ funasr/models/e2e_asr_paraformer.py | 820 ++++++++ funasr/models/e2e_uni_asr.py | 1076 ++++++++++ funasr/models/encoder/__init__.py | 0 funasr/models/encoder/abs_encoder.py | 21 + funasr/models/encoder/conformer_encoder.py | 598 ++++++ funasr/models/encoder/rnn_encoder.py | 115 ++ funasr/models/encoder/sanm_encoder.py | 595 ++++++ funasr/models/encoder/transformer_encoder.py | 684 +++++++ funasr/models/frontend/__init__.py | 0 funasr/models/frontend/abs_frontend.py | 17 + funasr/models/frontend/default.py | 133 ++ funasr/models/frontend/fused.py | 146 ++ funasr/models/frontend/s3prl.py | 143 ++ funasr/models/frontend/windowing.py | 81 + funasr/models/postencoder/__init__.py | 0 funasr/models/postencoder/abs_postencoder.py | 17 + .../hugging_face_transformers_postencoder.py | 115 ++ funasr/models/predictor/__init__.py | 0 funasr/models/predictor/cif.py | 266 +++ funasr/models/preencoder/__init__.py | 0 funasr/models/preencoder/abs_preencoder.py | 17 + funasr/models/preencoder/linear.py | 38 + funasr/models/preencoder/sinc.py | 282 +++ funasr/models/specaug/__init__.py | 0 funasr/models/specaug/abs_specaug.py | 18 + funasr/models/specaug/specaug.py | 184 ++ funasr/modules/add_sos_eos.py | 31 + funasr/modules/attention.py | 625 ++++++ funasr/modules/beam_search/__init__.py | 0 .../modules/beam_search/batch_beam_search.py | 348 ++++ .../batch_beam_search_online_sim.py | 270 +++ funasr/modules/beam_search/beam_search.py | 1400 +++++++++++++ funasr/modules/dynamic_conv.py | 125 ++ funasr/modules/dynamic_conv2d.py | 138 ++ funasr/modules/e2e_asr_common.py | 249 +++ funasr/modules/embedding.py | 408 ++++ funasr/modules/frontends/__init__.py | 1 + funasr/modules/frontends/beamformer.py | 84 + funasr/modules/frontends/dnn_beamformer.py | 172 ++ funasr/modules/frontends/dnn_wpe.py | 93 + funasr/modules/frontends/feature_transform.py | 263 +++ funasr/modules/frontends/frontend.py | 151 ++ funasr/modules/frontends/mask_estimator.py | 77 + funasr/modules/layer_norm.py | 42 + funasr/modules/lightconv.py | 112 + funasr/modules/lightconv2d.py | 124 ++ funasr/modules/mask.py | 35 + funasr/modules/multi_layer_conv.py | 105 + funasr/modules/nets_utils.py | 508 +++++ funasr/modules/positionwise_feed_forward.py | 58 + funasr/modules/repeat.py | 33 + funasr/modules/rnn/__init__.py | 1 + funasr/modules/rnn/argument.py | 156 ++ funasr/modules/rnn/attentions.py | 1808 +++++++++++++++++ funasr/modules/rnn/decoders.py | 1211 +++++++++++ funasr/modules/rnn/encoders.py | 372 ++++ funasr/modules/scorers/__init__.py | 1 + funasr/modules/scorers/ctc.py | 158 ++ funasr/modules/scorers/ctc_prefix_score.py | 359 ++++ funasr/modules/scorers/length_bonus.py | 61 + funasr/modules/scorers/scorer_interface.py | 188 ++ .../modules/streaming_utils/chunk_utilis.py | 390 ++++ funasr/modules/streaming_utils/utils.py | 47 + funasr/modules/subsampling.py | 304 +++ funasr/modules/subsampling_without_posenc.py | 61 + funasr/optimizers/__init__.py | 0 funasr/optimizers/sgd.py | 32 + funasr/samplers/__init__.py | 0 funasr/samplers/abs_sampler.py | 19 + funasr/samplers/build_batch_sampler.py | 167 ++ funasr/samplers/folded_batch_sampler.py | 156 ++ funasr/samplers/length_batch_sampler.py | 143 ++ funasr/samplers/num_elements_batch_sampler.py | 160 ++ funasr/samplers/sorted_batch_sampler.py | 95 + funasr/samplers/unsorted_batch_sampler.py | 91 + funasr/schedulers/__init__.py | 0 funasr/schedulers/abs_scheduler.py | 84 + funasr/schedulers/noam_lr.py | 65 + funasr/schedulers/warmup_lr.py | 50 + funasr/tasks/__init__.py | 0 funasr/tasks/abs_task.py | 1795 ++++++++++++++++ funasr/tasks/asr.py | 879 ++++++++ funasr/tasks/lm.py | 211 ++ funasr/text/__init__.py | 0 funasr/text/abs_tokenizer.py | 14 + funasr/text/build_tokenizer.py | 63 + funasr/text/char_tokenizer.py | 62 + funasr/text/cleaner.py | 48 + funasr/text/korean_cleaner.py | 77 + funasr/text/phoneme_tokenizer.py | 528 +++++ funasr/text/sentencepiece_tokenizer.py | 38 + funasr/text/token_id_converter.py | 60 + funasr/text/word_tokenizer.py | 58 + funasr/torch_utils/__init__.py | 0 funasr/torch_utils/add_gradient_noise.py | 31 + funasr/torch_utils/device_funcs.py | 71 + funasr/torch_utils/forward_adaptor.py | 33 + funasr/torch_utils/initialize.py | 102 + funasr/torch_utils/load_pretrained_model.py | 125 ++ funasr/torch_utils/model_summary.py | 70 + funasr/torch_utils/pytorch_version.py | 16 + funasr/torch_utils/recursive_op.py | 47 + funasr/torch_utils/set_all_random_seed.py | 10 + funasr/train/__init__.py | 0 funasr/train/abs_espnet_model.py | 55 + funasr/train/class_choices.py | 95 + funasr/train/distributed_utils.py | 384 ++++ funasr/train/reporter.py | 540 +++++ funasr/train/trainer.py | 814 ++++++++ funasr/utils/__init__.py | 0 funasr/utils/build_dataclass.py | 17 + funasr/utils/cli_utils.py | 65 + funasr/utils/config_argparse.py | 47 + funasr/utils/get_default_kwargs.py | 57 + funasr/utils/griffin_lim.py | 192 ++ funasr/utils/nested_dict_action.py | 106 + funasr/utils/sized_dict.py | 75 + funasr/utils/types.py | 149 ++ funasr/utils/yaml_no_alias_safe_dump.py | 14 + funasr/version.txt | 1 + image/dingding.jpg | Bin 0 -> 192919 bytes image/funasr_logo.jpg | Bin 0 -> 639403 bytes image/wechat.png | Bin 0 -> 179218 bytes setup.py | 146 ++ 314 files changed, 42966 insertions(+), 1 deletion(-) create mode 100644 README.md create mode 100644 egs/aishell/conformer/README.md create mode 100644 egs/aishell/conformer/conf/decode_asr_transformer.yaml create mode 100644 egs/aishell/conformer/conf/train_asr_conformer.yaml create mode 100755 egs/aishell/conformer/local/aishell_data_prep.sh create mode 100755 egs/aishell/conformer/local/prepare_data.sh create mode 100755 egs/aishell/conformer/path.sh create mode 100755 egs/aishell/conformer/run.sh create mode 120000 egs/aishell/conformer/utils create mode 100644 egs/aishell/paraformer/README.md create mode 100644 egs/aishell/paraformer/conf/decode_asr_transformer.yaml create mode 100644 egs/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml create mode 100644 egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml create mode 100644 egs/aishell/paraformer/conf/train_asr_paraformer_conformer_20e_6d_1280_320.yaml create mode 100644 egs/aishell/paraformer/conf/train_asr_paraformer_sanm_tf_40e_12d_1280_320_lfr6.yaml create mode 100644 egs/aishell/paraformer/conf/train_asr_paraformer_sanm_tf_50e_16d_2048_512_lfr6.yaml create mode 100644 egs/aishell/paraformer/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml create mode 100755 egs/aishell/paraformer/local/aishell_data_prep.sh create mode 100755 egs/aishell/paraformer/local/prepare_data.sh create mode 100755 egs/aishell/paraformer/path.sh create mode 100755 egs/aishell/paraformer/run.sh create mode 120000 egs/aishell/paraformer/utils create mode 100644 egs/aishell/paraformer2/README.md create mode 100644 egs/aishell/paraformer2/conf/decode_asr_transformer.yaml create mode 100644 egs/aishell/paraformer2/conf/decode_asr_transformer_noctc_1best.yaml create mode 100644 egs/aishell/paraformer2/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml create mode 100644 egs/aishell/paraformer2/conf/train_asr_paraformer_conformer_20e_6d_1280_320.yaml create mode 100644 egs/aishell/paraformer2/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml create mode 100755 egs/aishell/paraformer2/local/aishell_data_prep.sh create mode 100755 egs/aishell/paraformer2/local/extract_embeds.sh create mode 100755 egs/aishell/paraformer2/path.sh create mode 100755 egs/aishell/paraformer2/run.sh create mode 120000 egs/aishell/paraformer2/utils create mode 100644 egs/aishell/tranformer/conf/decode_asr_transformer.yaml create mode 100644 egs/aishell/tranformer/conf/train_asr_conformer.yaml create mode 100644 egs/aishell/tranformer/conf/train_asr_transformer.yaml create mode 100755 egs/aishell/tranformer/local/aishell_data_prep.sh create mode 100755 egs/aishell/tranformer/local/prepare_data.sh create mode 100755 egs/aishell/tranformer/path.sh create mode 100755 egs/aishell/tranformer/run.sh create mode 100644 egs/aishell/tranformer/utils/__init__.py create mode 100755 egs/aishell/tranformer/utils/apply_cmvn.py create mode 100755 egs/aishell/tranformer/utils/apply_cmvn.sh create mode 100755 egs/aishell/tranformer/utils/apply_lfr_and_cmvn.py create mode 100755 egs/aishell/tranformer/utils/apply_lfr_and_cmvn.sh create mode 100755 egs/aishell/tranformer/utils/combine_cmvn_file.py create mode 100755 egs/aishell/tranformer/utils/compute_cmvn.py create mode 100755 egs/aishell/tranformer/utils/compute_cmvn.sh create mode 100755 egs/aishell/tranformer/utils/compute_fbank.py create mode 100755 egs/aishell/tranformer/utils/compute_fbank.sh create mode 100755 egs/aishell/tranformer/utils/compute_wer.py create mode 100755 egs/aishell/tranformer/utils/easy_asr_infer.sh create mode 100755 egs/aishell/tranformer/utils/error_rate_zh create mode 100755 egs/aishell/tranformer/utils/extract_embeds.py create mode 100755 egs/aishell/tranformer/utils/filter_scp.pl create mode 100755 egs/aishell/tranformer/utils/fix_data.sh create mode 100755 egs/aishell/tranformer/utils/fix_data_feat.sh create mode 100755 egs/aishell/tranformer/utils/gen_ark_list.sh create mode 100755 egs/aishell/tranformer/utils/parse_options.sh create mode 100755 egs/aishell/tranformer/utils/print_args.py create mode 100755 egs/aishell/tranformer/utils/proc_conf_oss.py create mode 100755 egs/aishell/tranformer/utils/proce_text.py create mode 100755 egs/aishell/tranformer/utils/run.pl create mode 100755 egs/aishell/tranformer/utils/shuffle_list.pl create mode 100755 egs/aishell/tranformer/utils/split_data.py create mode 100755 egs/aishell/tranformer/utils/split_scp.pl create mode 100755 egs/aishell/tranformer/utils/subset_data_dir_tr_cv.sh create mode 100755 egs/aishell/tranformer/utils/text2token.py create mode 100755 egs/aishell/tranformer/utils/text_tokenize.py create mode 100755 egs/aishell/tranformer/utils/text_tokenize.sh create mode 100755 egs/aishell/tranformer/utils/textnorm_zh.py create mode 100644 egs_modelscope/aishell/paraformer/README.md create mode 100644 egs_modelscope/aishell/paraformer/RESULTS.md create mode 100644 egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml create mode 100644 egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml create mode 100644 egs_modelscope/aishell/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml create mode 100755 egs_modelscope/aishell/paraformer/local/aishell_data_prep.sh create mode 120000 egs_modelscope/aishell/paraformer/modelscope_utils create mode 100755 egs_modelscope/aishell/paraformer/paraformer_large_finetune.sh create mode 100755 egs_modelscope/aishell/paraformer/paraformer_large_infer.sh create mode 100755 egs_modelscope/aishell/paraformer/path.sh create mode 120000 egs_modelscope/aishell/paraformer/utils create mode 100644 egs_modelscope/aishell2/paraformer/README.md create mode 100644 egs_modelscope/aishell2/paraformer/RESULTS.md create mode 100644 egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml create mode 100644 egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml create mode 100644 egs_modelscope/aishell2/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml create mode 100755 egs_modelscope/aishell2/paraformer/local/aishell2_data_prep.sh create mode 120000 egs_modelscope/aishell2/paraformer/modelscope_utils create mode 100755 egs_modelscope/aishell2/paraformer/paraformer_large_finetune.sh create mode 100755 egs_modelscope/aishell2/paraformer/paraformer_large_infer.sh create mode 100755 egs_modelscope/aishell2/paraformer/path.sh create mode 120000 egs_modelscope/aishell2/paraformer/utils create mode 100644 egs_modelscope/common/README.md create mode 100644 egs_modelscope/common/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml create mode 100644 egs_modelscope/common/conf/decode_asr_transformer_noctc_1best.yaml create mode 100644 egs_modelscope/common/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml create mode 100755 egs_modelscope/common/modelscope_common_finetune.sh create mode 100755 egs_modelscope/common/modelscope_common_infer.sh create mode 100755 egs_modelscope/common/modelscope_common_infer_after_finetune.sh create mode 100755 egs_modelscope/common/modelscope_utils/download_model.py create mode 100755 egs_modelscope/common/modelscope_utils/modelscope_infer.sh create mode 100644 egs_modelscope/common/modelscope_utils/update_config.py create mode 100755 egs_modelscope/common/path.sh create mode 120000 egs_modelscope/common/utils create mode 100644 egs_modelscope/speechio/paraformer/README.md create mode 100644 egs_modelscope/speechio/paraformer/RESULTS.md create mode 120000 egs_modelscope/speechio/paraformer/modelscope_utils create mode 100755 egs_modelscope/speechio/paraformer/paraformer_large_infer.sh create mode 100755 egs_modelscope/speechio/paraformer/path.sh create mode 120000 egs_modelscope/speechio/paraformer/utils create mode 100644 egs_modelscope/wenetspeech/paraformer/README.md create mode 100644 egs_modelscope/wenetspeech/paraformer/RESULTS.md create mode 120000 egs_modelscope/wenetspeech/paraformer/modelscope_utils create mode 100755 egs_modelscope/wenetspeech/paraformer/paraformer_large_infer.sh create mode 100755 egs_modelscope/wenetspeech/paraformer/path.sh create mode 120000 egs_modelscope/wenetspeech/paraformer/utils create mode 100644 funasr/__init__.py create mode 100644 funasr/bin/__init__.py create mode 100755 funasr/bin/aggregate_stats_dirs.py create mode 100755 funasr/bin/asr_inference.py create mode 100755 funasr/bin/asr_inference_launch.py create mode 100755 funasr/bin/asr_inference_paraformer.py create mode 100755 funasr/bin/asr_inference_uniasr.py create mode 100755 funasr/bin/asr_train.py create mode 100755 funasr/bin/asr_train_paraformer.py create mode 100755 funasr/bin/asr_train_uniasr.py create mode 100755 funasr/bin/lm_calc_perplexity.py create mode 100755 funasr/bin/lm_train.py create mode 100755 funasr/bin/modelscope_infer.py create mode 100644 funasr/datasets/__init__.py create mode 100644 funasr/datasets/collate_fn.py create mode 100644 funasr/datasets/dataset.py create mode 100644 funasr/datasets/iterable_dataset.py create mode 100644 funasr/datasets/large_datasets/__init__.py create mode 100644 funasr/datasets/large_datasets/build_dataloader.py create mode 100644 funasr/datasets/large_datasets/datapipes/__init__.py create mode 100644 funasr/datasets/large_datasets/datapipes/batch.py create mode 100644 funasr/datasets/large_datasets/datapipes/filter.py create mode 100644 funasr/datasets/large_datasets/datapipes/map.py create mode 100644 funasr/datasets/large_datasets/dataset.py create mode 100644 funasr/datasets/large_datasets/utils/__init__.py create mode 100644 funasr/datasets/large_datasets/utils/filter.py create mode 100644 funasr/datasets/large_datasets/utils/low_frame_rate.py create mode 100644 funasr/datasets/large_datasets/utils/padding.py create mode 100644 funasr/datasets/large_datasets/utils/tokenize.py create mode 100644 funasr/datasets/preprocessor.py create mode 100644 funasr/fileio/__init__.py create mode 100644 funasr/fileio/datadir_writer.py create mode 100644 funasr/fileio/npy_scp.py create mode 100644 funasr/fileio/rand_gen_dataset.py create mode 100644 funasr/fileio/read_text.py create mode 100644 funasr/fileio/sound_scp.py create mode 100644 funasr/iterators/__init__.py create mode 100644 funasr/iterators/abs_iter_factory.py create mode 100644 funasr/iterators/chunk_iter_factory.py create mode 100644 funasr/iterators/multiple_iter_factory.py create mode 100644 funasr/iterators/sequence_iter_factory.py create mode 100644 funasr/layers/__init__.py create mode 100644 funasr/layers/abs_normalize.py create mode 100644 funasr/layers/complex_utils.py create mode 100644 funasr/layers/global_mvn.py create mode 100644 funasr/layers/inversible_interface.py create mode 100644 funasr/layers/label_aggregation.py create mode 100644 funasr/layers/log_mel.py create mode 100644 funasr/layers/mask_along_axis.py create mode 100644 funasr/layers/sinc_conv.py create mode 100644 funasr/layers/stft.py create mode 100644 funasr/layers/time_warp.py create mode 100644 funasr/layers/utterance_mvn.py create mode 100644 funasr/lm/__init__.py create mode 100644 funasr/lm/abs_model.py create mode 100644 funasr/lm/espnet_model.py create mode 100644 funasr/lm/seq_rnn_lm.py create mode 100644 funasr/lm/transformer_lm.py create mode 100644 funasr/losses/label_smoothing_loss.py create mode 100644 funasr/main_funcs/__init__.py create mode 100644 funasr/main_funcs/average_nbest_models.py create mode 100644 funasr/main_funcs/calculate_all_attentions.py create mode 100644 funasr/main_funcs/collect_stats.py create mode 100644 funasr/main_funcs/pack_funcs.py create mode 100644 funasr/models/ctc.py create mode 100644 funasr/models/decoder/__init__.py create mode 100644 funasr/models/decoder/abs_decoder.py create mode 100644 funasr/models/decoder/rnn_decoder.py create mode 100644 funasr/models/decoder/sanm_decoder.py create mode 100644 funasr/models/decoder/transformer_decoder.py create mode 100644 funasr/models/e2e_asr.py create mode 100644 funasr/models/e2e_asr_common.py create mode 100644 funasr/models/e2e_asr_paraformer.py create mode 100644 funasr/models/e2e_uni_asr.py create mode 100644 funasr/models/encoder/__init__.py create mode 100644 funasr/models/encoder/abs_encoder.py create mode 100644 funasr/models/encoder/conformer_encoder.py create mode 100644 funasr/models/encoder/rnn_encoder.py create mode 100644 funasr/models/encoder/sanm_encoder.py create mode 100644 funasr/models/encoder/transformer_encoder.py create mode 100644 funasr/models/frontend/__init__.py create mode 100644 funasr/models/frontend/abs_frontend.py create mode 100644 funasr/models/frontend/default.py create mode 100644 funasr/models/frontend/fused.py create mode 100644 funasr/models/frontend/s3prl.py create mode 100644 funasr/models/frontend/windowing.py create mode 100644 funasr/models/postencoder/__init__.py create mode 100644 funasr/models/postencoder/abs_postencoder.py create mode 100644 funasr/models/postencoder/hugging_face_transformers_postencoder.py create mode 100644 funasr/models/predictor/__init__.py create mode 100644 funasr/models/predictor/cif.py create mode 100644 funasr/models/preencoder/__init__.py create mode 100644 funasr/models/preencoder/abs_preencoder.py create mode 100644 funasr/models/preencoder/linear.py create mode 100644 funasr/models/preencoder/sinc.py create mode 100644 funasr/models/specaug/__init__.py create mode 100644 funasr/models/specaug/abs_specaug.py create mode 100644 funasr/models/specaug/specaug.py create mode 100644 funasr/modules/add_sos_eos.py create mode 100644 funasr/modules/attention.py create mode 100644 funasr/modules/beam_search/__init__.py create mode 100644 funasr/modules/beam_search/batch_beam_search.py create mode 100644 funasr/modules/beam_search/batch_beam_search_online_sim.py create mode 100644 funasr/modules/beam_search/beam_search.py create mode 100644 funasr/modules/dynamic_conv.py create mode 100644 funasr/modules/dynamic_conv2d.py create mode 100644 funasr/modules/e2e_asr_common.py create mode 100644 funasr/modules/embedding.py create mode 100644 funasr/modules/frontends/__init__.py create mode 100644 funasr/modules/frontends/beamformer.py create mode 100644 funasr/modules/frontends/dnn_beamformer.py create mode 100644 funasr/modules/frontends/dnn_wpe.py create mode 100644 funasr/modules/frontends/feature_transform.py create mode 100644 funasr/modules/frontends/frontend.py create mode 100644 funasr/modules/frontends/mask_estimator.py create mode 100644 funasr/modules/layer_norm.py create mode 100644 funasr/modules/lightconv.py create mode 100644 funasr/modules/lightconv2d.py create mode 100644 funasr/modules/mask.py create mode 100644 funasr/modules/multi_layer_conv.py create mode 100644 funasr/modules/nets_utils.py create mode 100644 funasr/modules/positionwise_feed_forward.py create mode 100644 funasr/modules/repeat.py create mode 100644 funasr/modules/rnn/__init__.py create mode 100644 funasr/modules/rnn/argument.py create mode 100644 funasr/modules/rnn/attentions.py create mode 100644 funasr/modules/rnn/decoders.py create mode 100644 funasr/modules/rnn/encoders.py create mode 100644 funasr/modules/scorers/__init__.py create mode 100644 funasr/modules/scorers/ctc.py create mode 100644 funasr/modules/scorers/ctc_prefix_score.py create mode 100644 funasr/modules/scorers/length_bonus.py create mode 100644 funasr/modules/scorers/scorer_interface.py create mode 100644 funasr/modules/streaming_utils/chunk_utilis.py create mode 100644 funasr/modules/streaming_utils/utils.py create mode 100644 funasr/modules/subsampling.py create mode 100644 funasr/modules/subsampling_without_posenc.py create mode 100644 funasr/optimizers/__init__.py create mode 100644 funasr/optimizers/sgd.py create mode 100644 funasr/samplers/__init__.py create mode 100644 funasr/samplers/abs_sampler.py create mode 100644 funasr/samplers/build_batch_sampler.py create mode 100644 funasr/samplers/folded_batch_sampler.py create mode 100644 funasr/samplers/length_batch_sampler.py create mode 100644 funasr/samplers/num_elements_batch_sampler.py create mode 100644 funasr/samplers/sorted_batch_sampler.py create mode 100644 funasr/samplers/unsorted_batch_sampler.py create mode 100644 funasr/schedulers/__init__.py create mode 100644 funasr/schedulers/abs_scheduler.py create mode 100644 funasr/schedulers/noam_lr.py create mode 100644 funasr/schedulers/warmup_lr.py create mode 100644 funasr/tasks/__init__.py create mode 100644 funasr/tasks/abs_task.py create mode 100644 funasr/tasks/asr.py create mode 100644 funasr/tasks/lm.py create mode 100644 funasr/text/__init__.py create mode 100644 funasr/text/abs_tokenizer.py create mode 100644 funasr/text/build_tokenizer.py create mode 100644 funasr/text/char_tokenizer.py create mode 100644 funasr/text/cleaner.py create mode 100644 funasr/text/korean_cleaner.py create mode 100644 funasr/text/phoneme_tokenizer.py create mode 100644 funasr/text/sentencepiece_tokenizer.py create mode 100644 funasr/text/token_id_converter.py create mode 100644 funasr/text/word_tokenizer.py create mode 100644 funasr/torch_utils/__init__.py create mode 100644 funasr/torch_utils/add_gradient_noise.py create mode 100644 funasr/torch_utils/device_funcs.py create mode 100644 funasr/torch_utils/forward_adaptor.py create mode 100644 funasr/torch_utils/initialize.py create mode 100644 funasr/torch_utils/load_pretrained_model.py create mode 100644 funasr/torch_utils/model_summary.py create mode 100644 funasr/torch_utils/pytorch_version.py create mode 100644 funasr/torch_utils/recursive_op.py create mode 100644 funasr/torch_utils/set_all_random_seed.py create mode 100644 funasr/train/__init__.py create mode 100644 funasr/train/abs_espnet_model.py create mode 100644 funasr/train/class_choices.py create mode 100644 funasr/train/distributed_utils.py create mode 100644 funasr/train/reporter.py create mode 100644 funasr/train/trainer.py create mode 100644 funasr/utils/__init__.py create mode 100644 funasr/utils/build_dataclass.py create mode 100644 funasr/utils/cli_utils.py create mode 100644 funasr/utils/config_argparse.py create mode 100644 funasr/utils/get_default_kwargs.py create mode 100644 funasr/utils/griffin_lim.py create mode 100644 funasr/utils/nested_dict_action.py create mode 100644 funasr/utils/sized_dict.py create mode 100644 funasr/utils/types.py create mode 100644 funasr/utils/yaml_no_alias_safe_dump.py create mode 100644 funasr/version.txt create mode 100644 image/dingding.jpg create mode 100644 image/funasr_logo.jpg create mode 100644 image/wechat.png create mode 100644 setup.py diff --git a/LICENSE b/LICENSE index 3f0f5bf7a..303aaf82d 100644 --- a/LICENSE +++ b/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 000000000..795a1308e --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +
+ +# FunASR: A Fundamental End-to-End Speech Recognition Toolkit + +FunASR hopes to build a bridge between academic research and industrial applications on speech recognition. By supporting the training & finetuning of the industrial-grade speech recognition model released on [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition), researchers and developers can conduct research and production of speech recognition models more conveniently, and promote the development of speech recognition ecology. ASR for Fun！ + +## Installation(Training and Developing) + +- Clone the repo: +``` sh +git clone https://github.com/alibaba/FunASR.git +``` + +- Install Conda: +``` sh +wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh +sh Miniconda3-latest-Linux-x86_64.sh +conda create -n funasr python=3.7 +conda activate funasr +``` + +- Install Pytorch (version >= 1.7.0): + +| cuda | | +|:-----:| --- | +| 9.2 | conda install pytorch==1.7.0 torchvision==0.8.0 torchaudio==0.7.0 cudatoolkit=9.2 -c pytorch | +| 10.2 | conda install pytorch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 cudatoolkit=10.2 -c pytorch | +| 11.1 | conda install pytorch==1.8.0 torchvision==0.9.0 torchaudio==0.8.0 cudatoolkit=11.1 -c pytorch | + +For more versions, please see https://pytorch.org/get-started/locally/ + +- Install ModelScope: +``` sh +pip install "modelscope[audio]" -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html +``` + +- Install other packages: + +``` sh +pip install --editable ./ +``` + +## Contact + +If you have any questions about FunASR, please contact us by + +- email: [funasr@list.alibaba-inc.com](funasr@list.alibaba-inc.com) + +- Dingding group: +
+ + +## Acknowledge + +1. We borrowed a lot of code from [Kaldi](http://kaldi-asr.org/) for data preparation. +2. We borrowed a lot of code from [ESPnet](https://github.com/espnet/espnet). FunASR follows up the training and finetuning pipelines of ESPnet. +3. We referred [Wenet](https://github.com/wenet-e2e/wenet) for building dataloader for large scale data training. + +## License +This project is licensed under the [The MIT License](https://opensource.org/licenses/MIT). FunASR also contains various third-party components and some code modified from other repos under other open source licenses. + +## Citations + +``` bibtex +@inproceedings{gao2020universal, + title={Universal ASR: Unifying Streaming and Non-Streaming ASR Using a Single Encoder-Decoder Model}, + author={Gao, Zhifu and Zhang, Shiliang and Lei, Ming and McLoughlin, Ian}, + booktitle={arXiv preprint arXiv:2010.14099}, + year={2010} +} + +@inproceedings{gao2022paraformer, + title={Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition}, + author={Gao, Zhifu and Zhang, Shiliang and McLoughlin, Ian and Yan, Zhijie}, + booktitle={INTERSPEECH}, + year={2022} +} +``` diff --git a/egs/aishell/conformer/README.md b/egs/aishell/conformer/README.md new file mode 100644 index 000000000..a67b183ed --- /dev/null +++ b/egs/aishell/conformer/README.md @@ -0,0 +1,17 @@ + +# Conformer Result + +## Training Config +- Feature info: using 80 dims fbank, global cmvn, speed perturb(0.9, 1.0, 1.1), specaugment +- Train info: lr 5e-4, batch_size 25000, 2 gpu(Tesla V100), acc_grad 1, 50 epochs +- Train config: conf/train_asr_transformer.yaml +- LM config: LM was not used +- Model size: 46M + +## Results (CER) +- Decode config: conf/decode_asr_transformer.yaml (ctc weight:0.5) + +| testset | CER(%) | +|:-----------:|:-------:| +| dev | 4.42 | +| test | 4.87 | \ No newline at end of file diff --git a/egs/aishell/conformer/conf/decode_asr_transformer.yaml b/egs/aishell/conformer/conf/decode_asr_transformer.yaml new file mode 100644 index 000000000..a147fa79d --- /dev/null +++ b/egs/aishell/conformer/conf/decode_asr_transformer.yaml @@ -0,0 +1,6 @@ +beam_size: 10 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.5 +lm_weight: 0.7 diff --git a/egs/aishell/conformer/conf/train_asr_conformer.yaml b/egs/aishell/conformer/conf/train_asr_conformer.yaml new file mode 100644 index 000000000..ddf217ec0 --- /dev/null +++ b/egs/aishell/conformer/conf/train_asr_conformer.yaml @@ -0,0 +1,80 @@ +# network architecture +# encoder related +encoder: conformer +encoder_conf: + output_size: 256 # dimension of attention + attention_heads: 4 + linear_units: 2048 # the number of units of position-wise feed forward + num_blocks: 12 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + macaron_style: true + use_cnn_module: true + cnn_module_kernel: 15 + +# decoder related +decoder: transformer +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +# hybrid CTC/attention +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: false + +# minibatch related +batch_type: length +batch_bins: 25000 +num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 50 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug +specaug_conf: + apply_time_warp: true + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + num_freq_mask: 2 + apply_time_mask: true + time_mask_width_range: + - 0 + - 40 + num_time_mask: 2 + +log_interval: 50 +normalize: None diff --git a/egs/aishell/conformer/local/aishell_data_prep.sh b/egs/aishell/conformer/local/aishell_data_prep.sh new file mode 100755 index 000000000..83f489b3c --- /dev/null +++ b/egs/aishell/conformer/local/aishell_data_prep.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2017 Xingyu Na +# Apache 2.0 + +#. ./path.sh || exit 1; + +if [ $# != 3 ]; then + echo "Usage: $0 " + echo " $0 /export/a05/xna/data/data_aishell/wav /export/a05/xna/data/data_aishell/transcript data" + exit 1; +fi + +aishell_audio_dir=$1 +aishell_text=$2/aishell_transcript_v0.8.txt +output_dir=$3 + +train_dir=$output_dir/data/local/train +dev_dir=$output_dir/data/local/dev +test_dir=$output_dir/data/local/test +tmp_dir=$output_dir/data/local/tmp + +mkdir -p $train_dir +mkdir -p $dev_dir +mkdir -p $test_dir +mkdir -p $tmp_dir + +# data directory check +if [ ! -d $aishell_audio_dir ] || [ ! -f $aishell_text ]; then + echo "Error: $0 requires two directory arguments" + exit 1; +fi + +# find wav audio file for train, dev and test resp. +find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist +n=`cat $tmp_dir/wav.flist | wc -l` +[ $n -ne 141925 ] && \ + echo Warning: expected 141925 data data files, found $n + +grep -i "wav/train" $tmp_dir/wav.flist > $train_dir/wav.flist || exit 1; +grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1; +grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1; + +rm -r $tmp_dir + +# Transcriptions preparation +for dir in $train_dir $dev_dir $test_dir; do + echo Preparing $dir transcriptions + sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list + paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all + utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt + awk '{print $1}' $dir/transcripts.txt > $dir/utt.list + utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp + sort -u $dir/transcripts.txt > $dir/text +done + +mkdir -p $output_dir/data/train $output_dir/data/dev $output_dir/data/test + +for f in wav.scp text; do + cp $train_dir/$f $output_dir/data/train/$f || exit 1; + cp $dev_dir/$f $output_dir/data/dev/$f || exit 1; + cp $test_dir/$f $output_dir/data/test/$f || exit 1; +done + +echo "$0: AISHELL data preparation succeeded" +exit 0; diff --git a/egs/aishell/conformer/local/prepare_data.sh b/egs/aishell/conformer/local/prepare_data.sh new file mode 100755 index 000000000..77791f9c1 --- /dev/null +++ b/egs/aishell/conformer/local/prepare_data.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG) +# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU) +# Apache 2.0 + +# transform raw AISHELL-2 data to kaldi format + +. ./path.sh || exit 1; + +tmp= +dir= + +if [ $# != 3 ]; then + echo "Usage: $0 " + echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train" + exit 1; +fi + +corpus=$1 +tmp=$2 +dir=$3 + +echo "prepare_data.sh: Preparing data in $corpus" + +mkdir -p $tmp +mkdir -p $dir + +# corpus check +if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then + echo "Error: $0 requires wav.scp and trans.txt under $corpus directory." + exit 1; +fi + +# validate utt-key list, IC0803W0380 is a bad utterance +awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list +awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list +utils/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list + +# wav.scp +awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp +utils/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp + +# text +utils/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text + +# copy prepared resources from tmp_dir to target dir +mkdir -p $dir +for f in wav.scp text; do + cp $tmp/$f $dir/$f || exit 1; +done + +echo "local/prepare_data.sh succeeded" +exit 0; diff --git a/egs/aishell/conformer/path.sh b/egs/aishell/conformer/path.sh new file mode 100755 index 000000000..7972642d0 --- /dev/null +++ b/egs/aishell/conformer/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs/aishell/conformer/run.sh b/egs/aishell/conformer/run.sh new file mode 100755 index 000000000..16ebc6759 --- /dev/null +++ b/egs/aishell/conformer/run.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; + +# machines configuration +CUDA_VISIBLE_DEVICES="0,1" +gpu_num=2 +count=1 +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding +# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob +njob=8 +train_cmd=utils/run.pl + +# general configuration +feats_dir=".." #feature output dictionary, for large data +exp_dir="." +lang=zh +dumpdir=dump/fbank +feats_type=fbank +token_type=char +scp=feats.scp +type=kaldi_ark +stage=0 +stop_stage=4 + +# feature configuration +feats_dim=80 +sample_frequency=16000 +nj=32 +speed_perturb="0.9,1.0,1.1" + +# data +data_aishell= + +# exp tag +tag="" + +. utils/parse_options.sh || exit 1; + +# Set bash to 'debug' mode, it will exit on : +# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands', +set -e +set -u +set -o pipefail + +train_set=train +valid_set=dev +test_sets="dev test" + +asr_config=conf/train_asr_conformer.yaml +model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}" + +inference_config=conf/decode_asr_transformer.yaml +inference_asr_model=valid.acc.ave_10best.pth + +# you can set gpu num for decoding here +gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] +else + inference_nj=$njob +fi + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + echo "stage 0: Data preparation" + # Data preparation + local/aishell_data_prep.sh ${data_aishell}/data_aishell/wav ${data_aishell}/data_aishell/transcript ${feats_dir} + for x in train dev test; do + cp ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org + paste -d " " <(cut -f 1 -d" " ${feats_dir}/data/${x}/text.org) <(cut -f 2- -d" " ${feats_dir}/data/${x}/text.org | tr -d " ") \ + > ${feats_dir}/data/${x}/text + utils/text2token.py -n 1 -s 1 ${feats_dir}/data/${x}/text > ${feats_dir}/data/${x}/text.org + mv ${feats_dir}/data/${x}/text.org ${feats_dir}/data/${x}/text + done +fi + +feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir} +feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir} +feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir} +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + echo "stage 1: Feature Generation" + # compute fbank features + fbankdir=${feats_dir}/fbank + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \ + ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train + utils/fix_data_feat.sh ${fbankdir}/train + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/dev ${exp_dir}/exp/make_fbank/dev ${fbankdir}/dev + utils/fix_data_feat.sh ${fbankdir}/dev + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/test ${exp_dir}/exp/make_fbank/test ${fbankdir}/test + utils/fix_data_feat.sh ${fbankdir}/test + + # compute global cmvn + utils/compute_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/train ${exp_dir}/exp/make_fbank/train + + # apply cmvn + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/train ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/train ${feat_train_dir} + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/dev ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir} + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/test ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/test ${feat_test_dir} + + cp ${fbankdir}/train/text ${fbankdir}/train/speech_shape ${fbankdir}/train/text_shape ${feat_train_dir} + cp ${fbankdir}/dev/text ${fbankdir}/dev/speech_shape ${fbankdir}/dev/text_shape ${feat_dev_dir} + cp ${fbankdir}/test/text ${fbankdir}/test/speech_shape ${fbankdir}/test/text_shape ${feat_test_dir} + + utils/fix_data_feat.sh ${feat_train_dir} + utils/fix_data_feat.sh ${feat_dev_dir} + utils/fix_data_feat.sh ${feat_test_dir} +fi + +token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt +echo "dictionary: ${token_list}" +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + echo "stage 2: Dictionary Preparation" + mkdir -p ${feats_dir}/data/${lang}_token_list/char/ + + echo "make a dictionary" + echo "" > ${token_list} + echo "" >> ${token_list} + echo "" >> ${token_list} + utils/text2token.py -s 1 -n 1 --space "" ${feats_dir}/data/train/text | cut -f 2- -d" " | tr " " "\n" \ + | sort | uniq | grep -a -v -e '^\s*$' | awk '{print $0}' >> ${token_list} + num_token=$(cat ${token_list} | wc -l) + echo "" >> ${token_list} + vocab_size=$(cat ${token_list} | wc -l) + awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char + awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev + cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/train + cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev +fi + +# Training Stage +world_size=$gpu_num # run on one machine +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + mkdir -p ${exp_dir}/exp/${model_dir} + mkdir -p ${exp_dir}/exp/${model_dir}/log + INIT_FILE=$exp_dir/ddp_init + if [ -f $INIT_FILE ];then + rm -f $INIT_FILE + fi + init_method=file://$(readlink -f $INIT_FILE) + echo "$0: init method is $init_method" + for ((i = 0; i < $gpu_num; ++i)); do + { + rank=$i + local_rank=$i + gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1]) + asr_train.py \ + --gpu_id $gpu_id \ + --use_preprocessor true \ + --token_type char \ + --token_list $token_list \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \ + --resume true \ + --output_dir ${exp_dir}/exp/${model_dir} \ + --config $asr_config \ + --input_size $feats_dim \ + --ngpu $gpu_num \ + --num_worker_count $count \ + --multiprocessing_distributed true \ + --dist_init_method $init_method \ + --dist_world_size $world_size \ + --dist_rank $rank \ + --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1 + } & + done + wait +fi + +# Testing Stage +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + utils/easy_asr_infer.sh \ + --lang zh \ + --datadir ${feats_dir} \ + --feats_type ${feats_type} \ + --feats_dim ${feats_dim} \ + --token_type ${token_type} \ + --gpu_inference ${gpu_inference} \ + --inference_config "${inference_config}" \ + --test_sets "${test_sets}" \ + --token_list $token_list \ + --asr_exp ${exp_dir}/${model_dir} \ + --stage 12 \ + --stop_stage 12 \ + --scp $scp \ + --text text \ + --inference_nj $inference_nj \ + --njob $njob \ + --inference_asr_model $inference_asr_model \ + --gpuid_list $gpuid_list \ + --mode asr +fi + diff --git a/egs/aishell/conformer/utils b/egs/aishell/conformer/utils new file mode 120000 index 000000000..40e14f57f --- /dev/null +++ b/egs/aishell/conformer/utils @@ -0,0 +1 @@ +../tranformer/utils \ No newline at end of file diff --git a/egs/aishell/paraformer/README.md b/egs/aishell/paraformer/README.md new file mode 100644 index 000000000..c0385db86 --- /dev/null +++ b/egs/aishell/paraformer/README.md @@ -0,0 +1,24 @@ +# Paraformer +pretrained model in [ModelScope](https://www.modelscope.cn/home)：[speech_paraformer_asr_nat-aishell1-pytorch](https://www.modelscope.cn/models/damo/speech_paraformer_asr_nat-aishell1-pytorch/summary) + +## Training Config +- Feature info: using 80 dims fbank, global cmvn, speed perturb(0.9, 1.0, 1.1), specaugment +- Train info: lr 5e-4, batch_size 25000, 2 gpu(Tesla V100), acc_grad 1, 50 epochs +- Train config: conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml +- LM config: LM was not used + +## Results (CER) + +- Decode config: conf/decode_asr_transformer_noctc_1best.yaml (ctc weight:0.0) + +| testset | CER(%) | +|:-----------:|:-------:| +| dev | 4.66 | +| test | 5.11 | + +- Decode config: conf/decode_asr_transformer.yaml (ctc weight:0.5) + +| testset | CER(%) | +|:-----------:|:-------:| +| dev | 4.52 | +| test | 4.94 | \ No newline at end of file diff --git a/egs/aishell/paraformer/conf/decode_asr_transformer.yaml b/egs/aishell/paraformer/conf/decode_asr_transformer.yaml new file mode 100644 index 000000000..a147fa79d --- /dev/null +++ b/egs/aishell/paraformer/conf/decode_asr_transformer.yaml @@ -0,0 +1,6 @@ +beam_size: 10 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.5 +lm_weight: 0.7 diff --git a/egs/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml b/egs/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml new file mode 100644 index 000000000..5436b12e4 --- /dev/null +++ b/egs/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml @@ -0,0 +1,6 @@ +beam_size: 1 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.0 +lm_weight: 0.15 diff --git a/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml b/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml new file mode 100644 index 000000000..b5ab916fc --- /dev/null +++ b/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml @@ -0,0 +1,91 @@ +# network architecture +# encoder related +encoder: conformer +encoder_conf: + output_size: 256 # dimension of attention + attention_heads: 4 + linear_units: 2048 # the number of units of position-wise feed forward + num_blocks: 12 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + macaron_style: true + use_cnn_module: true + cnn_module_kernel: 15 + +# decoder related +decoder: paraformer_decoder_san +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +model: paraformer +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 + length_normalized_loss: false + predictor_weight: 1.0 + sampling_ratio: 0.4 + +# minibatch related +batch_type: length +batch_bins: 25000 +num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 50 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug +specaug_conf: + apply_time_warp: true + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + num_freq_mask: 2 + apply_time_mask: true + time_mask_width_range: + - 0 + - 40 + num_time_mask: 2 + +predictor: cif_predictor +predictor_conf: + idim: 256 + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + + +log_interval: 50 +normalize: None \ No newline at end of file diff --git a/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_20e_6d_1280_320.yaml b/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_20e_6d_1280_320.yaml new file mode 100644 index 000000000..2b5e2d1a3 --- /dev/null +++ b/egs/aishell/paraformer/conf/train_asr_paraformer_conformer_20e_6d_1280_320.yaml @@ -0,0 +1,92 @@ +# network architecture +# encoder related +encoder: conformer +encoder_conf: + output_size: 320 # dimension of attention + attention_heads: 4 + linear_units: 1280 # the number of units of position-wise feed forward + num_blocks: 20 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + macaron_style: true + use_cnn_module: true + cnn_module_kernel: 15 + +# decoder related +decoder: paraformer_decoder_san +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +# hybrid CTC/attention +model: paraformer +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: false + predictor_weight: 1.0 + sampling_ratio: 0.4 + +# minibatch related +batch_type: length +batch_bins: 25000 +num_workers: 16 + +# optimization related +accum_grad: 4 +grad_clip: 5 +max_epoch: 50 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug +specaug_conf: + apply_time_warp: true + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + num_freq_mask: 2 + apply_time_mask: true + time_mask_width_range: + - 0 + - 40 + num_time_mask: 2 + +predictor: cif_predictor +predictor_conf: + idim: 256 + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + + +log_interval: 50 +normalize: None \ No newline at end of file diff --git a/egs/aishell/paraformer/conf/train_asr_paraformer_sanm_tf_40e_12d_1280_320_lfr6.yaml b/egs/aishell/paraformer/conf/train_asr_paraformer_sanm_tf_40e_12d_1280_320_lfr6.yaml new file mode 100644 index 000000000..864350755 --- /dev/null +++ b/egs/aishell/paraformer/conf/train_asr_paraformer_sanm_tf_40e_12d_1280_320_lfr6.yaml @@ -0,0 +1,114 @@ +# network architecture +# encoder related +encoder: sanm +encoder_conf: + output_size: 320 # dimension of attention + attention_heads: 4 + linear_units: 1280 # the number of units of position-wise feed forward + num_blocks: 40 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.1 + input_layer: pe # encoder architecture type + pos_enc_class: SinusoidalPositionEncoder + normalize_before: true + kernel_size: 11 + sanm_shfit: 0 + selfattention_layer_type: sanm + +# decoder related +decoder: paraformer_decoder_sanm +decoder_conf: + attention_heads: 4 + linear_units: 1280 + num_blocks: 12 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.1 + src_attention_dropout_rate: 0.1 + att_layer_num: 6 + kernel_size: 11 + sanm_shfit: 0 + + +predictor: cif_predictor +predictor_conf: + idim: 320 + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + +# hybrid CTC/attention +model: paraformer +model_conf: + ctc_weight: 0.0 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: true + predictor_weight: 1.0 + predictor_bias: 0 + sampling_ratio: 0.75 + + +# minibatch related +# dataset_type: small +batch_type: length +batch_bins: 6000 +num_workers: 16 +# dataset_type: large +dataset_conf: + filter_conf: + min_length: 10 + max_length: 250 + min_token_length: 1 + max_token_length: 200 + shuffle: True + shuffle_conf: + shuffle_size: 10240 + sort_size: 500 + batch_conf: + batch_type: token + batch_size: 6000 + num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 20 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 5 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 15000 + +specaug: specaug_lfr +specaug_conf: + apply_time_warp: false + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + lfr_rate: 6 + num_freq_mask: 1 + apply_time_mask: true + time_mask_width_range: + - 0 + - 12 + num_time_mask: 1 + +unused_parameters: true +log_interval: 50 +normalize: None +split_with_space: true \ No newline at end of file diff --git a/egs/aishell/paraformer/conf/train_asr_paraformer_sanm_tf_50e_16d_2048_512_lfr6.yaml b/egs/aishell/paraformer/conf/train_asr_paraformer_sanm_tf_50e_16d_2048_512_lfr6.yaml new file mode 100644 index 000000000..67983b379 --- /dev/null +++ b/egs/aishell/paraformer/conf/train_asr_paraformer_sanm_tf_50e_16d_2048_512_lfr6.yaml @@ -0,0 +1,114 @@ +# network architecture +# encoder related +encoder: sanm +encoder_conf: + output_size: 512 # dimension of attention + attention_heads: 4 + linear_units: 2048 # the number of units of position-wise feed forward + num_blocks: 50 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.1 + input_layer: pe # encoder architecture type + pos_enc_class: SinusoidalPositionEncoder + normalize_before: true + kernel_size: 11 + sanm_shfit: 0 + selfattention_layer_type: sanm + +# decoder related +decoder: paraformer_decoder_sanm +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 16 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.1 + src_attention_dropout_rate: 0.1 + att_layer_num: 16 + kernel_size: 11 + sanm_shfit: 0 + + +predictor: cif_predictor_v2 +predictor_conf: + idim: 512 + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + +# hybrid CTC/attention +model: paraformer +model_conf: + ctc_weight: 0.0 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: true + predictor_weight: 1.0 + predictor_bias: 1 + sampling_ratio: 0.75 + + +# minibatch related +# dataset_type: small +batch_type: length +batch_bins: 10000 +num_workers: 16 +# dataset_type: large +dataset_conf: + filter_conf: + min_length: 10 + max_length: 250 + min_token_length: 1 + max_token_length: 200 + shuffle: true + shuffle_conf: + shuffle_size: 10240 + sort_size: 500 + batch_conf: + batch_type: 'token' + batch_size: 6000 + num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 20 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 5 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug_lfr +specaug_conf: + apply_time_warp: false + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + lfr_rate: 6 + num_freq_mask: 1 + apply_time_mask: true + time_mask_width_range: + - 0 + - 12 + num_time_mask: 1 + +unused_parameters: true +log_interval: 50 +normalize: None +split_with_space: true diff --git a/egs/aishell/paraformer/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml b/egs/aishell/paraformer/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml new file mode 100644 index 000000000..f369b3d19 --- /dev/null +++ b/egs/aishell/paraformer/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml @@ -0,0 +1,99 @@ +# network architecture +# encoder related +encoder: conformer +encoder_conf: + output_size: 256 # dimension of attention + attention_heads: 4 + linear_units: 2048 # the number of units of position-wise feed forward + num_blocks: 12 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + macaron_style: true + use_cnn_module: true + cnn_module_kernel: 15 + +# decoder related +decoder: paraformer_decoder_san +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +# hybrid CTC/attention +model: paraformer_bert +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: false + predictor_weight: 1.0 + sampling_ratio: 0.4 + embeds_id: 3 + embed_dims: 768 + embeds_loss_weight: 2.0 + + + +# minibatch related +#batch_type: length +#batch_bins: 40000 +batch_type: numel +batch_bins: 2000000 +num_workers: 16 + +# optimization related +accum_grad: 4 +grad_clip: 5 +max_epoch: 50 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug +specaug_conf: + apply_time_warp: true + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + num_freq_mask: 2 + apply_time_mask: true + time_mask_width_range: + - 0 + - 40 + num_time_mask: 2 + +predictor: cif_predictor +predictor_conf: + idim: 256 + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + + +log_interval: 50 +normalize: None \ No newline at end of file diff --git a/egs/aishell/paraformer/local/aishell_data_prep.sh b/egs/aishell/paraformer/local/aishell_data_prep.sh new file mode 100755 index 000000000..83f489b3c --- /dev/null +++ b/egs/aishell/paraformer/local/aishell_data_prep.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2017 Xingyu Na +# Apache 2.0 + +#. ./path.sh || exit 1; + +if [ $# != 3 ]; then + echo "Usage: $0 " + echo " $0 /export/a05/xna/data/data_aishell/wav /export/a05/xna/data/data_aishell/transcript data" + exit 1; +fi + +aishell_audio_dir=$1 +aishell_text=$2/aishell_transcript_v0.8.txt +output_dir=$3 + +train_dir=$output_dir/data/local/train +dev_dir=$output_dir/data/local/dev +test_dir=$output_dir/data/local/test +tmp_dir=$output_dir/data/local/tmp + +mkdir -p $train_dir +mkdir -p $dev_dir +mkdir -p $test_dir +mkdir -p $tmp_dir + +# data directory check +if [ ! -d $aishell_audio_dir ] || [ ! -f $aishell_text ]; then + echo "Error: $0 requires two directory arguments" + exit 1; +fi + +# find wav audio file for train, dev and test resp. +find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist +n=`cat $tmp_dir/wav.flist | wc -l` +[ $n -ne 141925 ] && \ + echo Warning: expected 141925 data data files, found $n + +grep -i "wav/train" $tmp_dir/wav.flist > $train_dir/wav.flist || exit 1; +grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1; +grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1; + +rm -r $tmp_dir + +# Transcriptions preparation +for dir in $train_dir $dev_dir $test_dir; do + echo Preparing $dir transcriptions + sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list + paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all + utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt + awk '{print $1}' $dir/transcripts.txt > $dir/utt.list + utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp + sort -u $dir/transcripts.txt > $dir/text +done + +mkdir -p $output_dir/data/train $output_dir/data/dev $output_dir/data/test + +for f in wav.scp text; do + cp $train_dir/$f $output_dir/data/train/$f || exit 1; + cp $dev_dir/$f $output_dir/data/dev/$f || exit 1; + cp $test_dir/$f $output_dir/data/test/$f || exit 1; +done + +echo "$0: AISHELL data preparation succeeded" +exit 0; diff --git a/egs/aishell/paraformer/local/prepare_data.sh b/egs/aishell/paraformer/local/prepare_data.sh new file mode 100755 index 000000000..77791f9c1 --- /dev/null +++ b/egs/aishell/paraformer/local/prepare_data.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG) +# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU) +# Apache 2.0 + +# transform raw AISHELL-2 data to kaldi format + +. ./path.sh || exit 1; + +tmp= +dir= + +if [ $# != 3 ]; then + echo "Usage: $0 " + echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train" + exit 1; +fi + +corpus=$1 +tmp=$2 +dir=$3 + +echo "prepare_data.sh: Preparing data in $corpus" + +mkdir -p $tmp +mkdir -p $dir + +# corpus check +if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then + echo "Error: $0 requires wav.scp and trans.txt under $corpus directory." + exit 1; +fi + +# validate utt-key list, IC0803W0380 is a bad utterance +awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list +awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list +utils/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list + +# wav.scp +awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp +utils/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp + +# text +utils/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text + +# copy prepared resources from tmp_dir to target dir +mkdir -p $dir +for f in wav.scp text; do + cp $tmp/$f $dir/$f || exit 1; +done + +echo "local/prepare_data.sh succeeded" +exit 0; diff --git a/egs/aishell/paraformer/path.sh b/egs/aishell/paraformer/path.sh new file mode 100755 index 000000000..7972642d0 --- /dev/null +++ b/egs/aishell/paraformer/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs/aishell/paraformer/run.sh b/egs/aishell/paraformer/run.sh new file mode 100755 index 000000000..bebb646e0 --- /dev/null +++ b/egs/aishell/paraformer/run.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; + +# machines configuration +CUDA_VISIBLE_DEVICES="0,1" +gpu_num=2 +count=1 +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding +# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob +njob=8 +train_cmd=utils/run.pl + +# general configuration +feats_dir=".." #feature output dictionary, for large data +exp_dir="." +lang=zh +dumpdir=dump/fbank +feats_type=fbank +token_type=char +scp=feats.scp +type=kaldi_ark +stage=0 +stop_stage=4 + +# feature configuration +feats_dim=80 +sample_frequency=16000 +nj=32 +speed_perturb="0.9,1.0,1.1" + +# data +data_aishell= + +# exp tag +tag="" + +. utils/parse_options.sh || exit 1; + +# Set bash to 'debug' mode, it will exit on : +# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands', +set -e +set -u +set -o pipefail + +train_set=train +valid_set=dev +test_sets="dev test" + +asr_config=conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml +model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}" + +inference_config=conf/decode_asr_transformer_noctc_1best.yaml +inference_asr_model=valid.acc.ave_10best.pth + +# you can set gpu num for decoding here +gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] +else + inference_nj=$njob +fi + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + echo "stage 0: Data preparation" + # Data preparation + local/aishell_data_prep.sh ${data_aishell}/data_aishell/wav ${data_aishell}/data_aishell/transcript ${feats_dir} + for x in train dev test; do + cp ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org + paste -d " " <(cut -f 1 -d" " ${feats_dir}/data/${x}/text.org) <(cut -f 2- -d" " ${feats_dir}/data/${x}/text.org | tr -d " ") \ + > ${feats_dir}/data/${x}/text + utils/text2token.py -n 1 -s 1 ${feats_dir}/data/${x}/text > ${feats_dir}/data/${x}/text.org + mv ${feats_dir}/data/${x}/text.org ${feats_dir}/data/${x}/text + done +fi + +feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir} +feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir} +feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir} +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + echo "stage 1: Feature Generation" + # compute fbank features + fbankdir=${feats_dir}/fbank + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \ + ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train + utils/fix_data_feat.sh ${fbankdir}/train + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/dev ${exp_dir}/exp/make_fbank/dev ${fbankdir}/dev + utils/fix_data_feat.sh ${fbankdir}/dev + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/test ${exp_dir}/exp/make_fbank/test ${fbankdir}/test + utils/fix_data_feat.sh ${fbankdir}/test + + # compute global cmvn + utils/compute_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/train ${exp_dir}/exp/make_fbank/train + + # apply cmvn + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/train ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/train ${feat_train_dir} + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/dev ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir} + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/test ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/test ${feat_test_dir} + + cp ${fbankdir}/train/text ${fbankdir}/train/speech_shape ${fbankdir}/train/text_shape ${feat_train_dir} + cp ${fbankdir}/dev/text ${fbankdir}/dev/speech_shape ${fbankdir}/dev/text_shape ${feat_dev_dir} + cp ${fbankdir}/test/text ${fbankdir}/test/speech_shape ${fbankdir}/test/text_shape ${feat_test_dir} + + utils/fix_data_feat.sh ${feat_train_dir} + utils/fix_data_feat.sh ${feat_dev_dir} + utils/fix_data_feat.sh ${feat_test_dir} +fi + +token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt +echo "dictionary: ${token_list}" +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + echo "stage 2: Dictionary Preparation" + mkdir -p ${feats_dir}/data/${lang}_token_list/char/ + + echo "make a dictionary" + echo "" > ${token_list} + echo "" >> ${token_list} + echo "" >> ${token_list} + utils/text2token.py -s 1 -n 1 --space "" ${feats_dir}/data/train/text | cut -f 2- -d" " | tr " " "\n" \ + | sort | uniq | grep -a -v -e '^\s*$' | awk '{print $0}' >> ${token_list} + num_token=$(cat ${token_list} | wc -l) + echo "" >> ${token_list} + vocab_size=$(cat ${token_list} | wc -l) + awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char + awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev + cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/train + cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev +fi + +# Training Stage +world_size=$gpu_num # run on one machine +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + mkdir -p ${exp_dir}/exp/${model_dir} + mkdir -p ${exp_dir}/exp/log + INIT_FILE=$exp_dir/ddp_init + if [ -f $INIT_FILE ];then + rm -f $INIT_FILE + fi + init_method=file://$(readlink -f $INIT_FILE) + echo "$0: init method is $init_method" + for ((i = 0; i < $gpu_num; ++i)); do + { + rank=$i + local_rank=$i + gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1]) + asr_train_paraformer.py \ + --gpu_id $gpu_id \ + --use_preprocessor true \ + --token_type char \ + --token_list $token_list \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \ + --resume true \ + --output_dir ${exp_dir}/exp/${model_dir} \ + --config $asr_config \ + --input_size $feats_dim \ + --ngpu $gpu_num \ + --num_worker_count $count \ + --multiprocessing_distributed true \ + --dist_init_method $init_method \ + --dist_world_size $world_size \ + --dist_rank $rank \ + --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1 + } & + done + wait +fi + +# Testing Stage +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + utils/easy_asr_infer.sh \ + --lang zh \ + --datadir ${feats_dir} \ + --feats_type ${feats_type} \ + --feats_dim ${feats_dim} \ + --token_type ${token_type} \ + --gpu_inference ${gpu_inference} \ + --inference_config "${inference_config}" \ + --test_sets "${test_sets}" \ + --token_list $token_list \ + --asr_exp ${exp_dir}/${model_dir} \ + --stage 12 \ + --stop_stage 12 \ + --scp $scp \ + --text text \ + --inference_nj $inference_nj \ + --njob $njob \ + --inference_asr_model $inference_asr_model \ + --gpuid_list $gpuid_list \ + --mode paraformer +fi + diff --git a/egs/aishell/paraformer/utils b/egs/aishell/paraformer/utils new file mode 120000 index 000000000..40e14f57f --- /dev/null +++ b/egs/aishell/paraformer/utils @@ -0,0 +1 @@ +../tranformer/utils \ No newline at end of file diff --git a/egs/aishell/paraformer2/README.md b/egs/aishell/paraformer2/README.md new file mode 100644 index 000000000..ffce9493e --- /dev/null +++ b/egs/aishell/paraformer2/README.md @@ -0,0 +1,18 @@ +# ParaformerBert + specaug + speed perturbation + specaugmentation +## Environments +- date: `Mon Nov 21 13:25:30 CST 2022` +- python version: `3.7.12` +- FunASR version: `0.1.0` +- pytorch version: `pytorch 1.7.0` + +## Config files +- train config: conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml +- model size: 46M +- lm config: LM was not used +- decode config: conf/decode_asr_transformer_noctc_1best.yaml (CTC was not used) + +## Results (CER) +| testset | CER(%) | +|:-----------:|:-------:| +| dev | 4.30 | +| test | 4.80 | \ No newline at end of file diff --git a/egs/aishell/paraformer2/conf/decode_asr_transformer.yaml b/egs/aishell/paraformer2/conf/decode_asr_transformer.yaml new file mode 100644 index 000000000..a147fa79d --- /dev/null +++ b/egs/aishell/paraformer2/conf/decode_asr_transformer.yaml @@ -0,0 +1,6 @@ +beam_size: 10 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.5 +lm_weight: 0.7 diff --git a/egs/aishell/paraformer2/conf/decode_asr_transformer_noctc_1best.yaml b/egs/aishell/paraformer2/conf/decode_asr_transformer_noctc_1best.yaml new file mode 100644 index 000000000..5436b12e4 --- /dev/null +++ b/egs/aishell/paraformer2/conf/decode_asr_transformer_noctc_1best.yaml @@ -0,0 +1,6 @@ +beam_size: 1 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.0 +lm_weight: 0.15 diff --git a/egs/aishell/paraformer2/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml b/egs/aishell/paraformer2/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml new file mode 100644 index 000000000..779c7a913 --- /dev/null +++ b/egs/aishell/paraformer2/conf/train_asr_paraformer_conformer_12e_6d_2048_256.yaml @@ -0,0 +1,92 @@ +# network architecture +# encoder related +encoder: conformer +encoder_conf: + output_size: 256 # dimension of attention + attention_heads: 4 + linear_units: 2048 # the number of units of position-wise feed forward + num_blocks: 12 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + macaron_style: true + use_cnn_module: true + cnn_module_kernel: 15 + +# decoder related +decoder: paraformer_decoder_san +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +# hybrid CTC/attention +model: paraformer +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: false + predictor_weight: 1.0 + sampling_ratio: 0.4 + +# minibatch related +batch_type: length +batch_bins: 25000 +num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 50 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug +specaug_conf: + apply_time_warp: true + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + num_freq_mask: 2 + apply_time_mask: true + time_mask_width_range: + - 0 + - 40 + num_time_mask: 2 + +predictor: cif_predictor +predictor_conf: + idim: 256 + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + + +log_interval: 50 +normalize: None \ No newline at end of file diff --git a/egs/aishell/paraformer2/conf/train_asr_paraformer_conformer_20e_6d_1280_320.yaml b/egs/aishell/paraformer2/conf/train_asr_paraformer_conformer_20e_6d_1280_320.yaml new file mode 100644 index 000000000..29b9ca6d3 --- /dev/null +++ b/egs/aishell/paraformer2/conf/train_asr_paraformer_conformer_20e_6d_1280_320.yaml @@ -0,0 +1,94 @@ +# network architecture +# encoder related +encoder: conformer +encoder_conf: + output_size: 320 # dimension of attention + attention_heads: 4 + linear_units: 1280 # the number of units of position-wise feed forward + num_blocks: 20 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + macaron_style: true + use_cnn_module: true + cnn_module_kernel: 15 + +# decoder related +decoder: paraformer_decoder_san +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +# hybrid CTC/attention +model: paraformer +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: false + predictor_weight: 1.0 + sampling_ratio: 0.4 + +# minibatch related +#batch_type: length +#batch_bins: 40000 +batch_type: numel +batch_bins: 2000000 +num_workers: 16 + +# optimization related +accum_grad: 4 +grad_clip: 5 +max_epoch: 50 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug +specaug_conf: + apply_time_warp: true + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + num_freq_mask: 2 + apply_time_mask: true + time_mask_width_range: + - 0 + - 40 + num_time_mask: 2 + +predictor: cif_predictor +predictor_conf: + idim: 256 + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + + +log_interval: 50 +normalize: None \ No newline at end of file diff --git a/egs/aishell/paraformer2/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml b/egs/aishell/paraformer2/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml new file mode 100644 index 000000000..7562a49fb --- /dev/null +++ b/egs/aishell/paraformer2/conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml @@ -0,0 +1,100 @@ +# network architecture +# encoder related +encoder: conformer +encoder_conf: + output_size: 256 # dimension of attention + attention_heads: 4 + linear_units: 2048 # the number of units of position-wise feed forward + num_blocks: 12 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + macaron_style: true + use_cnn_module: true + cnn_module_kernel: 15 + +# decoder related +decoder: paraformer_decoder_san +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +# hybrid CTC/attention +model: paraformer_bert +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: false + predictor_weight: 1.0 + sampling_ratio: 0.4 + embeds_id: 3 + embed_dims: 768 + embeds_loss_weight: 2.0 + + + +# minibatch related +#batch_type: length +#batch_bins: 40000 +batch_type: numel +batch_bins: 2000000 +num_workers: 16 + +# optimization related +accum_grad: 4 +grad_clip: 5 +max_epoch: 50 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug +specaug_conf: + apply_time_warp: true + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + num_freq_mask: 2 + apply_time_mask: true + time_mask_width_range: + - 0 + - 40 + num_time_mask: 2 + +predictor: cif_predictor +predictor_conf: + idim: 256 + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + + +log_interval: 50 +normalize: None +allow_variable_data_keys: true \ No newline at end of file diff --git a/egs/aishell/paraformer2/local/aishell_data_prep.sh b/egs/aishell/paraformer2/local/aishell_data_prep.sh new file mode 100755 index 000000000..b6ea36b72 --- /dev/null +++ b/egs/aishell/paraformer2/local/aishell_data_prep.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Copyright 2017 Xingyu Na +# Apache 2.0 + +#. ./path.sh || exit 1; + +if [ $# != 2 ]; then + echo "Usage: $0 " + echo " $0 /export/a05/xna/data/data_aishell/wav /export/a05/xna/data/data_aishell/transcript" + exit 1; +fi + +aishell_audio_dir=$1 +aishell_text=$2/aishell_transcript_v0.8.txt + +train_dir=data/local/train +dev_dir=data/local/dev +test_dir=data/local/test +tmp_dir=data/local/tmp + +mkdir -p $train_dir +mkdir -p $dev_dir +mkdir -p $test_dir +mkdir -p $tmp_dir + +# data directory check +if [ ! -d $aishell_audio_dir ] || [ ! -f $aishell_text ]; then + echo "Error: $0 requires two directory arguments" + exit 1; +fi + +# find wav audio file for train, dev and test resp. +find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist +n=`cat $tmp_dir/wav.flist | wc -l` +[ $n -ne 141925 ] && \ + echo Warning: expected 141925 data data files, found $n + +grep -i "wav/train" $tmp_dir/wav.flist > $train_dir/wav.flist || exit 1; +grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1; +grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1; + +rm -r $tmp_dir + +# Transcriptions preparation +for dir in $train_dir $dev_dir $test_dir; do + echo Preparing $dir transcriptions + sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list + paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all + utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt + awk '{print $1}' $dir/transcripts.txt > $dir/utt.list + utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp + sort -u $dir/transcripts.txt > $dir/text +done + +mkdir -p data/train data/dev data/test + +for f in wav.scp text; do + cp $train_dir/$f data/train/$f || exit 1; + cp $dev_dir/$f data/dev/$f || exit 1; + cp $test_dir/$f data/test/$f || exit 1; +done + +echo "$0: AISHELL data preparation succeeded" +exit 0; diff --git a/egs/aishell/paraformer2/local/extract_embeds.sh b/egs/aishell/paraformer2/local/extract_embeds.sh new file mode 100755 index 000000000..6d9939077 --- /dev/null +++ b/egs/aishell/paraformer2/local/extract_embeds.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +stage=1 +stop_stage=3 + +bert_model_root="../../huggingface_models" +bert_model_name="bert-base-chinese" +#bert_model_name="chinese-roberta-wwm-ext" +#bert_model_name="mengzi-bert-base" +raw_dataset_path=~/Funasr_data/aishell-1 +model_path=${bert_model_root}/${bert_model_name} + +. utils/parse_options.sh || exit 1; + +nj=32 + +for data_set in train dev test;do + scp=$raw_dataset_path/dump/fbank/${data_set}/text + local_scp_dir_raw=$raw_dataset_path/embeds/$bert_model_name/${data_set} + local_scp_dir=$local_scp_dir_raw/split$nj + local_records_dir=$local_scp_dir_raw/ark + + mkdir -p $local_records_dir + mkdir -p $local_scp_dir + + split_scps="" + for JOB in $(seq ${nj}); do + split_scps="$split_scps $local_scp_dir/data.$JOB.text" + done + + utils/split_scp.pl $scp ${split_scps} + + + for num in {0..7};do + tmp=`expr $num \* 4` + + if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + for idx in {1..4}; do + JOB=`expr $tmp + $idx` + echo "proces jobid=$JOB" + { + + beg=0 + gpu=`expr $beg + $idx` + echo ${local_scp_dir}/log.${JOB} + python utils/extract_embeds.py $local_scp_dir/data.$JOB.text ${local_records_dir}/embeds.${JOB}.ark ${local_records_dir}/embeds.${JOB}.scp ${local_records_dir}/embeds.${JOB}.shape ${gpu} ${model_path} &> ${local_scp_dir}/log.${JOB} + } & + done + wait + fi + done + + if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + for JOB in $(seq ${nj}); do + cat ${local_records_dir}/embeds.${JOB}.scp || exit 1; + done > ${local_scp_dir_raw}/embeds.scp + + sed 's#nfs#data\/volume1#g' ${local_scp_dir_raw}/embeds.scp > ${local_scp_dir_raw}/embeds.scp.pai + + for JOB in $(seq ${nj}); do + cat ${local_records_dir}/embeds.${JOB}.shape || exit 1; + done > ${local_scp_dir_raw}/embeds.shape + fi +done + +echo "embeds is in: ${local_scp_dir_raw}" +echo "success" \ No newline at end of file diff --git a/egs/aishell/paraformer2/path.sh b/egs/aishell/paraformer2/path.sh new file mode 100755 index 000000000..7972642d0 --- /dev/null +++ b/egs/aishell/paraformer2/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs/aishell/paraformer2/run.sh b/egs/aishell/paraformer2/run.sh new file mode 100755 index 000000000..15d659c4e --- /dev/null +++ b/egs/aishell/paraformer2/run.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; + +# machines configuration +CUDA_VISIBLE_DEVICES="0,1" +gpu_num=2 +count=1 +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding +# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob +njob=8 +train_cmd=utils/run.pl + +# general configuration +feats_dir=".." #feature output dictionary, for large data +lang=zh +dumpdir=dump/fbank +feats_type=fbank +token_type=char +scp=feats.scp +type=kaldi_ark +stage=0 +stop_stage=4 + +skip_extract_embed=false +bert_model_root="../../huggingface_models" +bert_model_name="bert-base-chinese" + +# feature configuration +feats_dim=80 +sample_frequency=16000 +nj=32 +speed_perturb="0.9,1.0,1.1" + +# data +data_aishell= + +# exp tag +tag="" + +. utils/parse_options.sh || exit 1; + +# Set bash to 'debug' mode, it will exit on : +# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands', +set -e +set -u +set -o pipefail + +train_set=train +valid_set=dev +test_sets="dev test" + +asr_config=conf/train_asr_paraformerbert_conformer_12e_6d_2048_256.yaml +run_dir="exp" +model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}" +exp_dir=$run_dir/$model_dir + +inference_config=conf/decode_asr_transformer.yaml +inference_asr_model=valid.acc.ave_10best.pth + +# you can set gpu num for decoding here +gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] +else + inference_nj=$njob +fi + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + echo "stage 0: Data preparation" + # Data preparation + local/aishell_data_prep.sh ${data_aishell}/data_aishell/wav ${data_aishell}/data_aishell/transcript + for x in train dev test; do + cp data/${x}/text data/${x}/text.org + paste -d " " <(cut -f 1 -d" " data/${x}/text.org) <(cut -f 2- -d" " data/${x}/text.org | tr -d " ") \ + > data/${x}/text + utils/text2token.py -n 1 -s 1 data/${x}/text > data/${x}/text.org + mv data/${x}/text.org data/${x}/text + done +fi + +feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir} +feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir} +feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir} +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + echo "stage 1: Feature Generation" + # compute fbank features + fbankdir=${feats_dir}/fbank + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \ + data/train exp/make_fbank/train ${fbankdir}/train + utils/fix_data_feat.sh ${fbankdir}/train + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + data/dev exp/make_fbank/dev ${fbankdir}/dev + utils/fix_data_feat.sh ${fbankdir}/dev + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + data/test exp/make_fbank/test ${fbankdir}/test + utils/fix_data_feat.sh ${fbankdir}/test + + # compute global cmvn + utils/compute_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/train exp/make_fbank/train + + # apply cmvn + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/train ${fbankdir}/train/cmvn.json exp/make_fbank/train ${feat_train_dir} + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/dev ${fbankdir}/train/cmvn.json exp/make_fbank/dev ${feat_dev_dir} + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/test ${fbankdir}/train/cmvn.json exp/make_fbank/test ${feat_test_dir} + + cp ${fbankdir}/train/text ${fbankdir}/train/speech_shape ${fbankdir}/train/text_shape ${feat_train_dir} + cp ${fbankdir}/dev/text ${fbankdir}/dev/speech_shape ${fbankdir}/dev/text_shape ${feat_dev_dir} + cp ${fbankdir}/test/text ${fbankdir}/test/speech_shape ${fbankdir}/test/text_shape ${feat_test_dir} + + utils/fix_data_feat.sh ${feat_train_dir} + utils/fix_data_feat.sh ${feat_dev_dir} + utils/fix_data_feat.sh ${feat_test_dir} +fi + +token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt +echo "dictionary: ${token_list}" +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + echo "stage 2: Dictionary Preparation" + mkdir -p data/${lang}_token_list/char/ + + echo "make a dictionary" + echo "" > ${token_list} + echo "" >> ${token_list} + echo "" >> ${token_list} + utils/text2token.py -s 1 -n 1 --space "" data/train/text | cut -f 2- -d" " | tr " " "\n" \ + | sort | uniq | grep -a -v -e '^\s*$' | awk '{print $0}' >> ${token_list} + num_token=$(cat ${token_list} | wc -l) + echo "" >> ${token_list} + vocab_size=$(cat ${token_list} | wc -l) + awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char + awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char + mkdir -p asr_stats_fbank_zh_char/train + mkdir -p asr_stats_fbank_zh_char/dev + cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char asr_stats_fbank_zh_char/train + cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char asr_stats_fbank_zh_char/dev +fi + +if ! "${skip_extract_embed}"; then + local/extract_embeds.sh \ + --bert_model_root ${bert_model_root} \ + --bert_model_name ${bert_model_name} \ + --raw_dataset_path ${feats_dir} +fi + +# Training Stage +world_size=$gpu_num # run on one machine +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + mkdir -p $exp_dir + mkdir -p $exp_dir/log + INIT_FILE=$exp_dir/ddp_init + if [ -f $INIT_FILE ];then + rm -f $INIT_FILE + fi + init_method=file://$(readlink -f $INIT_FILE) + echo "$0: init method is $init_method" + for ((i = 0; i < $gpu_num; ++i)); do + { + rank=$i + local_rank=$i + gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1]) + asr_train_paraformer.py \ + --gpu_id $gpu_id \ + --use_preprocessor true \ + --token_type char \ + --token_list $token_list \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \ + --train_data_path_and_name_and_type ${feats_dir}/embeds/${bert_model_name}/${train_set}/embeds.scp,embed,${type} \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \ + --train_shape_file ${feats_dir}/embeds/${bert_model_name}/${train_set}/embeds.shape \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \ + --valid_data_path_and_name_and_type ${feats_dir}/embeds/${bert_model_name}/${valid_set}/embeds.scp,embed,${type} \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \ + --valid_shape_file ${feats_dir}/embeds/${bert_model_name}/${valid_set}/embeds.shape \ + --resume true \ + --output_dir $exp_dir \ + --config $asr_config \ + --input_size $feats_dim \ + --ngpu $gpu_num \ + --num_worker_count $count \ + --multiprocessing_distributed true \ + --dist_init_method $init_method \ + --dist_world_size $world_size \ + --dist_rank $rank \ + --allow_variable_data_keys true \ + --local_rank $local_rank 1> $exp_dir/log/train.log.$i 2>&1 + } & + done + wait +fi + +# Testing Stage +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + utils/easy_asr_infer.sh \ + --lang zh \ + --datadir ${feats_dir} \ + --feats_type ${feats_type} \ + --feats_dim ${feats_dim} \ + --token_type ${token_type} \ + --gpu_inference ${gpu_inference} \ + --inference_config "${inference_config}" \ + --test_sets "${test_sets}" \ + --token_list $token_list \ + --asr_exp $exp_dir \ + --stage 12 \ + --stop_stage 12 \ + --scp $scp \ + --text text \ + --inference_nj $inference_nj \ + --njob $njob \ + --inference_asr_model $inference_asr_model \ + --gpuid_list $gpuid_list \ + --gpu_inference ${gpu_inference} \ + --mode paraformer +fi + diff --git a/egs/aishell/paraformer2/utils b/egs/aishell/paraformer2/utils new file mode 120000 index 000000000..40e14f57f --- /dev/null +++ b/egs/aishell/paraformer2/utils @@ -0,0 +1 @@ +../tranformer/utils \ No newline at end of file diff --git a/egs/aishell/tranformer/conf/decode_asr_transformer.yaml b/egs/aishell/tranformer/conf/decode_asr_transformer.yaml new file mode 100644 index 000000000..a147fa79d --- /dev/null +++ b/egs/aishell/tranformer/conf/decode_asr_transformer.yaml @@ -0,0 +1,6 @@ +beam_size: 10 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.5 +lm_weight: 0.7 diff --git a/egs/aishell/tranformer/conf/train_asr_conformer.yaml b/egs/aishell/tranformer/conf/train_asr_conformer.yaml new file mode 100644 index 000000000..ddf217ec0 --- /dev/null +++ b/egs/aishell/tranformer/conf/train_asr_conformer.yaml @@ -0,0 +1,80 @@ +# network architecture +# encoder related +encoder: conformer +encoder_conf: + output_size: 256 # dimension of attention + attention_heads: 4 + linear_units: 2048 # the number of units of position-wise feed forward + num_blocks: 12 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + macaron_style: true + use_cnn_module: true + cnn_module_kernel: 15 + +# decoder related +decoder: transformer +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +# hybrid CTC/attention +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: false + +# minibatch related +batch_type: length +batch_bins: 25000 +num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 50 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug +specaug_conf: + apply_time_warp: true + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + num_freq_mask: 2 + apply_time_mask: true + time_mask_width_range: + - 0 + - 40 + num_time_mask: 2 + +log_interval: 50 +normalize: None diff --git a/egs/aishell/tranformer/conf/train_asr_transformer.yaml b/egs/aishell/tranformer/conf/train_asr_transformer.yaml new file mode 100644 index 000000000..ce987e7c9 --- /dev/null +++ b/egs/aishell/tranformer/conf/train_asr_transformer.yaml @@ -0,0 +1,70 @@ +# network architecture +# encoder related +encoder: transformer +encoder_conf: + output_size: 256 # dimension of attention + attention_heads: 4 + linear_units: 2048 # the number of units of position-wise feed forward + num_blocks: 12 # the number of encoder blocks + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.0 + input_layer: conv2d # encoder architecture type + normalize_before: true + +# decoder related +decoder: transformer +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.0 + src_attention_dropout_rate: 0.0 + +# hybrid CTC/attention +model_conf: + ctc_weight: 0.3 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: false + +# minibatch related +batch_type: length +batch_bins: 32000 +num_workers: 8 + +# optimization related +accum_grad: 1 +grad_clip: 5 +patience: 3 +max_epoch: 20 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +# NoamLR is deprecated. Use WarmupLR. +# The following is equivalent setting for NoamLR: +# +# optim: adam +# optim_conf: +# lr: 10. +# scheduler: noamlr +# scheduler_conf: +# model_size: 256 +# warmup_steps: 25000 +# +optim: adam +optim_conf: + lr: 0.002 +scheduler: warmuplr # pytorch v1.1.0+ required +scheduler_conf: + warmup_steps: 25000 + +log_interval: 50 +normalize: None diff --git a/egs/aishell/tranformer/local/aishell_data_prep.sh b/egs/aishell/tranformer/local/aishell_data_prep.sh new file mode 100755 index 000000000..83f489b3c --- /dev/null +++ b/egs/aishell/tranformer/local/aishell_data_prep.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2017 Xingyu Na +# Apache 2.0 + +#. ./path.sh || exit 1; + +if [ $# != 3 ]; then + echo "Usage: $0 " + echo " $0 /export/a05/xna/data/data_aishell/wav /export/a05/xna/data/data_aishell/transcript data" + exit 1; +fi + +aishell_audio_dir=$1 +aishell_text=$2/aishell_transcript_v0.8.txt +output_dir=$3 + +train_dir=$output_dir/data/local/train +dev_dir=$output_dir/data/local/dev +test_dir=$output_dir/data/local/test +tmp_dir=$output_dir/data/local/tmp + +mkdir -p $train_dir +mkdir -p $dev_dir +mkdir -p $test_dir +mkdir -p $tmp_dir + +# data directory check +if [ ! -d $aishell_audio_dir ] || [ ! -f $aishell_text ]; then + echo "Error: $0 requires two directory arguments" + exit 1; +fi + +# find wav audio file for train, dev and test resp. +find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist +n=`cat $tmp_dir/wav.flist | wc -l` +[ $n -ne 141925 ] && \ + echo Warning: expected 141925 data data files, found $n + +grep -i "wav/train" $tmp_dir/wav.flist > $train_dir/wav.flist || exit 1; +grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1; +grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1; + +rm -r $tmp_dir + +# Transcriptions preparation +for dir in $train_dir $dev_dir $test_dir; do + echo Preparing $dir transcriptions + sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list + paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all + utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt + awk '{print $1}' $dir/transcripts.txt > $dir/utt.list + utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp + sort -u $dir/transcripts.txt > $dir/text +done + +mkdir -p $output_dir/data/train $output_dir/data/dev $output_dir/data/test + +for f in wav.scp text; do + cp $train_dir/$f $output_dir/data/train/$f || exit 1; + cp $dev_dir/$f $output_dir/data/dev/$f || exit 1; + cp $test_dir/$f $output_dir/data/test/$f || exit 1; +done + +echo "$0: AISHELL data preparation succeeded" +exit 0; diff --git a/egs/aishell/tranformer/local/prepare_data.sh b/egs/aishell/tranformer/local/prepare_data.sh new file mode 100755 index 000000000..77791f9c1 --- /dev/null +++ b/egs/aishell/tranformer/local/prepare_data.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG) +# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU) +# Apache 2.0 + +# transform raw AISHELL-2 data to kaldi format + +. ./path.sh || exit 1; + +tmp= +dir= + +if [ $# != 3 ]; then + echo "Usage: $0 " + echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train" + exit 1; +fi + +corpus=$1 +tmp=$2 +dir=$3 + +echo "prepare_data.sh: Preparing data in $corpus" + +mkdir -p $tmp +mkdir -p $dir + +# corpus check +if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then + echo "Error: $0 requires wav.scp and trans.txt under $corpus directory." + exit 1; +fi + +# validate utt-key list, IC0803W0380 is a bad utterance +awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list +awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list +utils/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list + +# wav.scp +awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp +utils/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp + +# text +utils/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text + +# copy prepared resources from tmp_dir to target dir +mkdir -p $dir +for f in wav.scp text; do + cp $tmp/$f $dir/$f || exit 1; +done + +echo "local/prepare_data.sh succeeded" +exit 0; diff --git a/egs/aishell/tranformer/path.sh b/egs/aishell/tranformer/path.sh new file mode 100755 index 000000000..7972642d0 --- /dev/null +++ b/egs/aishell/tranformer/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs/aishell/tranformer/run.sh b/egs/aishell/tranformer/run.sh new file mode 100755 index 000000000..16ebc6759 --- /dev/null +++ b/egs/aishell/tranformer/run.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; + +# machines configuration +CUDA_VISIBLE_DEVICES="0,1" +gpu_num=2 +count=1 +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding +# for gpu decoding, inference_nj=ngpu*njob; for cpu decoding, inference_nj=njob +njob=8 +train_cmd=utils/run.pl + +# general configuration +feats_dir=".." #feature output dictionary, for large data +exp_dir="." +lang=zh +dumpdir=dump/fbank +feats_type=fbank +token_type=char +scp=feats.scp +type=kaldi_ark +stage=0 +stop_stage=4 + +# feature configuration +feats_dim=80 +sample_frequency=16000 +nj=32 +speed_perturb="0.9,1.0,1.1" + +# data +data_aishell= + +# exp tag +tag="" + +. utils/parse_options.sh || exit 1; + +# Set bash to 'debug' mode, it will exit on : +# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands', +set -e +set -u +set -o pipefail + +train_set=train +valid_set=dev +test_sets="dev test" + +asr_config=conf/train_asr_conformer.yaml +model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}" + +inference_config=conf/decode_asr_transformer.yaml +inference_asr_model=valid.acc.ave_10best.pth + +# you can set gpu num for decoding here +gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] +else + inference_nj=$njob +fi + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + echo "stage 0: Data preparation" + # Data preparation + local/aishell_data_prep.sh ${data_aishell}/data_aishell/wav ${data_aishell}/data_aishell/transcript ${feats_dir} + for x in train dev test; do + cp ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org + paste -d " " <(cut -f 1 -d" " ${feats_dir}/data/${x}/text.org) <(cut -f 2- -d" " ${feats_dir}/data/${x}/text.org | tr -d " ") \ + > ${feats_dir}/data/${x}/text + utils/text2token.py -n 1 -s 1 ${feats_dir}/data/${x}/text > ${feats_dir}/data/${x}/text.org + mv ${feats_dir}/data/${x}/text.org ${feats_dir}/data/${x}/text + done +fi + +feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir} +feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir} +feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir} +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + echo "stage 1: Feature Generation" + # compute fbank features + fbankdir=${feats_dir}/fbank + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \ + ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train + utils/fix_data_feat.sh ${fbankdir}/train + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/dev ${exp_dir}/exp/make_fbank/dev ${fbankdir}/dev + utils/fix_data_feat.sh ${fbankdir}/dev + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/test ${exp_dir}/exp/make_fbank/test ${fbankdir}/test + utils/fix_data_feat.sh ${fbankdir}/test + + # compute global cmvn + utils/compute_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/train ${exp_dir}/exp/make_fbank/train + + # apply cmvn + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/train ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/train ${feat_train_dir} + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/dev ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir} + utils/apply_cmvn.sh --cmd "$train_cmd" --nj $nj \ + ${fbankdir}/test ${fbankdir}/train/cmvn.json ${exp_dir}/exp/make_fbank/test ${feat_test_dir} + + cp ${fbankdir}/train/text ${fbankdir}/train/speech_shape ${fbankdir}/train/text_shape ${feat_train_dir} + cp ${fbankdir}/dev/text ${fbankdir}/dev/speech_shape ${fbankdir}/dev/text_shape ${feat_dev_dir} + cp ${fbankdir}/test/text ${fbankdir}/test/speech_shape ${fbankdir}/test/text_shape ${feat_test_dir} + + utils/fix_data_feat.sh ${feat_train_dir} + utils/fix_data_feat.sh ${feat_dev_dir} + utils/fix_data_feat.sh ${feat_test_dir} +fi + +token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt +echo "dictionary: ${token_list}" +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + echo "stage 2: Dictionary Preparation" + mkdir -p ${feats_dir}/data/${lang}_token_list/char/ + + echo "make a dictionary" + echo "" > ${token_list} + echo "" >> ${token_list} + echo "" >> ${token_list} + utils/text2token.py -s 1 -n 1 --space "" ${feats_dir}/data/train/text | cut -f 2- -d" " | tr " " "\n" \ + | sort | uniq | grep -a -v -e '^\s*$' | awk '{print $0}' >> ${token_list} + num_token=$(cat ${token_list} | wc -l) + echo "" >> ${token_list} + vocab_size=$(cat ${token_list} | wc -l) + awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char + awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev + cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/train + cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev +fi + +# Training Stage +world_size=$gpu_num # run on one machine +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + mkdir -p ${exp_dir}/exp/${model_dir} + mkdir -p ${exp_dir}/exp/${model_dir}/log + INIT_FILE=$exp_dir/ddp_init + if [ -f $INIT_FILE ];then + rm -f $INIT_FILE + fi + init_method=file://$(readlink -f $INIT_FILE) + echo "$0: init method is $init_method" + for ((i = 0; i < $gpu_num; ++i)); do + { + rank=$i + local_rank=$i + gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1]) + asr_train.py \ + --gpu_id $gpu_id \ + --use_preprocessor true \ + --token_type char \ + --token_list $token_list \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \ + --resume true \ + --output_dir ${exp_dir}/exp/${model_dir} \ + --config $asr_config \ + --input_size $feats_dim \ + --ngpu $gpu_num \ + --num_worker_count $count \ + --multiprocessing_distributed true \ + --dist_init_method $init_method \ + --dist_world_size $world_size \ + --dist_rank $rank \ + --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1 + } & + done + wait +fi + +# Testing Stage +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + utils/easy_asr_infer.sh \ + --lang zh \ + --datadir ${feats_dir} \ + --feats_type ${feats_type} \ + --feats_dim ${feats_dim} \ + --token_type ${token_type} \ + --gpu_inference ${gpu_inference} \ + --inference_config "${inference_config}" \ + --test_sets "${test_sets}" \ + --token_list $token_list \ + --asr_exp ${exp_dir}/${model_dir} \ + --stage 12 \ + --stop_stage 12 \ + --scp $scp \ + --text text \ + --inference_nj $inference_nj \ + --njob $njob \ + --inference_asr_model $inference_asr_model \ + --gpuid_list $gpuid_list \ + --mode asr +fi + diff --git a/egs/aishell/tranformer/utils/__init__.py b/egs/aishell/tranformer/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/egs/aishell/tranformer/utils/apply_cmvn.py b/egs/aishell/tranformer/utils/apply_cmvn.py new file mode 100755 index 000000000..b5c5086b3 --- /dev/null +++ b/egs/aishell/tranformer/utils/apply_cmvn.py @@ -0,0 +1,79 @@ +from kaldiio import ReadHelper +from kaldiio import WriteHelper + +import argparse +import json +import math +import numpy as np + + +def get_parser(): + parser = argparse.ArgumentParser( + description="apply cmvn", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--ark-file", + "-a", + default=False, + required=True, + type=str, + help="fbank ark file", + ) + parser.add_argument( + "--cmvn-file", + "-c", + default=False, + required=True, + type=str, + help="cmvn file", + ) + parser.add_argument( + "--ark-index", + "-i", + default=1, + required=True, + type=int, + help="ark index", + ) + parser.add_argument( + "--output-dir", + "-o", + default=False, + required=True, + type=str, + help="output dir", + ) + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + ark_file = args.output_dir + "/feats." + str(args.ark_index) + ".ark" + scp_file = args.output_dir + "/feats." + str(args.ark_index) + ".scp" + ark_writer = WriteHelper('ark,scp:{},{}'.format(ark_file, scp_file)) + + with open(args.cmvn_file) as f: + cmvn_stats = json.load(f) + + means = cmvn_stats['mean_stats'] + vars = cmvn_stats['var_stats'] + total_frames = cmvn_stats['total_frames'] + + for i in range(len(means)): + means[i] /= total_frames + vars[i] = vars[i] / total_frames - means[i] * means[i] + if vars[i] < 1.0e-20: + vars[i] = 1.0e-20 + vars[i] = 1.0 / math.sqrt(vars[i]) + + with ReadHelper('ark:{}'.format(args.ark_file)) as ark_reader: + for key, mat in ark_reader: + mat = (mat - means) * vars + ark_writer(key, mat) + + +if __name__ == '__main__': + main() diff --git a/egs/aishell/tranformer/utils/apply_cmvn.sh b/egs/aishell/tranformer/utils/apply_cmvn.sh new file mode 100755 index 000000000..f8fd1d140 --- /dev/null +++ b/egs/aishell/tranformer/utils/apply_cmvn.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; +# Begin configuration section. +nj=32 +cmd=./utils/run.pl + +echo "$0 $@" + +. utils/parse_options.sh || exit 1; + +fbankdir=$1 +cmvn_file=$2 +logdir=$3 +output_dir=$4 + +dump_dir=${output_dir}/ark; mkdir -p ${dump_dir} +mkdir -p ${logdir} + +$cmd JOB=1:$nj $logdir/apply_cmvn.JOB.log \ + python utils/apply_cmvn.py -a $fbankdir/ark/feats.JOB.ark \ + -c $cmvn_file -i JOB -o ${dump_dir} \ + || exit 1; + +for n in $(seq $nj); do + cat ${dump_dir}/feats.$n.scp || exit 1 +done > ${output_dir}/feats.scp || exit 1 + +echo "$0: Succeeded apply cmvn" diff --git a/egs/aishell/tranformer/utils/apply_lfr_and_cmvn.py b/egs/aishell/tranformer/utils/apply_lfr_and_cmvn.py new file mode 100755 index 000000000..50d18d1a4 --- /dev/null +++ b/egs/aishell/tranformer/utils/apply_lfr_and_cmvn.py @@ -0,0 +1,143 @@ +from kaldiio import ReadHelper, WriteHelper + +import argparse +import numpy as np + + +def build_LFR_features(inputs, m=7, n=6): + LFR_inputs = [] + T = inputs.shape[0] + T_lfr = int(np.ceil(T / n)) + left_padding = np.tile(inputs[0], ((m - 1) // 2, 1)) + inputs = np.vstack((left_padding, inputs)) + T = T + (m - 1) // 2 + for i in range(T_lfr): + if m <= T - i * n: + LFR_inputs.append(np.hstack(inputs[i * n:i * n + m])) + else: + num_padding = m - (T - i * n) + frame = np.hstack(inputs[i * n:]) + for _ in range(num_padding): + frame = np.hstack((frame, inputs[-1])) + LFR_inputs.append(frame) + return np.vstack(LFR_inputs) + + +def build_CMVN_features(inputs, mvn_file): # noqa + with open(mvn_file, 'r', encoding='utf-8') as f: + lines = f.readlines() + + add_shift_list = [] + rescale_list = [] + for i in range(len(lines)): + line_item = lines[i].split() + if line_item[0] == '': + line_item = lines[i + 1].split() + if line_item[0] == '': + add_shift_line = line_item[3:(len(line_item) - 1)] + add_shift_list = list(add_shift_line) + continue + elif line_item[0] == '': + line_item = lines[i + 1].split() + if line_item[0] == '': + rescale_line = line_item[3:(len(line_item) - 1)] + rescale_list = list(rescale_line) + continue + + for j in range(inputs.shape[0]): + for k in range(inputs.shape[1]): + add_shift_value = add_shift_list[k] + rescale_value = rescale_list[k] + inputs[j, k] = float(inputs[j, k]) + float(add_shift_value) + inputs[j, k] = float(inputs[j, k]) * float(rescale_value) + + return inputs + + +def get_parser(): + parser = argparse.ArgumentParser( + description="apply low_frame_rate and cmvn", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--ark-file", + "-a", + default=False, + required=True, + type=str, + help="fbank ark file", + ) + parser.add_argument( + "--lfr", + "-f", + default=True, + type=str, + help="low frame rate", + ) + parser.add_argument( + "--lfr-m", + "-m", + default=7, + type=int, + help="number of frames to stack", + ) + parser.add_argument( + "--lfr-n", + "-n", + default=6, + type=int, + help="number of frames to skip", + ) + parser.add_argument( + "--cmvn-file", + "-c", + default=False, + required=True, + type=str, + help="global cmvn file", + ) + parser.add_argument( + "--ark-index", + "-i", + default=1, + required=True, + type=int, + help="ark index", + ) + parser.add_argument( + "--output-dir", + "-o", + default=False, + required=True, + type=str, + help="output dir", + ) + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + dump_ark_file = args.output_dir + "/feats." + str(args.ark_index) + ".ark" + dump_scp_file = args.output_dir + "/feats." + str(args.ark_index) + ".scp" + shape_file = args.output_dir + "/len." + str(args.ark_index) + ark_writer = WriteHelper('ark,scp:{},{}'.format(dump_ark_file, dump_scp_file)) + + shape_writer = open(shape_file, 'w') + with ReadHelper('ark:{}'.format(args.ark_file)) as ark_reader: + for key, mat in ark_reader: + if args.lfr: + lfr = build_LFR_features(mat, args.lfr_m, args.lfr_n) + else: + lfr = mat + cmvn = build_CMVN_features(lfr, args.cmvn_file) + dims = cmvn.shape[1] + lens = cmvn.shape[0] + shape_writer.write(key + " " + str(lens) + "," + str(dims) + '\n') + ark_writer(key, cmvn) + + +if __name__ == '__main__': + main() + diff --git a/egs/aishell/tranformer/utils/apply_lfr_and_cmvn.sh b/egs/aishell/tranformer/utils/apply_lfr_and_cmvn.sh new file mode 100755 index 000000000..3119fdb8f --- /dev/null +++ b/egs/aishell/tranformer/utils/apply_lfr_and_cmvn.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + + +# Begin configuration section. +nj=32 +cmd=utils/run.pl + +# feature configuration +lfr=True +lfr_m=7 +lfr_n=6 + +echo "$0 $@" + +. utils/parse_options.sh || exit 1; + +fbankdir=$1 +cmvn_file=$2 +logdir=$3 +output_dir=$4 + +dump_dir=${output_dir}/ark; mkdir -p ${dump_dir} +mkdir -p ${logdir} + +$cmd JOB=1:$nj $logdir/apply_lfr_and_cmvn.JOB.log \ + python utils/apply_lfr_and_cmvn.py -a $fbankdir/ark/feats.JOB.ark \ + -f $lfr -m $lfr_m -n $lfr_n -c $cmvn_file -i JOB -o ${dump_dir} \ + || exit 1; + +for n in $(seq $nj); do + cat ${dump_dir}/feats.$n.scp || exit 1 +done > ${output_dir}/feats.scp || exit 1 + +for n in $(seq $nj); do + cat ${dump_dir}/len.$n || exit 1 +done > ${output_dir}/speech_shape || exit 1 + +echo "$0: Succeeded apply low frame rate and cmvn" diff --git a/egs/aishell/tranformer/utils/combine_cmvn_file.py b/egs/aishell/tranformer/utils/combine_cmvn_file.py new file mode 100755 index 000000000..e16174c60 --- /dev/null +++ b/egs/aishell/tranformer/utils/combine_cmvn_file.py @@ -0,0 +1,66 @@ +import argparse +import json +import numpy as np + +def get_parser(): + parser = argparse.ArgumentParser( + description="combine cmvn file", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--cmvn-dir", + "-c", + default=False, + required=True, + type=str, + help="cmvn dir", + ) + + parser.add_argument( + "--nj", + "-n", + default=1, + required=True, + type=int, + help="num of cmvn file", + ) + parser.add_argument( + "--output-dir", + "-o", + default=False, + required=True, + type=str, + help="output dir", + ) + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + total_means = 0.0 + total_vars = 0.0 + total_frames = 0 + + cmvn_file = args.output_dir + "/cmvn.json" + + for i in range(1, args.nj+1): + with open(args.cmvn_dir + "/cmvn." + str(i) + ".json", "r") as fin: + cmvn_stats = json.load(fin) + + total_means += np.array(cmvn_stats["mean_stats"]) + total_vars += np.array(cmvn_stats["var_stats"]) + total_frames += cmvn_stats["total_frames"] + + cmvn_info = { + 'mean_stats': list(total_means.tolist()), + 'var_stats': list(total_vars.tolist()), + 'total_frames': total_frames + } + with open(cmvn_file, 'w') as fout: + fout.write(json.dumps(cmvn_info)) + + +if __name__ == '__main__': + main() diff --git a/egs/aishell/tranformer/utils/compute_cmvn.py b/egs/aishell/tranformer/utils/compute_cmvn.py new file mode 100755 index 000000000..988d6dc9e --- /dev/null +++ b/egs/aishell/tranformer/utils/compute_cmvn.py @@ -0,0 +1,67 @@ +from kaldiio import ReadHelper + +import argparse +import numpy as np +import json + + +def get_parser(): + parser = argparse.ArgumentParser( + description="computer global cmvn", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--ark-file", + "-a", + default=False, + required=True, + type=str, + help="fbank ark file", + ) + parser.add_argument( + "--ark-index", + "-i", + default=1, + required=True, + type=int, + help="ark index", + ) + parser.add_argument( + "--output-dir", + "-o", + default=False, + required=True, + type=str, + help="output dir", + ) + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + ark_file = args.ark_file + "/feats." + str(args.ark_index) + ".ark" + cmvn_file = args.output_dir + "/cmvn." + str(args.ark_index) + ".json" + + mean_stats = 0.0 + var_stats = 0.0 + total_frames = 0 + + with ReadHelper('ark:{}'.format(ark_file)) as ark_reader: + for key, mat in ark_reader: + mean_stats += np.sum(mat, axis=0) + var_stats += np.sum(np.square(mat), axis=0) + total_frames += mat.shape[0] + + cmvn_info = { + 'mean_stats': list(mean_stats.tolist()), + 'var_stats': list(var_stats.tolist()), + 'total_frames': total_frames + } + with open(cmvn_file, 'w') as fout: + fout.write(json.dumps(cmvn_info)) + + +if __name__ == '__main__': + main() diff --git a/egs/aishell/tranformer/utils/compute_cmvn.sh b/egs/aishell/tranformer/utils/compute_cmvn.sh new file mode 100755 index 000000000..3a3019016 --- /dev/null +++ b/egs/aishell/tranformer/utils/compute_cmvn.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; +# Begin configuration section. +nj=32 +cmd=./utils/run.pl + +echo "$0 $@" + +. utils/parse_options.sh || exit 1; + +fbankdir=$1 +logdir=$2 + +output_dir=${fbankdir}/cmvn; mkdir -p ${output_dir} +mkdir -p ${logdir} + +$cmd JOB=1:$nj $logdir/cmvn.JOB.log \ + python utils/compute_cmvn.py -a $fbankdir/ark -i JOB -o ${output_dir} \ + || exit 1; + +python utils/combine_cmvn_file.py -c ${output_dir} -n $nj -o $fbankdir + +echo "$0: Succeeded compute global cmvn" diff --git a/egs/aishell/tranformer/utils/compute_fbank.py b/egs/aishell/tranformer/utils/compute_fbank.py new file mode 100755 index 000000000..d03b5a826 --- /dev/null +++ b/egs/aishell/tranformer/utils/compute_fbank.py @@ -0,0 +1,153 @@ +from kaldiio import WriteHelper + +import argparse +import numpy as np +import json +import torch +import torchaudio +import torchaudio.compliance.kaldi as kaldi + + +def compute_fbank(wav_file, + num_mel_bins=80, + frame_length=25, + frame_shift=10, + dither=0.0, + resample_rate=16000, + speed=1.0): + + waveform, sample_rate = torchaudio.load(wav_file) + if resample_rate != sample_rate: + waveform = torchaudio.transforms.Resample(orig_freq=sample_rate, + new_freq=resample_rate)(waveform) + if speed != 1.0: + waveform, _ = torchaudio.sox_effects.apply_effects_tensor( + waveform, resample_rate, + [['speed', str(speed)], ['rate', str(resample_rate)]] + ) + + waveform = waveform * (1 << 15) + mat = kaldi.fbank(waveform, + num_mel_bins=num_mel_bins, + frame_length=frame_length, + frame_shift=frame_shift, + dither=dither, + energy_floor=0.0, + window_type='hamming', + sample_frequency=resample_rate) + + return mat.numpy() + + +def get_parser(): + parser = argparse.ArgumentParser( + description="computer features", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--wav-lists", + "-w", + default=False, + required=True, + type=str, + help="input wav lists", + ) + parser.add_argument( + "--text-files", + "-t", + default=False, + required=True, + type=str, + help="input text files", + ) + parser.add_argument( + "--dims", + "-d", + default=80, + type=int, + help="feature dims", + ) + parser.add_argument( + "--sample-frequency", + "-s", + default=16000, + type=int, + help="sample frequency", + ) + parser.add_argument( + "--speed-perturb", + "-p", + default="1.0", + type=str, + help="speed perturb", + ) + parser.add_argument( + "--ark-index", + "-a", + default=1, + required=True, + type=int, + help="ark index", + ) + parser.add_argument( + "--output-dir", + "-o", + default=False, + required=True, + type=str, + help="output dir", + ) + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + ark_file = args.output_dir + "/ark/feats." + str(args.ark_index) + ".ark" + scp_file = args.output_dir + "/ark/feats." + str(args.ark_index) + ".scp" + text_file = args.output_dir + "/txt/text." + str(args.ark_index) + ".txt" + feats_shape_file = args.output_dir + "/ark/len." + str(args.ark_index) + text_shape_file = args.output_dir + "/txt/len." + str(args.ark_index) + + ark_writer = WriteHelper('ark,scp:{},{}'.format(ark_file, scp_file)) + text_writer = open(text_file, 'w') + feats_shape_writer = open(feats_shape_file, 'w') + text_shape_writer = open(text_shape_file, 'w') + + speed_perturb_list = args.speed_perturb.split(',') + + for speed in speed_perturb_list: + with open(args.wav_lists, 'r', encoding='utf-8') as wavfile: + with open(args.text_files, 'r', encoding='utf-8') as textfile: + for wav, text in zip(wavfile, textfile): + s_w = wav.strip().split() + wav_id = s_w[0] + wav_file = s_w[1] + + s_t = text.strip().split() + text_id = s_t[0] + txt = s_t[1:] + fbank = compute_fbank(wav_file, + num_mel_bins=args.dims, + resample_rate=args.sample_frequency, + speed=float(speed) + ) + feats_dims = fbank.shape[1] + feats_lens = fbank.shape[0] + txt_lens = len(txt) + if speed == "1.0": + wav_id_sp = wav_id + else: + wav_id_sp = wav_id + "_sp" + speed + + feats_shape_writer.write(wav_id_sp + " " + str(feats_lens) + "," + str(feats_dims) + '\n') + text_shape_writer.write(wav_id_sp + " " + str(txt_lens) + '\n') + + text_writer.write(wav_id_sp + " " + " ".join(txt) + '\n') + ark_writer(wav_id_sp, fbank) + + +if __name__ == '__main__': + main() + diff --git a/egs/aishell/tranformer/utils/compute_fbank.sh b/egs/aishell/tranformer/utils/compute_fbank.sh new file mode 100755 index 000000000..b456b4d83 --- /dev/null +++ b/egs/aishell/tranformer/utils/compute_fbank.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; +# Begin configuration section. +nj=32 +cmd=./utils/run.pl + +# feature configuration +feat_dims=80 +sample_frequency=16000 +speed_perturb="1.0" + +echo "$0 $@" + +. utils/parse_options.sh || exit 1; + +data=$1 +logdir=$2 +fbankdir=$3 + +[ ! -f $data/wav.scp ] && echo "$0: no such file $data/wav.scp" && exit 1; +[ ! -f $data/text ] && echo "$0: no such file $data/text" && exit 1; + +python utils/split_data.py $data $data $nj + +ark_dir=${fbankdir}/ark; mkdir -p ${ark_dir} +text_dir=${fbankdir}/txt; mkdir -p ${text_dir} +mkdir -p ${logdir} + +$cmd JOB=1:$nj $logdir/make_fbank.JOB.log \ + python utils/compute_fbank.py -w $data/split${nj}/JOB/wav.scp -t $data/split${nj}/JOB/text \ + -d $feat_dims -s $sample_frequency -p ${speed_perturb} -a JOB -o ${fbankdir} \ + || exit 1; + +for n in $(seq $nj); do + cat ${ark_dir}/feats.$n.scp || exit 1 +done > $fbankdir/feats.scp || exit 1 + +for n in $(seq $nj); do + cat ${text_dir}/text.$n.txt || exit 1 +done > $fbankdir/text || exit 1 + +for n in $(seq $nj); do + cat ${ark_dir}/len.$n || exit 1 +done > $fbankdir/speech_shape || exit 1 + +for n in $(seq $nj); do + cat ${text_dir}/len.$n || exit 1 +done > $fbankdir/text_shape || exit 1 + +echo "$0: Succeeded compute FBANK features" diff --git a/egs/aishell/tranformer/utils/compute_wer.py b/egs/aishell/tranformer/utils/compute_wer.py new file mode 100755 index 000000000..349a3f609 --- /dev/null +++ b/egs/aishell/tranformer/utils/compute_wer.py @@ -0,0 +1,157 @@ +import os +import numpy as np +import sys + +def compute_wer(ref_file, + hyp_file, + cer_detail_file): + rst = { + 'Wrd': 0, + 'Corr': 0, + 'Ins': 0, + 'Del': 0, + 'Sub': 0, + 'Snt': 0, + 'Err': 0.0, + 'S.Err': 0.0, + 'wrong_words': 0, + 'wrong_sentences': 0 + } + + hyp_dict = {} + ref_dict = {} + with open(hyp_file, 'r') as hyp_reader: + for line in hyp_reader: + key = line.strip().split()[0] + value = line.strip().split()[1:] + hyp_dict[key] = value + with open(ref_file, 'r') as ref_reader: + for line in ref_reader: + key = line.strip().split()[0] + value = line.strip().split()[1:] + ref_dict[key] = value + + cer_detail_writer = open(cer_detail_file, 'w') + for hyp_key in hyp_dict: + if hyp_key in ref_dict: + out_item = compute_wer_by_line(hyp_dict[hyp_key], ref_dict[hyp_key]) + rst['Wrd'] += out_item['nwords'] + rst['Corr'] += out_item['cor'] + rst['wrong_words'] += out_item['wrong'] + rst['Ins'] += out_item['ins'] + rst['Del'] += out_item['del'] + rst['Sub'] += out_item['sub'] + rst['Snt'] += 1 + if out_item['wrong'] > 0: + rst['wrong_sentences'] += 1 + cer_detail_writer.write(hyp_key + print_cer_detail(out_item) + '\n') + cer_detail_writer.write("ref:" + '\t' + "".join(ref_dict[hyp_key]) + '\n') + cer_detail_writer.write("hyp:" + '\t' + "".join(hyp_dict[hyp_key]) + '\n') + + if rst['Wrd'] > 0: + rst['Err'] = round(rst['wrong_words'] * 100 / rst['Wrd'], 2) + if rst['Snt'] > 0: + rst['S.Err'] = round(rst['wrong_sentences'] * 100 / rst['Snt'], 2) + + cer_detail_writer.write('\n') + cer_detail_writer.write("%WER " + str(rst['Err']) + " [ " + str(rst['wrong_words'])+ " / " + str(rst['Wrd']) + + ", " + str(rst['Ins']) + " ins, " + str(rst['Del']) + " del, " + str(rst['Sub']) + " sub ]" + '\n') + cer_detail_writer.write("%SER " + str(rst['S.Err']) + " [ " + str(rst['wrong_sentences']) + " / " + str(rst['Snt']) + " ]" + '\n') + cer_detail_writer.write("Scored " + str(len(hyp_dict)) + " sentences, " + str(len(hyp_dict) - rst['Snt']) + " not present in hyp." + '\n') + + +def compute_wer_by_line(hyp, + ref): + hyp = list(map(lambda x: x.lower(), hyp)) + ref = list(map(lambda x: x.lower(), ref)) + + len_hyp = len(hyp) + len_ref = len(ref) + + cost_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int16) + + ops_matrix = np.zeros((len_hyp + 1, len_ref + 1), dtype=np.int8) + + for i in range(len_hyp + 1): + cost_matrix[i][0] = i + for j in range(len_ref + 1): + cost_matrix[0][j] = j + + for i in range(1, len_hyp + 1): + for j in range(1, len_ref + 1): + if hyp[i - 1] == ref[j - 1]: + cost_matrix[i][j] = cost_matrix[i - 1][j - 1] + else: + substitution = cost_matrix[i - 1][j - 1] + 1 + insertion = cost_matrix[i - 1][j] + 1 + deletion = cost_matrix[i][j - 1] + 1 + + compare_val = [substitution, insertion, deletion] + + min_val = min(compare_val) + operation_idx = compare_val.index(min_val) + 1 + cost_matrix[i][j] = min_val + ops_matrix[i][j] = operation_idx + + match_idx = [] + i = len_hyp + j = len_ref + rst = { + 'nwords': len_ref, + 'cor': 0, + 'wrong': 0, + 'ins': 0, + 'del': 0, + 'sub': 0 + } + while i >= 0 or j >= 0: + i_idx = max(0, i) + j_idx = max(0, j) + + if ops_matrix[i_idx][j_idx] == 0: # correct + if i - 1 >= 0 and j - 1 >= 0: + match_idx.append((j - 1, i - 1)) + rst['cor'] += 1 + + i -= 1 + j -= 1 + + elif ops_matrix[i_idx][j_idx] == 2: # insert + i -= 1 + rst['ins'] += 1 + + elif ops_matrix[i_idx][j_idx] == 3: # delete + j -= 1 + rst['del'] += 1 + + elif ops_matrix[i_idx][j_idx] == 1: # substitute + i -= 1 + j -= 1 + rst['sub'] += 1 + + if i < 0 and j >= 0: + rst['del'] += 1 + elif j < 0 and i >= 0: + rst['ins'] += 1 + + match_idx.reverse() + wrong_cnt = cost_matrix[len_hyp][len_ref] + rst['wrong'] = wrong_cnt + + return rst + +def print_cer_detail(rst): + return ("(" + "nwords=" + str(rst['nwords']) + ",cor=" + str(rst['cor']) + + ",ins=" + str(rst['ins']) + ",del=" + str(rst['del']) + ",sub=" + + str(rst['sub']) + ") corr:" + '{:.2%}'.format(rst['cor']/rst['nwords']) + + ",cer:" + '{:.2%}'.format(rst['wrong']/rst['nwords'])) + +if __name__ == '__main__': + if len(sys.argv) != 4: + print("usage : python compute-wer.py test.ref test.hyp test.wer") + sys.exit(0) + + ref_file = sys.argv[1] + hyp_file = sys.argv[2] + cer_detail_file = sys.argv[3] + compute_wer(ref_file, hyp_file, cer_detail_file) diff --git a/egs/aishell/tranformer/utils/easy_asr_infer.sh b/egs/aishell/tranformer/utils/easy_asr_infer.sh new file mode 100755 index 000000000..1b8db3469 --- /dev/null +++ b/egs/aishell/tranformer/utils/easy_asr_infer.sh @@ -0,0 +1,407 @@ +#!/usr/bin/env bash + +# Set bash to 'debug' mode, it will exit on : +# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands', +set -e +set -u +set -o pipefail + +log() { + local fname=${BASH_SOURCE[1]##*/} + echo -e "$(date '+%Y-%m-%dT%H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" +} +min() { + local a b + a=$1 + for b in "$@"; do + if [ "${b}" -le "${a}" ]; then + a="${b}" + fi + done + echo "${a}" +} +SECONDS=0 + +# General configuration +stage=1 # Processes starts from the specified stage. +stop_stage=10000 # Processes is stopped at the specified stage. +skip_data_prep=true # Skip data preparation stages. +skip_train=false # Skip training stages. +skip_eval=false # Skip decoding and evaluation stages. +skip_upload=true # Skip packing and uploading stages. +skip_upload_hf=true # Skip uploading to hugging face stages. +cuda_cmd=utils/run.pl +decode_cmd=utils/run.pl +ngpu=1 # The number of gpus ("0" uses cpu, otherwise use gpu). +njob=1 # the number of jobs for each gpu +gpuid_list= +num_nodes=1 # The number of nodes. +nj=32 # The number of parallel jobs. +inference_nj=32 # The number of parallel jobs in decoding. +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding +datadir="./" +dumpdir=dump # Directory to dump features. +expdir=exp # Directory to save experiments. +python=python # Specify python to execute funasr commands. + +# Data preparation related +local_data_opts= # The options given to local/data.sh. + +# Speed perturbation related +speed_perturb_factors= # perturbation factors, e.g. "0.9 1.0 1.1" (separated by space). + +# Feature extraction related +feats_type=fbank # Feature type (raw or fbank_pitch). +feats_dim= +audio_format=flac # Audio format: wav, flac, wav.ark, flac.ark (only in feats_type=raw). +fs=16k # Sampling rate. +min_wav_duration=0.1 # Minimum duration in second. +max_wav_duration=20 # Maximum duration in second. + +# Tokenization related +token_type=bpe # Tokenization type (char or bpe). +nbpe=30 # The number of BPE vocabulary. +bpemode=unigram # Mode of BPE (unigram or bpe). +oov="" # Out of vocabulary symbol. +blank="" # CTC blank symbol +sos_eos="" # sos and eos symbole +bpe_input_sentence_size=100000000 # Size of input sentence for BPE. +bpe_nlsyms= # non-linguistic symbols list, separated by a comma, for BPE +bpe_char_cover=1.0 # character coverage when modeling BPE + +# Ngram model related +use_ngram=false +ngram_exp= +ngram_num=3 + +# Language model related +use_lm=false # Use language model for ASR decoding. +lm_tag= # Suffix to the result dir for language model training. +lm_exp= # Specify the directory path for LM experiment. + # If this option is specified, lm_tag is ignored. +lm_stats_dir= # Specify the directory path for LM statistics. +lm_config= # Config for language model training. +lm_args= # Arguments for language model training, e.g., "--max_epoch 10". + # Note that it will overwrite args in lm config. +use_word_lm=false # Whether to use word language model. +num_splits_lm=1 # Number of splitting for lm corpus. +# shellcheck disable=SC2034 +word_vocab_size=10000 # Size of word vocabulary. + +# ASR model related +asr_tag= # Suffix to the result dir for asr model training. +asr_exp= # Specify the directory path for ASR experiment. + # If this option is specified, asr_tag is ignored. +asr_stats_dir= # Specify the directory path for ASR statistics. +asr_config= # Config for asr model training. +asr_args= # Arguments for asr model training, e.g., "--max_epoch 10". + # Note that it will overwrite args in asr config. +pretrained_model= # Pretrained model to load +ignore_init_mismatch=false # Ignore initial mismatch +feats_normalize=global_mvn # Normalizaton layer type. +num_splits_asr=1 # Number of splitting for lm corpus. + +# Upload model related +hf_repo= + +# Decoding related +use_k2=false # Whether to use k2 based decoder +k2_ctc_decoding=true +use_nbest_rescoring=true # use transformer-decoder + # and transformer language model for nbest rescoring +num_paths=1000 # The 3rd argument of k2.random_paths. +nll_batch_size=100 # Affect GPU memory usage when computing nll + # during nbest rescoring +k2_config=./conf/decode_asr_transformer_with_k2.yaml + +use_streaming=false # Whether to use streaming decoding + +use_maskctc=false # Whether to use maskctc decoding + +batch_size=1 +inference_tag= # Suffix to the result dir for decoding. +inference_config= # Config for decoding. +inference_args= # Arguments for decoding, e.g., "--lm_weight 0.1". + # Note that it will overwrite args in inference config. +inference_lm=valid.loss.ave.pth # Language model path for decoding. +inference_ngram=${ngram_num}gram.bin +inference_asr_model=valid.acc.ave.pth # ASR model path for decoding. + # e.g. + # inference_asr_model=train.loss.best.pth + # inference_asr_model=3epoch.pth + # inference_asr_model=valid.acc.best.pth + # inference_asr_model=valid.loss.ave.pth +download_model= # Download a model from Model Zoo and use it for decoding. + +# [Task dependent] Set the datadir name created by local/data.sh +train_set= # Name of training set. +valid_set= # Name of validation set used for monitoring/tuning network training. +test_sets= # Names of test sets. Multiple items (e.g., both dev and eval sets) can be specified. +bpe_train_text= # Text file path of bpe training set. +lm_train_text= # Text file path of language model training set. +lm_dev_text= # Text file path of language model development set. +lm_test_text= # Text file path of language model evaluation set. +nlsyms_txt=none # Non-linguistic symbol list if existing. +cleaner=none # Text cleaner. +g2p=none # g2p method (needed if token_type=phn). +lang=noinfo # The language type of corpus. +score_opts= # The options given to sclite scoring +local_score_opts= # The options given to local/score.sh. +asr_speech_fold_length=800 # fold_length for speech data during ASR training. +asr_text_fold_length=150 # fold_length for text data during ASR training. +lm_fold_length=150 # fold_length for LM training. + +oss_path= +token_list= +scp= +text= + +mode= + +help_message=$(cat << EOF +Usage: $0 --train-set "" --valid-set "" --test_sets "" + +Options: + # General configuration + --stage # Processes starts from the specified stage (default="${stage}"). + --stop_stage # Processes is stopped at the specified stage (default="${stop_stage}"). + --skip_data_prep # Skip data preparation stages (default="${skip_data_prep}"). + --skip_train # Skip training stages (default="${skip_train}"). + --skip_eval # Skip decoding and evaluation stages (default="${skip_eval}"). + --skip_upload # Skip packing and uploading stages (default="${skip_upload}"). + --ngpu # The number of gpus ("0" uses cpu, otherwise use gpu, default="${ngpu}"). + --num_nodes # The number of nodes (default="${num_nodes}"). + --nj # The number of parallel jobs (default="${nj}"). + --inference_nj # The number of parallel jobs in decoding (default="${inference_nj}"). + --gpu_inference # Whether to perform gpu decoding (default="${gpu_inference}"). + --dumpdir # Directory to dump features (default="${dumpdir}"). + --expdir # Directory to save experiments (default="${expdir}"). + --python # Specify python to execute espnet commands (default="${python}"). + + # Data preparation related + --local_data_opts # The options given to local/data.sh (default="${local_data_opts}"). + + # Speed perturbation related + --speed_perturb_factors # speed perturbation factors, e.g. "0.9 1.0 1.1" (separated by space, default="${speed_perturb_factors}"). + + # Feature extraction related + --feats_type # Feature type (raw, fbank_pitch or extracted, default="${feats_type}"). + --audio_format # Audio format: wav, flac, wav.ark, flac.ark (only in feats_type=raw, default="${audio_format}"). + --fs # Sampling rate (default="${fs}"). + --min_wav_duration # Minimum duration in second (default="${min_wav_duration}"). + --max_wav_duration # Maximum duration in second (default="${max_wav_duration}"). + + # Tokenization related + --token_type # Tokenization type (char or bpe, default="${token_type}"). + --nbpe # The number of BPE vocabulary (default="${nbpe}"). + --bpemode # Mode of BPE (unigram or bpe, default="${bpemode}"). + --oov # Out of vocabulary symbol (default="${oov}"). + --blank # CTC blank symbol (default="${blank}"). + --sos_eos # sos and eos symbole (default="${sos_eos}"). + --bpe_input_sentence_size # Size of input sentence for BPE (default="${bpe_input_sentence_size}"). + --bpe_nlsyms # Non-linguistic symbol list for sentencepiece, separated by a comma. (default="${bpe_nlsyms}"). + --bpe_char_cover # Character coverage when modeling BPE (default="${bpe_char_cover}"). + + # Language model related + --lm_tag # Suffix to the result dir for language model training (default="${lm_tag}"). + --lm_exp # Specify the directory path for LM experiment. + # If this option is specified, lm_tag is ignored (default="${lm_exp}"). + --lm_stats_dir # Specify the directory path for LM statistics (default="${lm_stats_dir}"). + --lm_config # Config for language model training (default="${lm_config}"). + --lm_args # Arguments for language model training (default="${lm_args}"). + # e.g., --lm_args "--max_epoch 10" + # Note that it will overwrite args in lm config. + --use_word_lm # Whether to use word language model (default="${use_word_lm}"). + --word_vocab_size # Size of word vocabulary (default="${word_vocab_size}"). + --num_splits_lm # Number of splitting for lm corpus (default="${num_splits_lm}"). + + # ASR model related + --asr_tag # Suffix to the result dir for asr model training (default="${asr_tag}"). + --asr_exp # Specify the directory path for ASR experiment. + # If this option is specified, asr_tag is ignored (default="${asr_exp}"). + --asr_stats_dir # Specify the directory path for ASR statistics (default="${asr_stats_dir}"). + --asr_config # Config for asr model training (default="${asr_config}"). + --asr_args # Arguments for asr model training (default="${asr_args}"). + # e.g., --asr_args "--max_epoch 10" + # Note that it will overwrite args in asr config. + --pretrained_model= # Pretrained model to load (default="${pretrained_model}"). + --ignore_init_mismatch= # Ignore mismatch parameter init with pretrained model (default="${ignore_init_mismatch}"). + --feats_normalize # Normalizaton layer type (default="${feats_normalize}"). + --num_splits_asr # Number of splitting for lm corpus (default="${num_splits_asr}"). + + # Decoding related + --inference_tag # Suffix to the result dir for decoding (default="${inference_tag}"). + --inference_config # Config for decoding (default="${inference_config}"). + --inference_args # Arguments for decoding (default="${inference_args}"). + # e.g., --inference_args "--lm_weight 0.1" + # Note that it will overwrite args in inference config. + --inference_lm # Language model path for decoding (default="${inference_lm}"). + --inference_asr_model # ASR model path for decoding (default="${inference_asr_model}"). + --download_model # Download a model from Model Zoo and use it for decoding (default="${download_model}"). + --use_streaming # Whether to use streaming decoding (default="${use_streaming}"). + --use_maskctc # Whether to use maskctc decoding (default="${use_streaming}"). + + # [Task dependent] Set the datadir name created by local/data.sh + --train_set # Name of training set (required). + --valid_set # Name of validation set used for monitoring/tuning network training (required). + --test_sets # Names of test sets. + # Multiple items (e.g., both dev and eval sets) can be specified (required). + --bpe_train_text # Text file path of bpe training set. + --lm_train_text # Text file path of language model training set. + --lm_dev_text # Text file path of language model development set (default="${lm_dev_text}"). + --lm_test_text # Text file path of language model evaluation set (default="${lm_test_text}"). + --nlsyms_txt # Non-linguistic symbol list if existing (default="${nlsyms_txt}"). + --cleaner # Text cleaner (default="${cleaner}"). + --g2p # g2p method (default="${g2p}"). + --lang # The language type of corpus (default=${lang}). + --score_opts # The options given to sclite scoring (default="{score_opts}"). + --local_score_opts # The options given to local/score.sh (default="{local_score_opts}"). + --asr_speech_fold_length # fold_length for speech data during ASR training (default="${asr_speech_fold_length}"). + --asr_text_fold_length # fold_length for text data during ASR training (default="${asr_text_fold_length}"). + --lm_fold_length # fold_length for LM training (default="${lm_fold_length}"). +EOF +) + +log "$0 $*" +# Save command line args for logging (they will be lost after utils/parse_options.sh) +run_args=$(utils/print_args.py $0 "$@") +. utils/parse_options.sh + +if [ $# -ne 0 ]; then + log "${help_message}" + log "Error: No positional arguments are required." + exit 2 +fi + +# set absolute dump dir path +dumpdir=${datadir}/${dumpdir} + +if [ -z "${inference_tag}" ]; then + if [ -n "${inference_config}" ]; then + inference_tag="$(basename "${inference_config}" .yaml)" + else + inference_tag=inference + fi + + if "${use_k2}"; then + inference_tag+="_use_k2" + inference_tag+="_k2_ctc_decoding_${k2_ctc_decoding}" + inference_tag+="_use_nbest_rescoring_${use_nbest_rescoring}" + fi +fi + +# ========================== Main stages start from here. ========================== + +if [ ${stage} -le 12 ] && [ ${stop_stage} -ge 12 ]; then + log "Stage 12: Decoding: training_dir=${asr_exp}" + + if ${gpu_inference}; then + _cmd="${cuda_cmd}" + _ngpu=1 + else + _cmd="${decode_cmd}" + _ngpu=0 + fi + + _opts= + if [ -n "${inference_config}" ]; then + _opts+="--config ${inference_config} " + fi + + if "${use_lm}"; then + if "${use_word_lm}"; then + _opts+="--word_lm_train_config ${lm_exp}/config.yaml " + _opts+="--word_lm_file ${lm_exp}/${inference_lm} " + else + _opts+="--lm_train_config ${lm_exp}/config.yaml " + _opts+="--lm_file ${lm_exp}/${inference_lm} " + fi + fi + + if "${use_ngram}"; then + _opts+="--ngram_file ${ngram_exp}/${inference_ngram}" + inference_tag=${inference_tag}.${inference_ngram} + fi + + # 2. Generate run.sh + log "Generate '${asr_exp}/${inference_tag}/run.sh'. You can resume the process from stage 12 using this script" + mkdir -p "${asr_exp}/${inference_tag}"; echo "${run_args} --stage 12 \"\$@\"; exit \$?" > "${asr_exp}/${inference_tag}/run.sh"; chmod +x "${asr_exp}/${inference_tag}/run.sh" + + if "${use_streaming}"; then + asr_inference_tool="funasr.bin.asr_inference_streaming" + elif "${use_maskctc}"; then + asr_inference_tool="funasr.bin.asr_inference_maskctc" + else + asr_inference_tool="funasr.bin.asr_inference_launch" + fi + + for dset in ${test_sets}; do + if [ $feats_type == "ark_wav" ]; then + _data="${dumpdir}/wav/${dset}" + else + _data="${dumpdir}/$feats_type/${dset}" + fi + _dir="${asr_exp}/${inference_tag}/${inference_asr_model}/${dset}" + _logdir="${_dir}/logdir" + + if [ -d ${_dir} ]; then + #echo "${_dir} is already exists. if you want to decode again, please delete this dir first." + rm -r ${_dir} + fi + mkdir -p "${_logdir}" + + _scp=$scp + _type=kaldi_ark + + + # 1. Split the key file + key_file=${_data}/${_scp} + split_scps="" + if "${use_k2}"; then + # Now only _nj=1 is verified if using k2 + _nj=1 + else + _nj=$(min "${inference_nj}" "$(<${key_file} wc -l)") + fi + + for n in $(seq "${_nj}"); do + split_scps+=" ${_logdir}/keys.${n}.scp" + done + # shellcheck disable=SC2086 + utils/split_scp.pl "${key_file}" ${split_scps} + + # 2. Submit decoding jobs + log "Decoding started... log: '${_logdir}/asr_inference.*.log'" + # shellcheck disable=SC2086 + ${_cmd} --gpu "${_ngpu}" --max-jobs-run "${_nj}" JOB=1:"${_nj}" "${_logdir}"/asr_inference.JOB.log \ + ${python} -m ${asr_inference_tool} \ + --batch_size ${batch_size} \ + --ngpu "${_ngpu}" \ + --njob ${njob} \ + --gpuid_list ${gpuid_list} \ + --data_path_and_name_and_type "${_data}/${_scp},speech,${_type}" \ + --key_file "${_logdir}"/keys.JOB.scp \ + --asr_train_config "${asr_exp}"/config.yaml \ + --asr_model_file "${asr_exp}"/"${inference_asr_model}" \ + --output_dir "${_logdir}"/output.JOB \ + --mode $mode \ + ${_opts} ${inference_args} + + # 3. Concatenates the output files from each jobs + for f in token token_int score text; do + if [ -f "${_logdir}/output.1/1best_recog/${f}" ]; then + for i in $(seq "${_nj}"); do + cat "${_logdir}/output.${i}/1best_recog/${f}" + done | sort -k1 >"${_dir}/${f}" + fi + done + python utils/proce_text.py ${_dir}/text ${_dir}/${text}.proc + python utils/proce_text.py ${_data}/text ${_data}/${text}.proc + python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer + tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt + cat ${_dir}/text.cer.txt + done +fi + +log "Successfully finished. [elapsed=${SECONDS}s]" + diff --git a/egs/aishell/tranformer/utils/error_rate_zh b/egs/aishell/tranformer/utils/error_rate_zh new file mode 100755 index 000000000..6871a07fa --- /dev/null +++ b/egs/aishell/tranformer/utils/error_rate_zh @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +# coding=utf8 + +# Copyright 2021 Jiayu DU + +import sys +import argparse +import json +import logging +logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='[%(levelname)s] %(message)s') + +DEBUG = None + +def GetEditType(ref_token, hyp_token): + if ref_token == None and hyp_token != None: + return 'I' + elif ref_token != None and hyp_token == None: + return 'D' + elif ref_token == hyp_token: + return 'C' + elif ref_token != hyp_token: + return 'S' + else: + raise RuntimeError + +class AlignmentArc: + def __init__(self, src, dst, ref, hyp): + self.src = src + self.dst = dst + self.ref = ref + self.hyp = hyp + self.edit_type = GetEditType(ref, hyp) + +def similarity_score_function(ref_token, hyp_token): + return 0 if (ref_token == hyp_token) else -1.0 + +def insertion_score_function(token): + return -1.0 + +def deletion_score_function(token): + return -1.0 + +def EditDistance( + ref, + hyp, + similarity_score_function = similarity_score_function, + insertion_score_function = insertion_score_function, + deletion_score_function = deletion_score_function): + assert(len(ref) != 0) + class DPState: + def __init__(self): + self.score = -float('inf') + # backpointer + self.prev_r = None + self.prev_h = None + + def print_search_grid(S, R, H, fstream): + print(file=fstream) + for r in range(R): + for h in range(H): + print(F'[{r},{h}]:{S[r][h].score:4.3f}:({S[r][h].prev_r},{S[r][h].prev_h}) ', end='', file=fstream) + print(file=fstream) + + R = len(ref) + 1 + H = len(hyp) + 1 + + # Construct DP search space, a (R x H) grid + S = [ [] for r in range(R) ] + for r in range(R): + S[r] = [ DPState() for x in range(H) ] + + # initialize DP search grid origin, S(r = 0, h = 0) + S[0][0].score = 0.0 + S[0][0].prev_r = None + S[0][0].prev_h = None + + # initialize REF axis + for r in range(1, R): + S[r][0].score = S[r-1][0].score + deletion_score_function(ref[r-1]) + S[r][0].prev_r = r-1 + S[r][0].prev_h = 0 + + # initialize HYP axis + for h in range(1, H): + S[0][h].score = S[0][h-1].score + insertion_score_function(hyp[h-1]) + S[0][h].prev_r = 0 + S[0][h].prev_h = h-1 + + best_score = S[0][0].score + best_state = (0, 0) + + for r in range(1, R): + for h in range(1, H): + sub_or_cor_score = similarity_score_function(ref[r-1], hyp[h-1]) + new_score = S[r-1][h-1].score + sub_or_cor_score + if new_score >= S[r][h].score: + S[r][h].score = new_score + S[r][h].prev_r = r-1 + S[r][h].prev_h = h-1 + + del_score = deletion_score_function(ref[r-1]) + new_score = S[r-1][h].score + del_score + if new_score >= S[r][h].score: + S[r][h].score = new_score + S[r][h].prev_r = r - 1 + S[r][h].prev_h = h + + ins_score = insertion_score_function(hyp[h-1]) + new_score = S[r][h-1].score + ins_score + if new_score >= S[r][h].score: + S[r][h].score = new_score + S[r][h].prev_r = r + S[r][h].prev_h = h-1 + + best_score = S[R-1][H-1].score + best_state = (R-1, H-1) + + if DEBUG: + print_search_grid(S, R, H, sys.stderr) + + # Backtracing best alignment path, i.e. a list of arcs + # arc = (src, dst, ref, hyp, edit_type) + # src/dst = (r, h), where r/h refers to search grid state-id along Ref/Hyp axis + best_path = [] + r, h = best_state[0], best_state[1] + prev_r, prev_h = S[r][h].prev_r, S[r][h].prev_h + score = S[r][h].score + # loop invariant: + # 1. (prev_r, prev_h) -> (r, h) is a "forward arc" on best alignment path + # 2. score is the value of point(r, h) on DP search grid + while prev_r != None or prev_h != None: + src = (prev_r, prev_h) + dst = (r, h) + if (r == prev_r + 1 and h == prev_h + 1): # Substitution or correct + arc = AlignmentArc(src, dst, ref[prev_r], hyp[prev_h]) + elif (r == prev_r + 1 and h == prev_h): # Deletion + arc = AlignmentArc(src, dst, ref[prev_r], None) + elif (r == prev_r and h == prev_h + 1): # Insertion + arc = AlignmentArc(src, dst, None, hyp[prev_h]) + else: + raise RuntimeError + best_path.append(arc) + r, h = prev_r, prev_h + prev_r, prev_h = S[r][h].prev_r, S[r][h].prev_h + score = S[r][h].score + + best_path.reverse() + return (best_path, best_score) + +def PrettyPrintAlignment(alignment, stream = sys.stderr): + def get_token_str(token): + if token == None: + return "*" + return token + + def is_double_width_char(ch): + if (ch >= '\u4e00') and (ch <= '\u9fa5'): # codepoint ranges for Chinese chars + return True + # TODO: support other double-width-char language such as Japanese, Korean + else: + return False + + def display_width(token_str): + m = 0 + for c in token_str: + if is_double_width_char(c): + m += 2 + else: + m += 1 + return m + + R = ' REF : ' + H = ' HYP : ' + E = ' EDIT : ' + for arc in alignment: + r = get_token_str(arc.ref) + h = get_token_str(arc.hyp) + e = arc.edit_type if arc.edit_type != 'C' else '' + + nr, nh, ne = display_width(r), display_width(h), display_width(e) + n = max(nr, nh, ne) + 1 + + R += r + ' ' * (n-nr) + H += h + ' ' * (n-nh) + E += e + ' ' * (n-ne) + + print(R, file=stream) + print(H, file=stream) + print(E, file=stream) + +def CountEdits(alignment): + c, s, i, d = 0, 0, 0, 0 + for arc in alignment: + if arc.edit_type == 'C': + c += 1 + elif arc.edit_type == 'S': + s += 1 + elif arc.edit_type == 'I': + i += 1 + elif arc.edit_type == 'D': + d += 1 + else: + raise RuntimeError + return (c, s, i, d) + +def ComputeTokenErrorRate(c, s, i, d): + return 100.0 * (s + d + i) / (s + d + c) + +def ComputeSentenceErrorRate(num_err_utts, num_utts): + assert(num_utts != 0) + return 100.0 * num_err_utts / num_utts + + +class EvaluationResult: + def __init__(self): + self.num_ref_utts = 0 + self.num_hyp_utts = 0 + self.num_eval_utts = 0 # seen in both ref & hyp + self.num_hyp_without_ref = 0 + + self.C = 0 + self.S = 0 + self.I = 0 + self.D = 0 + self.token_error_rate = 0.0 + + self.num_utts_with_error = 0 + self.sentence_error_rate = 0.0 + + def to_json(self): + return json.dumps(self.__dict__) + + def to_kaldi(self): + info = ( + F'%WER {self.token_error_rate:.2f} [ {self.S + self.D + self.I} / {self.C + self.S + self.D}, {self.I} ins, {self.D} del, {self.S} sub ]\n' + F'%SER {self.sentence_error_rate:.2f} [ {self.num_utts_with_error} / {self.num_eval_utts} ]\n' + ) + return info + + def to_sclite(self): + return "TODO" + + def to_espnet(self): + return "TODO" + + def to_summary(self): + #return json.dumps(self.__dict__, indent=4) + summary = ( + '==================== Overall Statistics ====================\n' + F'num_ref_utts: {self.num_ref_utts}\n' + F'num_hyp_utts: {self.num_hyp_utts}\n' + F'num_hyp_without_ref: {self.num_hyp_without_ref}\n' + F'num_eval_utts: {self.num_eval_utts}\n' + F'sentence_error_rate: {self.sentence_error_rate:.2f}%\n' + F'token_error_rate: {self.token_error_rate:.2f}%\n' + F'token_stats:\n' + F' - tokens:{self.C + self.S + self.D:>7}\n' + F' - edits: {self.S + self.I + self.D:>7}\n' + F' - cor: {self.C:>7}\n' + F' - sub: {self.S:>7}\n' + F' - ins: {self.I:>7}\n' + F' - del: {self.D:>7}\n' + '============================================================\n' + ) + return summary + + +class Utterance: + def __init__(self, uid, text): + self.uid = uid + self.text = text + + +def LoadUtterances(filepath, format): + utts = {} + if format == 'text': # utt_id word1 word2 ... + with open(filepath, 'r', encoding='utf8') as f: + for line in f: + line = line.strip() + if line: + cols = line.split(maxsplit=1) + assert(len(cols) == 2 or len(cols) == 1) + uid = cols[0] + text = cols[1] if len(cols) == 2 else '' + if utts.get(uid) != None: + raise RuntimeError(F'Found duplicated utterence id {uid}') + utts[uid] = Utterance(uid, text) + else: + raise RuntimeError(F'Unsupported text format {format}') + return utts + + +def tokenize_text(text, tokenizer): + if tokenizer == 'whitespace': + return text.split() + elif tokenizer == 'char': + return [ ch for ch in ''.join(text.split()) ] + else: + raise RuntimeError(F'ERROR: Unsupported tokenizer {tokenizer}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + # optional + parser.add_argument('--tokenizer', choices=['whitespace', 'char'], default='whitespace', help='whitespace for WER, char for CER') + parser.add_argument('--ref-format', choices=['text'], default='text', help='reference format, first col is utt_id, the rest is text') + parser.add_argument('--hyp-format', choices=['text'], default='text', help='hypothesis format, first col is utt_id, the rest is text') + # required + parser.add_argument('--ref', type=str, required=True, help='input reference file') + parser.add_argument('--hyp', type=str, required=True, help='input hypothesis file') + + parser.add_argument('result_file', type=str) + args = parser.parse_args() + logging.info(args) + + ref_utts = LoadUtterances(args.ref, args.ref_format) + hyp_utts = LoadUtterances(args.hyp, args.hyp_format) + + r = EvaluationResult() + + # check valid utterances in hyp that have matched non-empty reference + eval_utts = [] + r.num_hyp_without_ref = 0 + for uid in sorted(hyp_utts.keys()): + if uid in ref_utts.keys(): # TODO: efficiency + if ref_utts[uid].text.strip(): # non-empty reference + eval_utts.append(uid) + else: + logging.warn(F'Found {uid} with empty reference, skipping...') + else: + logging.warn(F'Found {uid} without reference, skipping...') + r.num_hyp_without_ref += 1 + + r.num_hyp_utts = len(hyp_utts) + r.num_ref_utts = len(ref_utts) + r.num_eval_utts = len(eval_utts) + + with open(args.result_file, 'w+', encoding='utf8') as fo: + for uid in eval_utts: + ref = ref_utts[uid] + hyp = hyp_utts[uid] + + alignment, score = EditDistance( + tokenize_text(ref.text, args.tokenizer), + tokenize_text(hyp.text, args.tokenizer) + ) + + c, s, i, d = CountEdits(alignment) + utt_ter = ComputeTokenErrorRate(c, s, i, d) + + # utt-level evaluation result + print(F'{{"uid":{uid}, "score":{score}, "ter":{utt_ter:.2f}, "cor":{c}, "sub":{s}, "ins":{i}, "del":{d}}}', file=fo) + PrettyPrintAlignment(alignment, fo) + + r.C += c + r.S += s + r.I += i + r.D += d + + if utt_ter > 0: + r.num_utts_with_error += 1 + + # corpus level evaluation result + r.sentence_error_rate = ComputeSentenceErrorRate(r.num_utts_with_error, r.num_eval_utts) + r.token_error_rate = ComputeTokenErrorRate(r.C, r.S, r.I, r.D) + + print(r.to_summary(), file=fo) + + print(r.to_json()) + print(r.to_kaldi()) diff --git a/egs/aishell/tranformer/utils/extract_embeds.py b/egs/aishell/tranformer/utils/extract_embeds.py new file mode 100755 index 000000000..7b817d8ca --- /dev/null +++ b/egs/aishell/tranformer/utils/extract_embeds.py @@ -0,0 +1,47 @@ +from transformers import AutoTokenizer, AutoModel, pipeline +import numpy as np +import sys +import os +import torch +from kaldiio import WriteHelper +import re +text_file_json = sys.argv[1] +out_ark = sys.argv[2] +out_scp = sys.argv[3] +out_shape = sys.argv[4] +device = int(sys.argv[5]) +model_path = sys.argv[6] + +model = AutoModel.from_pretrained(model_path) +tokenizer = AutoTokenizer.from_pretrained(model_path) +extractor = pipeline(task="feature-extraction", model=model, tokenizer=tokenizer, device=device) + +with open(text_file_json, 'r') as f: + js = f.readlines() + + +f_shape = open(out_shape, "w") +with WriteHelper('ark,scp:{},{}'.format(out_ark, out_scp)) as writer: + with torch.no_grad(): + for idx, line in enumerate(js): + id, tokens = line.strip().split(" ", 1) + tokens = re.sub(" ", "", tokens.strip()) + tokens = ' '.join([j for j in tokens]) + token_num = len(tokens.split(" ")) + outputs = extractor(tokens) + outputs = np.array(outputs) + embeds = outputs[0, 1:-1, :] + + token_num_embeds, dim = embeds.shape + if token_num == token_num_embeds: + writer(id, embeds) + shape_line = "{} {},{}\n".format(id, token_num_embeds, dim) + f_shape.write(shape_line) + else: + print("{}, size has changed, {}, {}, {}".format(id, token_num, token_num_embeds, tokens)) + + + +f_shape.close() + + diff --git a/egs/aishell/tranformer/utils/filter_scp.pl b/egs/aishell/tranformer/utils/filter_scp.pl new file mode 100755 index 000000000..003530d53 --- /dev/null +++ b/egs/aishell/tranformer/utils/filter_scp.pl @@ -0,0 +1,87 @@ +#!/usr/bin/env perl +# Copyright 2010-2012 Microsoft Corporation +# Johns Hopkins University (author: Daniel Povey) + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# See the Apache 2 License for the specific language governing permissions and +# limitations under the License. + + +# This script takes a list of utterance-ids or any file whose first field +# of each line is an utterance-id, and filters an scp +# file (or any file whose "n-th" field is an utterance id), printing +# out only those lines whose "n-th" field is in id_list. The index of +# the "n-th" field is 1, by default, but can be changed by using +# the -f switch + +$exclude = 0; +$field = 1; +$shifted = 0; + +do { + $shifted=0; + if ($ARGV[0] eq "--exclude") { + $exclude = 1; + shift @ARGV; + $shifted=1; + } + if ($ARGV[0] eq "-f") { + $field = $ARGV[1]; + shift @ARGV; shift @ARGV; + $shifted=1 + } +} while ($shifted); + +if(@ARGV < 1 || @ARGV > 2) { + die "Usage: filter_scp.pl [--exclude] [-f ] id_list [in.scp] > out.scp \n" . + "Prints only the input lines whose f'th field (default: first) is in 'id_list'.\n" . + "Note: only the first field of each line in id_list matters. With --exclude, prints\n" . + "only the lines that were *not* in id_list.\n" . + "Caution: previously, the -f option was interpreted as a zero-based field index.\n" . + "If your older scripts (written before Oct 2014) stopped working and you used the\n" . + "-f option, add 1 to the argument.\n" . + "See also: scripts/filter_scp.pl .\n"; +} + + +$idlist = shift @ARGV; +open(F, "<$idlist") || die "Could not open id-list file $idlist"; +while() { + @A = split; + @A>=1 || die "Invalid id-list file line $_"; + $seen{$A[0]} = 1; +} + +if ($field == 1) { # Treat this as special case, since it is common. + while(<>) { + $_ =~ m/\s*(\S+)\s*/ || die "Bad line $_, could not get first field."; + # $1 is what we filter on. + if ((!$exclude && $seen{$1}) || ($exclude && !defined $seen{$1})) { + print $_; + } + } +} else { + while(<>) { + @A = split; + @A > 0 || die "Invalid scp file line $_"; + @A >= $field || die "Invalid scp file line $_"; + if ((!$exclude && $seen{$A[$field-1]}) || ($exclude && !defined $seen{$A[$field-1]})) { + print $_; + } + } +} + +# tests: +# the following should print "foo 1" +# ( echo foo 1; echo bar 2 ) | scripts/filter_scp.pl <(echo foo) +# the following should print "bar 2". +# ( echo foo 1; echo bar 2 ) | scripts/filter_scp.pl -f 2 <(echo 2) diff --git a/egs/aishell/tranformer/utils/fix_data.sh b/egs/aishell/tranformer/utils/fix_data.sh new file mode 100755 index 000000000..32cdde593 --- /dev/null +++ b/egs/aishell/tranformer/utils/fix_data.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +echo "$0 $@" +data_dir=$1 + +if [ ! -f ${data_dir}/wav.scp ]; then + echo "$0: wav.scp is not found" + exit 1; +fi + +if [ ! -f ${data_dir}/text ]; then + echo "$0: text is not found" + exit 1; +fi + + + +mkdir -p ${data_dir}/.backup + +awk '{print $1}' ${data_dir}/wav.scp > ${data_dir}/.backup/wav_id +awk '{print $1}' ${data_dir}/text > ${data_dir}/.backup/text_id + +sort ${data_dir}/.backup/wav_id ${data_dir}/.backup/text_id | uniq -d > ${data_dir}/.backup/id + +cp ${data_dir}/wav.scp ${data_dir}/.backup/wav.scp +cp ${data_dir}/text ${data_dir}/.backup/text + +mv ${data_dir}/wav.scp ${data_dir}/wav.scp.bak +mv ${data_dir}/text ${data_dir}/text.bak + +utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/wav.scp.bak > ${data_dir}/wav.scp +utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text.bak > ${data_dir}/text + +rm ${data_dir}/wav.scp.bak +rm ${data_dir}/text.bak diff --git a/egs/aishell/tranformer/utils/fix_data_feat.sh b/egs/aishell/tranformer/utils/fix_data_feat.sh new file mode 100755 index 000000000..2c92d7f71 --- /dev/null +++ b/egs/aishell/tranformer/utils/fix_data_feat.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +echo "$0 $@" +data_dir=$1 + +if [ ! -f ${data_dir}/feats.scp ]; then + echo "$0: feats.scp is not found" + exit 1; +fi + +if [ ! -f ${data_dir}/text ]; then + echo "$0: text is not found" + exit 1; +fi + +if [ ! -f ${data_dir}/speech_shape ]; then + echo "$0: feature lengths is not found" + exit 1; +fi + +if [ ! -f ${data_dir}/text_shape ]; then + echo "$0: text lengths is not found" + exit 1; +fi + +mkdir -p ${data_dir}/.backup + +awk '{print $1}' ${data_dir}/feats.scp > ${data_dir}/.backup/wav_id +awk '{print $1}' ${data_dir}/text > ${data_dir}/.backup/text_id + +sort ${data_dir}/.backup/wav_id ${data_dir}/.backup/text_id | uniq -d > ${data_dir}/.backup/id + +cp ${data_dir}/feats.scp ${data_dir}/.backup/feats.scp +cp ${data_dir}/text ${data_dir}/.backup/text +cp ${data_dir}/speech_shape ${data_dir}/.backup/speech_shape +cp ${data_dir}/text_shape ${data_dir}/.backup/text_shape + +mv ${data_dir}/feats.scp ${data_dir}/feats.scp.bak +mv ${data_dir}/text ${data_dir}/text.bak +mv ${data_dir}/speech_shape ${data_dir}/speech_shape.bak +mv ${data_dir}/text_shape ${data_dir}/text_shape.bak + +utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/feats.scp.bak > ${data_dir}/feats.scp +utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text.bak > ${data_dir}/text +utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/speech_shape.bak > ${data_dir}/speech_shape +utils/filter_scp.pl -f 1 ${data_dir}/.backup/id ${data_dir}/text_shape.bak > ${data_dir}/text_shape + +rm ${data_dir}/feats.scp.bak +rm ${data_dir}/text.bak +rm ${data_dir}/speech_shape.bak +rm ${data_dir}/text_shape.bak + diff --git a/egs/aishell/tranformer/utils/gen_ark_list.sh b/egs/aishell/tranformer/utils/gen_ark_list.sh new file mode 100755 index 000000000..be60f7be8 --- /dev/null +++ b/egs/aishell/tranformer/utils/gen_ark_list.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + + +# Begin configuration section. +nj=4 +cmd=./utils/run.pl + +echo "$0 $@" + +. utils/parse_options.sh || exit 1; + +data=$1 + +[ ! -d ${data}/ark ] && echo "$0: ark data is required" && exit 1; +[ ! -d ${data}/txt ] && echo "$0: txt data is required" && exit 1; + +for n in $(seq $nj); do + echo "$data/ark/feats.$n.ark $data/txt/text.$n" || exit 1 +done > $data/ark_txt.scp || exit 1 + diff --git a/egs/aishell/tranformer/utils/parse_options.sh b/egs/aishell/tranformer/utils/parse_options.sh new file mode 100755 index 000000000..71fb9e5ea --- /dev/null +++ b/egs/aishell/tranformer/utils/parse_options.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +# Copyright 2012 Johns Hopkins University (Author: Daniel Povey); +# Arnab Ghoshal, Karel Vesely + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# See the Apache 2 License for the specific language governing permissions and +# limitations under the License. + + +# Parse command-line options. +# To be sourced by another script (as in ". parse_options.sh"). +# Option format is: --option-name arg +# and shell variable "option_name" gets set to value "arg." +# The exception is --help, which takes no arguments, but prints the +# $help_message variable (if defined). + + +### +### The --config file options have lower priority to command line +### options, so we need to import them first... +### + +# Now import all the configs specified by command-line, in left-to-right order +for ((argpos=1; argpos<$#; argpos++)); do + if [ "${!argpos}" == "--config" ]; then + argpos_plus1=$((argpos+1)) + config=${!argpos_plus1} + [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1 + . $config # source the config file. + fi +done + + +### +### Now we process the command line options +### +while true; do + [ -z "${1:-}" ] && break; # break if there are no arguments + case "$1" in + # If the enclosing script is called with --help option, print the help + # message and exit. Scripts should put help messages in $help_message + --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2; + else printf "$help_message\n" 1>&2 ; fi; + exit 0 ;; + --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'" + exit 1 ;; + # If the first command-line argument begins with "--" (e.g. --foo-bar), + # then work out the variable name as $name, which will equal "foo_bar". + --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`; + # Next we test whether the variable in question is undefned-- if so it's + # an invalid option and we die. Note: $0 evaluates to the name of the + # enclosing script. + # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar + # is undefined. We then have to wrap this test inside "eval" because + # foo_bar is itself inside a variable ($name). + eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1; + + oldval="`eval echo \\$$name`"; + # Work out whether we seem to be expecting a Boolean argument. + if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then + was_bool=true; + else + was_bool=false; + fi + + # Set the variable to the right value-- the escaped quotes make it work if + # the option had spaces, like --cmd "queue.pl -sync y" + eval $name=\"$2\"; + + # Check that Boolean-valued arguments are really Boolean. + if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then + echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2 + exit 1; + fi + shift 2; + ;; + *) break; + esac +done + + +# Check for an empty argument to the --cmd option, which can easily occur as a +# result of scripting errors. +[ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1; + + +true; # so this script returns exit code 0. diff --git a/egs/aishell/tranformer/utils/print_args.py b/egs/aishell/tranformer/utils/print_args.py new file mode 100755 index 000000000..b0c61e5b4 --- /dev/null +++ b/egs/aishell/tranformer/utils/print_args.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +import sys + + +def get_commandline_args(no_executable=True): + extra_chars = [ + " ", + ";", + "&", + "|", + "<", + ">", + "?", + "*", + "~", + "`", + '"', + "'", + "\\", + "{", + "}", + "(", + ")", + ] + + # Escape the extra characters for shell + argv = [ + arg.replace("'", "'\\''") + if all(char not in arg for char in extra_chars) + else "'" + arg.replace("'", "'\\''") + "'" + for arg in sys.argv + ] + + if no_executable: + return " ".join(argv[1:]) + else: + return sys.executable + " " + " ".join(argv) + + +def main(): + print(get_commandline_args()) + + +if __name__ == "__main__": + main() diff --git a/egs/aishell/tranformer/utils/proc_conf_oss.py b/egs/aishell/tranformer/utils/proc_conf_oss.py new file mode 100755 index 000000000..c4a90c5c1 --- /dev/null +++ b/egs/aishell/tranformer/utils/proc_conf_oss.py @@ -0,0 +1,35 @@ +from pathlib import Path + +import torch +import yaml + + +class NoAliasSafeDumper(yaml.SafeDumper): + # Disable anchor/alias in yaml because looks ugly + def ignore_aliases(self, data): + return True + + +def yaml_no_alias_safe_dump(data, stream=None, **kwargs): + """Safe-dump in yaml with no anchor/alias""" + return yaml.dump( + data, stream, allow_unicode=True, Dumper=NoAliasSafeDumper, **kwargs + ) + + +def gen_conf(file, out_dir): + conf = torch.load(file)["config"] + conf["oss_bucket"] = "null" + print(conf) + output_dir = Path(out_dir) + output_dir.mkdir(parents=True, exist_ok=True) + with (output_dir / "config.yaml").open("w", encoding="utf-8") as f: + yaml_no_alias_safe_dump(conf, f, indent=4, sort_keys=False) + + +if __name__ == "__main__": + import sys + + in_f = sys.argv[1] + out_f = sys.argv[2] + gen_conf(in_f, out_f) diff --git a/egs/aishell/tranformer/utils/proce_text.py b/egs/aishell/tranformer/utils/proce_text.py new file mode 100755 index 000000000..9e517a4e1 --- /dev/null +++ b/egs/aishell/tranformer/utils/proce_text.py @@ -0,0 +1,31 @@ + +import sys +import re + +in_f = sys.argv[1] +out_f = sys.argv[2] + + +with open(in_f, "r", encoding="utf-8") as f: + lines = f.readlines() + +with open(out_f, "w", encoding="utf-8") as f: + for line in lines: + outs = line.strip().split(" ", 1) + if len(outs) == 2: + idx, text = outs + text = re.sub("", "", text) + text = re.sub("", "", text) + text = re.sub("@@", "", text) + text = re.sub("@", "", text) + text = re.sub("", "", text) + text = re.sub(" ", "", text) + text = text.lower() + else: + idx = outs[0] + text = " " + + text = [x for x in text] + text = " ".join(text) + out = "{} {}\n".format(idx, text) + f.write(out) diff --git a/egs/aishell/tranformer/utils/run.pl b/egs/aishell/tranformer/utils/run.pl new file mode 100755 index 000000000..483f95bc6 --- /dev/null +++ b/egs/aishell/tranformer/utils/run.pl @@ -0,0 +1,356 @@ +#!/usr/bin/env perl +use warnings; #sed replacement for -w perl parameter +# In general, doing +# run.pl some.log a b c is like running the command a b c in +# the bash shell, and putting the standard error and output into some.log. +# To run parallel jobs (backgrounded on the host machine), you can do (e.g.) +# run.pl JOB=1:4 some.JOB.log a b c JOB is like running the command a b c JOB +# and putting it in some.JOB.log, for each one. [Note: JOB can be any identifier]. +# If any of the jobs fails, this script will fail. + +# A typical example is: +# run.pl some.log my-prog "--opt=foo bar" foo \| other-prog baz +# and run.pl will run something like: +# ( my-prog '--opt=foo bar' foo | other-prog baz ) >& some.log +# +# Basically it takes the command-line arguments, quotes them +# as necessary to preserve spaces, and evaluates them with bash. +# In addition it puts the command line at the top of the log, and +# the start and end times of the command at the beginning and end. +# The reason why this is useful is so that we can create a different +# version of this program that uses a queueing system instead. + +#use Data::Dumper; + +@ARGV < 2 && die "usage: run.pl log-file command-line arguments..."; + +#print STDERR "COMMAND-LINE: " . Dumper(\@ARGV) . "\n"; +$job_pick = 'all'; +$max_jobs_run = -1; +$jobstart = 1; +$jobend = 1; +$ignored_opts = ""; # These will be ignored. + +# First parse an option like JOB=1:4, and any +# options that would normally be given to +# queue.pl, which we will just discard. + +for (my $x = 1; $x <= 2; $x++) { # This for-loop is to + # allow the JOB=1:n option to be interleaved with the + # options to qsub. + while (@ARGV >= 2 && $ARGV[0] =~ m:^-:) { + # parse any options that would normally go to qsub, but which will be ignored here. + my $switch = shift @ARGV; + if ($switch eq "-V") { + $ignored_opts .= "-V "; + } elsif ($switch eq "--max-jobs-run" || $switch eq "-tc") { + # we do support the option --max-jobs-run n, and its GridEngine form -tc n. + # if the command appears multiple times uses the smallest option. + if ( $max_jobs_run <= 0 ) { + $max_jobs_run = shift @ARGV; + } else { + my $new_constraint = shift @ARGV; + if ( ($new_constraint < $max_jobs_run) ) { + $max_jobs_run = $new_constraint; + } + } + + if (! ($max_jobs_run > 0)) { + die "run.pl: invalid option --max-jobs-run $max_jobs_run"; + } + } else { + my $argument = shift @ARGV; + if ($argument =~ m/^--/) { + print STDERR "run.pl: WARNING: suspicious argument '$argument' to $switch; starts with '-'\n"; + } + if ($switch eq "-sync" && $argument =~ m/^[yY]/) { + $ignored_opts .= "-sync "; # Note: in the + # corresponding code in queue.pl it says instead, just "$sync = 1;". + } elsif ($switch eq "-pe") { # e.g. -pe smp 5 + my $argument2 = shift @ARGV; + $ignored_opts .= "$switch $argument $argument2 "; + } elsif ($switch eq "--gpu") { + $using_gpu = $argument; + } elsif ($switch eq "--pick") { + if($argument =~ m/^(all|failed|incomplete)$/) { + $job_pick = $argument; + } else { + print STDERR "run.pl: ERROR: --pick argument must be one of 'all', 'failed' or 'incomplete'" + } + } else { + # Ignore option. + $ignored_opts .= "$switch $argument "; + } + } + } + if ($ARGV[0] =~ m/^([\w_][\w\d_]*)+=(\d+):(\d+)$/) { # e.g. JOB=1:20 + $jobname = $1; + $jobstart = $2; + $jobend = $3; + if ($jobstart > $jobend) { + die "run.pl: invalid job range $ARGV[0]"; + } + if ($jobstart <= 0) { + die "run.pl: invalid job range $ARGV[0], start must be strictly positive (this is required for GridEngine compatibility)."; + } + shift; + } elsif ($ARGV[0] =~ m/^([\w_][\w\d_]*)+=(\d+)$/) { # e.g. JOB=1. + $jobname = $1; + $jobstart = $2; + $jobend = $2; + shift; + } elsif ($ARGV[0] =~ m/.+\=.*\:.*$/) { + print STDERR "run.pl: Warning: suspicious first argument to run.pl: $ARGV[0]\n"; + } +} + +# Users found this message confusing so we are removing it. +# if ($ignored_opts ne "") { +# print STDERR "run.pl: Warning: ignoring options \"$ignored_opts\"\n"; +# } + +if ($max_jobs_run == -1) { # If --max-jobs-run option not set, + # then work out the number of processors if possible, + # and set it based on that. + $max_jobs_run = 0; + if ($using_gpu) { + if (open(P, "nvidia-smi -L |")) { + $max_jobs_run++ while (

); + close(P); + } + if ($max_jobs_run == 0) { + $max_jobs_run = 1; + print STDERR "run.pl: Warning: failed to detect number of GPUs from nvidia-smi, using ${max_jobs_run}\n"; + } + } elsif (open(P, ") { if (m/^processor/) { $max_jobs_run++; } } + if ($max_jobs_run == 0) { + print STDERR "run.pl: Warning: failed to detect any processors from /proc/cpuinfo\n"; + $max_jobs_run = 10; # reasonable default. + } + close(P); + } elsif (open(P, "sysctl -a |")) { # BSD/Darwin + while (

) { + if (m/hw\.ncpu\s*[:=]\s*(\d+)/) { # hw.ncpu = 4, or hw.ncpu: 4 + $max_jobs_run = $1; + last; + } + } + close(P); + if ($max_jobs_run == 0) { + print STDERR "run.pl: Warning: failed to detect any processors from sysctl -a\n"; + $max_jobs_run = 10; # reasonable default. + } + } else { + # allow at most 32 jobs at once, on non-UNIX systems; change this code + # if you need to change this default. + $max_jobs_run = 32; + } + # The just-computed value of $max_jobs_run is just the number of processors + # (or our best guess); and if it happens that the number of jobs we need to + # run is just slightly above $max_jobs_run, it will make sense to increase + # $max_jobs_run to equal the number of jobs, so we don't have a small number + # of leftover jobs. + $num_jobs = $jobend - $jobstart + 1; + if (!$using_gpu && + $num_jobs > $max_jobs_run && $num_jobs < 1.4 * $max_jobs_run) { + $max_jobs_run = $num_jobs; + } +} + +sub pick_or_exit { + # pick_or_exit ( $logfile ) + # Invoked before each job is started helps to run jobs selectively. + # + # Given the name of the output logfile decides whether the job must be + # executed (by returning from the subroutine) or not (by terminating the + # process calling exit) + # + # PRE: $job_pick is a global variable set by command line switch --pick + # and indicates which class of jobs must be executed. + # + # 1) If a failed job is not executed the process exit code will indicate + # failure, just as if the task was just executed and failed. + # + # 2) If a task is incomplete it will be executed. Incomplete may be either + # a job whose log file does not contain the accounting notes in the end, + # or a job whose log file does not exist. + # + # 3) If the $job_pick is set to 'all' (default behavior) a task will be + # executed regardless of the result of previous attempts. + # + # This logic could have been implemented in the main execution loop + # but a subroutine to preserve the current level of readability of + # that part of the code. + # + # Alexandre Felipe, (o.alexandre.felipe@gmail.com) 14th of August of 2020 + # + if($job_pick eq 'all'){ + return; # no need to bother with the previous log + } + open my $fh, "<", $_[0] or return; # job not executed yet + my $log_line; + my $cur_line; + while ($cur_line = <$fh>) { + if( $cur_line =~ m/# Ended \(code .*/ ) { + $log_line = $cur_line; + } + } + close $fh; + if (! defined($log_line)){ + return; # incomplete + } + if ( $log_line =~ m/# Ended \(code 0\).*/ ) { + exit(0); # complete + } elsif ( $log_line =~ m/# Ended \(code \d+(; signal \d+)?\).*/ ){ + if ($job_pick !~ m/^(failed|all)$/) { + exit(1); # failed but not going to run + } else { + return; # failed + } + } elsif ( $log_line =~ m/.*\S.*/ ) { + return; # incomplete jobs are always run + } +} + + +$logfile = shift @ARGV; + +if (defined $jobname && $logfile !~ m/$jobname/ && + $jobend > $jobstart) { + print STDERR "run.pl: you are trying to run a parallel job but " + . "you are putting the output into just one log file ($logfile)\n"; + exit(1); +} + +$cmd = ""; + +foreach $x (@ARGV) { + if ($x =~ m/^\S+$/) { $cmd .= $x . " "; } + elsif ($x =~ m:\":) { $cmd .= "'$x' "; } + else { $cmd .= "\"$x\" "; } +} + +#$Data::Dumper::Indent=0; +$ret = 0; +$numfail = 0; +%active_pids=(); + +use POSIX ":sys_wait_h"; +for ($jobid = $jobstart; $jobid <= $jobend; $jobid++) { + if (scalar(keys %active_pids) >= $max_jobs_run) { + + # Lets wait for a change in any child's status + # Then we have to work out which child finished + $r = waitpid(-1, 0); + $code = $?; + if ($r < 0 ) { die "run.pl: Error waiting for child process"; } # should never happen. + if ( defined $active_pids{$r} ) { + $jid=$active_pids{$r}; + $fail[$jid]=$code; + if ($code !=0) { $numfail++;} + delete $active_pids{$r}; + # print STDERR "Finished: $r/$jid " . Dumper(\%active_pids) . "\n"; + } else { + die "run.pl: Cannot find the PID of the child process that just finished."; + } + + # In theory we could do a non-blocking waitpid over all jobs running just + # to find out if only one or more jobs finished during the previous waitpid() + # However, we just omit this and will reap the next one in the next pass + # through the for(;;) cycle + } + $childpid = fork(); + if (!defined $childpid) { die "run.pl: Error forking in run.pl (writing to $logfile)"; } + if ($childpid == 0) { # We're in the child... this branch + # executes the job and returns (possibly with an error status). + if (defined $jobname) { + $cmd =~ s/$jobname/$jobid/g; + $logfile =~ s/$jobname/$jobid/g; + } + # exit if the job does not need to be executed + pick_or_exit( $logfile ); + + system("mkdir -p `dirname $logfile` 2>/dev/null"); + open(F, ">$logfile") || die "run.pl: Error opening log file $logfile"; + print F "# " . $cmd . "\n"; + print F "# Started at " . `date`; + $starttime = `date +'%s'`; + print F "#\n"; + close(F); + + # Pipe into bash.. make sure we're not using any other shell. + open(B, "|bash") || die "run.pl: Error opening shell command"; + print B "( " . $cmd . ") 2>>$logfile >> $logfile"; + close(B); # If there was an error, exit status is in $? + $ret = $?; + + $lowbits = $ret & 127; + $highbits = $ret >> 8; + if ($lowbits != 0) { $return_str = "code $highbits; signal $lowbits" } + else { $return_str = "code $highbits"; } + + $endtime = `date +'%s'`; + open(F, ">>$logfile") || die "run.pl: Error opening log file $logfile (again)"; + $enddate = `date`; + chop $enddate; + print F "# Accounting: time=" . ($endtime - $starttime) . " threads=1\n"; + print F "# Ended ($return_str) at " . $enddate . ", elapsed time " . ($endtime-$starttime) . " seconds\n"; + close(F); + exit($ret == 0 ? 0 : 1); + } else { + $pid[$jobid] = $childpid; + $active_pids{$childpid} = $jobid; + # print STDERR "Queued: " . Dumper(\%active_pids) . "\n"; + } +} + +# Now we have submitted all the jobs, lets wait until all the jobs finish +foreach $child (keys %active_pids) { + $jobid=$active_pids{$child}; + $r = waitpid($pid[$jobid], 0); + $code = $?; + if ($r == -1) { die "run.pl: Error waiting for child process"; } # should never happen. + if ($r != 0) { $fail[$jobid]=$code; $numfail++ if $code!=0; } # Completed successfully +} + +# Some sanity checks: +# The $fail array should not contain undefined codes +# The number of non-zeros in that array should be equal to $numfail +# We cannot do foreach() here, as the JOB ids do not start at zero +$failed_jids=0; +for ($jobid = $jobstart; $jobid <= $jobend; $jobid++) { + $job_return = $fail[$jobid]; + if (not defined $job_return ) { + # print Dumper(\@fail); + + die "run.pl: Sanity check failed: we have indication that some jobs are running " . + "even after we waited for all jobs to finish" ; + } + if ($job_return != 0 ){ $failed_jids++;} +} +if ($failed_jids != $numfail) { + die "run.pl: Sanity check failed: cannot find out how many jobs failed ($failed_jids x $numfail)." +} +if ($numfail > 0) { $ret = 1; } + +if ($ret != 0) { + $njobs = $jobend - $jobstart + 1; + if ($njobs == 1) { + if (defined $jobname) { + $logfile =~ s/$jobname/$jobstart/; # only one numbered job, so replace name with + # that job. + } + print STDERR "run.pl: job failed, log is in $logfile\n"; + if ($logfile =~ m/JOB/) { + print STDERR "run.pl: probably you forgot to put JOB=1:\$nj in your script."; + } + } + else { + $logfile =~ s/$jobname/*/g; + print STDERR "run.pl: $numfail / $njobs failed, log is in $logfile\n"; + } +} + + +exit ($ret); diff --git a/egs/aishell/tranformer/utils/shuffle_list.pl b/egs/aishell/tranformer/utils/shuffle_list.pl new file mode 100755 index 000000000..a116200f4 --- /dev/null +++ b/egs/aishell/tranformer/utils/shuffle_list.pl @@ -0,0 +1,44 @@ +#!/usr/bin/env perl + +# Copyright 2013 Johns Hopkins University (author: Daniel Povey) + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# See the Apache 2 License for the specific language governing permissions and +# limitations under the License. + + +if ($ARGV[0] eq "--srand") { + $n = $ARGV[1]; + $n =~ m/\d+/ || die "Bad argument to --srand option: \"$n\""; + srand($ARGV[1]); + shift; + shift; +} else { + srand(0); # Gives inconsistent behavior if we don't seed. +} + +if (@ARGV > 1 || $ARGV[0] =~ m/^-.+/) { # >1 args, or an option we + # don't understand. + print "Usage: shuffle_list.pl [--srand N] [input file] > output\n"; + print "randomizes the order of lines of input.\n"; + exit(1); +} + +@lines; +while (<>) { + push @lines, [ (rand(), $_)] ; +} + +@lines = sort { $a->[0] cmp $b->[0] } @lines; +foreach $l (@lines) { + print $l->[1]; +} \ No newline at end of file diff --git a/egs/aishell/tranformer/utils/split_data.py b/egs/aishell/tranformer/utils/split_data.py new file mode 100755 index 000000000..060eae6d3 --- /dev/null +++ b/egs/aishell/tranformer/utils/split_data.py @@ -0,0 +1,60 @@ +import os +import sys +import random + + +in_dir = sys.argv[1] +out_dir = sys.argv[2] +num_split = sys.argv[3] + + +def split_scp(scp, num): + assert len(scp) >= num + avg = len(scp) // num + out = [] + begin = 0 + + for i in range(num): + if i == num - 1: + out.append(scp[begin:]) + else: + out.append(scp[begin:begin+avg]) + begin += avg + + return out + + +os.path.exists("{}/wav.scp".format(in_dir)) +os.path.exists("{}/text".format(in_dir)) + +with open("{}/wav.scp".format(in_dir), 'r') as infile: + wav_list = infile.readlines() + +with open("{}/text".format(in_dir), 'r') as infile: + text_list = infile.readlines() + +assert len(wav_list) == len(text_list) + +x = list(zip(wav_list, text_list)) +random.shuffle(x) +wav_shuffle_list, text_shuffle_list = zip(*x) + +num_split = int(num_split) +wav_split_list = split_scp(wav_shuffle_list, num_split) +text_split_list = split_scp(text_shuffle_list, num_split) + +for idx, wav_list in enumerate(wav_split_list, 1): + path = out_dir + "/split" + str(num_split) + "/" + str(idx) + if not os.path.exists(path): + os.makedirs(path) + with open("{}/wav.scp".format(path), 'w') as wav_writer: + for line in wav_list: + wav_writer.write(line) + +for idx, text_list in enumerate(text_split_list, 1): + path = out_dir + "/split" + str(num_split) + "/" + str(idx) + if not os.path.exists(path): + os.makedirs(path) + with open("{}/text".format(path), 'w') as text_writer: + for line in text_list: + text_writer.write(line) diff --git a/egs/aishell/tranformer/utils/split_scp.pl b/egs/aishell/tranformer/utils/split_scp.pl new file mode 100755 index 000000000..0876dcb6d --- /dev/null +++ b/egs/aishell/tranformer/utils/split_scp.pl @@ -0,0 +1,246 @@ +#!/usr/bin/env perl + +# Copyright 2010-2011 Microsoft Corporation + +# See ../../COPYING for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# See the Apache 2 License for the specific language governing permissions and +# limitations under the License. + + +# This program splits up any kind of .scp or archive-type file. +# If there is no utt2spk option it will work on any text file and +# will split it up with an approximately equal number of lines in +# each but. +# With the --utt2spk option it will work on anything that has the +# utterance-id as the first entry on each line; the utt2spk file is +# of the form "utterance speaker" (on each line). +# It splits it into equal size chunks as far as it can. If you use the utt2spk +# option it will make sure these chunks coincide with speaker boundaries. In +# this case, if there are more chunks than speakers (and in some other +# circumstances), some of the resulting chunks will be empty and it will print +# an error message and exit with nonzero status. +# You will normally call this like: +# split_scp.pl scp scp.1 scp.2 scp.3 ... +# or +# split_scp.pl --utt2spk=utt2spk scp scp.1 scp.2 scp.3 ... +# Note that you can use this script to split the utt2spk file itself, +# e.g. split_scp.pl --utt2spk=utt2spk utt2spk utt2spk.1 utt2spk.2 ... + +# You can also call the scripts like: +# split_scp.pl -j 3 0 scp scp.0 +# [note: with this option, it assumes zero-based indexing of the split parts, +# i.e. the second number must be 0 <= n < num-jobs.] + +use warnings; + +$num_jobs = 0; +$job_id = 0; +$utt2spk_file = ""; +$one_based = 0; + +for ($x = 1; $x <= 3 && @ARGV > 0; $x++) { + if ($ARGV[0] eq "-j") { + shift @ARGV; + $num_jobs = shift @ARGV; + $job_id = shift @ARGV; + } + if ($ARGV[0] =~ /--utt2spk=(.+)/) { + $utt2spk_file=$1; + shift; + } + if ($ARGV[0] eq '--one-based') { + $one_based = 1; + shift @ARGV; + } +} + +if ($num_jobs != 0 && ($num_jobs < 0 || $job_id - $one_based < 0 || + $job_id - $one_based >= $num_jobs)) { + die "$0: Invalid job number/index values for '-j $num_jobs $job_id" . + ($one_based ? " --one-based" : "") . "'\n" +} + +$one_based + and $job_id--; + +if(($num_jobs == 0 && @ARGV < 2) || ($num_jobs > 0 && (@ARGV < 1 || @ARGV > 2))) { + die +"Usage: split_scp.pl [--utt2spk=] in.scp out1.scp out2.scp ... + or: split_scp.pl -j num-jobs job-id [--one-based] [--utt2spk=] in.scp [out.scp] + ... where 0 <= job-id < num-jobs, or 1 <= job-id <- num-jobs if --one-based.\n"; +} + +$error = 0; +$inscp = shift @ARGV; +if ($num_jobs == 0) { # without -j option + @OUTPUTS = @ARGV; +} else { + for ($j = 0; $j < $num_jobs; $j++) { + if ($j == $job_id) { + if (@ARGV > 0) { push @OUTPUTS, $ARGV[0]; } + else { push @OUTPUTS, "-"; } + } else { + push @OUTPUTS, "/dev/null"; + } + } +} + +if ($utt2spk_file ne "") { # We have the --utt2spk option... + open($u_fh, '<', $utt2spk_file) || die "$0: Error opening utt2spk file $utt2spk_file: $!\n"; + while(<$u_fh>) { + @A = split; + @A == 2 || die "$0: Bad line $_ in utt2spk file $utt2spk_file\n"; + ($u,$s) = @A; + $utt2spk{$u} = $s; + } + close $u_fh; + open($i_fh, '<', $inscp) || die "$0: Error opening input scp file $inscp: $!\n"; + @spkrs = (); + while(<$i_fh>) { + @A = split; + if(@A == 0) { die "$0: Empty or space-only line in scp file $inscp\n"; } + $u = $A[0]; + $s = $utt2spk{$u}; + defined $s || die "$0: No utterance $u in utt2spk file $utt2spk_file\n"; + if(!defined $spk_count{$s}) { + push @spkrs, $s; + $spk_count{$s} = 0; + $spk_data{$s} = []; # ref to new empty array. + } + $spk_count{$s}++; + push @{$spk_data{$s}}, $_; + } + # Now split as equally as possible .. + # First allocate spks to files by allocating an approximately + # equal number of speakers. + $numspks = @spkrs; # number of speakers. + $numscps = @OUTPUTS; # number of output files. + if ($numspks < $numscps) { + die "$0: Refusing to split data because number of speakers $numspks " . + "is less than the number of output .scp files $numscps\n"; + } + for($scpidx = 0; $scpidx < $numscps; $scpidx++) { + $scparray[$scpidx] = []; # [] is array reference. + } + for ($spkidx = 0; $spkidx < $numspks; $spkidx++) { + $scpidx = int(($spkidx*$numscps) / $numspks); + $spk = $spkrs[$spkidx]; + push @{$scparray[$scpidx]}, $spk; + $scpcount[$scpidx] += $spk_count{$spk}; + } + + # Now will try to reassign beginning + ending speakers + # to different scp's and see if it gets more balanced. + # Suppose objf we're minimizing is sum_i (num utts in scp[i] - average)^2. + # We can show that if considering changing just 2 scp's, we minimize + # this by minimizing the squared difference in sizes. This is + # equivalent to minimizing the absolute difference in sizes. This + # shows this method is bound to converge. + + $changed = 1; + while($changed) { + $changed = 0; + for($scpidx = 0; $scpidx < $numscps; $scpidx++) { + # First try to reassign ending spk of this scp. + if($scpidx < $numscps-1) { + $sz = @{$scparray[$scpidx]}; + if($sz > 0) { + $spk = $scparray[$scpidx]->[$sz-1]; + $count = $spk_count{$spk}; + $nutt1 = $scpcount[$scpidx]; + $nutt2 = $scpcount[$scpidx+1]; + if( abs( ($nutt2+$count) - ($nutt1-$count)) + < abs($nutt2 - $nutt1)) { # Would decrease + # size-diff by reassigning spk... + $scpcount[$scpidx+1] += $count; + $scpcount[$scpidx] -= $count; + pop @{$scparray[$scpidx]}; + unshift @{$scparray[$scpidx+1]}, $spk; + $changed = 1; + } + } + } + if($scpidx > 0 && @{$scparray[$scpidx]} > 0) { + $spk = $scparray[$scpidx]->[0]; + $count = $spk_count{$spk}; + $nutt1 = $scpcount[$scpidx-1]; + $nutt2 = $scpcount[$scpidx]; + if( abs( ($nutt2-$count) - ($nutt1+$count)) + < abs($nutt2 - $nutt1)) { # Would decrease + # size-diff by reassigning spk... + $scpcount[$scpidx-1] += $count; + $scpcount[$scpidx] -= $count; + shift @{$scparray[$scpidx]}; + push @{$scparray[$scpidx-1]}, $spk; + $changed = 1; + } + } + } + } + # Now print out the files... + for($scpidx = 0; $scpidx < $numscps; $scpidx++) { + $scpfile = $OUTPUTS[$scpidx]; + ($scpfile ne '-' ? open($f_fh, '>', $scpfile) + : open($f_fh, '>&', \*STDOUT)) || + die "$0: Could not open scp file $scpfile for writing: $!\n"; + $count = 0; + if(@{$scparray[$scpidx]} == 0) { + print STDERR "$0: eError: split_scp.pl producing empty .scp file " . + "$scpfile (too many splits and too few speakers?)\n"; + $error = 1; + } else { + foreach $spk ( @{$scparray[$scpidx]} ) { + print $f_fh @{$spk_data{$spk}}; + $count += $spk_count{$spk}; + } + $count == $scpcount[$scpidx] || die "Count mismatch [code error]"; + } + close($f_fh); + } +} else { + # This block is the "normal" case where there is no --utt2spk + # option and we just break into equal size chunks. + + open($i_fh, '<', $inscp) || die "$0: Error opening input scp file $inscp: $!\n"; + + $numscps = @OUTPUTS; # size of array. + @F = (); + while(<$i_fh>) { + push @F, $_; + } + $numlines = @F; + if($numlines == 0) { + print STDERR "$0: error: empty input scp file $inscp\n"; + $error = 1; + } + $linesperscp = int( $numlines / $numscps); # the "whole part".. + $linesperscp >= 1 || die "$0: You are splitting into too many pieces! [reduce \$nj ($numscps) to be smaller than the number of lines ($numlines) in $inscp]\n"; + $remainder = $numlines - ($linesperscp * $numscps); + ($remainder >= 0 && $remainder < $numlines) || die "bad remainder $remainder"; + # [just doing int() rounds down]. + $n = 0; + for($scpidx = 0; $scpidx < @OUTPUTS; $scpidx++) { + $scpfile = $OUTPUTS[$scpidx]; + ($scpfile ne '-' ? open($o_fh, '>', $scpfile) + : open($o_fh, '>&', \*STDOUT)) || + die "$0: Could not open scp file $scpfile for writing: $!\n"; + for($k = 0; $k < $linesperscp + ($scpidx < $remainder ? 1 : 0); $k++) { + print $o_fh $F[$n++]; + } + close($o_fh) || die "$0: Eror closing scp file $scpfile: $!\n"; + } + $n == $numlines || die "$n != $numlines [code error]"; +} + +exit ($error); diff --git a/egs/aishell/tranformer/utils/subset_data_dir_tr_cv.sh b/egs/aishell/tranformer/utils/subset_data_dir_tr_cv.sh new file mode 100755 index 000000000..e16cebdf1 --- /dev/null +++ b/egs/aishell/tranformer/utils/subset_data_dir_tr_cv.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +dev_num_utt=1000 + +echo "$0 $@" +. utils/parse_options.sh || exit 1; + +train_data=$1 +out_dir=$2 + +[ ! -f ${train_data}/wav.scp ] && echo "$0: no such file ${train_data}/wav.scp" && exit 1; +[ ! -f ${train_data}/text ] && echo "$0: no such file ${train_data}/text" && exit 1; + +mkdir -p ${out_dir}/train && mkdir -p ${out_dir}/dev + +cp ${train_data}/wav.scp ${out_dir}/train/wav.scp.bak +cp ${train_data}/text ${out_dir}/train/text.bak + +num_utt=$(wc -l <${out_dir}/train/wav.scp.bak) + +utils/shuffle_list.pl --srand 1 ${out_dir}/train/wav.scp.bak > ${out_dir}/train/wav.scp.shuf +head -n ${dev_num_utt} ${out_dir}/train/wav.scp.shuf > ${out_dir}/dev/wav.scp +tail -n $((${num_utt}-${dev_num_utt})) ${out_dir}/train/wav.scp.shuf > ${out_dir}/train/wav.scp + +utils/shuffle_list.pl --srand 1 ${out_dir}/train/text.bak > ${out_dir}/train/text.shuf +head -n ${dev_num_utt} ${out_dir}/train/text.shuf > ${out_dir}/dev/text +tail -n $((${num_utt}-${dev_num_utt})) ${out_dir}/train/text.shuf > ${out_dir}/train/text + +rm ${out_dir}/train/wav.scp.bak ${out_dir}/train/text.bak +rm ${out_dir}/train/wav.scp.shuf ${out_dir}/train/text.shuf diff --git a/egs/aishell/tranformer/utils/text2token.py b/egs/aishell/tranformer/utils/text2token.py new file mode 100755 index 000000000..56c39138f --- /dev/null +++ b/egs/aishell/tranformer/utils/text2token.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + + +import argparse +import codecs +import re +import sys + +is_python2 = sys.version_info[0] == 2 + + +def exist_or_not(i, match_pos): + start_pos = None + end_pos = None + for pos in match_pos: + if pos[0] <= i < pos[1]: + start_pos = pos[0] + end_pos = pos[1] + break + + return start_pos, end_pos + + +def get_parser(): + parser = argparse.ArgumentParser( + description="convert raw text to tokenized text", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--nchar", + "-n", + default=1, + type=int, + help="number of characters to split, i.e., \ + aabb -> a a b b with -n 1 and aa bb with -n 2", + ) + parser.add_argument( + "--skip-ncols", "-s", default=0, type=int, help="skip first n columns" + ) + parser.add_argument("--space", default="", type=str, help="space symbol") + parser.add_argument( + "--non-lang-syms", + "-l", + default=None, + type=str, + help="list of non-linguistic symobles, e.g., etc.", + ) + parser.add_argument("text", type=str, default=False, nargs="?", help="input text") + parser.add_argument( + "--trans_type", + "-t", + type=str, + default="char", + choices=["char", "phn"], + help="""Transcript type. char/phn. e.g., for TIMIT FADG0_SI1279 - + If trans_type is char, + read from SI1279.WRD file -> "bricks are an alternative" + Else if trans_type is phn, + read from SI1279.PHN file -> "sil b r ih sil k s aa r er n aa l + sil t er n ih sil t ih v sil" """, + ) + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + rs = [] + if args.non_lang_syms is not None: + with codecs.open(args.non_lang_syms, "r", encoding="utf-8") as f: + nls = [x.rstrip() for x in f.readlines()] + rs = [re.compile(re.escape(x)) for x in nls] + + if args.text: + f = codecs.open(args.text, encoding="utf-8") + else: + f = codecs.getreader("utf-8")(sys.stdin if is_python2 else sys.stdin.buffer) + + sys.stdout = codecs.getwriter("utf-8")( + sys.stdout if is_python2 else sys.stdout.buffer + ) + line = f.readline() + n = args.nchar + while line: + x = line.split() + print(" ".join(x[: args.skip_ncols]), end=" ") + a = " ".join(x[args.skip_ncols :]) + + # get all matched positions + match_pos = [] + for r in rs: + i = 0 + while i >= 0: + m = r.search(a, i) + if m: + match_pos.append([m.start(), m.end()]) + i = m.end() + else: + break + + if args.trans_type == "phn": + a = a.split(" ") + else: + if len(match_pos) > 0: + chars = [] + i = 0 + while i < len(a): + start_pos, end_pos = exist_or_not(i, match_pos) + if start_pos is not None: + chars.append(a[start_pos:end_pos]) + i = end_pos + else: + chars.append(a[i]) + i += 1 + a = chars + + a = [a[j : j + n] for j in range(0, len(a), n)] + + a_flat = [] + for z in a: + a_flat.append("".join(z)) + + a_chars = [z.replace(" ", args.space) for z in a_flat] + if args.trans_type == "phn": + a_chars = [z.replace("sil", args.space) for z in a_chars] + print(" ".join(a_chars)) + line = f.readline() + + +if __name__ == "__main__": + main() diff --git a/egs/aishell/tranformer/utils/text_tokenize.py b/egs/aishell/tranformer/utils/text_tokenize.py new file mode 100755 index 000000000..962ea11bc --- /dev/null +++ b/egs/aishell/tranformer/utils/text_tokenize.py @@ -0,0 +1,106 @@ +import re +import argparse + + +def load_dict(seg_file): + seg_dict = {} + with open(seg_file, 'r') as infile: + for line in infile: + s = line.strip().split() + key = s[0] + value = s[1:] + seg_dict[key] = " ".join(value) + return seg_dict + + +def forward_segment(text, dic): + word_list = [] + i = 0 + while i < len(text): + longest_word = text[i] + for j in range(i + 1, len(text) + 1): + word = text[i:j] + if word in dic: + if len(word) > len(longest_word): + longest_word = word + word_list.append(longest_word) + i += len(longest_word) + return word_list + + +def tokenize(txt, + seg_dict): + out_txt = "" + pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])") + for word in txt: + if pattern.match(word): + if word in seg_dict: + out_txt += seg_dict[word] + " " + else: + out_txt += "" + " " + else: + continue + return out_txt.strip() + + +def get_parser(): + parser = argparse.ArgumentParser( + description="text tokenize", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--text-file", + "-t", + default=False, + required=True, + type=str, + help="input text", + ) + parser.add_argument( + "--seg-file", + "-s", + default=False, + required=True, + type=str, + help="seg file", + ) + parser.add_argument( + "--txt-index", + "-i", + default=1, + required=True, + type=int, + help="txt index", + ) + parser.add_argument( + "--output-dir", + "-o", + default=False, + required=True, + type=str, + help="output dir", + ) + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args() + + txt_writer = open("{}/text.{}.txt".format(args.output_dir, args.txt_index), 'w') + shape_writer = open("{}/len.{}".format(args.output_dir, args.txt_index), 'w') + seg_dict = load_dict(args.seg_file) + with open(args.text_file, 'r') as infile: + for line in infile: + s = line.strip().split() + text_id = s[0] + text_list = forward_segment("".join(s[1:]).lower(), seg_dict) + text = tokenize(text_list, seg_dict) + lens = len(text.strip().split()) + txt_writer.write(text_id + " " + text + '\n') + shape_writer.write(text_id + " " + str(lens) + '\n') + + +if __name__ == '__main__': + main() + diff --git a/egs/aishell/tranformer/utils/text_tokenize.sh b/egs/aishell/tranformer/utils/text_tokenize.sh new file mode 100755 index 000000000..6b74fef80 --- /dev/null +++ b/egs/aishell/tranformer/utils/text_tokenize.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + + +# Begin configuration section. +nj=32 +cmd=utils/run.pl + +echo "$0 $@" + +. utils/parse_options.sh || exit 1; + +# tokenize configuration +text_dir=$1 +seg_file=$2 +logdir=$3 +output_dir=$4 + +txt_dir=${output_dir}/txt; mkdir -p ${output_dir}/txt +mkdir -p ${logdir} + +$cmd JOB=1:$nj $logdir/text_tokenize.JOB.log \ + python utils/text_tokenize.py -t ${text_dir}/txt/text.JOB.txt \ + -s ${seg_file} -i JOB -o ${txt_dir} \ + || exit 1; + +# concatenate the text files together. +for n in $(seq $nj); do + cat ${txt_dir}/text.$n.txt || exit 1 +done > ${output_dir}/text || exit 1 + +for n in $(seq $nj); do + cat ${txt_dir}/len.$n || exit 1 +done > ${output_dir}/text_shape || exit 1 + +echo "$0: Succeeded text tokenize" diff --git a/egs/aishell/tranformer/utils/textnorm_zh.py b/egs/aishell/tranformer/utils/textnorm_zh.py new file mode 100755 index 000000000..79feb83fd --- /dev/null +++ b/egs/aishell/tranformer/utils/textnorm_zh.py @@ -0,0 +1,834 @@ +#!/usr/bin/env python3 +# coding=utf-8 + +# Authors: +# 2019.5 Zhiyang Zhou (https://github.com/Joee1995/chn_text_norm.git) +# 2019.9 Jiayu DU +# +# requirements: +# - python 3.X +# notes: python 2.X WILL fail or produce misleading results + +import sys, os, argparse, codecs, string, re + +# ================================================================================ # +# basic constant +# ================================================================================ # +CHINESE_DIGIS = u'零一二三四五六七八九' +BIG_CHINESE_DIGIS_SIMPLIFIED = u'零壹贰叁肆伍陆柒捌玖' +BIG_CHINESE_DIGIS_TRADITIONAL = u'零壹貳參肆伍陸柒捌玖' +SMALLER_BIG_CHINESE_UNITS_SIMPLIFIED = u'十百千万' +SMALLER_BIG_CHINESE_UNITS_TRADITIONAL = u'拾佰仟萬' +LARGER_CHINESE_NUMERING_UNITS_SIMPLIFIED = u'亿兆京垓秭穰沟涧正载' +LARGER_CHINESE_NUMERING_UNITS_TRADITIONAL = u'億兆京垓秭穰溝澗正載' +SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED = u'十百千万' +SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL = u'拾佰仟萬' + +ZERO_ALT = u'〇' +ONE_ALT = u'幺' +TWO_ALTS = [u'两', u'兩'] + +POSITIVE = [u'正', u'正'] +NEGATIVE = [u'负', u'負'] +POINT = [u'点', u'點'] +# PLUS = [u'加', u'加'] +# SIL = [u'杠', u'槓'] + +FILLER_CHARS = ['呃', '啊'] +ER_WHITELIST = '(儿女|儿子|儿孙|女儿|儿媳|妻儿|' \ + '胎儿|婴儿|新生儿|婴幼儿|幼儿|少儿|小儿|儿歌|儿童|儿科|托儿所|孤儿|' \ + '儿戏|儿化|台儿庄|鹿儿岛|正儿八经|吊儿郎当|生儿育女|托儿带女|养儿防老|痴儿呆女|' \ + '佳儿佳妇|儿怜兽扰|儿无常父|儿不嫌母丑|儿行千里母担忧|儿大不由爷|苏乞儿)' + +# 中文数字系统类型 +NUMBERING_TYPES = ['low', 'mid', 'high'] + +CURRENCY_NAMES = '(人民币|美元|日元|英镑|欧元|马克|法郎|加拿大元|澳元|港币|先令|芬兰马克|爱尔兰镑|' \ + '里拉|荷兰盾|埃斯库多|比塞塔|印尼盾|林吉特|新西兰元|比索|卢布|新加坡元|韩元|泰铢)' +CURRENCY_UNITS = '((亿|千万|百万|万|千|百)|(亿|千万|百万|万|千|百|)元|(亿|千万|百万|万|千|百|)块|角|毛|分)' +COM_QUANTIFIERS = '(匹|张|座|回|场|尾|条|个|首|阙|阵|网|炮|顶|丘|棵|只|支|袭|辆|挑|担|颗|壳|窠|曲|墙|群|腔|' \ + '砣|座|客|贯|扎|捆|刀|令|打|手|罗|坡|山|岭|江|溪|钟|队|单|双|对|出|口|头|脚|板|跳|枝|件|贴|' \ + '针|线|管|名|位|身|堂|课|本|页|家|户|层|丝|毫|厘|分|钱|两|斤|担|铢|石|钧|锱|忽|(千|毫|微)克|' \ + '毫|厘|分|寸|尺|丈|里|寻|常|铺|程|(千|分|厘|毫|微)米|撮|勺|合|升|斗|石|盘|碗|碟|叠|桶|笼|盆|' \ + '盒|杯|钟|斛|锅|簋|篮|盘|桶|罐|瓶|壶|卮|盏|箩|箱|煲|啖|袋|钵|年|月|日|季|刻|时|周|天|秒|分|旬|' \ + '纪|岁|世|更|夜|春|夏|秋|冬|代|伏|辈|丸|泡|粒|颗|幢|堆|条|根|支|道|面|片|张|颗|块)' + +# punctuation information are based on Zhon project (https://github.com/tsroten/zhon.git) +CHINESE_PUNC_STOP = '！？｡。' +CHINESE_PUNC_NON_STOP = '＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏' +CHINESE_PUNC_LIST = CHINESE_PUNC_STOP + CHINESE_PUNC_NON_STOP + +# ================================================================================ # +# basic class +# ================================================================================ # +class ChineseChar(object): + """ + 中文字符 + 每个字符对应简体和繁体, + e.g. 简体 = '负', 繁体 = '負' + 转换时可转换为简体或繁体 + """ + + def __init__(self, simplified, traditional): + self.simplified = simplified + self.traditional = traditional + #self.__repr__ = self.__str__ + + def __str__(self): + return self.simplified or self.traditional or None + + def __repr__(self): + return self.__str__() + + +class ChineseNumberUnit(ChineseChar): + """ + 中文数字/数位字符 + 每个字符除繁简体外还有一个额外的大写字符 + e.g. '陆' 和 '陸' + """ + + def __init__(self, power, simplified, traditional, big_s, big_t): + super(ChineseNumberUnit, self).__init__(simplified, traditional) + self.power = power + self.big_s = big_s + self.big_t = big_t + + def __str__(self): + return '10^{}'.format(self.power) + + @classmethod + def create(cls, index, value, numbering_type=NUMBERING_TYPES[1], small_unit=False): + + if small_unit: + return ChineseNumberUnit(power=index + 1, + simplified=value[0], traditional=value[1], big_s=value[1], big_t=value[1]) + elif numbering_type == NUMBERING_TYPES[0]: + return ChineseNumberUnit(power=index + 8, + simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]) + elif numbering_type == NUMBERING_TYPES[1]: + return ChineseNumberUnit(power=(index + 2) * 4, + simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]) + elif numbering_type == NUMBERING_TYPES[2]: + return ChineseNumberUnit(power=pow(2, index + 3), + simplified=value[0], traditional=value[1], big_s=value[0], big_t=value[1]) + else: + raise ValueError( + 'Counting type should be in {0} ({1} provided).'.format(NUMBERING_TYPES, numbering_type)) + + +class ChineseNumberDigit(ChineseChar): + """ + 中文数字字符 + """ + + def __init__(self, value, simplified, traditional, big_s, big_t, alt_s=None, alt_t=None): + super(ChineseNumberDigit, self).__init__(simplified, traditional) + self.value = value + self.big_s = big_s + self.big_t = big_t + self.alt_s = alt_s + self.alt_t = alt_t + + def __str__(self): + return str(self.value) + + @classmethod + def create(cls, i, v): + return ChineseNumberDigit(i, v[0], v[1], v[2], v[3]) + + +class ChineseMath(ChineseChar): + """ + 中文数位字符 + """ + + def __init__(self, simplified, traditional, symbol, expression=None): + super(ChineseMath, self).__init__(simplified, traditional) + self.symbol = symbol + self.expression = expression + self.big_s = simplified + self.big_t = traditional + + +CC, CNU, CND, CM = ChineseChar, ChineseNumberUnit, ChineseNumberDigit, ChineseMath + + +class NumberSystem(object): + """ + 中文数字系统 + """ + pass + + +class MathSymbol(object): + """ + 用于中文数字系统的数学符号 (繁/简体), e.g. + positive = ['正', '正'] + negative = ['负', '負'] + point = ['点', '點'] + """ + + def __init__(self, positive, negative, point): + self.positive = positive + self.negative = negative + self.point = point + + def __iter__(self): + for v in self.__dict__.values(): + yield v + + +# class OtherSymbol(object): +# """ +# 其他符号 +# """ +# +# def __init__(self, sil): +# self.sil = sil +# +# def __iter__(self): +# for v in self.__dict__.values(): +# yield v + + +# ================================================================================ # +# basic utils +# ================================================================================ # +def create_system(numbering_type=NUMBERING_TYPES[1]): + """ + 根据数字系统类型返回创建相应的数字系统，默认为 mid + NUMBERING_TYPES = ['low', 'mid', 'high']: 中文数字系统类型 + low: '兆' = '亿' * '十' = $10^{9}$, '京' = '兆' * '十', etc. + mid: '兆' = '亿' * '万' = $10^{12}$, '京' = '兆' * '万', etc. + high: '兆' = '亿' * '亿' = $10^{16}$, '京' = '兆' * '兆', etc. + 返回对应的数字系统 + """ + + # chinese number units of '亿' and larger + all_larger_units = zip( + LARGER_CHINESE_NUMERING_UNITS_SIMPLIFIED, LARGER_CHINESE_NUMERING_UNITS_TRADITIONAL) + larger_units = [CNU.create(i, v, numbering_type, False) + for i, v in enumerate(all_larger_units)] + # chinese number units of '十, 百, 千, 万' + all_smaller_units = zip( + SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED, SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL) + smaller_units = [CNU.create(i, v, small_unit=True) + for i, v in enumerate(all_smaller_units)] + # digis + chinese_digis = zip(CHINESE_DIGIS, CHINESE_DIGIS, + BIG_CHINESE_DIGIS_SIMPLIFIED, BIG_CHINESE_DIGIS_TRADITIONAL) + digits = [CND.create(i, v) for i, v in enumerate(chinese_digis)] + digits[0].alt_s, digits[0].alt_t = ZERO_ALT, ZERO_ALT + digits[1].alt_s, digits[1].alt_t = ONE_ALT, ONE_ALT + digits[2].alt_s, digits[2].alt_t = TWO_ALTS[0], TWO_ALTS[1] + + # symbols + positive_cn = CM(POSITIVE[0], POSITIVE[1], '+', lambda x: x) + negative_cn = CM(NEGATIVE[0], NEGATIVE[1], '-', lambda x: -x) + point_cn = CM(POINT[0], POINT[1], '.', lambda x, + y: float(str(x) + '.' + str(y))) + # sil_cn = CM(SIL[0], SIL[1], '-', lambda x, y: float(str(x) + '-' + str(y))) + system = NumberSystem() + system.units = smaller_units + larger_units + system.digits = digits + system.math = MathSymbol(positive_cn, negative_cn, point_cn) + # system.symbols = OtherSymbol(sil_cn) + return system + + +def chn2num(chinese_string, numbering_type=NUMBERING_TYPES[1]): + + def get_symbol(char, system): + for u in system.units: + if char in [u.traditional, u.simplified, u.big_s, u.big_t]: + return u + for d in system.digits: + if char in [d.traditional, d.simplified, d.big_s, d.big_t, d.alt_s, d.alt_t]: + return d + for m in system.math: + if char in [m.traditional, m.simplified]: + return m + + def string2symbols(chinese_string, system): + int_string, dec_string = chinese_string, '' + for p in [system.math.point.simplified, system.math.point.traditional]: + if p in chinese_string: + int_string, dec_string = chinese_string.split(p) + break + return [get_symbol(c, system) for c in int_string], \ + [get_symbol(c, system) for c in dec_string] + + def correct_symbols(integer_symbols, system): + """ + 一百八 to 一百八十 + 一亿一千三百万 to 一亿 一千万 三百万 + """ + + if integer_symbols and isinstance(integer_symbols[0], CNU): + if integer_symbols[0].power == 1: + integer_symbols = [system.digits[1]] + integer_symbols + + if len(integer_symbols) > 1: + if isinstance(integer_symbols[-1], CND) and isinstance(integer_symbols[-2], CNU): + integer_symbols.append( + CNU(integer_symbols[-2].power - 1, None, None, None, None)) + + result = [] + unit_count = 0 + for s in integer_symbols: + if isinstance(s, CND): + result.append(s) + unit_count = 0 + elif isinstance(s, CNU): + current_unit = CNU(s.power, None, None, None, None) + unit_count += 1 + + if unit_count == 1: + result.append(current_unit) + elif unit_count > 1: + for i in range(len(result)): + if isinstance(result[-i - 1], CNU) and result[-i - 1].power < current_unit.power: + result[-i - 1] = CNU(result[-i - 1].power + + current_unit.power, None, None, None, None) + return result + + def compute_value(integer_symbols): + """ + Compute the value. + When current unit is larger than previous unit, current unit * all previous units will be used as all previous units. + e.g. '两千万' = 2000 * 10000 not 2000 + 10000 + """ + value = [0] + last_power = 0 + for s in integer_symbols: + if isinstance(s, CND): + value[-1] = s.value + elif isinstance(s, CNU): + value[-1] *= pow(10, s.power) + if s.power > last_power: + value[:-1] = list(map(lambda v: v * + pow(10, s.power), value[:-1])) + last_power = s.power + value.append(0) + return sum(value) + + system = create_system(numbering_type) + int_part, dec_part = string2symbols(chinese_string, system) + int_part = correct_symbols(int_part, system) + int_str = str(compute_value(int_part)) + dec_str = ''.join([str(d.value) for d in dec_part]) + if dec_part: + return '{0}.{1}'.format(int_str, dec_str) + else: + return int_str + + +def num2chn(number_string, numbering_type=NUMBERING_TYPES[1], big=False, + traditional=False, alt_zero=False, alt_one=False, alt_two=True, + use_zeros=True, use_units=True): + + def get_value(value_string, use_zeros=True): + + striped_string = value_string.lstrip('0') + + # record nothing if all zeros + if not striped_string: + return [] + + # record one digits + elif len(striped_string) == 1: + if use_zeros and len(value_string) != len(striped_string): + return [system.digits[0], system.digits[int(striped_string)]] + else: + return [system.digits[int(striped_string)]] + + # recursively record multiple digits + else: + result_unit = next(u for u in reversed( + system.units) if u.power < len(striped_string)) + result_string = value_string[:-result_unit.power] + return get_value(result_string) + [result_unit] + get_value(striped_string[-result_unit.power:]) + + system = create_system(numbering_type) + + int_dec = number_string.split('.') + if len(int_dec) == 1: + int_string = int_dec[0] + dec_string = "" + elif len(int_dec) == 2: + int_string = int_dec[0] + dec_string = int_dec[1] + else: + raise ValueError( + "invalid input num string with more than one dot: {}".format(number_string)) + + if use_units and len(int_string) > 1: + result_symbols = get_value(int_string) + else: + result_symbols = [system.digits[int(c)] for c in int_string] + dec_symbols = [system.digits[int(c)] for c in dec_string] + if dec_string: + result_symbols += [system.math.point] + dec_symbols + + if alt_two: + liang = CND(2, system.digits[2].alt_s, system.digits[2].alt_t, + system.digits[2].big_s, system.digits[2].big_t) + for i, v in enumerate(result_symbols): + if isinstance(v, CND) and v.value == 2: + next_symbol = result_symbols[i + + 1] if i < len(result_symbols) - 1 else None + previous_symbol = result_symbols[i - 1] if i > 0 else None + if isinstance(next_symbol, CNU) and isinstance(previous_symbol, (CNU, type(None))): + if next_symbol.power != 1 and ((previous_symbol is None) or (previous_symbol.power != 1)): + result_symbols[i] = liang + + # if big is True, '两' will not be used and `alt_two` has no impact on output + if big: + attr_name = 'big_' + if traditional: + attr_name += 't' + else: + attr_name += 's' + else: + if traditional: + attr_name = 'traditional' + else: + attr_name = 'simplified' + + result = ''.join([getattr(s, attr_name) for s in result_symbols]) + + # if not use_zeros: + # result = result.strip(getattr(system.digits[0], attr_name)) + + if alt_zero: + result = result.replace( + getattr(system.digits[0], attr_name), system.digits[0].alt_s) + + if alt_one: + result = result.replace( + getattr(system.digits[1], attr_name), system.digits[1].alt_s) + + for i, p in enumerate(POINT): + if result.startswith(p): + return CHINESE_DIGIS[0] + result + + # ^10, 11, .., 19 + if len(result) >= 2 and result[1] in [SMALLER_CHINESE_NUMERING_UNITS_SIMPLIFIED[0], + SMALLER_CHINESE_NUMERING_UNITS_TRADITIONAL[0]] and \ + result[0] in [CHINESE_DIGIS[1], BIG_CHINESE_DIGIS_SIMPLIFIED[1], BIG_CHINESE_DIGIS_TRADITIONAL[1]]: + result = result[1:] + + return result + + +# ================================================================================ # +# different types of rewriters +# ================================================================================ # +class Cardinal: + """ + CARDINAL类 + """ + + def __init__(self, cardinal=None, chntext=None): + self.cardinal = cardinal + self.chntext = chntext + + def chntext2cardinal(self): + return chn2num(self.chntext) + + def cardinal2chntext(self): + return num2chn(self.cardinal) + +class Digit: + """ + DIGIT类 + """ + + def __init__(self, digit=None, chntext=None): + self.digit = digit + self.chntext = chntext + + # def chntext2digit(self): + # return chn2num(self.chntext) + + def digit2chntext(self): + return num2chn(self.digit, alt_two=False, use_units=False) + + +class TelePhone: + """ + TELEPHONE类 + """ + + def __init__(self, telephone=None, raw_chntext=None, chntext=None): + self.telephone = telephone + self.raw_chntext = raw_chntext + self.chntext = chntext + + # def chntext2telephone(self): + # sil_parts = self.raw_chntext.split('') + # self.telephone = '-'.join([ + # str(chn2num(p)) for p in sil_parts + # ]) + # return self.telephone + + def telephone2chntext(self, fixed=False): + + if fixed: + sil_parts = self.telephone.split('-') + self.raw_chntext = ''.join([ + num2chn(part, alt_two=False, use_units=False) for part in sil_parts + ]) + self.chntext = self.raw_chntext.replace('', '') + else: + sp_parts = self.telephone.strip('+').split() + self.raw_chntext = ''.join([ + num2chn(part, alt_two=False, use_units=False) for part in sp_parts + ]) + self.chntext = self.raw_chntext.replace('', '') + return self.chntext + + +class Fraction: + """ + FRACTION类 + """ + + def __init__(self, fraction=None, chntext=None): + self.fraction = fraction + self.chntext = chntext + + def chntext2fraction(self): + denominator, numerator = self.chntext.split('分之') + return chn2num(numerator) + '/' + chn2num(denominator) + + def fraction2chntext(self): + numerator, denominator = self.fraction.split('/') + return num2chn(denominator) + '分之' + num2chn(numerator) + + +class Date: + """ + DATE类 + """ + + def __init__(self, date=None, chntext=None): + self.date = date + self.chntext = chntext + + # def chntext2date(self): + # chntext = self.chntext + # try: + # year, other = chntext.strip().split('年', maxsplit=1) + # year = Digit(chntext=year).digit2chntext() + '年' + # except ValueError: + # other = chntext + # year = '' + # if other: + # try: + # month, day = other.strip().split('月', maxsplit=1) + # month = Cardinal(chntext=month).chntext2cardinal() + '月' + # except ValueError: + # day = chntext + # month = '' + # if day: + # day = Cardinal(chntext=day[:-1]).chntext2cardinal() + day[-1] + # else: + # month = '' + # day = '' + # date = year + month + day + # self.date = date + # return self.date + + def date2chntext(self): + date = self.date + try: + year, other = date.strip().split('年', 1) + year = Digit(digit=year).digit2chntext() + '年' + except ValueError: + other = date + year = '' + if other: + try: + month, day = other.strip().split('月', 1) + month = Cardinal(cardinal=month).cardinal2chntext() + '月' + except ValueError: + day = date + month = '' + if day: + day = Cardinal(cardinal=day[:-1]).cardinal2chntext() + day[-1] + else: + month = '' + day = '' + chntext = year + month + day + self.chntext = chntext + return self.chntext + + +class Money: + """ + MONEY类 + """ + + def __init__(self, money=None, chntext=None): + self.money = money + self.chntext = chntext + + # def chntext2money(self): + # return self.money + + def money2chntext(self): + money = self.money + pattern = re.compile(r'(\d+(\.\d+)?)') + matchers = pattern.findall(money) + if matchers: + for matcher in matchers: + money = money.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext()) + self.chntext = money + return self.chntext + + +class Percentage: + """ + PERCENTAGE类 + """ + + def __init__(self, percentage=None, chntext=None): + self.percentage = percentage + self.chntext = chntext + + def chntext2percentage(self): + return chn2num(self.chntext.strip().strip('百分之')) + '%' + + def percentage2chntext(self): + return '百分之' + num2chn(self.percentage.strip().strip('%')) + + +def remove_erhua(text, er_whitelist): + """ + 去除儿化音词中的儿: + 他女儿在那边儿 -> 他女儿在那边 + """ + + er_pattern = re.compile(er_whitelist) + new_str='' + while re.search('儿',text): + a = re.search('儿',text).span() + remove_er_flag = 0 + + if er_pattern.search(text): + b = er_pattern.search(text).span() + if b[0] <= a[0]: + remove_er_flag = 1 + + if remove_er_flag == 0 : + new_str = new_str + text[0:a[0]] + text = text[a[1]:] + else: + new_str = new_str + text[0:b[1]] + text = text[b[1]:] + + text = new_str + text + return text + +# ================================================================================ # +# NSW Normalizer +# ================================================================================ # +class NSWNormalizer: + def __init__(self, raw_text): + self.raw_text = '^' + raw_text + '$' + self.norm_text = '' + + def _particular(self): + text = self.norm_text + pattern = re.compile(r"(([a-zA-Z]+)二([a-zA-Z]+))") + matchers = pattern.findall(text) + if matchers: + # print('particular') + for matcher in matchers: + text = text.replace(matcher[0], matcher[1]+'2'+matcher[2], 1) + self.norm_text = text + return self.norm_text + + def normalize(self): + text = self.raw_text + + # 规范化日期 + pattern = re.compile(r"\D+((([089]\d|(19|20)\d{2})年)?(\d{1,2}月(\d{1,2}[日号])?)?)") + matchers = pattern.findall(text) + if matchers: + #print('date') + for matcher in matchers: + text = text.replace(matcher[0], Date(date=matcher[0]).date2chntext(), 1) + + # 规范化金钱 + pattern = re.compile(r"\D+((\d+(\.\d+)?)[多余几]?" + CURRENCY_UNITS + r"(\d" + CURRENCY_UNITS + r"?)?)") + matchers = pattern.findall(text) + if matchers: + #print('money') + for matcher in matchers: + text = text.replace(matcher[0], Money(money=matcher[0]).money2chntext(), 1) + + # 规范化固话/手机号码 + # 手机 + # http://www.jihaoba.com/news/show/13680 + # 移动：139、138、137、136、135、134、159、158、157、150、151、152、188、187、182、183、184、178、198 + # 联通：130、131、132、156、155、186、185、176 + # 电信：133、153、189、180、181、177 + pattern = re.compile(r"\D((\+?86 ?)?1([38]\d|5[0-35-9]|7[678]|9[89])\d{8})\D") + matchers = pattern.findall(text) + if matchers: + #print('telephone') + for matcher in matchers: + text = text.replace(matcher[0], TelePhone(telephone=matcher[0]).telephone2chntext(), 1) + # 固话 + pattern = re.compile(r"\D((0(10|2[1-3]|[3-9]\d{2})-?)?[1-9]\d{6,7})\D") + matchers = pattern.findall(text) + if matchers: + # print('fixed telephone') + for matcher in matchers: + text = text.replace(matcher[0], TelePhone(telephone=matcher[0]).telephone2chntext(fixed=True), 1) + + # 规范化分数 + pattern = re.compile(r"(\d+/\d+)") + matchers = pattern.findall(text) + if matchers: + #print('fraction') + for matcher in matchers: + text = text.replace(matcher, Fraction(fraction=matcher).fraction2chntext(), 1) + + # 规范化百分数 + text = text.replace('％', '%') + pattern = re.compile(r"(\d+(\.\d+)?%)") + matchers = pattern.findall(text) + if matchers: + #print('percentage') + for matcher in matchers: + text = text.replace(matcher[0], Percentage(percentage=matcher[0]).percentage2chntext(), 1) + + # 规范化纯数+量词 + pattern = re.compile(r"(\d+(\.\d+)?)[多余几]?" + COM_QUANTIFIERS) + matchers = pattern.findall(text) + if matchers: + #print('cardinal+quantifier') + for matcher in matchers: + text = text.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext(), 1) + + # 规范化数字编号 + pattern = re.compile(r"(\d{4,32})") + matchers = pattern.findall(text) + if matchers: + #print('digit') + for matcher in matchers: + text = text.replace(matcher, Digit(digit=matcher).digit2chntext(), 1) + + # 规范化纯数 + pattern = re.compile(r"(\d+(\.\d+)?)") + matchers = pattern.findall(text) + if matchers: + #print('cardinal') + for matcher in matchers: + text = text.replace(matcher[0], Cardinal(cardinal=matcher[0]).cardinal2chntext(), 1) + + self.norm_text = text + self._particular() + + return self.norm_text.lstrip('^').rstrip('$') + + +def nsw_test_case(raw_text): + print('I:' + raw_text) + print('O:' + NSWNormalizer(raw_text).normalize()) + print('') + + +def nsw_test(): + nsw_test_case('固话：0595-23865596或23880880。') + nsw_test_case('固话：0595-23865596或23880880。') + nsw_test_case('手机：+86 19859213959或15659451527。') + nsw_test_case('分数：32477/76391。') + nsw_test_case('百分数：80.03%。') + nsw_test_case('编号：31520181154418。') + nsw_test_case('纯数：2983.07克或12345.60米。') + nsw_test_case('日期：1999年2月20日或09年3月15号。') + nsw_test_case('金钱：12块5，34.5元，20.1万') + nsw_test_case('特殊：O2O或B2C。') + nsw_test_case('3456万吨') + nsw_test_case('2938个') + nsw_test_case('938') + nsw_test_case('今天吃了115个小笼包231个馒头') + nsw_test_case('有62％的概率') + + +if __name__ == '__main__': + #nsw_test() + + p = argparse.ArgumentParser() + p.add_argument('ifile', help='input filename, assume utf-8 encoding') + p.add_argument('ofile', help='output filename') + p.add_argument('--to_upper', action='store_true', help='convert to upper case') + p.add_argument('--to_lower', action='store_true', help='convert to lower case') + p.add_argument('--has_key', action='store_true', help="input text has Kaldi's key as first field.") + p.add_argument('--remove_fillers', type=bool, default=True, help='remove filler chars such as "呃, 啊"') + p.add_argument('--remove_erhua', type=bool, default=True, help='remove erhua chars such as "这儿"') + p.add_argument('--log_interval', type=int, default=10000, help='log interval in number of processed lines') + args = p.parse_args() + + ifile = codecs.open(args.ifile, 'r', 'utf8') + ofile = codecs.open(args.ofile, 'w+', 'utf8') + + n = 0 + for l in ifile: + key = '' + text = '' + if args.has_key: + cols = l.split(maxsplit=1) + key = cols[0] + if len(cols) == 2: + text = cols[1].strip() + else: + text = '' + else: + text = l.strip() + + # cases + if args.to_upper and args.to_lower: + sys.stderr.write('text norm: to_upper OR to_lower?') + exit(1) + if args.to_upper: + text = text.upper() + if args.to_lower: + text = text.lower() + + # Filler chars removal + if args.remove_fillers: + for ch in FILLER_CHARS: + text = text.replace(ch, '') + + if args.remove_erhua: + text = remove_erhua(text, ER_WHITELIST) + + # NSW(Non-Standard-Word) normalization + text = NSWNormalizer(text).normalize() + + # Punctuations removal + old_chars = CHINESE_PUNC_LIST + string.punctuation # includes all CN and EN punctuations + new_chars = ' ' * len(old_chars) + del_chars = '' + text = text.translate(str.maketrans(old_chars, new_chars, del_chars)) + + # + if args.has_key: + ofile.write(key + '\t' + text + '\n') + else: + ofile.write(text + '\n') + + n += 1 + if n % args.log_interval == 0: + sys.stderr.write("text norm: {} lines done.\n".format(n)) + + sys.stderr.write("text norm: {} lines done in total.\n".format(n)) + + ifile.close() + ofile.close() diff --git a/egs_modelscope/aishell/paraformer/README.md b/egs_modelscope/aishell/paraformer/README.md new file mode 100644 index 000000000..48a5621b1 --- /dev/null +++ b/egs_modelscope/aishell/paraformer/README.md @@ -0,0 +1,38 @@ +# ModelScope: Paraformer-large Model + +## Highlight + +### ModelScope: Paraformer-Large Model +- Fast: Non-autoregressive (NAR) model, the Paraformer can achieve comparable performance to the state-of-the-art AR transformer, with more than 10x speedup. +- Accurate: SOTA in a lot of public ASR tasks, with a very significant relative improvement, capable of industrial implementation. +- Convenient: Quickly and easily download Paraformer-large from Modelscope for finetuning and inference. + - Support finetuning and inference on AISHELL-1 and AISHELL-2. + - Support inference on AISHELL-1, AISHELL-2, Wenetspeech, SpeechIO and other audio. + +## How to finetune and infer using a pretrained ModelScope Paraformer-large Model + +### Finetune +- Modify finetune training related parameters in `conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml` +- Setting parameters in `paraformer_large_finetune.sh` + - data_aishell: please set the aishell data path + - tag: exp tag + - init_model_name: speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning +- Then you can run the pipeline to finetune with our model download from modelscope and infer after finetune: +```sh + sh ./paraformer_large_finetune.sh +``` + +### Inference + +Or you can download the model from ModelScope for inference directly. + +- Setting parameters in `paraformer_large_infer.sh` + - ori_data: please set the aishell raw data path + - data_dir: data output dictionary + - exp_dir: the result path + - model_name: speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope + - test_sets: please set the testsets name +- Then you can run the pipeline to infer with: +```sh + sh ./paraformer_large_infer.sh +``` diff --git a/egs_modelscope/aishell/paraformer/RESULTS.md b/egs_modelscope/aishell/paraformer/RESULTS.md new file mode 100644 index 000000000..516750453 --- /dev/null +++ b/egs_modelscope/aishell/paraformer/RESULTS.md @@ -0,0 +1,24 @@ +# Paraformer-Large +- Model link: +- Model size: 220M +- Train config: conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml + +# Environments +- date: `Tue Nov 22 18:48:39 CST 2022` +- python version: `3.7.12` +- FunASR version: `0.1.0` +- pytorch version: `pytorch 1.7.0` +- Git hash: `` +- Commit date: `` + +# Beachmark Results + +## AISHELL-1 +- Decode config: conf/decode_asr_transformer_noctc_1best.yaml + - Decode without CTC + - Decode without LM + +| testset | CER(%)| +|:---------:|:-----:| +| dev | 1.75 | +| test | 1.95 | diff --git a/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml b/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml new file mode 100644 index 000000000..22f02d913 --- /dev/null +++ b/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml @@ -0,0 +1,6 @@ +beam_size: 10 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.0 +lm_weight: 0.15 diff --git a/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml b/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml new file mode 100644 index 000000000..e6231927c --- /dev/null +++ b/egs_modelscope/aishell/paraformer/conf/decode_asr_transformer_noctc_1best.yaml @@ -0,0 +1,6 @@ +beam_size: 1 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.0 +lm_weight: 0.0 diff --git a/egs_modelscope/aishell/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml b/egs_modelscope/aishell/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml new file mode 100644 index 000000000..e9210f373 --- /dev/null +++ b/egs_modelscope/aishell/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml @@ -0,0 +1,91 @@ +# network architecture +# encoder related +encoder_conf: + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.1 + +# decoder related +decoder_conf: + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.1 + src_attention_dropout_rate: 0.1 + +predictor_conf: + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + +# hybrid CTC/attention +model_conf: + ctc_weight: 0.0 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: true + predictor_weight: 1.0 + predictor_bias: 1 + sampling_ratio: 0.75 + +# minibatch related +# dataset_type: small +batch_type: length +batch_bins: 2000 +num_workers: 16 +# dataset_type: large +dataset_conf: + filter_conf: + min_length: 10 + max_length: 250 + min_token_length: 1 + max_token_length: 200 + shuffle: true + shuffle_conf: + shuffle_size: 10240 + sort_size: 500 + batch_conf: + batch_type: 'token' + batch_size: 6000 + num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 20 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug_lfr +specaug_conf: + apply_time_warp: false + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + lfr_rate: 6 + num_freq_mask: 1 + apply_time_mask: true + time_mask_width_range: + - 0 + - 12 + num_time_mask: 1 + +unused_parameters: true +log_interval: 50 +normalize: None +split_with_space: true diff --git a/egs_modelscope/aishell/paraformer/local/aishell_data_prep.sh b/egs_modelscope/aishell/paraformer/local/aishell_data_prep.sh new file mode 100755 index 000000000..83f489b3c --- /dev/null +++ b/egs_modelscope/aishell/paraformer/local/aishell_data_prep.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# Copyright 2017 Xingyu Na +# Apache 2.0 + +#. ./path.sh || exit 1; + +if [ $# != 3 ]; then + echo "Usage: $0 " + echo " $0 /export/a05/xna/data/data_aishell/wav /export/a05/xna/data/data_aishell/transcript data" + exit 1; +fi + +aishell_audio_dir=$1 +aishell_text=$2/aishell_transcript_v0.8.txt +output_dir=$3 + +train_dir=$output_dir/data/local/train +dev_dir=$output_dir/data/local/dev +test_dir=$output_dir/data/local/test +tmp_dir=$output_dir/data/local/tmp + +mkdir -p $train_dir +mkdir -p $dev_dir +mkdir -p $test_dir +mkdir -p $tmp_dir + +# data directory check +if [ ! -d $aishell_audio_dir ] || [ ! -f $aishell_text ]; then + echo "Error: $0 requires two directory arguments" + exit 1; +fi + +# find wav audio file for train, dev and test resp. +find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist +n=`cat $tmp_dir/wav.flist | wc -l` +[ $n -ne 141925 ] && \ + echo Warning: expected 141925 data data files, found $n + +grep -i "wav/train" $tmp_dir/wav.flist > $train_dir/wav.flist || exit 1; +grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1; +grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1; + +rm -r $tmp_dir + +# Transcriptions preparation +for dir in $train_dir $dev_dir $test_dir; do + echo Preparing $dir transcriptions + sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list + paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all + utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt + awk '{print $1}' $dir/transcripts.txt > $dir/utt.list + utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp + sort -u $dir/transcripts.txt > $dir/text +done + +mkdir -p $output_dir/data/train $output_dir/data/dev $output_dir/data/test + +for f in wav.scp text; do + cp $train_dir/$f $output_dir/data/train/$f || exit 1; + cp $dev_dir/$f $output_dir/data/dev/$f || exit 1; + cp $test_dir/$f $output_dir/data/test/$f || exit 1; +done + +echo "$0: AISHELL data preparation succeeded" +exit 0; diff --git a/egs_modelscope/aishell/paraformer/modelscope_utils b/egs_modelscope/aishell/paraformer/modelscope_utils new file mode 120000 index 000000000..fc97768c8 --- /dev/null +++ b/egs_modelscope/aishell/paraformer/modelscope_utils @@ -0,0 +1 @@ +../../common/modelscope_utils \ No newline at end of file diff --git a/egs_modelscope/aishell/paraformer/paraformer_large_finetune.sh b/egs_modelscope/aishell/paraformer/paraformer_large_finetune.sh new file mode 100755 index 000000000..a68338fb9 --- /dev/null +++ b/egs_modelscope/aishell/paraformer/paraformer_large_finetune.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; + +# machines configuration +CUDA_VISIBLE_DEVICES="0,1" # set gpus, e.g., CUDA_VISIBLE_DEVICES="0,1" +gpu_num=2 +count=1 +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding +njob=4 # the number of jobs for each gpu +train_cmd=utils/run.pl + +# general configuration +feats_dir="." #feature output dictionary, for large data +exp_dir="." +lang=zh +dumpdir=dump/fbank +feats_type=fbank +token_type=char +scp=feats.scp +type=kaldi_ark +stage=0 +stop_stage=4 + +# feature configuration +feats_dim=560 +sample_frequency=16000 +nj=32 +speed_perturb="1.0" +lfr=True +lfr_m=7 +lfr_n=6 + +init_model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning +cmvn_file=init_model/${init_model_name}/am.mvn +seg_file=init_model/${init_model_name}/seg_dict +vocab=init_model/${init_model_name}/tokens.txt + +# data +data_aishell= + +# exp tag +tag="" + +# Set bash to 'debug' mode, it will exit on : +# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands', +set -e +set -u +set -o pipefail + +train_set=train +valid_set=dev +test_sets="dev test" + +asr_config=conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml +init_param="init_model/${init_model_name}/${init_model_name}" + +inference_config=conf/decode_asr_transformer_noctc_1best.yaml +inference_asr_model=valid.acc.ave_10best.pth + +. utils/parse_options.sh || exit 1; + +# download model from modelscope +python modelscope_utils/download_model.py --model_name ${init_model_name} + +if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${init_model_name} ]; then + echo "${HOME}/.cache/modelscope/hub/damo/${init_model_name} must exist" + exit 1 +else + if [ -d init_model/${init_model_name} ]; then + echo "init_model/${init_model_name} is already exists. if you want to decode again, please delete init_model/${init_model_name} first." + else + mkdir -p init_model/${init_model_name} + cp -r ${HOME}/.cache/modelscope/hub/damo/${init_model_name}/* init_model/${init_model_name} + fi +fi + +model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}" + +# you can set gpu num for decoding here +gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +inference_nj=$[${ngpu}*${njob}] + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + echo "stage 0: Data preparation" + # Data preparation + local/aishell_data_prep.sh ${data_aishell}/data_aishell/wav ${data_aishell}/data_aishell/transcript ${feats_dir} + for x in train dev test; do + cp ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org + paste -d " " <(cut -f 1 -d" " ${feats_dir}/data/${x}/text.org) <(cut -f 2- -d" " ${feats_dir}/data/${x}/text.org | tr -d " ") \ + > ${feats_dir}/data/${x}/text + rm ${feats_dir}/data/${x}/text.org + done +fi + +feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir} +feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir} +feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir} +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + echo "Feature Generation" + # compute fbank features + fbankdir=${feats_dir}/fbank + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \ + ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train + utils/fix_data_feat.sh ${fbankdir}/train + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/dev ${exp_dir}/exp/make_fbank/dev ${fbankdir}/dev + utils/fix_data_feat.sh ${fbankdir}/dev + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/test ${exp_dir}/exp/make_fbank/test ${fbankdir}/test + utils/fix_data_feat.sh ${fbankdir}/test + + echo "apply low_frame_rate and cmvn" + [ ! -f ${cmvn_file} ] && echo "$0: cmvn file is required" && exit 1; + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/train ${cmvn_file} ${exp_dir}/exp/make_fbank/train ${feat_train_dir} + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/dev ${cmvn_file} ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir} + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/test ${cmvn_file} ${exp_dir}/exp/make_fbank/test ${feat_test_dir} + + echo "Text Tokenize" + # 我爱reading->我 爱 read@@ ing + utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/train ${seg_file} ${feat_train_dir}/log ${feat_train_dir} + utils/fix_data_feat.sh ${feat_train_dir} + utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/dev ${seg_file} ${feat_dev_dir}/log ${feat_dev_dir} + utils/fix_data_feat.sh ${feat_dev_dir} + cp ${fbankdir}/test/text ${feat_test_dir} +fi + +token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt +echo "dictionary: ${token_list}" +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + echo "stage 2: Dictionary Preparation" + mkdir -p ${feats_dir}/data/${lang}_token_list/char/ + cp $vocab ${token_list} + + vocab_size=$(wc -l <${token_list}) + awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char + awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev + cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/train + cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev +fi + +# Training Stage +world_size=$gpu_num # run on one machine +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # update asr train config.yaml + python modelscope_utils/update_config.py --modelscope_config init_model/${init_model_name}/asr_train_config.yaml --finetune_config ${asr_config} --output_config init_model/${init_model_name}/asr_finetune_config.yaml + finetune_config=init_model/${init_model_name}/asr_finetune_config.yaml + + mkdir -p ${exp_dir}/exp/${model_dir} + mkdir -p ${exp_dir}/exp/${model_dir}/log + INIT_FILE=$exp_dir/ddp_init + if [ -f $INIT_FILE ];then + rm -f $INIT_FILE + fi + init_method=file://$(readlink -f $INIT_FILE) + echo "$0: init method is $init_method" + for ((i = 0; i < $gpu_num; ++i)); do + { + rank=$i + local_rank=$i + gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1]) + asr_train_paraformer.py \ + --gpu_id $gpu_id \ + --use_preprocessor true \ + --token_type $token_type \ + --token_list $token_list \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \ + --resume true \ + --output_dir ${exp_dir}/exp/${model_dir} \ + --init_param $init_param \ + --config $finetune_config \ + --input_size $feats_dim \ + --ngpu $gpu_num \ + --num_worker_count $count \ + --multiprocessing_distributed true \ + --dist_init_method $init_method \ + --dist_world_size $world_size \ + --dist_rank $rank \ + --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1 + } & + done + wait +fi + +# Testing Stage +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + ./utils/easy_asr_infer.sh \ + --lang zh \ + --datadir ${feats_dir} \ + --feats_type ${feats_type} \ + --feats_dim ${feats_dim} \ + --token_type ${token_type} \ + --gpu_inference ${gpu_inference} \ + --inference_config "${inference_config}" \ + --test_sets "${test_sets}" \ + --token_list $token_list \ + --asr_exp ${exp_dir}/exp/${model_dir} \ + --stage 12 \ + --stop_stage 12 \ + --scp $scp \ + --text text \ + --inference_nj $inference_nj \ + --njob $njob \ + --inference_asr_model $inference_asr_model \ + --gpuid_list $gpuid_list \ + --mode paraformer +fi + diff --git a/egs_modelscope/aishell/paraformer/paraformer_large_infer.sh b/egs_modelscope/aishell/paraformer/paraformer_large_infer.sh new file mode 100755 index 000000000..8e2c8f33d --- /dev/null +++ b/egs_modelscope/aishell/paraformer/paraformer_large_infer.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +ori_data= +data_dir= +exp_dir= +model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch +inference_nj=32 +gpuid_list="0,1" # set gpus, e.g., gpuid_list="0,1" +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +njob=4 # the number of jobs for each gpu +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] +else + inference_nj=$njob +fi + +# LM configs +use_lm=false +beam_size=1 +lm_weight=0.0 + +test_sets="dev test" + +. utils/parse_options.sh + +aishell_audio_dir=$ori_data/data_aishell/wav +aishell_text=$ori_data/data_aishell/transcript/aishell_transcript_v0.8.txt +dev_dir=${data_dir}/aishell/dev +test_dir=${data_dir}/aishell/test +tmp_dir=${data_dir}/aishell/tmp + +mkdir -p ${dev_dir} +mkdir -p ${test_dir} +mkdir -p ${tmp_dir} + +find $aishell_audio_dir -iname "*.wav" > $tmp_dir/wav.flist +grep -i "wav/dev" $tmp_dir/wav.flist > $dev_dir/wav.flist || exit 1; +grep -i "wav/test" $tmp_dir/wav.flist > $test_dir/wav.flist || exit 1; + +rm -r $tmp_dir + +for dir in $dev_dir $test_dir; do + sed -e 's/\.wav//' $dir/wav.flist | awk -F '/' '{print $NF}' > $dir/utt.list + paste -d' ' $dir/utt.list $dir/wav.flist > $dir/wav.scp_all + utils/filter_scp.pl -f 1 $dir/utt.list $aishell_text > $dir/transcripts.txt + awk '{print $1}' $dir/transcripts.txt > $dir/utt.list + utils/filter_scp.pl -f 1 $dir/utt.list $dir/wav.scp_all | sort -u > $dir/wav.scp + sort -u $dir/transcripts.txt > $dir/text +done + +mkdir -p ${exp_dir}/aishell + +modelscope_utils/modelscope_infer.sh \ + --data_dir ${data_dir}/aishell \ + --exp_dir ${exp_dir}/aishell \ + --test_sets "${test_sets}" \ + --model_name ${model_name} \ + --inference_nj ${inference_nj} \ + --gpuid_list ${gpuid_list} \ + --njob ${njob} \ + --gpu_inference ${gpu_inference} \ + --use_lm ${use_lm} \ + --beam_size ${beam_size} \ + --lm_weight ${lm_weight} diff --git a/egs_modelscope/aishell/paraformer/path.sh b/egs_modelscope/aishell/paraformer/path.sh new file mode 100755 index 000000000..7972642d0 --- /dev/null +++ b/egs_modelscope/aishell/paraformer/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs_modelscope/aishell/paraformer/utils b/egs_modelscope/aishell/paraformer/utils new file mode 120000 index 000000000..37d976175 --- /dev/null +++ b/egs_modelscope/aishell/paraformer/utils @@ -0,0 +1 @@ +../../../egs/aishell/tranformer/utils/ \ No newline at end of file diff --git a/egs_modelscope/aishell2/paraformer/README.md b/egs_modelscope/aishell2/paraformer/README.md new file mode 100644 index 000000000..46bd3ad71 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/README.md @@ -0,0 +1,39 @@ +# ModelScope: Paraformer-large Model + +## Highlight + +### ModelScope: Paraformer-Large Model +- Fast: Non-autoregressive (NAR) model, the Paraformer can achieve comparable performance to the state-of-the-art AR transformer, with more than 10x speedup. +- Accurate: SOTA in a lot of public ASR tasks, with a very significant relative improvement, capable of industrial implementation. +- Convenient: Quickly and easily download Paraformer-large from Modelscope for finetuning and inference. + - Support finetuning and inference on AISHELL-1 and AISHELL-2. + - Support inference on AISHELL-1, AISHELL-2, Wenetspeech, SpeechIO and other audio. + +## How to finetune and infer using a pretrained ModelScope Paraformer-large Model + +### Finetune +- Modify finetune training related parameters in `conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml` +- Setting parameters in `paraformer_large_finetune.sh` + - tr_dir: please set the aishell2 train data path + - dev_tst_dir: please set the aishell2 dev/test data path + - tag: exp tag + - init_model_name: speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning +- Then you can run the pipeline to finetune with our model download from modelscope and infer after finetune: +```sh + sh ./paraformer_large_finetune.sh +``` + +### Inference + +Or you can download the model from ModelScope for inference directly. + +- Setting parameters in `paraformer_large_infer.sh` + - ori_data: please set the aishell2 dev/test raw data path + - data_dir: data output dictionary + - exp_dir: the result path + - model_name: speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope + - test_sets: please set the testsets name +- Then you can run the pipeline to infer with: +```sh + sh ./paraformer_large_infer.sh +``` diff --git a/egs_modelscope/aishell2/paraformer/RESULTS.md b/egs_modelscope/aishell2/paraformer/RESULTS.md new file mode 100644 index 000000000..a265a749c --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/RESULTS.md @@ -0,0 +1,26 @@ +# Paraformer-Large +- Model link: +- Model size: 220M +- Train config: conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml + +# Environments +- date: `Tue Nov 22 18:48:39 CST 2022` +- python version: `3.7.12` +- FunASR version: `0.1.0` +- pytorch version: `pytorch 1.7.0` +- Git hash: `` +- Commit date: `` + +# Beachmark Results + +## AISHELL-2 +- Decode config: conf/decode_asr_transformer_noctc_1best.yaml + - Decode without CTC + - Decode without LM + +| testset | CER(%)| +|:------------:|:-----:| +| dev_ios | 2.80 | +| test_android | 3.13 | +| test_ios | 2.85 | +| test_mic | 3.06 | diff --git a/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml b/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml new file mode 100644 index 000000000..22f02d913 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml @@ -0,0 +1,6 @@ +beam_size: 10 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.0 +lm_weight: 0.15 diff --git a/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml b/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml new file mode 100644 index 000000000..e6231927c --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/conf/decode_asr_transformer_noctc_1best.yaml @@ -0,0 +1,6 @@ +beam_size: 1 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.0 +lm_weight: 0.0 diff --git a/egs_modelscope/aishell2/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml b/egs_modelscope/aishell2/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml new file mode 100644 index 000000000..e9210f373 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml @@ -0,0 +1,91 @@ +# network architecture +# encoder related +encoder_conf: + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.1 + +# decoder related +decoder_conf: + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.1 + src_attention_dropout_rate: 0.1 + +predictor_conf: + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + +# hybrid CTC/attention +model_conf: + ctc_weight: 0.0 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: true + predictor_weight: 1.0 + predictor_bias: 1 + sampling_ratio: 0.75 + +# minibatch related +# dataset_type: small +batch_type: length +batch_bins: 2000 +num_workers: 16 +# dataset_type: large +dataset_conf: + filter_conf: + min_length: 10 + max_length: 250 + min_token_length: 1 + max_token_length: 200 + shuffle: true + shuffle_conf: + shuffle_size: 10240 + sort_size: 500 + batch_conf: + batch_type: 'token' + batch_size: 6000 + num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 20 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug_lfr +specaug_conf: + apply_time_warp: false + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + lfr_rate: 6 + num_freq_mask: 1 + apply_time_mask: true + time_mask_width_range: + - 0 + - 12 + num_time_mask: 1 + +unused_parameters: true +log_interval: 50 +normalize: None +split_with_space: true diff --git a/egs_modelscope/aishell2/paraformer/local/aishell2_data_prep.sh b/egs_modelscope/aishell2/paraformer/local/aishell2_data_prep.sh new file mode 100755 index 000000000..77791f9c1 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/local/aishell2_data_prep.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Copyright 2018 AIShell-Foundation(Authors:Jiayu DU, Xingyu NA, Bengu WU, Hao ZHENG) +# 2018 Beijing Shell Shell Tech. Co. Ltd. (Author: Hui BU) +# Apache 2.0 + +# transform raw AISHELL-2 data to kaldi format + +. ./path.sh || exit 1; + +tmp= +dir= + +if [ $# != 3 ]; then + echo "Usage: $0 " + echo " $0 /export/AISHELL-2/iOS/train data/local/train data/train" + exit 1; +fi + +corpus=$1 +tmp=$2 +dir=$3 + +echo "prepare_data.sh: Preparing data in $corpus" + +mkdir -p $tmp +mkdir -p $dir + +# corpus check +if [ ! -d $corpus ] || [ ! -f $corpus/wav.scp ] || [ ! -f $corpus/trans.txt ]; then + echo "Error: $0 requires wav.scp and trans.txt under $corpus directory." + exit 1; +fi + +# validate utt-key list, IC0803W0380 is a bad utterance +awk '{print $1}' $corpus/wav.scp | grep -v 'IC0803W0380' > $tmp/wav_utt.list +awk '{print $1}' $corpus/trans.txt > $tmp/trans_utt.list +utils/filter_scp.pl -f 1 $tmp/wav_utt.list $tmp/trans_utt.list > $tmp/utt.list + +# wav.scp +awk -F'\t' -v path_prefix=$corpus '{printf("%s\t%s/%s\n",$1,path_prefix,$2)}' $corpus/wav.scp > $tmp/tmp_wav.scp +utils/filter_scp.pl -f 1 $tmp/utt.list $tmp/tmp_wav.scp | sort -k 1 | uniq > $tmp/wav.scp + +# text +utils/filter_scp.pl -f 1 $tmp/utt.list $corpus/trans.txt | sort -k 1 | uniq > $tmp/text + +# copy prepared resources from tmp_dir to target dir +mkdir -p $dir +for f in wav.scp text; do + cp $tmp/$f $dir/$f || exit 1; +done + +echo "local/prepare_data.sh succeeded" +exit 0; diff --git a/egs_modelscope/aishell2/paraformer/modelscope_utils b/egs_modelscope/aishell2/paraformer/modelscope_utils new file mode 120000 index 000000000..fc97768c8 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/modelscope_utils @@ -0,0 +1 @@ +../../common/modelscope_utils \ No newline at end of file diff --git a/egs_modelscope/aishell2/paraformer/paraformer_large_finetune.sh b/egs_modelscope/aishell2/paraformer/paraformer_large_finetune.sh new file mode 100755 index 000000000..d4b5dde73 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/paraformer_large_finetune.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; + +# machines configuration +CUDA_VISIBLE_DEVICES="0,1" # set gpus, e.g., CUDA_VISIBLE_DEVICES="0,1" +gpu_num=2 +count=1 +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding +njob=4 # the number of jobs for each gpu +train_cmd=utils/run.pl + +# general configuration +feats_dir="." #feature output dictionary, for large data +exp_dir="." +lang=zh +dumpdir=dump/fbank +feats_type=fbank +token_type=char +scp=feats.scp +type=kaldi_ark +stage=0 +stop_stage=4 + +# feature configuration +feats_dim=560 +sample_frequency=16000 +nj=100 +speed_perturb="1.0" +lfr=True +lfr_m=7 +lfr_n=6 + +init_model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning +cmvn_file=init_model/${init_model_name}/am.mvn +seg_file=init_model/${init_model_name}/seg_dict +vocab=init_model/${init_model_name}/tokens.txt + +# data +tr_dir= +dev_tst_dir= + +# exp tag +tag="" + +# Set bash to 'debug' mode, it will exit on : +# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands', +set -e +set -u +set -o pipefail + +train_set=train +valid_set=dev_ios +test_sets="dev_ios test_android test_ios test_mic" + +asr_config=conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml +init_param="init_model/${init_model_name}/${init_model_name}" + +inference_config=conf/decode_asr_transformer_noctc_1best.yaml +inference_asr_model=valid.acc.ave_10best.pth + +. utils/parse_options.sh || exit 1; + +# download model from modelscope +python modelscope_utils/download_model.py --model_name ${init_model_name} + +if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${init_model_name} ]; then + echo "${HOME}/.cache/modelscope/hub/damo/${init_model_name} must exist" + exit 1 +else + if [ -d init_model/${init_model_name} ]; then + echo "init_model/${init_model_name} is already exists. if you want to decode again, please delete init_model/${init_model_name} first." + else + mkdir -p init_model/${init_model_name} + cp -r ${HOME}/.cache/modelscope/hub/damo/${init_model_name}/* init_model/${init_model_name} + fi +fi + +model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}" + +# you can set gpu num for decoding here +gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +inference_nj=$[${ngpu}*${njob}] + +if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then + echo "stage 0: Data preparation" + # For training set + local/aishell2_data_prep.sh ${tr_dir} ${feats_dir}/data/local/train ${feats_dir}/data/train || exit 1; + # # For dev and test set + for x in Android iOS Mic; do + local/aishell2_data_prep.sh ${dev_tst_dir}/${x}/dev ${feats_dir}/data/local/dev_${x,,} ${feats_dir}/data/dev_${x,,} || exit 1; + local/aishell2_data_prep.sh ${dev_tst_dir}/${x}/test ${feats_dir}/data/local/test_${x,,} ${feats_dir}/data/test_${x,,} || exit 1; + done + # Normalize text to capital letters + for x in train dev_android dev_ios dev_mic test_android test_ios test_mic; do + mv ${feats_dir}/data/${x}/text ${feats_dir}/data/${x}/text.org + paste -d " " <(cut -f 1 ${feats_dir}/data/${x}/text.org) <(cut -f 2- ${feats_dir}/data/${x}/text.org \ + | tr 'A-Z' 'a-z' | tr -d " ") \ + > ${feats_dir}/data/${x}/text + rm ${feats_dir}/data/${x}/text.org + done +fi + +feat_train_dir=${feats_dir}/${dumpdir}/${train_set}; mkdir -p ${feat_train_dir} +feat_dev_dir=${feats_dir}/${dumpdir}/${valid_set}; mkdir -p ${feat_dev_dir} +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + echo "Feature Generation" + # compute fbank features + fbankdir=${feats_dir}/fbank + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \ + ${feats_dir}/data/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train + utils/fix_data_feat.sh ${fbankdir}/train + for x in android ios mic; do + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/dev_${x} ${exp_dir}/exp/make_fbank/dev_${x} ${fbankdir}/dev_${x} + utils/fix_data_feat.sh ${fbankdir}/dev_${x} + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${feats_dir}/data/test_${x} ${exp_dir}/exp/make_fbank/test_${x} ${fbankdir}/test_${x} + utils/fix_data_feat.sh ${fbankdir}/test_${x} + done + + echo "apply low_frame_rate and cmvn" + [ ! -f ${cmvn_file} ] && echo "$0: cmvn file is required" && exit 1; + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/${train_set} ${cmvn_file} ${exp_dir}/exp/make_fbank/train ${feat_train_dir} + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/${valid_set} ${cmvn_file} ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir} + for x in android ios mic; do + feat_test_dir=${feats_dir}/${dumpdir}/test_${x}; mkdir ${feat_test_dir} + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/test_${x} ${cmvn_file} ${exp_dir}/exp/make_fbank/test_${x} ${feat_test_dir} + done + + echo "Text Tokenize" + # 我爱reading->我 爱 read@@ ing + utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/${train_set} ${seg_file} ${feat_train_dir}/log ${feat_train_dir} + utils/fix_data_feat.sh ${feat_train_dir} + utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/${valid_set} ${seg_file} ${feat_dev_dir}/log ${feat_dev_dir} + utils/fix_data_feat.sh ${feat_dev_dir} + for x in android ios mic; do + feat_test_dir=${feats_dir}/${dumpdir}/test_${x} + cp ${fbankdir}/test_${x}/text ${feat_test_dir} + done +fi + +token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt +echo "dictionary: ${token_list}" +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + echo "stage 2: Dictionary Preparation" + mkdir -p ${feats_dir}/data/${lang}_token_list/char/ + cp $vocab ${token_list} + + vocab_size=$(wc -l <${token_list}) + awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char + awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev_ios + cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${train_set} + cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/${valid_set} +fi + +# Training Stage +world_size=$gpu_num # run on one machine +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # update asr train config.yaml + python modelscope_utils/update_config.py --modelscope_config init_model/${init_model_name}/asr_train_config.yaml --finetune_config ${asr_config} --output_config init_model/${init_model_name}/asr_finetune_config.yaml + finetune_config=init_model/${init_model_name}/asr_finetune_config.yaml + + mkdir -p ${exp_dir}/exp/${model_dir} + mkdir -p ${exp_dir}/exp/${model_dir}/log + INIT_FILE=$exp_dir/ddp_init + if [ -f $INIT_FILE ];then + rm -f $INIT_FILE + fi + init_method=file://$(readlink -f $INIT_FILE) + echo "$0: init method is $init_method" + for ((i = 0; i < $gpu_num; ++i)); do + { + rank=$i + local_rank=$i + gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1]) + asr_train_paraformer.py \ + --gpu_id $gpu_id \ + --use_preprocessor true \ + --token_type $token_type \ + --token_list $token_list \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \ + --resume true \ + --output_dir ${exp_dir}/exp/${model_dir} \ + --init_param $init_param \ + --config $finetune_config \ + --input_size $feats_dim \ + --ngpu $gpu_num \ + --num_worker_count $count \ + --multiprocessing_distributed true \ + --dist_init_method $init_method \ + --dist_world_size $world_size \ + --dist_rank $rank \ + --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1 + } & + done + wait +fi + +# Testing Stage +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + ./utils/easy_asr_infer.sh \ + --lang zh \ + --datadir ${feats_dir} \ + --feats_type ${feats_type} \ + --feats_dim ${feats_dim} \ + --token_type ${token_type} \ + --gpu_inference ${gpu_inference} \ + --inference_config "${inference_config}" \ + --test_sets "${test_sets}" \ + --token_list $token_list \ + --asr_exp ${exp_dir}/exp/${model_dir} \ + --stage 12 \ + --stop_stage 12 \ + --scp $scp \ + --text text \ + --inference_nj $inference_nj \ + --njob $njob \ + --inference_asr_model $inference_asr_model \ + --gpuid_list $gpuid_list \ + --mode paraformer +fi + diff --git a/egs_modelscope/aishell2/paraformer/paraformer_large_infer.sh b/egs_modelscope/aishell2/paraformer/paraformer_large_infer.sh new file mode 100755 index 000000000..95b32fc75 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/paraformer_large_infer.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +ori_data= +data_dir= +exp_dir= +model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch +inference_nj=32 +gpuid_list="0,1" # set gpus, e.g., gpuid_list="0,1" +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +njob=4 # the number of jobs for each gpu +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] +else + inference_nj=$njob +fi + +use_lm=false +beam_size=1 +lm_weight=0.0 + +test_sets="dev_ios test_android test_ios test_mic" + +. utils/parse_options.sh + +for x in Android iOS Mic; do + local/aishell2_data_prep.sh ${ori_data}/${x}/dev ${data_dir}/aishell2/local/dev_${x,,} ${data_dir}/aishell2/dev_${x,,} || exit 1; + local/aishell2_data_prep.sh ${ori_data}/${x}/test ${data_dir}/aishell2/local/test_${x,,} ${data_dir}/aishell2/test_${x,,} || exit 1; +done +for x in dev_android dev_ios dev_mic test_android test_ios test_mic; do + mv ${data_dir}/aishell2/${x}/text ${data_dir}/aishell2/${x}/text.org + paste -d " " <(cut -f 1 ${data_dir}/aishell2/${x}/text.org) <(cut -f 2- ${data_dir}/aishell2/${x}/text.org \ + | tr 'A-Z' 'a-z' | tr -d " ") \ + > ${data_dir}/aishell2/${x}/text + rm ${data_dir}/aishell2/${x}/text.org +done + +mkdir -p ${exp_dir}/aishell2 + +modelscope_utils/modelscope_infer.sh \ + --data_dir ${data_dir}/aishell2 \ + --exp_dir ${exp_dir}/aishell2 \ + --test_sets "${test_sets}" \ + --model_name ${model_name} \ + --inference_nj ${inference_nj} \ + --gpuid_list ${gpuid_list} \ + --njob ${njob} \ + --gpu_inference ${gpu_inference} \ + --use_lm ${use_lm} \ + --beam_size ${beam_size} \ + --lm_weight ${lm_weight} diff --git a/egs_modelscope/aishell2/paraformer/path.sh b/egs_modelscope/aishell2/paraformer/path.sh new file mode 100755 index 000000000..7972642d0 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs_modelscope/aishell2/paraformer/utils b/egs_modelscope/aishell2/paraformer/utils new file mode 120000 index 000000000..37d976175 --- /dev/null +++ b/egs_modelscope/aishell2/paraformer/utils @@ -0,0 +1 @@ +../../../egs/aishell/tranformer/utils/ \ No newline at end of file diff --git a/egs_modelscope/common/README.md b/egs_modelscope/common/README.md new file mode 100644 index 000000000..f2049e2f0 --- /dev/null +++ b/egs_modelscope/common/README.md @@ -0,0 +1,27 @@ +# ModelScope Model + +## How to finetune and infer using a pretrained ModelScope Model + +### Finetune +- Modify finetune training related parameters in `conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml` +- Setting parameters in `modelscope_common_finetune.sh` + - dataset: the dataset dir needs to include files: train/wav.scp, train/text; optional dev/wav.scp, dev/text, test/wav.scp test/text + - tag: exp tag + - init_model_name: speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning +- Then you can run the pipeline to finetune with our model download from modelscope: +```sh + sh ./modelscope_common_finetune.sh +``` + +### Inference + +Or you can use the finetuned model for inference directly. + +- Setting parameters in `modelscope_common_infer.sh` + - data_dir: # wav list, ${data_dir}/wav.scp + - exp_dir: the result path + - model_name: speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope +- Then you can run the pipeline to infer with: +```sh + sh ./modelscope_common_infer.sh +``` diff --git a/egs_modelscope/common/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml b/egs_modelscope/common/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml new file mode 100644 index 000000000..22f02d913 --- /dev/null +++ b/egs_modelscope/common/conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml @@ -0,0 +1,6 @@ +beam_size: 10 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.0 +lm_weight: 0.15 diff --git a/egs_modelscope/common/conf/decode_asr_transformer_noctc_1best.yaml b/egs_modelscope/common/conf/decode_asr_transformer_noctc_1best.yaml new file mode 100644 index 000000000..e6231927c --- /dev/null +++ b/egs_modelscope/common/conf/decode_asr_transformer_noctc_1best.yaml @@ -0,0 +1,6 @@ +beam_size: 1 +penalty: 0.0 +maxlenratio: 0.0 +minlenratio: 0.0 +ctc_weight: 0.0 +lm_weight: 0.0 diff --git a/egs_modelscope/common/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml b/egs_modelscope/common/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml new file mode 100644 index 000000000..e9210f373 --- /dev/null +++ b/egs_modelscope/common/conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml @@ -0,0 +1,91 @@ +# network architecture +# encoder related +encoder_conf: + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.1 + +# decoder related +decoder_conf: + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.1 + src_attention_dropout_rate: 0.1 + +predictor_conf: + threshold: 1.0 + l_order: 1 + r_order: 1 + tail_threshold: 0.45 + +# hybrid CTC/attention +model_conf: + ctc_weight: 0.0 + lsm_weight: 0.1 # label smoothing option + length_normalized_loss: true + predictor_weight: 1.0 + predictor_bias: 1 + sampling_ratio: 0.75 + +# minibatch related +# dataset_type: small +batch_type: length +batch_bins: 2000 +num_workers: 16 +# dataset_type: large +dataset_conf: + filter_conf: + min_length: 10 + max_length: 250 + min_token_length: 1 + max_token_length: 200 + shuffle: true + shuffle_conf: + shuffle_size: 10240 + sort_size: 500 + batch_conf: + batch_type: 'token' + batch_size: 6000 + num_workers: 16 + +# optimization related +accum_grad: 1 +grad_clip: 5 +max_epoch: 20 +val_scheduler_criterion: + - valid + - acc +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 + +optim: adam +optim_conf: + lr: 0.0005 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 30000 + +specaug: specaug_lfr +specaug_conf: + apply_time_warp: false + time_warp_window: 5 + time_warp_mode: bicubic + apply_freq_mask: true + freq_mask_width_range: + - 0 + - 30 + lfr_rate: 6 + num_freq_mask: 1 + apply_time_mask: true + time_mask_width_range: + - 0 + - 12 + num_time_mask: 1 + +unused_parameters: true +log_interval: 50 +normalize: None +split_with_space: true diff --git a/egs_modelscope/common/modelscope_common_finetune.sh b/egs_modelscope/common/modelscope_common_finetune.sh new file mode 100755 index 000000000..a43083f0c --- /dev/null +++ b/egs_modelscope/common/modelscope_common_finetune.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash + +. ./path.sh || exit 1; + +# machines configuration +CUDA_VISIBLE_DEVICES="0,1" # set gpus, e.g., CUDA_VISIBLE_DEVICES="0,1" +gpu_num=2 +count=1 +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding +njob=4 # the number of jobs for each gpu +train_cmd=utils/run.pl + +# general configuration +feats_dir="." #feature output dictionary, for large data +exp_dir="." +lang=zh +dumpdir=dump/fbank +feats_type=fbank +token_type=char +scp=feats.scp +type=kaldi_ark +stage=1 +stop_stage=4 + +# feature configuration +feats_dim=560 +sample_frequency=16000 +nj=32 +speed_perturb="1.0" +lfr=True +lfr_m=7 +lfr_n=6 + +init_model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope during fine-tuning +cmvn_file=init_model/${init_model_name}/am.mvn +seg_file=init_model/${init_model_name}/seg_dict +vocab=init_model/${init_model_name}/tokens.txt + +# data +dataset= # dataset (include train/wav.scp, train/text, dev/wav.scp, dev/text, optional test/wav.scp test/text) + +# exp tag +tag="" + +# Set bash to 'debug' mode, it will exit on : +# -e 'error', -u 'undefined variable', -o ... 'error in pipeline', -x 'print commands', +set -e +set -u +set -o pipefail + +train_set=train +valid_set=dev +test_sets="dev test" + +asr_config=conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml +init_param="init_model/${init_model_name}/${init_model_name}" + +inference_config=conf/decode_asr_transformer_noctc_1best.yaml +inference_asr_model=valid.acc.ave_10best.pth + +. utils/parse_options.sh || exit 1; + +# download model from modelscope +python modelscope_utils/download_model.py --model_name ${init_model_name} + +if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${init_model_name} ]; then + echo "${HOME}/.cache/modelscope/hub/damo/${init_model_name} must exist" + exit 1 +else + if [ -d init_model/${init_model_name} ]; then + echo "init_model/${init_model_name} is already exists. if you want to decode again, please delete init_model/${init_model_name} first." + else + mkdir -p init_model/${init_model_name} + cp -r ${HOME}/.cache/modelscope/hub/damo/${init_model_name}/* init_model/${init_model_name} + fi +fi + +model_dir="baseline_$(basename "${asr_config}" .yaml)_${feats_type}_${lang}_${token_type}_${tag}" + +# you can set gpu num for decoding here +gpuid_list=$CUDA_VISIBLE_DEVICES # set gpus for decoding, the same as training stage by default +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +inference_nj=$[${ngpu}*${njob}] + +[ ! -d ${dataset} ] && echo "$0: Training data is required" && exit 1; +[ ! -f ${dataset}/train/wav.scp ] && [ ! -f ${dataset}/train/text ] && echo "$0: Training data wav.scp or text is not found" && exit 1; + +if [ ! -d "${dataset}/dev" ]; then + utils/fix_data.sh ${dataset}/train + utils/subset_data_dir_tr_cv.sh --dev-num-utt 1000 ${dataset}/train ${dataset} +fi +if [ ! -d "${dataset}/test" ]; then + test_sets="dev" +fi + +feat_train_dir=${feats_dir}/${dumpdir}/train; mkdir -p ${feat_train_dir} +feat_dev_dir=${feats_dir}/${dumpdir}/dev; mkdir -p ${feat_dev_dir} +feat_test_dir=${feats_dir}/${dumpdir}/test; mkdir -p ${feat_test_dir} + +if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then + echo "Feature Generation" + # compute fbank features + fbankdir=${feats_dir}/fbank + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj --speed_perturb ${speed_perturb} \ + ${dataset}/train ${exp_dir}/exp/make_fbank/train ${fbankdir}/train + utils/fix_data_feat.sh ${fbankdir}/train + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${dataset}/dev ${exp_dir}/exp/make_fbank/dev ${fbankdir}/dev + utils/fix_data_feat.sh ${fbankdir}/dev + if [ -d "${dataset}/test" ]; then + utils/compute_fbank.sh --cmd "$train_cmd" --nj $nj \ + ${dataset}/test ${exp_dir}/exp/make_fbank/test ${fbankdir}/test + utils/fix_data_feat.sh ${fbankdir}/test + fi + + echo "apply low_frame_rate and cmvn" + [ ! -f ${cmvn_file} ] && echo "$0: cmvn file is required" && exit 1; + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/train ${cmvn_file} ${exp_dir}/exp/make_fbank/train ${feat_train_dir} + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/dev ${cmvn_file} ${exp_dir}/exp/make_fbank/dev ${feat_dev_dir} + if [ -d "${dataset}/test" ]; then + utils/apply_lfr_and_cmvn.sh --cmd "$train_cmd" --nj $nj \ + --lfr $lfr --lfr-m $lfr_m --lfr-n $lfr_n \ + ${fbankdir}/test ${cmvn_file} ${exp_dir}/exp/make_fbank/test ${feat_test_dir} + fi + + echo "Text Tokenize" + # 我爱reading->我 爱 read@@ ing + utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/train ${seg_file} ${feat_train_dir}/log ${feat_train_dir} + utils/fix_data_feat.sh ${feat_train_dir} + utils/text_tokenize.sh --cmd "$train_cmd" --nj $nj ${fbankdir}/dev ${seg_file} ${feat_dev_dir}/log ${feat_dev_dir} + utils/fix_data_feat.sh ${feat_dev_dir} + if [ -d "${dataset}/test" ]; then + cp ${fbankdir}/test/text ${feat_test_dir} + fi +fi + +token_list=${feats_dir}/data/${lang}_token_list/char/tokens.txt +echo "dictionary: ${token_list}" +if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then + echo "stage 2: Dictionary Preparation" + mkdir -p ${feats_dir}/data/${lang}_token_list/char/ + cp $vocab ${token_list} + + vocab_size=$(wc -l <${token_list}) + awk -v v=,${vocab_size} '{print $0v}' ${feat_train_dir}/text_shape > ${feat_train_dir}/text_shape.char + awk -v v=,${vocab_size} '{print $0v}' ${feat_dev_dir}/text_shape > ${feat_dev_dir}/text_shape.char + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/train + mkdir -p ${feats_dir}/asr_stats_fbank_zh_char/dev + cp ${feat_train_dir}/speech_shape ${feat_train_dir}/text_shape ${feat_train_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/train + cp ${feat_dev_dir}/speech_shape ${feat_dev_dir}/text_shape ${feat_dev_dir}/text_shape.char ${feats_dir}/asr_stats_fbank_zh_char/dev +fi + +# Training Stage +world_size=$gpu_num # run on one machine +if [ ${stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then + # update asr train config.yaml + python modelscope_utils/update_config.py --modelscope_config init_model/${init_model_name}/asr_train_config.yaml --finetune_config ${asr_config} --output_config init_model/${init_model_name}/asr_finetune_config.yaml + finetune_config=init_model/${init_model_name}/asr_finetune_config.yaml + + mkdir -p ${exp_dir}/exp/${model_dir} + mkdir -p ${exp_dir}/exp/${model_dir}/log + INIT_FILE=$exp_dir/ddp_init + if [ -f $INIT_FILE ];then + rm -f $INIT_FILE + fi + init_method=file://$(readlink -f $INIT_FILE) + echo "$0: init method is $init_method" + for ((i = 0; i < $gpu_num; ++i)); do + { + rank=$i + local_rank=$i + gpu_id=$(echo $CUDA_VISIBLE_DEVICES | cut -d',' -f$[$i+1]) + asr_train_paraformer.py \ + --gpu_id $gpu_id \ + --use_preprocessor true \ + --token_type $token_type \ + --token_list $token_list \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/${scp},speech,${type} \ + --train_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${train_set}/text,text,text \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/speech_shape \ + --train_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${train_set}/text_shape.char \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/${scp},speech,${type} \ + --valid_data_path_and_name_and_type ${feats_dir}/${dumpdir}/${valid_set}/text,text,text \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/speech_shape \ + --valid_shape_file ${feats_dir}/asr_stats_fbank_zh_char/${valid_set}/text_shape.char \ + --resume true \ + --output_dir ${exp_dir}/exp/${model_dir} \ + --init_param $init_param \ + --config $finetune_config \ + --input_size $feats_dim \ + --ngpu $gpu_num \ + --num_worker_count $count \ + --multiprocessing_distributed true \ + --dist_init_method $init_method \ + --dist_world_size $world_size \ + --dist_rank $rank \ + --local_rank $local_rank 1> ${exp_dir}/exp/${model_dir}/log/train.log.$i 2>&1 + } & + done + wait +fi + +# Testing Stage +if [ ${stage} -le 4 ] && [ ${stop_stage} -ge 4 ]; then + ./utils/easy_asr_infer.sh \ + --lang zh \ + --datadir ${feats_dir} \ + --feats_type ${feats_type} \ + --feats_dim ${feats_dim} \ + --token_type ${token_type} \ + --gpu_inference ${gpu_inference} \ + --inference_config "${inference_config}" \ + --test_sets "${test_sets}" \ + --token_list $token_list \ + --asr_exp ${exp_dir}/exp/${model_dir} \ + --stage 12 \ + --stop_stage 12 \ + --scp $scp \ + --text text \ + --inference_nj $inference_nj \ + --njob $njob \ + --inference_asr_model $inference_asr_model \ + --gpuid_list $gpuid_list \ + --mode paraformer +fi + diff --git a/egs_modelscope/common/modelscope_common_infer.sh b/egs_modelscope/common/modelscope_common_infer.sh new file mode 100755 index 000000000..12b2cbcb2 --- /dev/null +++ b/egs_modelscope/common/modelscope_common_infer.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope +data_dir= # wav list, ${data_dir}/wav.scp +exp_dir="exp" +gpuid_list="0,1" +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +njob=4 +gpu_inference=true +decode_cmd=utils/run.pl + +. utils/parse_options.sh + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] + _ngpu=1 +else + inference_nj=${njob} + _ngpu=0 +fi + +# LM configs +use_lm=false +beam_size=1 +lm_weight=0.0 + +python modelscope_utils/download_model.py \ + --model_name ${model_name} + +if [ -d ${exp_dir} ]; then + echo "${exp_dir} is already exists. if you want to decode again, please delete ${exp_dir} first." + exit 1 +else + mkdir -p ${exp_dir}/${model_name} + cp ${HOME}/.cache/modelscope/hub/damo/${model_name}/* ${exp_dir}/${model_name}/. -r + _dir=${exp_dir}/decode_asr + _logdir=${_dir}/logdir + mkdir -p "${_dir}" + mkdir -p "${_logdir}" +fi + +for n in $(seq "${inference_nj}"); do + split_scps+=" ${_logdir}/keys.${n}.scp" +done +# shellcheck disable=SC2086 +utils/split_scp.pl "${data_dir}/wav.scp" ${split_scps} + +if "${use_lm}"; then + cp ${exp_dir}/${model_name}/decode_asr_transformer.yaml ${exp_dir}/${model_name}/decode_asr_transformer.yaml.back + cp ${exp_dir}/${model_name}/decode_asr_transformer_wav.yaml ${exp_dir}/${model_name}/decode_asr_transformer_wav.yaml.back + sed -i "s#beam_size: [0-9]*#beam_size: `echo $beam_size`#g" ${exp_dir}/${model_name}/decode_asr_transformer.yaml + sed -i "s#beam_size: [0-9]*#beam_size: `echo $beam_size`#g" ${exp_dir}/${model_name}/decode_asr_transformer_wav.yaml + sed -i "s#lm_weight: 0.[0-9]*#lm_weight: `echo $lm_weight`#g" ${exp_dir}/${model_name}/decode_asr_transformer.yaml + sed -i "s#lm_weight: 0.[0-9]*#lm_weight: `echo $lm_weight`#g" ${exp_dir}/${model_name}/decode_asr_transformer_wav.yaml +fi + +echo "Decoding started... log: '${_logdir}/asr_inference.*.log'" +# shellcheck disable=SC2086 +${decode_cmd} --max-jobs-run "${inference_nj}" JOB=1:"${inference_nj}" "${_logdir}"/asr_inference.JOB.log \ + python -m funasr.bin.modelscope_infer \ + --local_model_path ${exp_dir}/${model_name} \ + --wav_list ${_logdir}/keys.JOB.scp \ + --output_file ${_logdir}/text.JOB \ + --gpuid_list ${gpuid_list} \ + --njob ${njob} \ + --ngpu ${_ngpu} \ + + for i in $(seq ${inference_nj}); do + cat ${_logdir}/text.${i} + done | sort -k1 >${_dir}/text + +mv ${exp_dir}/${model_name}/decode_asr_transformer.yaml.back ${exp_dir}/${model_name}/decode_asr_transformer.yaml +mv ${exp_dir}/${model_name}/decode_asr_transformer_wav.yaml.back ${exp_dir}/${model_name}/decode_asr_transformer_wav.yaml + diff --git a/egs_modelscope/common/modelscope_common_infer_after_finetune.sh b/egs_modelscope/common/modelscope_common_infer_after_finetune.sh new file mode 100755 index 000000000..00dd28336 --- /dev/null +++ b/egs_modelscope/common/modelscope_common_infer_after_finetune.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +pretrained_model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # pre-trained model, download from modelscope +data_dir= # wav list, ${data_dir}/wav.scp +finetune_model_name= # fine-tuning model name +finetune_exp_dir= # fine-tuning model experiment result path +gpuid_list="0,1" +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +njob=4 +gpu_inference=true +decode_cmd=utils/run.pl + +. utils/parse_options.sh + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] + _ngpu=1 +else + inference_nj=${njob} + inference_nj=${njob} + _ngpu=0 +fi + +if [ ! -d ${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name} ]; then + echo "${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name} must exist." + exit 1 +else + exp_dir=${finetune_exp_dir}/${finetune_model_name}.modelscope + mkdir -p $exp_dir + cp ${finetune_exp_dir}/${finetune_model_name} ${exp_dir}/${finetune_model_name}.modelscope + cp ${HOME}/.cache/modelscope/hub/damo/${pretrained_model_name}/* ${exp_dir}/. -r +fi + +_dir=${exp_dir}/decode_asr +_logdir=${_dir}/logdir +if [ -d ${_dir} ]; then + echo "${_dir} is already exists. if you want to decode again, please delete ${_dir} first." +else + mkdir -p "${_dir}" + mkdir -p "${_logdir}" +fi + +for n in $(seq "${inference_nj}"); do + split_scps+=" ${_logdir}/keys.${n}.scp" +done +# shellcheck disable=SC2086 +utils/split_scp.pl "${data_dir}/wav.scp" ${split_scps} + +echo "Decoding started... log: '${_logdir}/asr_inference.*.log'" +# shellcheck disable=SC2086 +${decode_cmd} --max-jobs-run "${inference_nj}" JOB=1:"${inference_nj}" "${_logdir}"/asr_inference.JOB.log \ + python -m funasr.bin.modelscope_infer \ + --local_model_path ${exp_dir} \ + --wav_list ${_logdir}/keys.JOB.scp \ + --output_file ${_logdir}/text.JOB \ + --gpuid_list ${gpuid_list} \ + --njob ${njob} \ + --ngpu ${_ngpu} \ + + for i in $(seq ${inference_nj}); do + cat ${_logdir}/text.${i} + done | sort -k1 >${_dir}/text \ No newline at end of file diff --git a/egs_modelscope/common/modelscope_utils/download_model.py b/egs_modelscope/common/modelscope_utils/download_model.py new file mode 100755 index 000000000..5d5f70dd1 --- /dev/null +++ b/egs_modelscope/common/modelscope_utils/download_model.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import argparse + +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="download model configs", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--model_name", + type=str, + default="speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", + help="model name in modelscope") + args = parser.parse_args() + + inference_pipeline = pipeline( + task=Tasks.auto_speech_recognition, + model='damo/{}'.format(args.model_name), + model_revision='v1.0.0') diff --git a/egs_modelscope/common/modelscope_utils/modelscope_infer.sh b/egs_modelscope/common/modelscope_utils/modelscope_infer.sh new file mode 100755 index 000000000..1a56dce98 --- /dev/null +++ b/egs_modelscope/common/modelscope_utils/modelscope_infer.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +data_dir= +exp_dir= +model_name= +inference_nj=32 +gpuid_list="0,1,2,3" +njob=32 +gpu_inference=true + +test_sets="dev test" +decode_cmd=utils/run.pl + +# LM configs +use_lm=false +beam_size=1 +lm_weight=0.0 + +. utils/parse_options.sh + +if ${gpu_inference}; then + _ngpu=1 +else + _ngpu=0 +fi + +# download model from modelscope +python modelscope_utils/download_model.py \ + --model_name ${model_name} + +modelscope_dir=${HOME}/.cache/modelscope/hub/damo/${model_name} + + +for dset in ${test_sets}; do + _dir=${exp_dir}/${model_name}/decode_asr/${dset} + _logdir=${_dir}/logdir + _data=${data_dir}/${dset} + if [ -d ${_dir} ]; then + echo "${_dir} is already exists. if you want to decode again, please delete ${_dir} first." + exit 1 + else + mkdir -p "${_dir}" + mkdir -p "${_logdir}" + fi + + if "${use_lm}"; then + cp ${modelscope_dir}/decode_asr_transformer.yaml ${modelscope_dir}/decode_asr_transformer.yaml.back + cp ${modelscope_dir}/decode_asr_transformer_wav.yaml ${modelscope_dir}/decode_asr_transformer_wav.yaml.back + sed -i "s#beam_size: [0-9]*#beam_size: `echo $beam_size`#g" ${modelscope_dir}/decode_asr_transformer.yaml + sed -i "s#beam_size: [0-9]*#beam_size: `echo $beam_size`#g" ${modelscope_dir}/decode_asr_transformer_wav.yaml + sed -i "s#lm_weight: 0.[0-9]*#lm_weight: `echo $lm_weight`#g" ${modelscope_dir}/decode_asr_transformer.yaml + sed -i "s#lm_weight: 0.[0-9]*#lm_weight: `echo $lm_weight`#g" ${modelscope_dir}/decode_asr_transformer_wav.yaml + fi + + for n in $(seq "${inference_nj}"); do + split_scps+=" ${_logdir}/keys.${n}.scp" + done + # shellcheck disable=SC2086 + utils/split_scp.pl "${data_dir}/${dset}/wav.scp" ${split_scps} + + echo "Decoding started... log: '${_logdir}/asr_inference.*.log'" + # shellcheck disable=SC2086 + ${decode_cmd} --max-jobs-run "${inference_nj}" JOB=1:"${inference_nj}" "${_logdir}"/asr_inference.JOB.log \ + python -m funasr.bin.modelscope_infer \ + --model_name ${model_name} \ + --wav_list ${_logdir}/keys.JOB.scp \ + --output_file ${_logdir}/text.JOB \ + --gpuid_list ${gpuid_list} \ + --njob ${njob} \ + --ngpu ${_ngpu} \ + + for i in $(seq ${inference_nj}); do + cat ${_logdir}/text.${i} + done | sort -k1 >${_dir}/text + + python utils/proce_text.py ${_dir}/text ${_dir}/text.proc + python utils/proce_text.py ${_data}/text ${_data}/text.proc + python utils/compute_wer.py ${_data}/text.proc ${_dir}/text.proc ${_dir}/text.cer + tail -n 3 ${_dir}/text.cer > ${_dir}/text.cer.txt + cat ${_dir}/text.cer.txt +done + +if "${use_lm}"; then + mv ${modelscope_dir}/decode_asr_transformer.yaml.back ${modelscope_dir}/decode_asr_transformer.yaml + mv ${modelscope_dir}/decode_asr_transformer_wav.yaml.back ${modelscope_dir}/decode_asr_transformer_wav.yaml +fi diff --git a/egs_modelscope/common/modelscope_utils/update_config.py b/egs_modelscope/common/modelscope_utils/update_config.py new file mode 100644 index 000000000..88466edcd --- /dev/null +++ b/egs_modelscope/common/modelscope_utils/update_config.py @@ -0,0 +1,41 @@ +import yaml +import argparse + +def update_dct(fin_configs, root): + if root == {}: + return {} + for root_key, root_value in root.items(): + if not isinstance(root[root_key],dict): + fin_configs[root_key] = root[root_key] + else: + result = update_dct(fin_configs[root_key], root[root_key]) + fin_configs[root_key] = result + return fin_configs + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="update configs", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--modelscope_config", + type=str, + help="modelscope config file") + parser.add_argument("--finetune_config", + type=str, + help="finetune config file") + parser.add_argument("--output_config", + type=str, + help="output config file") + args = parser.parse_args() + + with open(args.modelscope_config) as f: + modelscope_configs = yaml.safe_load(f) + + with open(args.finetune_config) as f: + finetune_configs = yaml.safe_load(f) + + # update configs, e.g., lr, batch_size, ... + modelscope_configs = update_dct(modelscope_configs, finetune_configs) + + with open(args.output_config, "w") as f: + yaml.dump(modelscope_configs, f, indent=4) diff --git a/egs_modelscope/common/path.sh b/egs_modelscope/common/path.sh new file mode 100755 index 000000000..c340218c2 --- /dev/null +++ b/egs_modelscope/common/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs_modelscope/common/utils b/egs_modelscope/common/utils new file mode 120000 index 000000000..cbef564a5 --- /dev/null +++ b/egs_modelscope/common/utils @@ -0,0 +1 @@ +../../egs/aishell/tranformer/utils/ \ No newline at end of file diff --git a/egs_modelscope/speechio/paraformer/README.md b/egs_modelscope/speechio/paraformer/README.md new file mode 100644 index 000000000..669185f6e --- /dev/null +++ b/egs_modelscope/speechio/paraformer/README.md @@ -0,0 +1,24 @@ +# ModelScope: Paraformer-large Model + +## Highlight + +### ModelScope: Paraformer-Large Model +- Fast: Non-autoregressive (NAR) model, the Paraformer can achieve comparable performance to the state-of-the-art AR transformer, with more than 10x speedup. +- Accurate: SOTA in a lot of public ASR tasks, with a very significant relative improvement, capable of industrial implementation. +- Convenient: Quickly and easily download Paraformer-large from Modelscope for finetuning and inference. + - Support finetuning and inference on AISHELL-1 and AISHELL-2. + - Support inference on AISHELL-1, AISHELL-2, Wenetspeech, SpeechIO and other audio. + +## How to infer using a pretrained ModelScope Paraformer-large Model + +### Inference +- Setting parameters in `paraformer_large_infer.sh` + - ori_data: please set the speechio raw data path + - data_dir: data output dictionary + - exp_dir: the result path + - model_name: speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # base model, download from modelscope + - test_sets: please set the testsets name +- Then you can run the pipeline to infer with: +```sh + sh ./paraformer_large_infer.sh +``` diff --git a/egs_modelscope/speechio/paraformer/RESULTS.md b/egs_modelscope/speechio/paraformer/RESULTS.md new file mode 100644 index 000000000..9938e74fe --- /dev/null +++ b/egs_modelscope/speechio/paraformer/RESULTS.md @@ -0,0 +1,42 @@ +# Paraformer-Large +- Model link: +- Model size: 220M +- Train config: conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml + +# Environments +- date: `Tue Nov 22 18:48:39 CST 2022` +- python version: `3.7.12` +- FunASR version: `0.1.0` +- pytorch version: `pytorch 1.7.0` +- Git hash: `` +- Commit date: `` + +# Beachmark Results + + +## SpeechIO TIOBE +- Decode config 1: conf/decode_asr_transformer_noctc_1best.yaml + - Decode without CTC + - Decode without LM +- Decode config 2: conf/decode_asr_transformer_noctc_10best_lm_weight_0.15.yaml + - Decode without CTC + - Decode with Transformer-LM + - LM weight: 0.15 + +| testset | w/o LM | w/ LM | +|:------------------:|:----:|:----:| +|SPEECHIO_ASR_ZH00001| 0.49 | 0.35 | +|SPEECHIO_ASR_ZH00002| 3.23 | 2.86 | +|SPEECHIO_ASR_ZH00003| 1.13 | 0.80 | +|SPEECHIO_ASR_ZH00004| 1.33 | 1.10 | +|SPEECHIO_ASR_ZH00005| 1.41 | 1.18 | +|SPEECHIO_ASR_ZH00006| 5.25 | 4.85 | +|SPEECHIO_ASR_ZH00007| 5.51 | 4.97 | +|SPEECHIO_ASR_ZH00008| 3.69 | 3.18 | +|SPEECHIO_ASR_ZH00009| 3.02 | 2.78 | +|SPEECHIO_ASR_ZH000010| 3.35 | 2.99 | +|SPEECHIO_ASR_ZH000011| 1.54 | 1.25 | +|SPEECHIO_ASR_ZH000012| 2.06 | 1.68 | +|SPEECHIO_ASR_ZH000013| 2.57 | 2.25 | +|SPEECHIO_ASR_ZH000014| 3.86 | 3.08 | +|SPEECHIO_ASR_ZH000015| 3.34 | 2.67 | diff --git a/egs_modelscope/speechio/paraformer/modelscope_utils b/egs_modelscope/speechio/paraformer/modelscope_utils new file mode 120000 index 000000000..fc97768c8 --- /dev/null +++ b/egs_modelscope/speechio/paraformer/modelscope_utils @@ -0,0 +1 @@ +../../common/modelscope_utils \ No newline at end of file diff --git a/egs_modelscope/speechio/paraformer/paraformer_large_infer.sh b/egs_modelscope/speechio/paraformer/paraformer_large_infer.sh new file mode 100755 index 000000000..bcf8c331c --- /dev/null +++ b/egs_modelscope/speechio/paraformer/paraformer_large_infer.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +ori_data= +data_dir= +exp_dir= +model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch +inference_nj=32 +gpuid_list="0,1" # set gpus, e.g., gpuid_list="0,1" +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +njob=4 # the number of jobs for each gpu +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] +else + inference_nj=$njob +fi + +# LM configs +use_lm=false +beam_size=1 +lm_weight=0.0 + +test_sets="SPEECHIO_ASR_ZH00001 SPEECHIO_ASR_ZH00002 SPEECHIO_ASR_ZH00003 SPEECHIO_ASR_ZH00004 SPEECHIO_ASR_ZH00005 SPEECHIO_ASR_ZH00006 SPEECHIO_ASR_ZH00007 SPEECHIO_ASR_ZH00008 SPEECHIO_ASR_ZH00009 SPEECHIO_ASR_ZH00010 SPEECHIO_ASR_ZH00011 SPEECHIO_ASR_ZH00012 SPEECHIO_ASR_ZH00013 SPEECHIO_ASR_ZH00014 SPEECHIO_ASR_ZH00015" + +. utils/parse_options.sh + +for tset_name in ${test_sets}; do + test_dir=${data_dir}/speechio/${tset_name} + mkdir -p ${test_dir} + find ${ori_data}/${tset_name} -iname "*.wav" > ${test_dir}/wav.flist + sed -e 's/\.wav//' ${test_dir}/wav.flist | awk -F '/' '{print $NF}' > ${test_dir}/utt.list + paste -d' ' ${test_dir}/utt.list ${test_dir}/wav.flist > ${test_dir}/wav.scp + cp ${ori_data}/${tset_name}/trans.txt ${test_dir}/text + sed -i "s/\t/ /g" ${test_dir}/text +done + +mkdir -p ${exp_dir}/speechio + +modelscope_utils/modelscope_infer.sh \ + --data_dir ${data_dir}/speechio \ + --exp_dir ${exp_dir}/speechio \ + --test_sets "${test_sets}" \ + --model_name ${model_name} \ + --inference_nj ${inference_nj} \ + --gpuid_list ${gpuid_list} \ + --njob ${njob} \ + --gpu_inference ${gpu_inference} \ + --use_lm ${use_lm} \ + --beam_size ${beam_size} \ + --lm_weight ${lm_weight} + +# SpeechIO TIOBE textnorm +for tset_name in ${test_sets}; do + echo "$0 --> Normalizing REF text ..." + ./utils/textnorm_zh.py \ + --has_key --to_upper \ + ${ori_data}/${tset_name}/trans.txt \ + ${data_dir}/speechio/${tset_name}/ref.txt + + cp ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/text.proc ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/raw_rec.txt + sed -i "s###g" ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/raw_rec.txt + echo "$0 --> Normalizing HYP text ..." + ./utils/textnorm_zh.py \ + --has_key --to_upper \ + ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/raw_rec.txt \ + ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/rec.txt + grep -v $'\t$' ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/rec.txt > ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/rec_non_empty.txt + + echo "$0 --> computing WER/CER and alignment ..." + ./utils/error_rate_zh \ + --tokenizer char \ + --ref ${data_dir}/speechio/${tset_name}/ref.txt \ + --hyp ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/rec_non_empty.txt \ + ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/DETAILS.txt | tee ${exp_dir}/speechio/${model_name}/decode_asr/${tset_name}/RESULTS.txt +done + diff --git a/egs_modelscope/speechio/paraformer/path.sh b/egs_modelscope/speechio/paraformer/path.sh new file mode 100755 index 000000000..7972642d0 --- /dev/null +++ b/egs_modelscope/speechio/paraformer/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs_modelscope/speechio/paraformer/utils b/egs_modelscope/speechio/paraformer/utils new file mode 120000 index 000000000..37d976175 --- /dev/null +++ b/egs_modelscope/speechio/paraformer/utils @@ -0,0 +1 @@ +../../../egs/aishell/tranformer/utils/ \ No newline at end of file diff --git a/egs_modelscope/wenetspeech/paraformer/README.md b/egs_modelscope/wenetspeech/paraformer/README.md new file mode 100644 index 000000000..9dc5f3f4b --- /dev/null +++ b/egs_modelscope/wenetspeech/paraformer/README.md @@ -0,0 +1,24 @@ +# ModelScope: Paraformer-large Model + +## Highlight + +### ModelScope: Paraformer-Large Model +- Fast: Non-autoregressive (NAR) model, the Paraformer can achieve comparable performance to the state-of-the-art AR transformer, with more than 10x speedup. +- Accurate: SOTA in a lot of public ASR tasks, with a very significant relative improvement, capable of industrial implementation. +- Convenient: Quickly and easily download Paraformer-large from Modelscope for finetuning and inference. + - Support finetuning and inference on AISHELL-1 and AISHELL-2. + - Support inference on AISHELL-1, AISHELL-2, Wenetspeech, SpeechIO and other audio. + +## How to infer using a pretrained ModelScope Paraformer-large Model + +### Inference +- Setting parameters in `paraformer_large_infer.sh` + - ori_data: please set the wenetspeech raw data path + - data_dir: data output dictionary + - exp_dir: the result path + - model_name: speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch # base model, download from modelscope + - test_sets: please set the testsets name +- Then you can run the pipeline to infer with: +```sh + sh ./paraformer_large_infer.sh +``` diff --git a/egs_modelscope/wenetspeech/paraformer/RESULTS.md b/egs_modelscope/wenetspeech/paraformer/RESULTS.md new file mode 100644 index 000000000..a912c92a6 --- /dev/null +++ b/egs_modelscope/wenetspeech/paraformer/RESULTS.md @@ -0,0 +1,25 @@ +# Paraformer-Large +- Model link: +- Model size: 220M +- Train config: conf/train_asr_paraformer_sanm_50e_16d_2048_512_lfr6.yaml + +# Environments +- date: `Tue Nov 22 18:48:39 CST 2022` +- python version: `3.7.12` +- FunASR version: `0.1.0` +- pytorch version: `pytorch 1.7.0` +- Git hash: `` +- Commit date: `` + +# Beachmark Results + +## Wenetspeech +- Decode config: conf/decode_asr_transformer_noctc_1best.yaml + - Decode without CTC + - Decode without LM + +| testset | CER(%)| +|:---------:|:-----:| +| dev | 3.57 | +| test | 6.97 | +| test_net | 6.74 | diff --git a/egs_modelscope/wenetspeech/paraformer/modelscope_utils b/egs_modelscope/wenetspeech/paraformer/modelscope_utils new file mode 120000 index 000000000..fc97768c8 --- /dev/null +++ b/egs_modelscope/wenetspeech/paraformer/modelscope_utils @@ -0,0 +1 @@ +../../common/modelscope_utils \ No newline at end of file diff --git a/egs_modelscope/wenetspeech/paraformer/paraformer_large_infer.sh b/egs_modelscope/wenetspeech/paraformer/paraformer_large_infer.sh new file mode 100755 index 000000000..182a32488 --- /dev/null +++ b/egs_modelscope/wenetspeech/paraformer/paraformer_large_infer.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +ori_data= +data_dir= +exp_dir= +model_name=speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch +inference_nj=32 +gpuid_list="0,1" # set gpus, e.g., gpuid_list="0,1" +ngpu=$(echo $gpuid_list | awk -F "," '{print NF}') +njob=4 # the number of jobs for each gpu +gpu_inference=true # Whether to perform gpu decoding, set false for cpu decoding + +if ${gpu_inference}; then + inference_nj=$[${ngpu}*${njob}] +else + inference_nj=$njob +fi + +# LM configs +use_lm=false +beam_size=1 +lm_weight=0.0 + +test_sets="dev test_meeting test_net" + +. utils/parse_options.sh + +for tset_name in ${test_sets}; do + test_dir=${data_dir}/wenetspeech/${tset_name} + mkdir -p ${test_dir} + find ${ori_data}/${tset_name} -iname "*.wav" > ${test_dir}/wav.flist + sed -e 's/\.wav//' ${test_dir}/wav.flist | awk -F '/' '{print $NF}' > ${test_dir}/utt.list + paste -d' ' ${test_dir}/utt.list ${test_dir}/wav.flist > ${test_dir}/wav.scp + cp ${ori_data}/${tset_name}/trans.txt ${test_dir}/text + sed -i "s/\t/ /g" ${test_dir}/text +done + +mkdir -p ${exp_dir}/wenetspeech + +modelscope_utils/modelscope_infer.sh \ + --data_dir ${data_dir}/wenetspeech \ + --exp_dir ${exp_dir}/wenetspeech \ + --test_sets "${test_sets}" \ + --model_name ${model_name} \ + --inference_nj ${inference_nj} \ + --gpuid_list ${gpuid_list} \ + --njob ${njob} \ + --gpu_inference ${gpu_inference} \ + --use_lm ${use_lm} \ + --beam_size ${beam_size} \ + --lm_weight ${lm_weight} + diff --git a/egs_modelscope/wenetspeech/paraformer/path.sh b/egs_modelscope/wenetspeech/paraformer/path.sh new file mode 100755 index 000000000..7972642d0 --- /dev/null +++ b/egs_modelscope/wenetspeech/paraformer/path.sh @@ -0,0 +1,5 @@ +export FUNASR_DIR=$PWD/../../.. + +# NOTE(kan-bayashi): Use UTF-8 in Python to avoid UnicodeDecodeError when LC_ALL=C +export PYTHONIOENCODING=UTF-8 +export PATH=$FUNASR_DIR/funasr/bin:$PATH diff --git a/egs_modelscope/wenetspeech/paraformer/utils b/egs_modelscope/wenetspeech/paraformer/utils new file mode 120000 index 000000000..37d976175 --- /dev/null +++ b/egs_modelscope/wenetspeech/paraformer/utils @@ -0,0 +1 @@ +../../../egs/aishell/tranformer/utils/ \ No newline at end of file diff --git a/funasr/__init__.py b/funasr/__init__.py new file mode 100644 index 000000000..f297bc3e6 --- /dev/null +++ b/funasr/__init__.py @@ -0,0 +1,8 @@ +"""Initialize funasr package.""" + +import os + +dirname = os.path.dirname(__file__) +version_file = os.path.join(dirname, "version.txt") +with open(version_file, "r") as f: + __version__ = f.read().strip() diff --git a/funasr/bin/__init__.py b/funasr/bin/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/bin/aggregate_stats_dirs.py b/funasr/bin/aggregate_stats_dirs.py new file mode 100755 index 000000000..94cbdf888 --- /dev/null +++ b/funasr/bin/aggregate_stats_dirs.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +import argparse +import logging +import sys +from pathlib import Path +from typing import Iterable +from typing import Union + +import numpy as np + +from funasr.utils.cli_utils import get_commandline_args + + +def aggregate_stats_dirs( + input_dir: Iterable[Union[str, Path]], + output_dir: Union[str, Path], + log_level: str, + skip_sum_stats: bool, +): + logging.basicConfig( + level=log_level, + format="%(asctime)s (%(module)s:%(lineno)d) (levelname)s: %(message)s", + ) + + input_dirs = [Path(p) for p in input_dir] + output_dir = Path(output_dir) + + for mode in ["train", "valid"]: + with (input_dirs[0] / mode / "batch_keys").open("r", encoding="utf-8") as f: + batch_keys = [line.strip() for line in f if line.strip() != ""] + with (input_dirs[0] / mode / "stats_keys").open("r", encoding="utf-8") as f: + stats_keys = [line.strip() for line in f if line.strip() != ""] + (output_dir / mode).mkdir(parents=True, exist_ok=True) + + for key in batch_keys: + with (output_dir / mode / f"{key}_shape").open( + "w", encoding="utf-8" + ) as fout: + for idir in input_dirs: + with (idir / mode / f"{key}_shape").open( + "r", encoding="utf-8" + ) as fin: + # Read to the last in order to sort keys + # because the order can be changed if num_workers>=1 + lines = fin.readlines() + lines = sorted(lines, key=lambda x: x.split()[0]) + for line in lines: + fout.write(line) + + for key in stats_keys: + if not skip_sum_stats: + sum_stats = None + for idir in input_dirs: + stats = np.load(idir / mode / f"{key}_stats.npz") + if sum_stats is None: + sum_stats = dict(**stats) + else: + for k in stats: + sum_stats[k] += stats[k] + + np.savez(output_dir / mode / f"{key}_stats.npz", **sum_stats) + + # if --write_collected_feats=true + p = Path(mode) / "collect_feats" / f"{key}.scp" + scp = input_dirs[0] / p + if scp.exists(): + (output_dir / p).parent.mkdir(parents=True, exist_ok=True) + with (output_dir / p).open("w", encoding="utf-8") as fout: + for idir in input_dirs: + with (idir / p).open("r", encoding="utf-8") as fin: + for line in fin: + fout.write(line) + + +def get_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Aggregate statistics directories into one directory", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "--log_level", + type=lambda x: x.upper(), + default="INFO", + choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"), + help="The verbose level of logging", + ) + parser.add_argument( + "--skip_sum_stats", + default=False, + action="store_true", + help="Skip computing the sum of statistics.", + ) + + parser.add_argument("--input_dir", action="append", help="Input directories") + parser.add_argument("--output_dir", required=True, help="Output directory") + return parser + + +def main(cmd=None): + print(get_commandline_args(), file=sys.stderr) + parser = get_parser() + args = parser.parse_args(cmd) + kwargs = vars(args) + aggregate_stats_dirs(**kwargs) + + +if __name__ == "__main__": + main() diff --git a/funasr/bin/asr_inference.py b/funasr/bin/asr_inference.py new file mode 100755 index 000000000..6ee0ffef8 --- /dev/null +++ b/funasr/bin/asr_inference.py @@ -0,0 +1,548 @@ +#!/usr/bin/env python3 +# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import argparse +import logging +import sys +from pathlib import Path +from typing import Any +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import numpy as np +import torch +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.fileio.datadir_writer import DatadirWriter +from funasr.modules.beam_search.batch_beam_search import BatchBeamSearch +from funasr.modules.beam_search.batch_beam_search_online_sim import BatchBeamSearchOnlineSim +from funasr.modules.beam_search.beam_search import BeamSearch +from funasr.modules.beam_search.beam_search import Hypothesis +from funasr.modules.scorers.ctc import CTCPrefixScorer +from funasr.modules.scorers.length_bonus import LengthBonus +from funasr.modules.scorers.scorer_interface import BatchScorerInterface +from funasr.modules.subsampling import TooShortUttError +from funasr.tasks.asr import ASRTask +from funasr.tasks.lm import LMTask +from funasr.text.build_tokenizer import build_tokenizer +from funasr.text.token_id_converter import TokenIDConverter +from funasr.torch_utils.device_funcs import to_device +from funasr.torch_utils.set_all_random_seed import set_all_random_seed +from funasr.utils import config_argparse +from funasr.utils.cli_utils import get_commandline_args +from funasr.utils.types import str2bool +from funasr.utils.types import str2triple_str +from funasr.utils.types import str_or_none + + +class Speech2Text: + """Speech2Text class + + Examples: + >>> import soundfile + >>> speech2text = Speech2Text("asr_config.yml", "asr.pth") + >>> audio, rate = soundfile.read("speech.wav") + >>> speech2text(audio) + [(text, token, token_int, hypothesis object), ...] + + """ + + def __init__( + self, + asr_train_config: Union[Path, str] = None, + asr_model_file: Union[Path, str] = None, + lm_train_config: Union[Path, str] = None, + lm_file: Union[Path, str] = None, + token_type: str = None, + bpemodel: str = None, + device: str = "cpu", + maxlenratio: float = 0.0, + minlenratio: float = 0.0, + batch_size: int = 1, + dtype: str = "float32", + beam_size: int = 20, + ctc_weight: float = 0.5, + lm_weight: float = 1.0, + ngram_weight: float = 0.9, + penalty: float = 0.0, + nbest: int = 1, + streaming: bool = False, + **kwargs, + ): + assert check_argument_types() + + # 1. Build ASR model + scorers = {} + asr_model, asr_train_args = ASRTask.build_model_from_file( + asr_train_config, asr_model_file, device + ) + logging.info("asr_model: {}".format(asr_model)) + logging.info("asr_train_args: {}".format(asr_train_args)) + asr_model.to(dtype=getattr(torch, dtype)).eval() + + decoder = asr_model.decoder + + ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos) + token_list = asr_model.token_list + scorers.update( + decoder=decoder, + ctc=ctc, + length_bonus=LengthBonus(len(token_list)), + ) + + # 2. Build Language model + if lm_train_config is not None: + lm, lm_train_args = LMTask.build_model_from_file( + lm_train_config, lm_file, device + ) + scorers["lm"] = lm.lm + + # 3. Build ngram model + # ngram is not supported now + ngram = None + scorers["ngram"] = ngram + + # 4. Build BeamSearch object + # transducer is not supported now + beam_search_transducer = None + + weights = dict( + decoder=1.0 - ctc_weight, + ctc=ctc_weight, + lm=lm_weight, + ngram=ngram_weight, + length_bonus=penalty, + ) + beam_search = BeamSearch( + beam_size=beam_size, + weights=weights, + scorers=scorers, + sos=asr_model.sos, + eos=asr_model.eos, + vocab_size=len(token_list), + token_list=token_list, + pre_beam_score_key=None if ctc_weight == 1.0 else "full", + ) + + # TODO(karita): make all scorers batchfied + if batch_size == 1: + non_batch = [ + k + for k, v in beam_search.full_scorers.items() + if not isinstance(v, BatchScorerInterface) + ] + if len(non_batch) == 0: + if streaming: + beam_search.__class__ = BatchBeamSearchOnlineSim + beam_search.set_streaming_config(asr_train_config) + logging.info( + "BatchBeamSearchOnlineSim implementation is selected." + ) + else: + beam_search.__class__ = BatchBeamSearch + logging.info("BatchBeamSearch implementation is selected.") + else: + logging.warning( + f"As non-batch scorers {non_batch} are found, " + f"fall back to non-batch implementation." + ) + + beam_search.to(device=device, dtype=getattr(torch, dtype)).eval() + for scorer in scorers.values(): + if isinstance(scorer, torch.nn.Module): + scorer.to(device=device, dtype=getattr(torch, dtype)).eval() + logging.info(f"Beam_search: {beam_search}") + logging.info(f"Decoding device={device}, dtype={dtype}") + + # 5. [Optional] Build Text converter: e.g. bpe-sym -> Text + if token_type is None: + token_type = asr_train_args.token_type + if bpemodel is None: + bpemodel = asr_train_args.bpemodel + + if token_type is None: + tokenizer = None + elif token_type == "bpe": + if bpemodel is not None: + tokenizer = build_tokenizer(token_type=token_type, bpemodel=bpemodel) + else: + tokenizer = None + else: + tokenizer = build_tokenizer(token_type=token_type) + converter = TokenIDConverter(token_list=token_list) + logging.info(f"Text tokenizer: {tokenizer}") + + self.asr_model = asr_model + self.asr_train_args = asr_train_args + self.converter = converter + self.tokenizer = tokenizer + self.beam_search = beam_search + self.beam_search_transducer = beam_search_transducer + self.maxlenratio = maxlenratio + self.minlenratio = minlenratio + self.device = device + self.dtype = dtype + self.nbest = nbest + + @torch.no_grad() + def __call__( + self, speech: Union[torch.Tensor, np.ndarray] + ) -> List[ + Tuple[ + Optional[str], + List[str], + List[int], + Union[Hypothesis], + ] + ]: + """Inference + + Args: + data: Input speech data + Returns: + text, token, token_int, hyp + + """ + assert check_argument_types() + + # Input as audio signal + if isinstance(speech, np.ndarray): + speech = torch.tensor(speech) + + # data: (Nsamples,) -> (1, Nsamples) + speech = speech.unsqueeze(0).to(getattr(torch, self.dtype)) + # lengths: (1,) + lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1)) + batch = {"speech": speech, "speech_lengths": lengths} + + # a. To device + batch = to_device(batch, device=self.device) + + # b. Forward Encoder + enc, _ = self.asr_model.encode(**batch) + if isinstance(enc, tuple): + enc = enc[0] + assert len(enc) == 1, len(enc) + + # c. Passed the encoder result and the beam search + nbest_hyps = self.beam_search( + x=enc[0], maxlenratio=self.maxlenratio, minlenratio=self.minlenratio + ) + + nbest_hyps = nbest_hyps[: self.nbest] + + results = [] + for hyp in nbest_hyps: + assert isinstance(hyp, (Hypothesis)), type(hyp) + + # remove sos/eos and get results + last_pos = -1 + if isinstance(hyp.yseq, list): + token_int = hyp.yseq[1:last_pos] + else: + token_int = hyp.yseq[1:last_pos].tolist() + + # remove blank symbol id, which is assumed to be 0 + token_int = list(filter(lambda x: x != 0, token_int)) + + # Change integer-ids to tokens + token = self.converter.ids2tokens(token_int) + + if self.tokenizer is not None: + text = self.tokenizer.tokens2text(token) + else: + text = None + results.append((text, token, token_int, hyp)) + + assert check_return_type(results) + return results + + +def inference( + output_dir: str, + maxlenratio: float, + minlenratio: float, + batch_size: int, + dtype: str, + beam_size: int, + ngpu: int, + seed: int, + ctc_weight: float, + lm_weight: float, + ngram_weight: float, + penalty: float, + nbest: int, + num_workers: int, + log_level: Union[int, str], + data_path_and_name_and_type: Sequence[Tuple[str, str, str]], + key_file: Optional[str], + asr_train_config: Optional[str], + asr_model_file: Optional[str], + lm_train_config: Optional[str], + lm_file: Optional[str], + word_lm_train_config: Optional[str], + token_type: Optional[str], + bpemodel: Optional[str], + allow_variable_data_keys: bool, + streaming: bool, + **kwargs, +): + assert check_argument_types() + if batch_size > 1: + raise NotImplementedError("batch decoding is not implemented") + if word_lm_train_config is not None: + raise NotImplementedError("Word LM is not implemented") + if ngpu > 1: + raise NotImplementedError("only single GPU decoding is supported") + + logging.basicConfig( + level=log_level, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", + ) + + if ngpu >= 1: + device = "cuda" + else: + device = "cpu" + + # 1. Set random-seed + set_all_random_seed(seed) + + # 2. Build speech2text + speech2text_kwargs = dict( + asr_train_config=asr_train_config, + asr_model_file=asr_model_file, + lm_train_config=lm_train_config, + lm_file=lm_file, + token_type=token_type, + bpemodel=bpemodel, + device=device, + maxlenratio=maxlenratio, + minlenratio=minlenratio, + dtype=dtype, + beam_size=beam_size, + ctc_weight=ctc_weight, + lm_weight=lm_weight, + ngram_weight=ngram_weight, + penalty=penalty, + nbest=nbest, + streaming=streaming, + ) + logging.info("speech2text_kwargs: {}".format(speech2text_kwargs)) + speech2text = Speech2Text(**speech2text_kwargs) + + # 3. Build data-iterator + loader = ASRTask.build_streaming_iterator( + data_path_and_name_and_type, + dtype=dtype, + batch_size=batch_size, + key_file=key_file, + num_workers=num_workers, + preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False), + collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False), + allow_variable_data_keys=allow_variable_data_keys, + inference=True, + ) + + # 7 .Start for-loop + # FIXME(kamo): The output format should be discussed about + with DatadirWriter(output_dir) as writer: + for keys, batch in loader: + assert isinstance(batch, dict), type(batch) + assert all(isinstance(s, str) for s in keys), keys + _bs = len(next(iter(batch.values()))) + assert len(keys) == _bs, f"{len(keys)} != {_bs}" + batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")} + + # N-best list of (text, token, token_int, hyp_object) + try: + results = speech2text(**batch) + except TooShortUttError as e: + logging.warning(f"Utterance {keys} {e}") + hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[]) + results = [[" ", [""], [2], hyp]] * nbest + + # Only supporting batch_size==1 + key = keys[0] + for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results): + # Create a directory: outdir/{n}best_recog + ibest_writer = writer[f"{n}best_recog"] + + # Write the result to each file + ibest_writer["token"][key] = " ".join(token) + ibest_writer["token_int"][key] = " ".join(map(str, token_int)) + ibest_writer["score"][key] = str(hyp.score) + + if text is not None: + ibest_writer["text"][key] = text + + +def get_parser(): + parser = config_argparse.ArgumentParser( + description="ASR Decoding", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Note(kamo): Use '_' instead of '-' as separator. + # '-' is confusing if written in yaml. + parser.add_argument( + "--log_level", + type=lambda x: x.upper(), + default="INFO", + choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"), + help="The verbose level of logging", + ) + + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument( + "--ngpu", + type=int, + default=0, + help="The number of gpus. 0 indicates CPU mode", + ) + parser.add_argument( + "--gpuid_list", + type=str, + default="", + help="The visible gpus", + ) + parser.add_argument("--seed", type=int, default=0, help="Random seed") + parser.add_argument( + "--dtype", + default="float32", + choices=["float16", "float32", "float64"], + help="Data type", + ) + parser.add_argument( + "--num_workers", + type=int, + default=1, + help="The number of workers used for DataLoader", + ) + + group = parser.add_argument_group("Input data related") + group.add_argument( + "--data_path_and_name_and_type", + type=str2triple_str, + required=True, + action="append", + ) + group.add_argument("--key_file", type=str_or_none) + group.add_argument("--allow_variable_data_keys", type=str2bool, default=False) + + group = parser.add_argument_group("The model configuration related") + group.add_argument( + "--asr_train_config", + type=str, + help="ASR training configuration", + ) + group.add_argument( + "--asr_model_file", + type=str, + help="ASR model parameter file", + ) + group.add_argument( + "--lm_train_config", + type=str, + help="LM training configuration", + ) + group.add_argument( + "--lm_file", + type=str, + help="LM parameter file", + ) + group.add_argument( + "--word_lm_train_config", + type=str, + help="Word LM training configuration", + ) + group.add_argument( + "--word_lm_file", + type=str, + help="Word LM parameter file", + ) + group.add_argument( + "--ngram_file", + type=str, + help="N-gram parameter file", + ) + group.add_argument( + "--model_tag", + type=str, + help="Pretrained model tag. If specify this option, *_train_config and " + "*_file will be overwritten", + ) + + group = parser.add_argument_group("Beam-search related") + group.add_argument( + "--batch_size", + type=int, + default=1, + help="The batch size for inference", + ) + group.add_argument("--nbest", type=int, default=1, help="Output N-best hypotheses") + group.add_argument("--beam_size", type=int, default=20, help="Beam size") + group.add_argument("--penalty", type=float, default=0.0, help="Insertion penalty") + group.add_argument( + "--maxlenratio", + type=float, + default=0.0, + help="Input length ratio to obtain max output length. " + "If maxlenratio=0.0 (default), it uses a end-detect " + "function " + "to automatically find maximum hypothesis lengths." + "If maxlenratio<0.0, its absolute value is interpreted" + "as a constant max output length", + ) + group.add_argument( + "--minlenratio", + type=float, + default=0.0, + help="Input length ratio to obtain min output length", + ) + group.add_argument( + "--ctc_weight", + type=float, + default=0.5, + help="CTC weight in joint decoding", + ) + group.add_argument("--lm_weight", type=float, default=1.0, help="RNNLM weight") + group.add_argument("--ngram_weight", type=float, default=0.9, help="ngram weight") + group.add_argument("--streaming", type=str2bool, default=False) + + group = parser.add_argument_group("Text converter related") + group.add_argument( + "--token_type", + type=str_or_none, + default=None, + choices=["char", "bpe", None], + help="The token type for ASR model. " + "If not given, refers from the training args", + ) + group.add_argument( + "--bpemodel", + type=str_or_none, + default=None, + help="The model path of sentencepiece. " + "If not given, refers from the training args", + ) + + return parser + + +def main(cmd=None): + print(get_commandline_args(), file=sys.stderr) + parser = get_parser() + args = parser.parse_args(cmd) + kwargs = vars(args) + kwargs.pop("config", None) + inference(**kwargs) + + +if __name__ == "__main__": + main() diff --git a/funasr/bin/asr_inference_launch.py b/funasr/bin/asr_inference_launch.py new file mode 100755 index 000000000..9d328ad21 --- /dev/null +++ b/funasr/bin/asr_inference_launch.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import argparse +import logging +import os +import sys + +from funasr.utils import config_argparse +from funasr.utils.cli_utils import get_commandline_args +from funasr.utils.types import str2bool +from funasr.utils.types import str2triple_str +from funasr.utils.types import str_or_none + + +def get_parser(): + parser = config_argparse.ArgumentParser( + description="ASR Decoding", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Note(kamo): Use '_' instead of '-' as separator. + # '-' is confusing if written in yaml. + parser.add_argument( + "--log_level", + type=lambda x: x.upper(), + default="INFO", + choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"), + help="The verbose level of logging", + ) + + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument( + "--ngpu", + type=int, + default=0, + help="The number of gpus. 0 indicates CPU mode", + ) + parser.add_argument( + "--njob", + type=int, + default=1, + help="The number of jobs for each gpu", + ) + parser.add_argument( + "--gpuid_list", + type=str, + default="", + help="The visible gpus", + ) + parser.add_argument("--seed", type=int, default=0, help="Random seed") + parser.add_argument( + "--dtype", + default="float32", + choices=["float16", "float32", "float64"], + help="Data type", + ) + parser.add_argument( + "--num_workers", + type=int, + default=1, + help="The number of workers used for DataLoader", + ) + + group = parser.add_argument_group("Input data related") + group.add_argument( + "--data_path_and_name_and_type", + type=str2triple_str, + required=True, + action="append", + ) + group.add_argument("--key_file", type=str_or_none) + group.add_argument("--allow_variable_data_keys", type=str2bool, default=False) + + group = parser.add_argument_group("The model configuration related") + group.add_argument( + "--asr_train_config", + type=str, + help="ASR training configuration", + ) + group.add_argument( + "--asr_model_file", + type=str, + help="ASR model parameter file", + ) + group.add_argument( + "--lm_train_config", + type=str, + help="LM training configuration", + ) + group.add_argument( + "--lm_file", + type=str, + help="LM parameter file", + ) + group.add_argument( + "--word_lm_train_config", + type=str, + help="Word LM training configuration", + ) + group.add_argument( + "--word_lm_file", + type=str, + help="Word LM parameter file", + ) + group.add_argument( + "--ngram_file", + type=str, + help="N-gram parameter file", + ) + group.add_argument( + "--model_tag", + type=str, + help="Pretrained model tag. If specify this option, *_train_config and " + "*_file will be overwritten", + ) + + group = parser.add_argument_group("Beam-search related") + group.add_argument( + "--batch_size", + type=int, + default=1, + help="The batch size for inference", + ) + group.add_argument("--nbest", type=int, default=5, help="Output N-best hypotheses") + group.add_argument("--beam_size", type=int, default=20, help="Beam size") + group.add_argument("--penalty", type=float, default=0.0, help="Insertion penalty") + group.add_argument( + "--maxlenratio", + type=float, + default=0.0, + help="Input length ratio to obtain max output length. " + "If maxlenratio=0.0 (default), it uses a end-detect " + "function " + "to automatically find maximum hypothesis lengths." + "If maxlenratio<0.0, its absolute value is interpreted" + "as a constant max output length", + ) + group.add_argument( + "--minlenratio", + type=float, + default=0.0, + help="Input length ratio to obtain min output length", + ) + group.add_argument( + "--ctc_weight", + type=float, + default=0.5, + help="CTC weight in joint decoding", + ) + group.add_argument("--lm_weight", type=float, default=1.0, help="RNNLM weight") + group.add_argument("--ngram_weight", type=float, default=0.9, help="ngram weight") + group.add_argument("--streaming", type=str2bool, default=False) + + group = parser.add_argument_group("Text converter related") + group.add_argument( + "--token_type", + type=str_or_none, + default=None, + choices=["char", "bpe", None], + help="The token type for ASR model. " + "If not given, refers from the training args", + ) + group.add_argument( + "--bpemodel", + type=str_or_none, + default=None, + help="The model path of sentencepiece. " + "If not given, refers from the training args", + ) + group.add_argument("--token_num_relax", type=int, default=1, help="") + group.add_argument("--decoding_ind", type=int, default=0, help="") + group.add_argument("--decoding_mode", type=str, default="model1", help="") + group.add_argument( + "--ctc_weight2", + type=float, + default=0.0, + help="CTC weight in joint decoding", + ) + return parser + + +def main(cmd=None): + print(get_commandline_args(), file=sys.stderr) + parser = get_parser() + parser.add_argument( + "--mode", + type=str, + default="asr", + help="The decoding mode", + ) + args = parser.parse_args(cmd) + kwargs = vars(args) + kwargs.pop("config", None) + + # set logging messages + logging.basicConfig( + level=args.log_level, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", + ) + logging.info("Decoding args: {}".format(kwargs)) + + # gpu setting + if args.ngpu > 0: + jobid = int(args.output_dir.split(".")[-1]) + gpuid = args.gpuid_list.split(",")[(jobid - 1) // args.njob] + os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + os.environ["CUDA_VISIBLE_DEVICES"] = gpuid + + if args.mode == "asr": + from funasr.bin.asr_inference import inference + inference(**kwargs) + elif args.mode == "uniasr": + from funasr.bin.asr_inference_uniasr import inference + inference(**kwargs) + elif args.mode == "paraformer": + from funasr.bin.asr_inference_paraformer import inference + inference(**kwargs) + else: + logging.info("Unknown decoding mode: {}".format(args.mode)) + + +if __name__ == "__main__": + main() diff --git a/funasr/bin/asr_inference_paraformer.py b/funasr/bin/asr_inference_paraformer.py new file mode 100755 index 000000000..ed75010d1 --- /dev/null +++ b/funasr/bin/asr_inference_paraformer.py @@ -0,0 +1,528 @@ +#!/usr/bin/env python3 +import argparse +import logging +import sys +import time +from pathlib import Path +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import numpy as np +import torch +from typeguard import check_argument_types + +from funasr.fileio.datadir_writer import DatadirWriter +from funasr.modules.beam_search.beam_search import BeamSearchPara as BeamSearch +from funasr.modules.beam_search.beam_search import Hypothesis +from funasr.modules.scorers.ctc import CTCPrefixScorer +from funasr.modules.scorers.length_bonus import LengthBonus +from funasr.modules.subsampling import TooShortUttError +from funasr.tasks.asr import ASRTaskParaformer as ASRTask +from funasr.tasks.lm import LMTask +from funasr.text.build_tokenizer import build_tokenizer +from funasr.text.token_id_converter import TokenIDConverter +from funasr.torch_utils.device_funcs import to_device +from funasr.torch_utils.set_all_random_seed import set_all_random_seed +from funasr.utils import config_argparse +from funasr.utils.cli_utils import get_commandline_args +from funasr.utils.types import str2bool +from funasr.utils.types import str2triple_str +from funasr.utils.types import str_or_none + + +class Speech2Text: + """Speech2Text class + + Examples: + >>> import soundfile + >>> speech2text = Speech2Text("asr_config.yml", "asr.pth") + >>> audio, rate = soundfile.read("speech.wav") + >>> speech2text(audio) + [(text, token, token_int, hypothesis object), ...] + + """ + + def __init__( + self, + asr_train_config: Union[Path, str] = None, + asr_model_file: Union[Path, str] = None, + lm_train_config: Union[Path, str] = None, + lm_file: Union[Path, str] = None, + token_type: str = None, + bpemodel: str = None, + device: str = "cpu", + maxlenratio: float = 0.0, + minlenratio: float = 0.0, + dtype: str = "float32", + beam_size: int = 20, + ctc_weight: float = 0.5, + lm_weight: float = 1.0, + ngram_weight: float = 0.9, + penalty: float = 0.0, + nbest: int = 1, + **kwargs, + ): + assert check_argument_types() + + # 1. Build ASR model + scorers = {} + asr_model, asr_train_args = ASRTask.build_model_from_file( + asr_train_config, asr_model_file, device + ) + logging.info("asr_model: {}".format(asr_model)) + logging.info("asr_train_args: {}".format(asr_train_args)) + asr_model.to(dtype=getattr(torch, dtype)).eval() + + ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos) + token_list = asr_model.token_list + scorers.update( + ctc=ctc, + length_bonus=LengthBonus(len(token_list)), + ) + + # 2. Build Language model + if lm_train_config is not None: + lm, lm_train_args = LMTask.build_model_from_file( + lm_train_config, lm_file, device + ) + scorers["lm"] = lm.lm + + # 3. Build ngram model + # ngram is not supported now + ngram = None + scorers["ngram"] = ngram + + # 4. Build BeamSearch object + # transducer is not supported now + beam_search_transducer = None + + weights = dict( + decoder=1.0 - ctc_weight, + ctc=ctc_weight, + lm=lm_weight, + ngram=ngram_weight, + length_bonus=penalty, + ) + beam_search = BeamSearch( + beam_size=beam_size, + weights=weights, + scorers=scorers, + sos=asr_model.sos, + eos=asr_model.eos, + vocab_size=len(token_list), + token_list=token_list, + pre_beam_score_key=None if ctc_weight == 1.0 else "full", + ) + + beam_search.to(device=device, dtype=getattr(torch, dtype)).eval() + for scorer in scorers.values(): + if isinstance(scorer, torch.nn.Module): + scorer.to(device=device, dtype=getattr(torch, dtype)).eval() + logging.info(f"Beam_search: {beam_search}") + logging.info(f"Decoding device={device}, dtype={dtype}") + + # 5. [Optional] Build Text converter: e.g. bpe-sym -> Text + if token_type is None: + token_type = asr_train_args.token_type + if bpemodel is None: + bpemodel = asr_train_args.bpemodel + + if token_type is None: + tokenizer = None + elif token_type == "bpe": + if bpemodel is not None: + tokenizer = build_tokenizer(token_type=token_type, bpemodel=bpemodel) + else: + tokenizer = None + else: + tokenizer = build_tokenizer(token_type=token_type) + converter = TokenIDConverter(token_list=token_list) + logging.info(f"Text tokenizer: {tokenizer}") + + self.asr_model = asr_model + self.asr_train_args = asr_train_args + self.converter = converter + self.tokenizer = tokenizer + self.beam_search = beam_search + self.beam_search_transducer = beam_search_transducer + self.maxlenratio = maxlenratio + self.minlenratio = minlenratio + self.device = device + self.dtype = dtype + self.nbest = nbest + + @torch.no_grad() + def __call__( + self, speech: Union[torch.Tensor, np.ndarray] + ): + """Inference + + Args: + data: Input speech data + Returns: + text, token, token_int, hyp + + """ + assert check_argument_types() + + # Input as audio signal + if isinstance(speech, np.ndarray): + speech = torch.tensor(speech) + + # data: (Nsamples,) -> (1, Nsamples) + speech = speech.unsqueeze(0).to(getattr(torch, self.dtype)) + lfr_factor = max(1, (speech.size()[-1]//80)-1) + # lengths: (1,) + lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1)) + batch = {"speech": speech, "speech_lengths": lengths} + + # a. To device + batch = to_device(batch, device=self.device) + + # b. Forward Encoder + enc, enc_len = self.asr_model.encode(**batch) + if isinstance(enc, tuple): + enc = enc[0] + assert len(enc) == 1, len(enc) + + predictor_outs = self.asr_model.calc_predictor(enc, enc_len) + pre_acoustic_embeds, pre_token_length = predictor_outs[0], predictor_outs[1] + pre_token_length = torch.tensor([pre_acoustic_embeds.size(1)], device=pre_acoustic_embeds.device) + decoder_outs = self.asr_model.cal_decoder_with_predictor(enc, enc_len, pre_acoustic_embeds, pre_token_length) + decoder_out, ys_pad_lens = decoder_outs[0], decoder_outs[1] + + nbest_hyps = self.beam_search( + x=enc[0], am_scores=decoder_out[0], maxlenratio=self.maxlenratio, minlenratio=self.minlenratio + ) + + nbest_hyps = nbest_hyps[: self.nbest] + results = [] + for hyp in nbest_hyps: + assert isinstance(hyp, (Hypothesis)), type(hyp) + + # remove sos/eos and get results + last_pos = -1 + if isinstance(hyp.yseq, list): + token_int = hyp.yseq[1:last_pos] + else: + token_int = hyp.yseq[1:last_pos].tolist() + + # remove blank symbol id, which is assumed to be 0 + token_int = list(filter(lambda x: x != 0, token_int)) + + # Change integer-ids to tokens + token = self.converter.ids2tokens(token_int) + + if self.tokenizer is not None: + text = self.tokenizer.tokens2text(token) + else: + text = None + + results.append((text, token, token_int, hyp, speech.size(1), lfr_factor)) + + # assert check_return_type(results) + return results + + +def inference( + output_dir: str, + maxlenratio: float, + minlenratio: float, + batch_size: int, + dtype: str, + beam_size: int, + ngpu: int, + seed: int, + ctc_weight: float, + lm_weight: float, + ngram_weight: float, + penalty: float, + nbest: int, + num_workers: int, + log_level: Union[int, str], + data_path_and_name_and_type: Sequence[Tuple[str, str, str]], + key_file: Optional[str], + asr_train_config: Optional[str], + asr_model_file: Optional[str], + lm_train_config: Optional[str], + lm_file: Optional[str], + word_lm_train_config: Optional[str], + token_type: Optional[str], + bpemodel: Optional[str], + allow_variable_data_keys: bool, + **kwargs, +): + assert check_argument_types() + if batch_size > 1: + raise NotImplementedError("batch decoding is not implemented") + if word_lm_train_config is not None: + raise NotImplementedError("Word LM is not implemented") + if ngpu > 1: + raise NotImplementedError("only single GPU decoding is supported") + + logging.basicConfig( + level=log_level, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", + ) + + if ngpu >= 1: + device = "cuda" + else: + device = "cpu" + + # 1. Set random-seed + set_all_random_seed(seed) + + # 2. Build speech2text + speech2text_kwargs = dict( + asr_train_config=asr_train_config, + asr_model_file=asr_model_file, + lm_train_config=lm_train_config, + lm_file=lm_file, + token_type=token_type, + bpemodel=bpemodel, + device=device, + maxlenratio=maxlenratio, + minlenratio=minlenratio, + dtype=dtype, + beam_size=beam_size, + ctc_weight=ctc_weight, + lm_weight=lm_weight, + ngram_weight=ngram_weight, + penalty=penalty, + nbest=nbest, + ) + speech2text = Speech2Text(**speech2text_kwargs) + + # 3. Build data-iterator + loader = ASRTask.build_streaming_iterator( + data_path_and_name_and_type, + dtype=dtype, + batch_size=batch_size, + key_file=key_file, + num_workers=num_workers, + preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False), + collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False), + allow_variable_data_keys=allow_variable_data_keys, + inference=True, + ) + + forward_time_total = 0.0 + length_total = 0.0 + # 7 .Start for-loop + # FIXME(kamo): The output format should be discussed about + with DatadirWriter(output_dir) as writer: + for keys, batch in loader: + assert isinstance(batch, dict), type(batch) + assert all(isinstance(s, str) for s in keys), keys + _bs = len(next(iter(batch.values()))) + assert len(keys) == _bs, f"{len(keys)} != {_bs}" + batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")} + + logging.info("decoding, utt_id: {}".format(keys)) + # N-best list of (text, token, token_int, hyp_object) + + try: + time_beg = time.time() + results = speech2text(**batch) + time_end = time.time() + forward_time = time_end - time_beg + lfr_factor = results[0][-1] + length = results[0][-2] + results = [results[0][:-2]] + forward_time_total += forward_time + length_total += length + logging.info( + "decoding, feature length: {}, forward_time: {:.4f}, rtf: {:.4f}". + format(length, forward_time, 100 * forward_time / (length*lfr_factor))) + except TooShortUttError as e: + logging.warning(f"Utterance {keys} {e}") + hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[]) + results = [[" ", [""], [2], hyp]] * nbest + + # Only supporting batch_size==1 + key = keys[0] + for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results): + # Create a directory: outdir/{n}best_recog + ibest_writer = writer[f"{n}best_recog"] + + # Write the result to each file + ibest_writer["token"][key] = " ".join(token) + ibest_writer["token_int"][key] = " ".join(map(str, token_int)) + ibest_writer["score"][key] = str(hyp.score) + + if text is not None: + ibest_writer["text"][key] = text + + logging.info("decoding, predictions: {}".format(text)) + + logging.info("decoding, feature length total: {}, forward_time total: {:.4f}, rtf avg: {:.4f}". + format(length_total, forward_time_total, 100 * forward_time_total / (length_total*lfr_factor))) + + +def get_parser(): + parser = config_argparse.ArgumentParser( + description="ASR Decoding", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Note(kamo): Use '_' instead of '-' as separator. + # '-' is confusing if written in yaml. + parser.add_argument( + "--log_level", + type=lambda x: x.upper(), + default="INFO", + choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"), + help="The verbose level of logging", + ) + + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument( + "--ngpu", + type=int, + default=0, + help="The number of gpus. 0 indicates CPU mode", + ) + parser.add_argument("--seed", type=int, default=0, help="Random seed") + parser.add_argument( + "--dtype", + default="float32", + choices=["float16", "float32", "float64"], + help="Data type", + ) + parser.add_argument( + "--num_workers", + type=int, + default=1, + help="The number of workers used for DataLoader", + ) + + group = parser.add_argument_group("Input data related") + group.add_argument( + "--data_path_and_name_and_type", + type=str2triple_str, + required=True, + action="append", + ) + group.add_argument("--key_file", type=str_or_none) + group.add_argument("--allow_variable_data_keys", type=str2bool, default=False) + + group = parser.add_argument_group("The model configuration related") + group.add_argument( + "--asr_train_config", + type=str, + help="ASR training configuration", + ) + group.add_argument( + "--asr_model_file", + type=str, + help="ASR model parameter file", + ) + group.add_argument( + "--lm_train_config", + type=str, + help="LM training configuration", + ) + group.add_argument( + "--lm_file", + type=str, + help="LM parameter file", + ) + group.add_argument( + "--word_lm_train_config", + type=str, + help="Word LM training configuration", + ) + group.add_argument( + "--word_lm_file", + type=str, + help="Word LM parameter file", + ) + group.add_argument( + "--ngram_file", + type=str, + help="N-gram parameter file", + ) + group.add_argument( + "--model_tag", + type=str, + help="Pretrained model tag. If specify this option, *_train_config and " + "*_file will be overwritten", + ) + + group = parser.add_argument_group("Beam-search related") + group.add_argument( + "--batch_size", + type=int, + default=1, + help="The batch size for inference", + ) + group.add_argument("--nbest", type=int, default=1, help="Output N-best hypotheses") + group.add_argument("--beam_size", type=int, default=20, help="Beam size") + group.add_argument("--penalty", type=float, default=0.0, help="Insertion penalty") + group.add_argument( + "--maxlenratio", + type=float, + default=0.0, + help="Input length ratio to obtain max output length. " + "If maxlenratio=0.0 (default), it uses a end-detect " + "function " + "to automatically find maximum hypothesis lengths." + "If maxlenratio<0.0, its absolute value is interpreted" + "as a constant max output length", + ) + group.add_argument( + "--minlenratio", + type=float, + default=0.0, + help="Input length ratio to obtain min output length", + ) + group.add_argument( + "--ctc_weight", + type=float, + default=0.5, + help="CTC weight in joint decoding", + ) + group.add_argument("--lm_weight", type=float, default=1.0, help="RNNLM weight") + group.add_argument("--ngram_weight", type=float, default=0.9, help="ngram weight") + group.add_argument("--streaming", type=str2bool, default=False) + + group.add_argument( + "--frontend_conf", + default=None, + help="", + ) + + group = parser.add_argument_group("Text converter related") + group.add_argument( + "--token_type", + type=str_or_none, + default=None, + choices=["char", "bpe", None], + help="The token type for ASR model. " + "If not given, refers from the training args", + ) + group.add_argument( + "--bpemodel", + type=str_or_none, + default=None, + help="The model path of sentencepiece. " + "If not given, refers from the training args", + ) + + return parser + + +def main(cmd=None): + print(get_commandline_args(), file=sys.stderr) + parser = get_parser() + args = parser.parse_args(cmd) + kwargs = vars(args) + kwargs.pop("config", None) + inference(**kwargs) + + +if __name__ == "__main__": + main() diff --git a/funasr/bin/asr_inference_uniasr.py b/funasr/bin/asr_inference_uniasr.py new file mode 100755 index 000000000..796c5b303 --- /dev/null +++ b/funasr/bin/asr_inference_uniasr.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +import argparse +import logging +import sys +from pathlib import Path +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import numpy as np +import torch +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.fileio.datadir_writer import DatadirWriter +from funasr.modules.beam_search.beam_search import BeamSearchScama as BeamSearch +from funasr.modules.beam_search.beam_search import Hypothesis +from funasr.modules.scorers.ctc import CTCPrefixScorer +from funasr.modules.scorers.length_bonus import LengthBonus +from funasr.modules.subsampling import TooShortUttError +from funasr.tasks.asr import ASRTaskUniASR as ASRTask +from funasr.tasks.lm import LMTask +from funasr.text.build_tokenizer import build_tokenizer +from funasr.text.token_id_converter import TokenIDConverter +from funasr.torch_utils.device_funcs import to_device +from funasr.torch_utils.set_all_random_seed import set_all_random_seed +from funasr.utils import config_argparse +from funasr.utils.cli_utils import get_commandline_args +from funasr.utils.types import str2bool +from funasr.utils.types import str2triple_str +from funasr.utils.types import str_or_none + + +class Speech2Text: + """Speech2Text class + + Examples: + >>> import soundfile + >>> speech2text = Speech2Text("asr_config.yml", "asr.pth") + >>> audio, rate = soundfile.read("speech.wav") + >>> speech2text(audio) + [(text, token, token_int, hypothesis object), ...] + + """ + + def __init__( + self, + asr_train_config: Union[Path, str] = None, + asr_model_file: Union[Path, str] = None, + lm_train_config: Union[Path, str] = None, + lm_file: Union[Path, str] = None, + token_type: str = None, + bpemodel: str = None, + device: str = "cpu", + maxlenratio: float = 0.0, + minlenratio: float = 0.0, + dtype: str = "float32", + beam_size: int = 20, + ctc_weight: float = 0.5, + lm_weight: float = 1.0, + ngram_weight: float = 0.9, + penalty: float = 0.0, + nbest: int = 1, + token_num_relax: int = 1, + decoding_ind: int = 0, + decoding_mode: str = "model1", + **kwargs, + ): + assert check_argument_types() + + # 1. Build ASR model + scorers = {} + asr_model, asr_train_args = ASRTask.build_model_from_file( + asr_train_config, asr_model_file, device + ) + asr_model.to(dtype=getattr(torch, dtype)).eval() + if decoding_mode == "model1": + decoder = asr_model.decoder + else: + decoder = asr_model.decoder2 + + ctc = CTCPrefixScorer(ctc=asr_model.ctc, eos=asr_model.eos) + token_list = asr_model.token_list + scorers.update( + decoder=decoder, + ctc=ctc, + length_bonus=LengthBonus(len(token_list)), + ) + + # 2. Build Language model + if lm_train_config is not None: + lm, lm_train_args = LMTask.build_model_from_file( + lm_train_config, lm_file, device + ) + scorers["lm"] = lm.lm + + # 3. Build ngram model + # ngram is not supported now + ngram = None + scorers["ngram"] = ngram + + # 4. Build BeamSearch object + # transducer is not supported now + beam_search_transducer = None + + weights = dict( + decoder=1.0 - ctc_weight, + ctc=ctc_weight, + lm=lm_weight, + ngram=ngram_weight, + length_bonus=penalty, + ) + beam_search = BeamSearch( + beam_size=beam_size, + weights=weights, + scorers=scorers, + sos=asr_model.sos, + eos=asr_model.eos, + vocab_size=len(token_list), + token_list=token_list, + pre_beam_score_key=None if ctc_weight == 1.0 else "full", + ) + + beam_search.to(device=device, dtype=getattr(torch, dtype)).eval() + for scorer in scorers.values(): + if isinstance(scorer, torch.nn.Module): + scorer.to(device=device, dtype=getattr(torch, dtype)).eval() + logging.info(f"Beam_search: {beam_search}") + logging.info(f"Decoding device={device}, dtype={dtype}") + + # 5. [Optional] Build Text converter: e.g. bpe-sym -> Text + if token_type is None: + token_type = asr_train_args.token_type + if bpemodel is None: + bpemodel = asr_train_args.bpemodel + + if token_type is None: + tokenizer = None + elif token_type == "bpe": + if bpemodel is not None: + tokenizer = build_tokenizer(token_type=token_type, bpemodel=bpemodel) + else: + tokenizer = None + else: + tokenizer = build_tokenizer(token_type=token_type) + converter = TokenIDConverter(token_list=token_list) + logging.info(f"Text tokenizer: {tokenizer}") + + self.asr_model = asr_model + self.asr_train_args = asr_train_args + self.converter = converter + self.tokenizer = tokenizer + self.beam_search = beam_search + self.beam_search_transducer = beam_search_transducer + self.maxlenratio = maxlenratio + self.minlenratio = minlenratio + self.device = device + self.dtype = dtype + self.nbest = nbest + self.token_num_relax = token_num_relax + self.decoding_ind = decoding_ind + self.decoding_mode = decoding_mode + + @torch.no_grad() + def __call__( + self, speech: Union[torch.Tensor, np.ndarray] + ) -> List[ + Tuple[ + Optional[str], + List[str], + List[int], + Union[Hypothesis], + ] + ]: + """Inference + + Args: + data: Input speech data + Returns: + text, token, token_int, hyp + + """ + assert check_argument_types() + + # Input as audio signal + if isinstance(speech, np.ndarray): + speech = torch.tensor(speech) + + # data: (Nsamples,) -> (1, Nsamples) + speech = speech.unsqueeze(0).to(getattr(torch, self.dtype)) + # lengths: (1,) + lengths = speech.new_full([1], dtype=torch.long, fill_value=speech.size(1)) + batch = {"speech": speech, "speech_lengths": lengths} + + # a. To device + batch = to_device(batch, device=self.device) + # b. Forward Encoder + speech_raw = speech.clone().to(self.device) + enc, enc_len = self.asr_model.encode(**batch, ind=self.decoding_ind) + if isinstance(enc, tuple): + enc = enc[0] + assert len(enc) == 1, len(enc) + if self.decoding_mode == "model1": + predictor_outs = self.asr_model.calc_predictor_mask(enc, enc_len) + else: + enc, enc_len = self.asr_model.encode2(enc, enc_len, speech_raw, lengths, ind=self.decoding_ind) + predictor_outs = self.asr_model.calc_predictor_mask2(enc, enc_len) + + scama_mask = predictor_outs[4] + pre_token_length = predictor_outs[1] + pre_acoustic_embeds = predictor_outs[0] + maxlen = pre_token_length.sum().item() + self.token_num_relax + minlen = max(0, pre_token_length.sum().item() - self.token_num_relax) + # c. Passed the encoder result and the beam search + nbest_hyps = self.beam_search( + x=enc[0], scama_mask=scama_mask, pre_acoustic_embeds=pre_acoustic_embeds, maxlenratio=self.maxlenratio, + minlenratio=self.minlenratio, maxlen=int(maxlen), minlen=int(minlen), + ) + + nbest_hyps = nbest_hyps[: self.nbest] + + results = [] + for hyp in nbest_hyps: + assert isinstance(hyp, (Hypothesis)), type(hyp) + + # remove sos/eos and get results + last_pos = -1 + if isinstance(hyp.yseq, list): + token_int = hyp.yseq[1:last_pos] + else: + token_int = hyp.yseq[1:last_pos].tolist() + + # remove blank symbol id, which is assumed to be 0 + token_int = list(filter(lambda x: x != 0, token_int)) + + # Change integer-ids to tokens + token = self.converter.ids2tokens(token_int) + + if self.tokenizer is not None: + text = self.tokenizer.tokens2text(token) + else: + text = None + results.append((text, token, token_int, hyp)) + + assert check_return_type(results) + return results + + +def inference( + output_dir: str, + maxlenratio: float, + minlenratio: float, + batch_size: int, + dtype: str, + beam_size: int, + ngpu: int, + seed: int, + ctc_weight: float, + lm_weight: float, + ngram_weight: float, + penalty: float, + nbest: int, + num_workers: int, + log_level: Union[int, str], + data_path_and_name_and_type: Sequence[Tuple[str, str, str]], + key_file: Optional[str], + asr_train_config: Optional[str], + asr_model_file: Optional[str], + lm_train_config: Optional[str], + lm_file: Optional[str], + word_lm_train_config: Optional[str], + ngram_file: Optional[str], + token_type: Optional[str], + bpemodel: Optional[str], + allow_variable_data_keys: bool, + streaming: bool, + token_num_relax: int = 1, + decoding_ind: int = 0, + decoding_mode: str = "model1", + **kwargs, +): + assert check_argument_types() + if batch_size > 1: + raise NotImplementedError("batch decoding is not implemented") + if word_lm_train_config is not None: + raise NotImplementedError("Word LM is not implemented") + if ngpu > 1: + raise NotImplementedError("only single GPU decoding is supported") + + logging.basicConfig( + level=log_level, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", + ) + + if ngpu >= 1: + device = "cuda" + else: + device = "cpu" + + # 1. Set random-seed + set_all_random_seed(seed) + + # 2. Build speech2text + speech2text_kwargs = dict( + asr_train_config=asr_train_config, + asr_model_file=asr_model_file, + lm_train_config=lm_train_config, + lm_file=lm_file, + ngram_file=ngram_file, + token_type=token_type, + bpemodel=bpemodel, + device=device, + maxlenratio=maxlenratio, + minlenratio=minlenratio, + dtype=dtype, + beam_size=beam_size, + ctc_weight=ctc_weight, + lm_weight=lm_weight, + ngram_weight=ngram_weight, + penalty=penalty, + nbest=nbest, + streaming=streaming, + token_num_relax=token_num_relax, + decoding_ind=decoding_ind, + decoding_mode=decoding_mode, + ) + speech2text = Speech2Text(**speech2text_kwargs) + + # 3. Build data-iterator + loader = ASRTask.build_streaming_iterator( + data_path_and_name_and_type, + dtype=dtype, + batch_size=batch_size, + key_file=key_file, + num_workers=num_workers, + preprocess_fn=ASRTask.build_preprocess_fn(speech2text.asr_train_args, False), + collate_fn=ASRTask.build_collate_fn(speech2text.asr_train_args, False), + allow_variable_data_keys=allow_variable_data_keys, + inference=True, + ) + + # 7 .Start for-loop + # FIXME(kamo): The output format should be discussed about + with DatadirWriter(output_dir) as writer: + for keys, batch in loader: + assert isinstance(batch, dict), type(batch) + assert all(isinstance(s, str) for s in keys), keys + _bs = len(next(iter(batch.values()))) + assert len(keys) == _bs, f"{len(keys)} != {_bs}" + batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")} + + # N-best list of (text, token, token_int, hyp_object) + try: + results = speech2text(**batch) + except TooShortUttError as e: + logging.warning(f"Utterance {keys} {e}") + hyp = Hypothesis(score=0.0, scores={}, states={}, yseq=[]) + results = [[" ", [""], [2], hyp]] * nbest + + # Only supporting batch_size==1 + key = keys[0] + logging.info(f"Utterance: {key}") + for n, (text, token, token_int, hyp) in zip(range(1, nbest + 1), results): + # Create a directory: outdir/{n}best_recog + ibest_writer = writer[f"{n}best_recog"] + + # Write the result to each file + ibest_writer["token"][key] = " ".join(token) + ibest_writer["token_int"][key] = " ".join(map(str, token_int)) + ibest_writer["score"][key] = str(hyp.score) + + if text is not None: + ibest_writer["text"][key] = text + + +def get_parser(): + parser = config_argparse.ArgumentParser( + description="ASR Decoding", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Note(kamo): Use '_' instead of '-' as separator. + # '-' is confusing if written in yaml. + parser.add_argument( + "--log_level", + type=lambda x: x.upper(), + default="INFO", + choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"), + help="The verbose level of logging", + ) + + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument( + "--ngpu", + type=int, + default=0, + help="The number of gpus. 0 indicates CPU mode", + ) + parser.add_argument("--seed", type=int, default=0, help="Random seed") + parser.add_argument( + "--dtype", + default="float32", + choices=["float16", "float32", "float64"], + help="Data type", + ) + parser.add_argument( + "--num_workers", + type=int, + default=1, + help="The number of workers used for DataLoader", + ) + + group = parser.add_argument_group("Input data related") + group.add_argument( + "--data_path_and_name_and_type", + type=str2triple_str, + required=True, + action="append", + ) + group.add_argument("--key_file", type=str_or_none) + group.add_argument("--allow_variable_data_keys", type=str2bool, default=False) + + group = parser.add_argument_group("The model configuration related") + group.add_argument( + "--asr_train_config", + type=str, + help="ASR training configuration", + ) + group.add_argument( + "--asr_model_file", + type=str, + help="ASR model parameter file", + ) + group.add_argument( + "--lm_train_config", + type=str, + help="LM training configuration", + ) + group.add_argument( + "--lm_file", + type=str, + help="LM parameter file", + ) + group.add_argument( + "--word_lm_train_config", + type=str, + help="Word LM training configuration", + ) + group.add_argument( + "--word_lm_file", + type=str, + help="Word LM parameter file", + ) + group.add_argument( + "--ngram_file", + type=str, + help="N-gram parameter file", + ) + group.add_argument( + "--model_tag", + type=str, + help="Pretrained model tag. If specify this option, *_train_config and " + "*_file will be overwritten", + ) + + group = parser.add_argument_group("Beam-search related") + group.add_argument( + "--batch_size", + type=int, + default=1, + help="The batch size for inference", + ) + group.add_argument("--nbest", type=int, default=1, help="Output N-best hypotheses") + group.add_argument("--beam_size", type=int, default=20, help="Beam size") + group.add_argument("--penalty", type=float, default=0.0, help="Insertion penalty") + group.add_argument( + "--maxlenratio", + type=float, + default=0.0, + help="Input length ratio to obtain max output length. " + "If maxlenratio=0.0 (default), it uses a end-detect " + "function " + "to automatically find maximum hypothesis lengths." + "If maxlenratio<0.0, its absolute value is interpreted" + "as a constant max output length", + ) + group.add_argument( + "--minlenratio", + type=float, + default=0.0, + help="Input length ratio to obtain min output length", + ) + group.add_argument( + "--ctc_weight", + type=float, + default=0.5, + help="CTC weight in joint decoding", + ) + group.add_argument("--lm_weight", type=float, default=1.0, help="RNNLM weight") + group.add_argument("--ngram_weight", type=float, default=0.9, help="ngram weight") + group.add_argument("--streaming", type=str2bool, default=False) + + group = parser.add_argument_group("Text converter related") + group.add_argument( + "--token_type", + type=str_or_none, + default=None, + choices=["char", "bpe", None], + help="The token type for ASR model. " + "If not given, refers from the training args", + ) + group.add_argument( + "--bpemodel", + type=str_or_none, + default=None, + help="The model path of sentencepiece. " + "If not given, refers from the training args", + ) + group.add_argument("--token_num_relax", type=int, default=1, help="") + group.add_argument("--decoding_ind", type=int, default=0, help="") + group.add_argument("--decoding_mode", type=str, default="model1", help="") + group.add_argument( + "--ctc_weight2", + type=float, + default=0.0, + help="CTC weight in joint decoding", + ) + return parser + + +def main(cmd=None): + print(get_commandline_args(), file=sys.stderr) + parser = get_parser() + args = parser.parse_args(cmd) + kwargs = vars(args) + kwargs.pop("config", None) + inference(**kwargs) + + +if __name__ == "__main__": + main() diff --git a/funasr/bin/asr_train.py b/funasr/bin/asr_train.py new file mode 100755 index 000000000..bba50daf0 --- /dev/null +++ b/funasr/bin/asr_train.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import os + +from funasr.tasks.asr import ASRTask + + +# for ASR Training +def parse_args(): + parser = ASRTask.get_parser() + parser.add_argument( + "--gpu_id", + type=int, + default=0, + help="local gpu id.", + ) + args = parser.parse_args() + return args + + +def main(args=None, cmd=None): + # for ASR Training + ASRTask.main(args=args, cmd=cmd) + + +if __name__ == '__main__': + args = parse_args() + + # setup local gpu_id + os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_id) + + # DDP settings + if args.ngpu > 1: + args.distributed = True + else: + args.distributed = False + assert args.num_worker_count == 1 + + # re-compute batch size: when dataset type is small + if args.dataset_type == "small": + if args.batch_size is not None: + args.batch_size = args.batch_size * args.ngpu + if args.batch_bins is not None: + args.batch_bins = args.batch_bins * args.ngpu + + main(args=args) diff --git a/funasr/bin/asr_train_paraformer.py b/funasr/bin/asr_train_paraformer.py new file mode 100755 index 000000000..76943d5b7 --- /dev/null +++ b/funasr/bin/asr_train_paraformer.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import os + +from funasr.tasks.asr import ASRTaskParaformer as ASRTask + + +# for ASR Training +def parse_args(): + parser = ASRTask.get_parser() + parser.add_argument( + "--gpu_id", + type=int, + default=0, + help="local gpu id.", + ) + args = parser.parse_args() + return args + + +def main(args=None, cmd=None): + # for ASR Training + ASRTask.main(args=args, cmd=cmd) + + +if __name__ == '__main__': + args = parse_args() + + # setup local gpu_id + os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_id) + + # DDP settings + if args.ngpu > 1: + args.distributed = True + else: + args.distributed = False + assert args.num_worker_count == 1 + + # re-compute batch size: when dataset type is small + if args.dataset_type == "small": + if args.batch_size is not None: + args.batch_size = args.batch_size * args.ngpu + if args.batch_bins is not None: + args.batch_bins = args.batch_bins * args.ngpu + + main(args=args) diff --git a/funasr/bin/asr_train_uniasr.py b/funasr/bin/asr_train_uniasr.py new file mode 100755 index 000000000..a40b5032c --- /dev/null +++ b/funasr/bin/asr_train_uniasr.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import os + +from funasr.tasks.asr import ASRTaskUniASR + + +# for ASR Training +def parse_args(): + parser = ASRTaskUniASR.get_parser() + parser.add_argument( + "--gpu_id", + type=int, + default=0, + help="local gpu id.", + ) + args = parser.parse_args() + return args + + +def main(args=None, cmd=None): + # for ASR Training + ASRTaskUniASR.main(args=args, cmd=cmd) + + +if __name__ == '__main__': + args = parse_args() + + # setup local gpu_id + os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_id) + + # DDP settings + if args.ngpu > 1: + args.distributed = True + else: + args.distributed = False + assert args.num_worker_count == 1 + + # re-compute batch size: when dataset type is small + if args.dataset_type == "small": + if args.batch_size is not None: + args.batch_size = args.batch_size * args.ngpu + if args.batch_bins is not None: + args.batch_bins = args.batch_bins * args.ngpu + + main(args=args) diff --git a/funasr/bin/lm_calc_perplexity.py b/funasr/bin/lm_calc_perplexity.py new file mode 100755 index 000000000..27a8a71fc --- /dev/null +++ b/funasr/bin/lm_calc_perplexity.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +import argparse +import logging +from pathlib import Path +import sys +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import numpy as np +import torch +from torch.nn.parallel import data_parallel +from typeguard import check_argument_types + +from funasr.utils.cli_utils import get_commandline_args +from funasr.fileio.datadir_writer import DatadirWriter +from funasr.tasks.lm import LMTask +from funasr.torch_utils.device_funcs import to_device +from funasr.torch_utils.forward_adaptor import ForwardAdaptor +from funasr.torch_utils.set_all_random_seed import set_all_random_seed +from funasr.utils import config_argparse +from funasr.utils.types import float_or_none +from funasr.utils.types import str2bool +from funasr.utils.types import str2triple_str +from funasr.utils.types import str_or_none + + +def calc_perplexity( + output_dir: str, + batch_size: int, + dtype: str, + ngpu: int, + seed: int, + num_workers: int, + log_level: Union[int, str], + data_path_and_name_and_type: Sequence[Tuple[str, str, str]], + key_file: Optional[str], + train_config: Optional[str], + model_file: Optional[str], + log_base: Optional[float], + allow_variable_data_keys: bool, +): + assert check_argument_types() + logging.basicConfig( + level=log_level, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", + ) + + if ngpu >= 1: + device = "cuda" + else: + device = "cpu" + + # 1. Set random-seed + set_all_random_seed(seed) + + # 2. Build LM + model, train_args = LMTask.build_model_from_file(train_config, model_file, device) + # Wrape model to make model.nll() data-parallel + wrapped_model = ForwardAdaptor(model, "nll") + wrapped_model.to(dtype=getattr(torch, dtype)).eval() + logging.info(f"Model:\n{model}") + + # 3. Build data-iterator + loader = LMTask.build_streaming_iterator( + data_path_and_name_and_type, + dtype=dtype, + batch_size=batch_size, + key_file=key_file, + num_workers=num_workers, + preprocess_fn=LMTask.build_preprocess_fn(train_args, False), + collate_fn=LMTask.build_collate_fn(train_args, False), + allow_variable_data_keys=allow_variable_data_keys, + inference=True, + ) + + # 4. Start for-loop + with DatadirWriter(output_dir) as writer: + total_nll = 0.0 + total_ntokens = 0 + for keys, batch in loader: + assert isinstance(batch, dict), type(batch) + assert all(isinstance(s, str) for s in keys), keys + _bs = len(next(iter(batch.values()))) + assert len(keys) == _bs, f"{len(keys)} != {_bs}" + + with torch.no_grad(): + batch = to_device(batch, device) + if ngpu <= 1: + # NOTE(kamo): data_parallel also should work with ngpu=1, + # but for debuggability it's better to keep this block. + nll, lengths = wrapped_model(**batch) + else: + nll, lengths = data_parallel( + wrapped_model, (), range(ngpu), module_kwargs=batch + ) + + assert _bs == len(nll) == len(lengths), (_bs, len(nll), len(lengths)) + # nll: (B, L) -> (B,) + nll = nll.detach().cpu().numpy().sum(1) + # lengths: (B,) + lengths = lengths.detach().cpu().numpy() + total_nll += nll.sum() + total_ntokens += lengths.sum() + + for key, _nll, ntoken in zip(keys, nll, lengths): + if log_base is None: + utt_ppl = np.exp(_nll / ntoken) + else: + utt_ppl = log_base ** (_nll / ntoken / np.log(log_base)) + + # Write PPL of each utts for debugging or analysis + writer["utt2ppl"][key] = str(utt_ppl) + writer["utt2ntokens"][key] = str(ntoken) + + if log_base is None: + ppl = np.exp(total_nll / total_ntokens) + else: + ppl = log_base ** (total_nll / total_ntokens / np.log(log_base)) + + with (Path(output_dir) / "ppl").open("w", encoding="utf-8") as f: + f.write(f"{ppl}\n") + with (Path(output_dir) / "base").open("w", encoding="utf-8") as f: + if log_base is None: + _log_base = np.e + else: + _log_base = log_base + f.write(f"{_log_base}\n") + logging.info(f"PPL={ppl}") + + +def get_parser(): + parser = config_argparse.ArgumentParser( + description="Calc perplexity", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Note(kamo): Use '_' instead of '-' as separator. + # '-' is confusing if written in yaml. + parser.add_argument( + "--log_level", + type=lambda x: x.upper(), + default="INFO", + choices=("CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"), + help="The verbose level of logging", + ) + + parser.add_argument("--output_dir", type=str, required=True) + parser.add_argument( + "--ngpu", + type=int, + default=0, + help="The number of gpus. 0 indicates CPU mode", + ) + parser.add_argument("--seed", type=int, default=0, help="Random seed") + parser.add_argument( + "--dtype", + default="float32", + choices=["float16", "float32", "float64"], + help="Data type", + ) + parser.add_argument( + "--num_workers", + type=int, + default=1, + help="The number of workers used for DataLoader", + ) + parser.add_argument( + "--batch_size", + type=int, + default=1, + help="The batch size for inference", + ) + parser.add_argument( + "--log_base", + type=float_or_none, + default=None, + help="The base of logarithm for Perplexity. " + "If None, napier's constant is used.", + ) + + group = parser.add_argument_group("Input data related") + group.add_argument( + "--data_path_and_name_and_type", + type=str2triple_str, + required=True, + action="append", + ) + group.add_argument("--key_file", type=str_or_none) + group.add_argument("--allow_variable_data_keys", type=str2bool, default=False) + + group = parser.add_argument_group("The model configuration related") + group.add_argument("--train_config", type=str) + group.add_argument("--model_file", type=str) + + return parser + + +def main(cmd=None): + print(get_commandline_args(), file=sys.stderr) + parser = get_parser() + args = parser.parse_args(cmd) + kwargs = vars(args) + kwargs.pop("config", None) + calc_perplexity(**kwargs) + + +if __name__ == "__main__": + main() diff --git a/funasr/bin/lm_train.py b/funasr/bin/lm_train.py new file mode 100755 index 000000000..faa7a4596 --- /dev/null +++ b/funasr/bin/lm_train.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +from funasr.tasks.lm import LMTask + + +def get_parser(): + parser = LMTask.get_parser() + return parser + + +def main(cmd=None): + """LM training. + + Example: + + % python lm_train.py asr --print_config --optim adadelta + % python lm_train.py --config conf/train_asr.yaml + """ + LMTask.main(cmd=cmd) + + +if __name__ == "__main__": + main() diff --git a/funasr/bin/modelscope_infer.py b/funasr/bin/modelscope_infer.py new file mode 100755 index 000000000..440c88163 --- /dev/null +++ b/funasr/bin/modelscope_infer.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +import argparse +import logging +import os + +from modelscope.pipelines import pipeline +from modelscope.utils.constant import Tasks + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="decoding configs", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--model_name", + type=str, + default="speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", + help="model name in modelscope") + parser.add_argument("--local_model_path", + type=str, + default=None, + help="local model path, usually for fine-tuning") + parser.add_argument("--wav_list", + type=str, + help="input wav list") + parser.add_argument("--output_file", + type=str, + help="saving decoding results") + parser.add_argument( + "--njob", + type=int, + default=1, + help="The number of jobs for each gpu", + ) + parser.add_argument( + "--gpuid_list", + type=str, + default="", + help="The visible gpus", + ) + parser.add_argument( + "--ngpu", + type=int, + default=0, + help="The number of gpus. 0 indicates CPU mode", + ) + args = parser.parse_args() + + # set logging messages + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", + ) + logging.info("Decoding args: {}".format(args)) + + # gpu setting + if args.ngpu > 0: + jobid = int(args.output_file.split(".")[-1]) + gpuid = args.gpuid_list.split(",")[(jobid - 1) // args.njob] + os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + os.environ["CUDA_VISIBLE_DEVICES"] = gpuid + + if args.local_model_path is None: + inference_pipeline = pipeline( + task=Tasks.auto_speech_recognition, + model="damo/{}".format(args.model_name)) + else: + inference_pipeline = pipeline( + task=Tasks.auto_speech_recognition, + model=args.local_model_path) + + + with open(args.wav_list, 'r') as f_wav: + wav_lines = f_wav.readlines() + + with open(args.output_file, "w") as f_out: + for line in wav_lines: + wav_id, wav_path = line.strip().split() + logging.info("decoding, utt_id: ['{}']".format(wav_id)) + rec_result = inference_pipeline(audio_in=wav_path) + text = rec_result["text"] + f_out.write(wav_id + " " + text + "\n") + logging.info("best hypo: {} \n".format(text)) diff --git a/funasr/datasets/__init__.py b/funasr/datasets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/datasets/collate_fn.py b/funasr/datasets/collate_fn.py new file mode 100644 index 000000000..d52032f9e --- /dev/null +++ b/funasr/datasets/collate_fn.py @@ -0,0 +1,83 @@ +from typing import Collection +from typing import Dict +from typing import List +from typing import Tuple +from typing import Union + +import numpy as np +import torch +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.modules.nets_utils import pad_list + + +class CommonCollateFn: + """Functor class of common_collate_fn()""" + + def __init__( + self, + float_pad_value: Union[float, int] = 0.0, + int_pad_value: int = -32768, + not_sequence: Collection[str] = (), + max_sample_size=None + ): + assert check_argument_types() + self.float_pad_value = float_pad_value + self.int_pad_value = int_pad_value + self.not_sequence = set(not_sequence) + self.max_sample_size = max_sample_size + + def __repr__(self): + return ( + f"{self.__class__}(float_pad_value={self.float_pad_value}, " + f"int_pad_value={self.float_pad_value})" + ) + + def __call__( + self, data: Collection[Tuple[str, Dict[str, np.ndarray]]] + ) -> Tuple[List[str], Dict[str, torch.Tensor]]: + return common_collate_fn( + data, + float_pad_value=self.float_pad_value, + int_pad_value=self.int_pad_value, + not_sequence=self.not_sequence, + ) + + +def common_collate_fn( + data: Collection[Tuple[str, Dict[str, np.ndarray]]], + float_pad_value: Union[float, int] = 0.0, + int_pad_value: int = -32768, + not_sequence: Collection[str] = (), +) -> Tuple[List[str], Dict[str, torch.Tensor]]: + """Concatenate ndarray-list to an array and convert to torch.Tensor. + """ + assert check_argument_types() + uttids = [u for u, _ in data] + data = [d for _, d in data] + + assert all(set(data[0]) == set(d) for d in data), "dict-keys mismatching" + assert all( + not k.endswith("_lengths") for k in data[0] + ), f"*_lengths is reserved: {list(data[0])}" + + output = {} + for key in data[0]: + if data[0][key].dtype.kind == "i": + pad_value = int_pad_value + else: + pad_value = float_pad_value + + array_list = [d[key] for d in data] + tensor_list = [torch.from_numpy(a) for a in array_list] + tensor = pad_list(tensor_list, pad_value) + output[key] = tensor + + if key not in not_sequence: + lens = torch.tensor([d[key].shape[0] for d in data], dtype=torch.long) + output[key + "_lengths"] = lens + + output = (uttids, output) + assert check_return_type(output) + return output \ No newline at end of file diff --git a/funasr/datasets/dataset.py b/funasr/datasets/dataset.py new file mode 100644 index 000000000..2af93d0bc --- /dev/null +++ b/funasr/datasets/dataset.py @@ -0,0 +1,444 @@ +# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +from abc import ABC +from abc import abstractmethod +import collections +import copy +import functools +import logging +import numbers +import re +from typing import Any +from typing import Callable +from typing import Collection +from typing import Dict +from typing import Mapping +from typing import Tuple +from typing import Union + +import h5py +import humanfriendly +import kaldiio +import numpy as np +import torch +from torch.utils.data.dataset import Dataset +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.fileio.npy_scp import NpyScpReader +from funasr.fileio.rand_gen_dataset import FloatRandomGenerateDataset +from funasr.fileio.rand_gen_dataset import IntRandomGenerateDataset +from funasr.fileio.read_text import load_num_sequence_text +from funasr.fileio.read_text import read_2column_text +from funasr.fileio.sound_scp import SoundScpReader +from funasr.utils.sized_dict import SizedDict + + +class AdapterForSoundScpReader(collections.abc.Mapping): + def __init__(self, loader, dtype=None): + assert check_argument_types() + self.loader = loader + self.dtype = dtype + self.rate = None + + def keys(self): + return self.loader.keys() + + def __len__(self): + return len(self.loader) + + def __iter__(self): + return iter(self.loader) + + def __getitem__(self, key: str) -> np.ndarray: + retval = self.loader[key] + + if isinstance(retval, tuple): + assert len(retval) == 2, len(retval) + if isinstance(retval[0], int) and isinstance(retval[1], np.ndarray): + # sound scp case + rate, array = retval + elif isinstance(retval[1], int) and isinstance(retval[0], np.ndarray): + # Extended ark format case + array, rate = retval + else: + raise RuntimeError( + f"Unexpected type: {type(retval[0])}, {type(retval[1])}" + ) + + if self.rate is not None and self.rate != rate: + raise RuntimeError( + f"Sampling rates are mismatched: {self.rate} != {rate}" + ) + self.rate = rate + # Multichannel wave fie + # array: (NSample, Channel) or (Nsample) + if self.dtype is not None: + array = array.astype(self.dtype) + + else: + # Normal ark case + assert isinstance(retval, np.ndarray), type(retval) + array = retval + if self.dtype is not None: + array = array.astype(self.dtype) + + assert isinstance(array, np.ndarray), type(array) + return array + + +class H5FileWrapper: + def __init__(self, path: str): + self.path = path + self.h5_file = h5py.File(path, "r") + + def __repr__(self) -> str: + return str(self.h5_file) + + def __len__(self) -> int: + return len(self.h5_file) + + def __iter__(self): + return iter(self.h5_file) + + def __getitem__(self, key) -> np.ndarray: + value = self.h5_file[key] + return value[()] + + +def sound_loader(path, float_dtype=None): + # The file is as follows: + # utterance_id_A /some/where/a.wav + # utterance_id_B /some/where/a.flac + + # NOTE(kamo): SoundScpReader doesn't support pipe-fashion + # like Kaldi e.g. "cat a.wav |". + # NOTE(kamo): The audio signal is normalized to [-1,1] range. + loader = SoundScpReader(path, normalize=True, always_2d=False) + + # SoundScpReader.__getitem__() returns Tuple[int, ndarray], + # but ndarray is desired, so Adapter class is inserted here + return AdapterForSoundScpReader(loader, float_dtype) + + +def kaldi_loader(path, float_dtype=None, max_cache_fd: int = 0): + loader = kaldiio.load_scp(path, max_cache_fd=max_cache_fd) + return AdapterForSoundScpReader(loader, float_dtype) + + +def rand_int_loader(filepath, loader_type): + # e.g. rand_int_3_10 + try: + low, high = map(int, loader_type[len("rand_int_") :].split("_")) + except ValueError: + raise RuntimeError(f"e.g rand_int_3_10: but got {loader_type}") + return IntRandomGenerateDataset(filepath, low, high) + + +DATA_TYPES = { + "sound": dict( + func=sound_loader, + kwargs=["float_dtype"], + help="Audio format types which supported by sndfile wav, flac, etc." + "\n\n" + " utterance_id_a a.wav\n" + " utterance_id_b b.wav\n" + " ...", + ), + "kaldi_ark": dict( + func=kaldi_loader, + kwargs=["max_cache_fd"], + help="Kaldi-ark file type." + "\n\n" + " utterance_id_A /some/where/a.ark:123\n" + " utterance_id_B /some/where/a.ark:456\n" + " ...", + ), + "npy": dict( + func=NpyScpReader, + kwargs=[], + help="Npy file format." + "\n\n" + " utterance_id_A /some/where/a.npy\n" + " utterance_id_B /some/where/b.npy\n" + " ...", + ), + "text_int": dict( + func=functools.partial(load_num_sequence_text, loader_type="text_int"), + kwargs=[], + help="A text file in which is written a sequence of interger numbers " + "separated by space." + "\n\n" + " utterance_id_A 12 0 1 3\n" + " utterance_id_B 3 3 1\n" + " ...", + ), + "csv_int": dict( + func=functools.partial(load_num_sequence_text, loader_type="csv_int"), + kwargs=[], + help="A text file in which is written a sequence of interger numbers " + "separated by comma." + "\n\n" + " utterance_id_A 100,80\n" + " utterance_id_B 143,80\n" + " ...", + ), + "text_float": dict( + func=functools.partial(load_num_sequence_text, loader_type="text_float"), + kwargs=[], + help="A text file in which is written a sequence of float numbers " + "separated by space." + "\n\n" + " utterance_id_A 12. 3.1 3.4 4.4\n" + " utterance_id_B 3. 3.12 1.1\n" + " ...", + ), + "csv_float": dict( + func=functools.partial(load_num_sequence_text, loader_type="csv_float"), + kwargs=[], + help="A text file in which is written a sequence of float numbers " + "separated by comma." + "\n\n" + " utterance_id_A 12.,3.1,3.4,4.4\n" + " utterance_id_B 3.,3.12,1.1\n" + " ...", + ), + "text": dict( + func=read_2column_text, + kwargs=[], + help="Return text as is. The text must be converted to ndarray " + "by 'preprocess'." + "\n\n" + " utterance_id_A hello world\n" + " utterance_id_B foo bar\n" + " ...", + ), + "hdf5": dict( + func=H5FileWrapper, + kwargs=[], + help="A HDF5 file which contains arrays at the first level or the second level." + " >>> f = h5py.File('file.h5')\n" + " >>> array1 = f['utterance_id_A']\n" + " >>> array2 = f['utterance_id_B']\n", + ), + "rand_float": dict( + func=FloatRandomGenerateDataset, + kwargs=[], + help="Generate random float-ndarray which has the given shapes " + "in the file." + "\n\n" + " utterance_id_A 3,4\n" + " utterance_id_B 10,4\n" + " ...", + ), + "rand_int_\\d+_\\d+": dict( + func=rand_int_loader, + kwargs=["loader_type"], + help="e.g. 'rand_int_0_10'. Generate random int-ndarray which has the given " + "shapes in the path. " + "Give the lower and upper value by the file type. e.g. " + "rand_int_0_10 -> Generate integers from 0 to 10." + "\n\n" + " utterance_id_A 3,4\n" + " utterance_id_B 10,4\n" + " ...", + ), +} + + +class AbsDataset(Dataset, ABC): + @abstractmethod + def has_name(self, name) -> bool: + raise NotImplementedError + + @abstractmethod + def names(self) -> Tuple[str, ...]: + raise NotImplementedError + + @abstractmethod + def __getitem__(self, uid) -> Tuple[Any, Dict[str, np.ndarray]]: + raise NotImplementedError + + +class ESPnetDataset(AbsDataset): + """Pytorch Dataset class for ESPNet. + + Examples: + >>> dataset = ESPnetDataset([('wav.scp', 'input', 'sound'), + ... ('token_int', 'output', 'text_int')], + ... ) + ... uttid, data = dataset['uttid'] + {'input': per_utt_array, 'output': per_utt_array} + """ + + def __init__( + self, + path_name_type_list: Collection[Tuple[str, str, str]], + preprocess: Callable[ + [str, Dict[str, np.ndarray]], Dict[str, np.ndarray] + ] = None, + float_dtype: str = "float32", + int_dtype: str = "long", + max_cache_size: Union[float, int, str] = 0.0, + max_cache_fd: int = 0, + ): + assert check_argument_types() + if len(path_name_type_list) == 0: + raise ValueError( + '1 or more elements are required for "path_name_type_list"' + ) + + path_name_type_list = copy.deepcopy(path_name_type_list) + self.preprocess = preprocess + + self.float_dtype = float_dtype + self.int_dtype = int_dtype + self.max_cache_fd = max_cache_fd + + self.loader_dict = {} + self.debug_info = {} + for path, name, _type in path_name_type_list: + if name in self.loader_dict: + raise RuntimeError(f'"{name}" is duplicated for data-key') + + loader = self._build_loader(path, _type) + self.loader_dict[name] = loader + self.debug_info[name] = path, _type + if len(self.loader_dict[name]) == 0: + raise RuntimeError(f"{path} has no samples") + + # TODO(kamo): Should check consistency of each utt-keys? + + if isinstance(max_cache_size, str): + max_cache_size = humanfriendly.parse_size(max_cache_size) + self.max_cache_size = max_cache_size + if max_cache_size > 0: + self.cache = SizedDict(shared=True) + else: + self.cache = None + + def _build_loader( + self, path: str, loader_type: str + ) -> Mapping[str, Union[np.ndarray, torch.Tensor, str, numbers.Number]]: + """Helper function to instantiate Loader. + + Args: + path: The file path + loader_type: loader_type. sound, npy, text_int, text_float, etc + """ + for key, dic in DATA_TYPES.items(): + # e.g. loader_type="sound" + # -> return DATA_TYPES["sound"]["func"](path) + if re.match(key, loader_type): + kwargs = {} + for key2 in dic["kwargs"]: + if key2 == "loader_type": + kwargs["loader_type"] = loader_type + elif key2 == "float_dtype": + kwargs["float_dtype"] = self.float_dtype + elif key2 == "int_dtype": + kwargs["int_dtype"] = self.int_dtype + elif key2 == "max_cache_fd": + kwargs["max_cache_fd"] = self.max_cache_fd + else: + raise RuntimeError(f"Not implemented keyword argument: {key2}") + + func = dic["func"] + try: + return func(path, **kwargs) + except Exception: + if hasattr(func, "__name__"): + name = func.__name__ + else: + name = str(func) + logging.error(f"An error happened with {name}({path})") + raise + else: + raise RuntimeError(f"Not supported: loader_type={loader_type}") + + def has_name(self, name) -> bool: + return name in self.loader_dict + + def names(self) -> Tuple[str, ...]: + return tuple(self.loader_dict) + + def __iter__(self): + return iter(next(iter(self.loader_dict.values()))) + + def __repr__(self): + _mes = self.__class__.__name__ + _mes += "(" + for name, (path, _type) in self.debug_info.items(): + _mes += f'\n {name}: {{"path": "{path}", "type": "{_type}"}}' + _mes += f"\n preprocess: {self.preprocess})" + return _mes + + def __getitem__(self, uid: Union[str, int]) -> Tuple[str, Dict[str, np.ndarray]]: + assert check_argument_types() + + # Change integer-id to string-id + if isinstance(uid, int): + d = next(iter(self.loader_dict.values())) + uid = list(d)[uid] + + if self.cache is not None and uid in self.cache: + data = self.cache[uid] + return uid, data + + data = {} + # 1. Load data from each loaders + for name, loader in self.loader_dict.items(): + try: + value = loader[uid] + if isinstance(value, (list, tuple)): + value = np.array(value) + if not isinstance( + value, (np.ndarray, torch.Tensor, str, numbers.Number) + ): + raise TypeError( + f"Must be ndarray, torch.Tensor, str or Number: {type(value)}" + ) + except Exception: + path, _type = self.debug_info[name] + logging.error( + f"Error happened with path={path}, type={_type}, id={uid}" + ) + raise + + # torch.Tensor is converted to ndarray + if isinstance(value, torch.Tensor): + value = value.numpy() + elif isinstance(value, numbers.Number): + value = np.array([value]) + data[name] = value + + # 2. [Option] Apply preprocessing + # e.g. funasr.train.preprocessor:CommonPreprocessor + if self.preprocess is not None: + data = self.preprocess(uid, data) + + # 3. Force data-precision + for name in data: + value = data[name] + if not isinstance(value, np.ndarray): + raise RuntimeError( + f"All values must be converted to np.ndarray object " + f'by preprocessing, but "{name}" is still {type(value)}.' + ) + + # Cast to desired type + if value.dtype.kind == "f": + value = value.astype(self.float_dtype) + elif value.dtype.kind == "i": + value = value.astype(self.int_dtype) + else: + raise NotImplementedError(f"Not supported dtype: {value.dtype}") + data[name] = value + + if self.cache is not None and self.cache.size < self.max_cache_size: + self.cache[uid] = data + + retval = uid, data + assert check_return_type(retval) + return retval diff --git a/funasr/datasets/iterable_dataset.py b/funasr/datasets/iterable_dataset.py new file mode 100644 index 000000000..319dd7ffe --- /dev/null +++ b/funasr/datasets/iterable_dataset.py @@ -0,0 +1,237 @@ +"""Iterable dataset module.""" +import copy +from io import StringIO +from pathlib import Path +from typing import Callable +from typing import Collection +from typing import Dict +from typing import Iterator +from typing import Tuple +from typing import Union + +import kaldiio +import numpy as np +import soundfile +import torch +from torch.utils.data.dataset import IterableDataset +from typeguard import check_argument_types + +from funasr.datasets.dataset import ESPnetDataset + + +def load_kaldi(input): + retval = kaldiio.load_mat(input) + if isinstance(retval, tuple): + assert len(retval) == 2, len(retval) + if isinstance(retval[0], int) and isinstance(retval[1], np.ndarray): + # sound scp case + rate, array = retval + elif isinstance(retval[1], int) and isinstance(retval[0], np.ndarray): + # Extended ark format case + array, rate = retval + else: + raise RuntimeError(f"Unexpected type: {type(retval[0])}, {type(retval[1])}") + + # Multichannel wave fie + # array: (NSample, Channel) or (Nsample) + + else: + # Normal ark case + assert isinstance(retval, np.ndarray), type(retval) + array = retval + return array + + +DATA_TYPES = { + "sound": lambda x: soundfile.read(x)[0], + "kaldi_ark": load_kaldi, + "npy": np.load, + "text_int": lambda x: np.loadtxt( + StringIO(x), ndmin=1, dtype=np.long, delimiter=" " + ), + "csv_int": lambda x: np.loadtxt(StringIO(x), ndmin=1, dtype=np.long, delimiter=","), + "text_float": lambda x: np.loadtxt( + StringIO(x), ndmin=1, dtype=np.float32, delimiter=" " + ), + "csv_float": lambda x: np.loadtxt( + StringIO(x), ndmin=1, dtype=np.float32, delimiter="," + ), + "text": lambda x: x, +} + + +class IterableESPnetDataset(IterableDataset): + """Pytorch Dataset class for ESPNet. + + Examples: + >>> dataset = IterableESPnetDataset([('wav.scp', 'input', 'sound'), + ... ('token_int', 'output', 'text_int')], + ... ) + >>> for uid, data in dataset: + ... data + {'input': per_utt_array, 'output': per_utt_array} + """ + + def __init__( + self, + path_name_type_list: Collection[Tuple[str, str, str]], + preprocess: Callable[ + [str, Dict[str, np.ndarray]], Dict[str, np.ndarray] + ] = None, + float_dtype: str = "float32", + int_dtype: str = "long", + key_file: str = None, + ): + assert check_argument_types() + if len(path_name_type_list) == 0: + raise ValueError( + '1 or more elements are required for "path_name_type_list"' + ) + + path_name_type_list = copy.deepcopy(path_name_type_list) + self.preprocess = preprocess + + self.float_dtype = float_dtype + self.int_dtype = int_dtype + self.key_file = key_file + + self.debug_info = {} + non_iterable_list = [] + self.path_name_type_list = [] + + for path, name, _type in path_name_type_list: + if name in self.debug_info: + raise RuntimeError(f'"{name}" is duplicated for data-key') + self.debug_info[name] = path, _type + if _type not in DATA_TYPES: + non_iterable_list.append((path, name, _type)) + else: + self.path_name_type_list.append((path, name, _type)) + + if len(non_iterable_list) != 0: + # Some types doesn't support iterable mode + self.non_iterable_dataset = ESPnetDataset( + path_name_type_list=non_iterable_list, + preprocess=preprocess, + float_dtype=float_dtype, + int_dtype=int_dtype, + ) + else: + self.non_iterable_dataset = None + + if Path(Path(path_name_type_list[0][0]).parent, "utt2category").exists(): + self.apply_utt2category = True + else: + self.apply_utt2category = False + + def has_name(self, name) -> bool: + return name in self.debug_info + + def names(self) -> Tuple[str, ...]: + return tuple(self.debug_info) + + def __repr__(self): + _mes = self.__class__.__name__ + _mes += "(" + for name, (path, _type) in self.debug_info.items(): + _mes += f'\n {name}: {{"path": "{path}", "type": "{_type}"}}' + _mes += f"\n preprocess: {self.preprocess})" + return _mes + + def __iter__(self) -> Iterator[Tuple[Union[str, int], Dict[str, np.ndarray]]]: + if self.key_file is not None: + uid_iter = ( + line.rstrip().split(maxsplit=1)[0] + for line in open(self.key_file, encoding="utf-8") + ) + elif len(self.path_name_type_list) != 0: + uid_iter = ( + line.rstrip().split(maxsplit=1)[0] + for line in open(self.path_name_type_list[0][0], encoding="utf-8") + ) + else: + uid_iter = iter(self.non_iterable_dataset) + + files = [open(lis[0], encoding="utf-8") for lis in self.path_name_type_list] + + worker_info = torch.utils.data.get_worker_info() + + linenum = 0 + count = 0 + for count, uid in enumerate(uid_iter, 1): + # If num_workers>=1, split keys + if worker_info is not None: + if (count - 1) % worker_info.num_workers != worker_info.id: + continue + + # 1. Read a line from each file + while True: + keys = [] + values = [] + for f in files: + linenum += 1 + try: + line = next(f) + except StopIteration: + raise RuntimeError(f"{uid} is not found in the files") + sps = line.rstrip().split(maxsplit=1) + if len(sps) != 2: + raise RuntimeError( + f"This line doesn't include a space:" + f" {f}:L{linenum}: {line})" + ) + key, value = sps + keys.append(key) + values.append(value) + + for k_idx, k in enumerate(keys): + if k != keys[0]: + raise RuntimeError( + f"Keys are mismatched. Text files (idx={k_idx}) is " + f"not sorted or not having same keys at L{linenum}" + ) + + # If the key is matched, break the loop + if len(keys) == 0 or keys[0] == uid: + break + + # 2. Load the entry from each line and create a dict + data = {} + # 2.a. Load data streamingly + for value, (path, name, _type) in zip(values, self.path_name_type_list): + func = DATA_TYPES[_type] + # Load entry + array = func(value) + data[name] = array + if self.non_iterable_dataset is not None: + # 2.b. Load data from non-iterable dataset + _, from_non_iterable = self.non_iterable_dataset[uid] + data.update(from_non_iterable) + + # 3. [Option] Apply preprocessing + # e.g. funasr.train.preprocessor:CommonPreprocessor + if self.preprocess is not None: + data = self.preprocess(uid, data) + + # 4. Force data-precision + for name in data: + value = data[name] + if not isinstance(value, np.ndarray): + raise RuntimeError( + f"All values must be converted to np.ndarray object " + f'by preprocessing, but "{name}" is still {type(value)}.' + ) + + # Cast to desired type + if value.dtype.kind == "f": + value = value.astype(self.float_dtype) + elif value.dtype.kind == "i": + value = value.astype(self.int_dtype) + else: + raise NotImplementedError(f"Not supported dtype: {value.dtype}") + data[name] = value + + yield uid, data + + if count == 0: + raise RuntimeError("No iteration") diff --git a/funasr/datasets/large_datasets/__init__.py b/funasr/datasets/large_datasets/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/datasets/large_datasets/build_dataloader.py b/funasr/datasets/large_datasets/build_dataloader.py new file mode 100644 index 000000000..37fbb7cb4 --- /dev/null +++ b/funasr/datasets/large_datasets/build_dataloader.py @@ -0,0 +1,41 @@ +import logging + +import yaml + +from torch.utils.data import DataLoader +from funasr.datasets.large_datasets.dataset import Dataset +from funasr.iterators.abs_iter_factory import AbsIterFactory + + +def read_symbol_table(symbol_table_file): + if isinstance(symbol_table_file, str): + symbol_table = {} + with open(symbol_table_file, "r", encoding="utf8") as fin: + for i, line in enumerate(fin): + char = line.strip() + symbol_table[char] = i + else: + assert isinstance(symbol_table_file, list) + symbol_table = {} + for i, char in enumerate(symbol_table_file): + symbol_table[char] = i + return symbol_table + + +class ArkDataLoader(AbsIterFactory): + def __init__(self, data_list, dict_file, config_file, mode="train"): + symbol_table = read_symbol_table(dict_file) + with open(config_file, "r") as fin: + configs = yaml.load(fin, Loader=yaml.FullLoader) + self.dataset_conf = configs["dataset_conf"] + logging.info("dataloader config: {}".format(self.dataset_conf)) + self.dataset = Dataset(data_list, symbol_table, + self.dataset_conf, mode=mode) + + def build_iter(self, epoch, shuffle=True): + self.dataset.set_epoch(epoch) + data_loader = DataLoader(self.dataset, + batch_size=None, + pin_memory=True, + num_workers=self.dataset_conf.get("num_workers", 8)) + return data_loader diff --git a/funasr/datasets/large_datasets/datapipes/__init__.py b/funasr/datasets/large_datasets/datapipes/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/datasets/large_datasets/datapipes/batch.py b/funasr/datasets/large_datasets/datapipes/batch.py new file mode 100644 index 000000000..9c85d5edc --- /dev/null +++ b/funasr/datasets/large_datasets/datapipes/batch.py @@ -0,0 +1,148 @@ +import random + +from itertools import count +from functools import partial +from torch.utils.data import IterableDataset +from funasr.datasets.large_datasets.datapipes.map import MapperIterDataPipe + +tiebreaker = count() + + +def _default_len_fn(token): + return len(token), next(tiebreaker) + + +def _token_len_fn(token, len_fn): + return len_fn(token), next(tiebreaker), token + + +class MaxTokenBucketizerIterDataPipe(IterableDataset): + + def __init__( + self, + datapipe, + batch_size=8000, + len_fn=_default_len_fn, + buffer_size=10240, + sort_size=500 + ): + assert batch_size > 0, "Batch size is required to be larger than 0!" + assert buffer_size >= -1, "Buffer size is required to be larger than -1!" + assert sort_size > 0, "Sort size is required to be larger than 0!" + + datapipe = MapperIterDataPipe(datapipe, fn=partial(_token_len_fn, len_fn=len_fn)) + self.datapipe = datapipe + self.batch_size = batch_size + self.buffer_size = buffer_size + self.sort_size = sort_size + + def set_epoch(self, epoch): + self.epoch = epoch + + def __iter__(self): + buffer = [] + batch = [] + bucket = [] + max_lengths = 0 + batch_lengths = 0 + + if self.buffer_size == -1: + for d in self.datapipe: + if d[0] > self.batch_size: + continue + buffer.append(d) + buffer.sort() + for sample in buffer: + length, _, token = sample + if length > max_lengths: + max_lengths = length + batch_lengths = max_lengths * (len(batch) + 1) + if batch_lengths > self.batch_size: + bucket.append(batch) + batch = [] + max_lengths = length + batch.append(token) + random.shuffle(bucket) + if bucket: + for batch_sample in bucket: + yield batch_sample + if batch: + yield batch + + elif self.buffer_size == 0: + for d in self.datapipe: + if d[0] > self.batch_size: + continue + length, _, token = d + if length > self.batch_size: + continue + if length > max_lengths: + max_lengths = length + batch_lengths = max_lengths * (len(batch) + 1) + if batch_lengths > self.batch_size: + yield batch + batch = [] + max_lengths = length + batch.append(token) + if batch: + yield batch + + else: + for d in self.datapipe: + if d[0] > self.batch_size: + continue + buffer.append(d) + if len(buffer) == self.buffer_size: + random.shuffle(buffer) + for sample in buffer: + bucket.append(sample) + if len(bucket) == self.sort_size: + bucket.sort() + for x in bucket: + length, _, token = x + if length > max_lengths: + max_lengths = length + batch_lengths = max_lengths * (len(batch) + 1) + if batch_lengths > self.batch_size: + yield batch + batch = [] + max_lengths = length + batch.append(token) + bucket = [] + buffer = [] + + if buffer: + random.shuffle(buffer) + for sample in buffer: + bucket.append(sample) + if len(bucket) == self.sort_size: + bucket.sort() + for x in bucket: + length, _, token = x + if length > max_lengths: + max_lengths = length + batch_lengths = max_lengths * (len(batch) + 1) + if batch_lengths > self.batch_size: + yield batch + batch = [] + max_lengths = length + batch.append(token) + bucket = [] + buffer = [] + + if bucket: + bucket.sort() + for x in bucket: + length, _, token = x + if length > max_lengths: + max_lengths = length + batch_lengths = max_lengths * (len(batch) + 1) + if batch_lengths > self.batch_size: + yield batch + batch = [] + max_lengths = length + batch.append(token) + bucket = [] + + if batch: + yield batch diff --git a/funasr/datasets/large_datasets/datapipes/filter.py b/funasr/datasets/large_datasets/datapipes/filter.py new file mode 100644 index 000000000..e79934d18 --- /dev/null +++ b/funasr/datasets/large_datasets/datapipes/filter.py @@ -0,0 +1,24 @@ +from torch.utils.data import IterableDataset + +def default_fn(data): + return data + + +class FilterIterDataPipe(IterableDataset): + + def __init__(self, + datapipe, + fn=default_fn): + self.datapipe = datapipe + self.fn = fn + + def set_epoch(self, epoch): + self.epoch = epoch + + def __iter__(self): + assert callable(self.fn) + for data in self.datapipe: + if self.fn(data): + yield data + else: + continue \ No newline at end of file diff --git a/funasr/datasets/large_datasets/datapipes/map.py b/funasr/datasets/large_datasets/datapipes/map.py new file mode 100644 index 000000000..6e0168de0 --- /dev/null +++ b/funasr/datasets/large_datasets/datapipes/map.py @@ -0,0 +1,22 @@ +from torch.utils.data import IterableDataset + + +def default_fn(data): + return data + + +class MapperIterDataPipe(IterableDataset): + + def __init__(self, + datapipe, + fn=default_fn): + self.datapipe = datapipe + self.fn = fn + + def set_epoch(self, epoch): + self.epoch = epoch + + def __iter__(self): + assert callable(self.fn) + for data in self.datapipe: + yield self.fn(data) diff --git a/funasr/datasets/large_datasets/dataset.py b/funasr/datasets/large_datasets/dataset.py new file mode 100644 index 000000000..60c5abd1b --- /dev/null +++ b/funasr/datasets/large_datasets/dataset.py @@ -0,0 +1,175 @@ +import os +import random +from functools import partial + +import torch +import torch.distributed as dist +from kaldiio import ReadHelper +from torch.utils.data import IterableDataset + +from funasr.datasets.large_datasets.datapipes.batch import MaxTokenBucketizerIterDataPipe +from funasr.datasets.large_datasets.datapipes.filter import FilterIterDataPipe +from funasr.datasets.large_datasets.datapipes.map import MapperIterDataPipe +from funasr.datasets.large_datasets.utils.filter import filter +from funasr.datasets.large_datasets.utils.padding import padding +from funasr.datasets.large_datasets.utils.tokenize import tokenize + + +def read_lists(list_file): + lists = [] + with open(list_file, 'r', encoding='utf8') as fin: + for line in fin: + parts = line.strip() + lists.append(parts) + return lists + + +class AudioDataset(IterableDataset): + def __init__(self, scp_lists, data_names, data_types, shuffle=True, mode="train"): + self.scp_lists = scp_lists + self.data_names = data_names + self.data_types = data_types + self.shuffle = shuffle + self.mode = mode + self.epoch = -1 + self.rank = 0 + self.world_size = 1 + self.worker_id = 0 + self.num_workers = 1 + + def set_epoch(self, epoch): + self.epoch = epoch + + def get_rank_data_list(self, data_index): + assert dist.is_available() + if dist.is_initialized(): + self.rank = dist.get_rank() + self.world_size = dist.get_world_size() + else: + self.rank = 0 + self.world_size = 1 + + if self.mode == "train": + if self.shuffle: + random.seed(self.epoch) + random.shuffle(data_index) + return data_index[self.rank::self.world_size] + + return data_index + + def get_worker_data_list(self, rank_data_index): + worker_info = torch.utils.data.get_worker_info() + if worker_info is None: + self.worker_id = 0 + self.num_workers = 1 + else: + self.worker_id = worker_info.id + self.num_workers = worker_info.num_workers + + return rank_data_index[self.worker_id::self.num_workers] + + def close_reader(self, reader_list): + for reader in reader_list: + reader.close() + + def __iter__(self): + data_index = list(range(len(self.scp_lists))) + rank_data_index = self.get_rank_data_list(data_index) + worker_data_index = self.get_worker_data_list(rank_data_index) + + for index in worker_data_index: + data = dict(scp=self.scp_lists[index]) + + assert 'scp' in data + scp = data['scp'] + data_file_list = scp.strip().split() + data_name_list = self.data_names.split(",") + data_type_list = self.data_types.split(",") + + for file in data_file_list: + assert os.path.exists(file), "{} not exists".format(file) + + assert len(data_file_list) == len(data_name_list) == len(data_type_list), \ + "The item number of data, data_names, data_types must be the same " + + reader_list = [] + for data_file, data_type in zip(data_file_list, data_type_list): + if data_type == "kaldi_ark": + ark_reader = ReadHelper('ark:{}'.format(data_file)) + reader_list.append(ark_reader) + elif data_type == "text": + text_reader = open(data_file, "r") + reader_list.append(text_reader) + else: + raise TypeError("Data type {} is not supported".format(data_type)) + + for items in zip(*reader_list): + sample_dict = {} + for item, (data_name, data_type) in zip(items, zip(data_name_list, data_type_list)): + if data_type == "kaldi_ark": + key, mat = item + sample_dict[data_name] = mat + if data_name == "speech": + sample_dict["key"] = key + else: + text = item + sample_dict[data_name] = text.strip().split()[1:] + yield sample_dict + + self.close_reader(reader_list) + + +def len_fn_example(data): + return len(data) + + +def len_fn_token(data): + assert "speech" in data + return data["speech"].shape[0] + + +def Dataset(data_list_file, + dict, + conf, + mode="train"): + scp_lists = read_lists(data_list_file) + shuffle = conf.get('shuffle', True) + data_names = conf.get("data_names", "speech,text") + data_types = conf.get("data_types", "kaldi_ark,text") + dataset = AudioDataset(scp_lists, data_names, data_types, shuffle=shuffle, mode=mode) + + filter_conf = conf.get('filter_conf', {}) + filter_fn = partial(filter, **filter_conf) + dataset = FilterIterDataPipe(dataset, fn=filter_fn) + + vocab = {'vocab': dict} + tokenize_fn = partial(tokenize, **vocab) + dataset = MapperIterDataPipe(dataset, fn=tokenize_fn) + + if shuffle: + buffer_conf = conf.get('shuffle_conf', {}) + buffer_size = buffer_conf['shuffle_size'] + sort_size = buffer_conf['sort_size'] + else: + buffer_size = 0 + sort_size = 1 + + batch_conf = conf.get('batch_conf', {}) + batch_size = batch_conf['batch_size'] + batch_type = batch_conf['batch_type'] + + assert batch_type in ["example", "token"] + if batch_type == 'example': + len_fn = len_fn_example + else: + len_fn = len_fn_token + + dataset = MaxTokenBucketizerIterDataPipe(dataset, + batch_size=batch_size, + len_fn=len_fn, + buffer_size=buffer_size, + sort_size=sort_size) + + dataset = MapperIterDataPipe(dataset, fn=padding) + + return dataset diff --git a/funasr/datasets/large_datasets/utils/__init__.py b/funasr/datasets/large_datasets/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/datasets/large_datasets/utils/filter.py b/funasr/datasets/large_datasets/utils/filter.py new file mode 100644 index 000000000..5dc911f56 --- /dev/null +++ b/funasr/datasets/large_datasets/utils/filter.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + + +def filter(data, + min_length=10, + max_length=10000, + min_token_length=0, + max_token_length=200): + assert "speech" in data + assert "text" in data + + num_frames = data["speech"].shape[0] + num_tokens = len(data['text']) + + return min_length < num_frames < max_length and min_token_length < num_tokens < max_token_length \ No newline at end of file diff --git a/funasr/datasets/large_datasets/utils/low_frame_rate.py b/funasr/datasets/large_datasets/utils/low_frame_rate.py new file mode 100644 index 000000000..76eb2da93 --- /dev/null +++ b/funasr/datasets/large_datasets/utils/low_frame_rate.py @@ -0,0 +1,30 @@ +import numpy as np + + +def build_LFR_features(data, m, n): + """ + Actually, this implements stacking frames and skipping frames. + if m = 1 and n = 1, just return the origin features. + if m = 1 and n > 1, it works like skipping. + if m > 1 and n = 1, it works like stacking but only support right frames. + if m > 1 and n > 1, it works like LFR. + + Args: + inputs_batch: inputs is T x D np.ndarray + m: number of frames to stack + n: number of frames to skip + """ + + LFR_inputs = [] + T = data.shape[0] + T_lfr = int(np.ceil(T / n)) + for i in range(T_lfr): + if m <= T - i * n: + LFR_inputs.append(np.hstack(data[i*n:i*n+m])) + else: + num_padding = m - (T - i * n) + frame = np.hstack(data[i*n:]) + for _ in range(num_padding): + frame = np.hstack((frame, data[-1])) + LFR_inputs.append(frame) + return np.vstack(LFR_inputs) diff --git a/funasr/datasets/large_datasets/utils/padding.py b/funasr/datasets/large_datasets/utils/padding.py new file mode 100644 index 000000000..2e91e78c1 --- /dev/null +++ b/funasr/datasets/large_datasets/utils/padding.py @@ -0,0 +1,35 @@ +import numpy as np +import torch +from torch.nn.utils.rnn import pad_sequence + + +def padding(data, float_pad_value=0.0, int_pad_value=-1): + assert isinstance(data, list) + assert "key" in data[0] + assert "speech" in data[0] + assert "text" in data[0] + + keys = [x["key"] for x in data] + + batch = {} + data_names = data[0].keys() + for data_name in data_names: + if data_name == "key": + continue + else: + if data[0][data_name].dtype.kind == "i": + pad_value = int_pad_value + tensor_type = torch.int64 + else: + pad_value = float_pad_value + tensor_type = torch.float32 + + tensor_list = [torch.tensor(np.copy(d[data_name]), dtype=tensor_type) for d in data] + tensor_lengths = torch.tensor([len(d[data_name]) for d in data], dtype=torch.int32) + tensor_pad = pad_sequence(tensor_list, + batch_first=True, + padding_value=pad_value) + batch[data_name] = tensor_pad + batch[data_name + "_lengths"] = tensor_lengths + + return keys, batch diff --git a/funasr/datasets/large_datasets/utils/tokenize.py b/funasr/datasets/large_datasets/utils/tokenize.py new file mode 100644 index 000000000..937e14482 --- /dev/null +++ b/funasr/datasets/large_datasets/utils/tokenize.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import numpy as np + +def tokenize(data, + vocab=None): + assert "text" in data + assert isinstance(vocab, dict) + text = data["text"] + token = [] + for x in text: + if x in vocab: + token.append(vocab[x]) + else: + token.append(vocab['']) + + data["text"] = np.array(token) + return data diff --git a/funasr/datasets/preprocessor.py b/funasr/datasets/preprocessor.py new file mode 100644 index 000000000..80d1adcfe --- /dev/null +++ b/funasr/datasets/preprocessor.py @@ -0,0 +1,496 @@ +from abc import ABC +from abc import abstractmethod +from pathlib import Path +from typing import Collection +from typing import Dict +from typing import Iterable +from typing import List +from typing import Union + +import numpy as np +import scipy.signal +import soundfile +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.text.build_tokenizer import build_tokenizer +from funasr.text.cleaner import TextCleaner +from funasr.text.token_id_converter import TokenIDConverter + + +class AbsPreprocessor(ABC): + def __init__(self, train: bool): + self.train = train + + @abstractmethod + def __call__( + self, uid: str, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + raise NotImplementedError + + +def framing( + x, + frame_length: int = 512, + frame_shift: int = 256, + centered: bool = True, + padded: bool = True, +): + if x.size == 0: + raise ValueError("Input array size is zero") + if frame_length < 1: + raise ValueError("frame_length must be a positive integer") + if frame_length > x.shape[-1]: + raise ValueError("frame_length is greater than input length") + if 0 >= frame_shift: + raise ValueError("frame_shift must be greater than 0") + + if centered: + pad_shape = [(0, 0) for _ in range(x.ndim - 1)] + [ + (frame_length // 2, frame_length // 2) + ] + x = np.pad(x, pad_shape, mode="constant", constant_values=0) + + if padded: + # Pad to integer number of windowed segments + # I.e make x.shape[-1] = frame_length + (nseg-1)*nstep, + # with integer nseg + nadd = (-(x.shape[-1] - frame_length) % frame_shift) % frame_length + pad_shape = [(0, 0) for _ in range(x.ndim - 1)] + [(0, nadd)] + x = np.pad(x, pad_shape, mode="constant", constant_values=0) + + # Created strided array of data segments + if frame_length == 1 and frame_length == frame_shift: + result = x[..., None] + else: + shape = x.shape[:-1] + ( + (x.shape[-1] - frame_length) // frame_shift + 1, + frame_length, + ) + strides = x.strides[:-1] + (frame_shift * x.strides[-1], x.strides[-1]) + result = np.lib.stride_tricks.as_strided(x, shape=shape, strides=strides) + return result + + +def detect_non_silence( + x: np.ndarray, + threshold: float = 0.01, + frame_length: int = 1024, + frame_shift: int = 512, + window: str = "boxcar", +) -> np.ndarray: + """Power based voice activity detection. + + Args: + x: (Channel, Time) + >>> x = np.random.randn(1000) + >>> detect = detect_non_silence(x) + >>> assert x.shape == detect.shape + >>> assert detect.dtype == np.bool + """ + if x.shape[-1] < frame_length: + return np.full(x.shape, fill_value=True, dtype=np.bool) + + if x.dtype.kind == "i": + x = x.astype(np.float64) + # framed_w: (C, T, F) + framed_w = framing( + x, + frame_length=frame_length, + frame_shift=frame_shift, + centered=False, + padded=True, + ) + framed_w *= scipy.signal.get_window(window, frame_length).astype(framed_w.dtype) + # power: (C, T) + power = (framed_w**2).mean(axis=-1) + # mean_power: (C, 1) + mean_power = np.mean(power, axis=-1, keepdims=True) + if np.all(mean_power == 0): + return np.full(x.shape, fill_value=True, dtype=np.bool) + # detect_frames: (C, T) + detect_frames = power / mean_power > threshold + # detects: (C, T, F) + detects = np.broadcast_to( + detect_frames[..., None], detect_frames.shape + (frame_shift,) + ) + # detects: (C, TF) + detects = detects.reshape(*detect_frames.shape[:-1], -1) + # detects: (C, TF) + return np.pad( + detects, + [(0, 0)] * (x.ndim - 1) + [(0, x.shape[-1] - detects.shape[-1])], + mode="edge", + ) + + +class CommonPreprocessor(AbsPreprocessor): + def __init__( + self, + train: bool, + token_type: str = None, + token_list: Union[Path, str, Iterable[str]] = None, + bpemodel: Union[Path, str, Iterable[str]] = None, + text_cleaner: Collection[str] = None, + g2p_type: str = None, + unk_symbol: str = "", + space_symbol: str = "", + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + delimiter: str = None, + rir_scp: str = None, + rir_apply_prob: float = 1.0, + noise_scp: str = None, + noise_apply_prob: float = 1.0, + noise_db_range: str = "3_10", + speech_volume_normalize: float = None, + speech_name: str = "speech", + text_name: str = "text", + split_with_space: bool = False, + ): + super().__init__(train) + self.train = train + self.speech_name = speech_name + self.text_name = text_name + self.speech_volume_normalize = speech_volume_normalize + self.rir_apply_prob = rir_apply_prob + self.noise_apply_prob = noise_apply_prob + self.split_with_space = split_with_space + + if token_type is not None: + if token_list is None: + raise ValueError("token_list is required if token_type is not None") + self.text_cleaner = TextCleaner(text_cleaner) + + self.tokenizer = build_tokenizer( + token_type=token_type, + bpemodel=bpemodel, + delimiter=delimiter, + space_symbol=space_symbol, + non_linguistic_symbols=non_linguistic_symbols, + g2p_type=g2p_type, + ) + self.token_id_converter = TokenIDConverter( + token_list=token_list, + unk_symbol=unk_symbol, + ) + else: + self.text_cleaner = None + self.tokenizer = None + self.token_id_converter = None + + if train and rir_scp is not None: + self.rirs = [] + with open(rir_scp, "r", encoding="utf-8") as f: + for line in f: + sps = line.strip().split(None, 1) + if len(sps) == 1: + self.rirs.append(sps[0]) + else: + self.rirs.append(sps[1]) + else: + self.rirs = None + + if train and noise_scp is not None: + self.noises = [] + with open(noise_scp, "r", encoding="utf-8") as f: + for line in f: + sps = line.strip().split(None, 1) + if len(sps) == 1: + self.noises.append(sps[0]) + else: + self.noises.append(sps[1]) + sps = noise_db_range.split("_") + if len(sps) == 1: + self.noise_db_low, self.noise_db_high = float(sps[0]) + elif len(sps) == 2: + self.noise_db_low, self.noise_db_high = float(sps[0]), float(sps[1]) + else: + raise ValueError( + "Format error: '{noise_db_range}' e.g. -3_4 -> [-3db,4db]" + ) + else: + self.noises = None + + def _speech_process( + self, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, Union[str, np.ndarray]]: + assert check_argument_types() + if self.speech_name in data: + if self.train and (self.rirs is not None or self.noises is not None): + speech = data[self.speech_name] + nsamples = len(speech) + + # speech: (Nmic, Time) + if speech.ndim == 1: + speech = speech[None, :] + else: + speech = speech.T + # Calc power on non shlence region + power = (speech[detect_non_silence(speech)] ** 2).mean() + + # 1. Convolve RIR + if self.rirs is not None and self.rir_apply_prob >= np.random.random(): + rir_path = np.random.choice(self.rirs) + if rir_path is not None: + rir, _ = soundfile.read( + rir_path, dtype=np.float64, always_2d=True + ) + + # rir: (Nmic, Time) + rir = rir.T + + # speech: (Nmic, Time) + # Note that this operation doesn't change the signal length + speech = scipy.signal.convolve(speech, rir, mode="full")[ + :, : speech.shape[1] + ] + # Reverse mean power to the original power + power2 = (speech[detect_non_silence(speech)] ** 2).mean() + speech = np.sqrt(power / max(power2, 1e-10)) * speech + + # 2. Add Noise + if ( + self.noises is not None + and self.noise_apply_prob >= np.random.random() + ): + noise_path = np.random.choice(self.noises) + if noise_path is not None: + noise_db = np.random.uniform( + self.noise_db_low, self.noise_db_high + ) + with soundfile.SoundFile(noise_path) as f: + if f.frames == nsamples: + noise = f.read(dtype=np.float64, always_2d=True) + elif f.frames < nsamples: + offset = np.random.randint(0, nsamples - f.frames) + # noise: (Time, Nmic) + noise = f.read(dtype=np.float64, always_2d=True) + # Repeat noise + noise = np.pad( + noise, + [(offset, nsamples - f.frames - offset), (0, 0)], + mode="wrap", + ) + else: + offset = np.random.randint(0, f.frames - nsamples) + f.seek(offset) + # noise: (Time, Nmic) + noise = f.read( + nsamples, dtype=np.float64, always_2d=True + ) + if len(noise) != nsamples: + raise RuntimeError(f"Something wrong: {noise_path}") + # noise: (Nmic, Time) + noise = noise.T + + noise_power = (noise**2).mean() + scale = ( + 10 ** (-noise_db / 20) + * np.sqrt(power) + / np.sqrt(max(noise_power, 1e-10)) + ) + speech = speech + scale * noise + + speech = speech.T + ma = np.max(np.abs(speech)) + if ma > 1.0: + speech /= ma + data[self.speech_name] = speech + + if self.speech_volume_normalize is not None: + speech = data[self.speech_name] + ma = np.max(np.abs(speech)) + data[self.speech_name] = speech * self.speech_volume_normalize / ma + assert check_return_type(data) + return data + + def _text_process( + self, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + if self.text_name in data and self.tokenizer is not None: + text = data[self.text_name] + text = self.text_cleaner(text) + if self.split_with_space: + tokens = text.strip().split(" ") + else: + tokens = self.tokenizer.text2tokens(text) + text_ints = self.token_id_converter.tokens2ids(tokens) + data[self.text_name] = np.array(text_ints, dtype=np.int64) + assert check_return_type(data) + return data + + def __call__( + self, uid: str, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + assert check_argument_types() + + data = self._speech_process(data) + data = self._text_process(data) + return data + + +class CommonPreprocessor_multi(AbsPreprocessor): + def __init__( + self, + train: bool, + token_type: str = None, + token_list: Union[Path, str, Iterable[str]] = None, + bpemodel: Union[Path, str, Iterable[str]] = None, + text_cleaner: Collection[str] = None, + g2p_type: str = None, + unk_symbol: str = "", + space_symbol: str = "", + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + delimiter: str = None, + speech_name: str = "speech", + text_name: List[str] = ["text"], + ): + super().__init__(train) + self.train = train + self.speech_name = speech_name + self.text_name = text_name + + if token_type is not None: + if token_list is None: + raise ValueError("token_list is required if token_type is not None") + self.text_cleaner = TextCleaner(text_cleaner) + + self.tokenizer = build_tokenizer( + token_type=token_type, + bpemodel=bpemodel, + delimiter=delimiter, + space_symbol=space_symbol, + non_linguistic_symbols=non_linguistic_symbols, + g2p_type=g2p_type, + ) + self.token_id_converter = TokenIDConverter( + token_list=token_list, + unk_symbol=unk_symbol, + ) + else: + self.text_cleaner = None + self.tokenizer = None + self.token_id_converter = None + + def _text_process( + self, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + for text_n in self.text_name: + if text_n in data and self.tokenizer is not None: + text = data[text_n] + text = self.text_cleaner(text) + tokens = self.tokenizer.text2tokens(text) + text_ints = self.token_id_converter.tokens2ids(tokens) + data[text_n] = np.array(text_ints, dtype=np.int64) + assert check_return_type(data) + return data + + def __call__( + self, uid: str, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + assert check_argument_types() + + if self.speech_name in data: + # Nothing now: candidates: + # - STFT + # - Fbank + # - CMVN + # - Data augmentation + pass + + data = self._text_process(data) + return data + + +class MutliTokenizerCommonPreprocessor(CommonPreprocessor): + def __init__( + self, + train: bool, + token_type: List[str] = [None], + token_list: List[Union[Path, str, Iterable[str]]] = [None], + bpemodel: List[Union[Path, str, Iterable[str]]] = [None], + text_cleaner: Collection[str] = None, + g2p_type: str = None, + unk_symbol: str = "", + space_symbol: str = "", + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + delimiter: str = None, + rir_scp: str = None, + rir_apply_prob: float = 1.0, + noise_scp: str = None, + noise_apply_prob: float = 1.0, + noise_db_range: str = "3_10", + speech_volume_normalize: float = None, + speech_name: str = "speech", + text_name: List[str] = ["text"], + ): + # TODO(jiatong): sync with Kamo and Jing on interface for preprocessor + super().__init__( + train=train, + token_type=token_type[0], + token_list=token_list[0], + bpemodel=bpemodel[0], + text_cleaner=text_cleaner, + g2p_type=g2p_type, + unk_symbol=unk_symbol, + space_symbol=space_symbol, + non_linguistic_symbols=non_linguistic_symbols, + delimiter=delimiter, + speech_name=speech_name, + text_name=text_name[0], + rir_scp=rir_scp, + rir_apply_prob=rir_apply_prob, + noise_scp=noise_scp, + noise_apply_prob=noise_apply_prob, + noise_db_range=noise_db_range, + speech_volume_normalize=speech_volume_normalize, + ) + + assert ( + len(token_type) == len(token_list) == len(bpemodel) == len(text_name) + ), "token_type, token_list, bpemodel, or processing text_name mismatched" + self.num_tokenizer = len(token_type) + self.tokenizer = [] + self.token_id_converter = [] + + for i in range(self.num_tokenizer): + if token_type[i] is not None: + if token_list[i] is None: + raise ValueError("token_list is required if token_type is not None") + + self.tokenizer.append( + build_tokenizer( + token_type=token_type[i], + bpemodel=bpemodel[i], + delimiter=delimiter, + space_symbol=space_symbol, + non_linguistic_symbols=non_linguistic_symbols, + g2p_type=g2p_type, + ) + ) + self.token_id_converter.append( + TokenIDConverter( + token_list=token_list[i], + unk_symbol=unk_symbol, + ) + ) + else: + self.tokenizer.append(None) + self.token_id_converter.append(None) + + self.text_cleaner = TextCleaner(text_cleaner) + self.text_name = text_name # override the text_name from CommonPreprocessor + + def _text_process( + self, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + for i in range(self.num_tokenizer): + text_name = self.text_name[i] + if text_name in data and self.tokenizer[i] is not None: + text = data[text_name] + text = self.text_cleaner(text) + tokens = self.tokenizer[i].text2tokens(text) + text_ints = self.token_id_converter[i].tokens2ids(tokens) + data[text_name] = np.array(text_ints, dtype=np.int64) + assert check_return_type(data) + return data diff --git a/funasr/fileio/__init__.py b/funasr/fileio/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/fileio/datadir_writer.py b/funasr/fileio/datadir_writer.py new file mode 100644 index 000000000..bafdf984f --- /dev/null +++ b/funasr/fileio/datadir_writer.py @@ -0,0 +1,77 @@ +from pathlib import Path +from typing import Union +import warnings + +from typeguard import check_argument_types +from typeguard import check_return_type + + +class DatadirWriter: + """Writer class to create kaldi like data directory. + + Examples: + >>> with DatadirWriter("output") as writer: + ... # output/sub.txt is created here + ... subwriter = writer["sub.txt"] + ... # Write "uttidA some/where/a.wav" + ... subwriter["uttidA"] = "some/where/a.wav" + ... subwriter["uttidB"] = "some/where/b.wav" + + """ + + def __init__(self, p: Union[Path, str]): + assert check_argument_types() + self.path = Path(p) + self.chilidren = {} + self.fd = None + self.has_children = False + self.keys = set() + + def __enter__(self): + return self + + def __getitem__(self, key: str) -> "DatadirWriter": + assert check_argument_types() + if self.fd is not None: + raise RuntimeError("This writer points out a file") + + if key not in self.chilidren: + w = DatadirWriter((self.path / key)) + self.chilidren[key] = w + self.has_children = True + + retval = self.chilidren[key] + assert check_return_type(retval) + return retval + + def __setitem__(self, key: str, value: str): + assert check_argument_types() + if self.has_children: + raise RuntimeError("This writer points out a directory") + if key in self.keys: + warnings.warn(f"Duplicated: {key}") + + if self.fd is None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.fd = self.path.open("w", encoding="utf-8") + + self.keys.add(key) + self.fd.write(f"{key} {value}\n") + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + if self.has_children: + prev_child = None + for child in self.chilidren.values(): + child.close() + if prev_child is not None and prev_child.keys != child.keys: + warnings.warn( + f"Ids are mismatching between " + f"{prev_child.path} and {child.path}" + ) + prev_child = child + + elif self.fd is not None: + self.fd.close() diff --git a/funasr/fileio/npy_scp.py b/funasr/fileio/npy_scp.py new file mode 100644 index 000000000..26666b678 --- /dev/null +++ b/funasr/fileio/npy_scp.py @@ -0,0 +1,97 @@ +import collections.abc +from pathlib import Path +from typing import Union + +import numpy as np +from typeguard import check_argument_types + +from funasr.fileio.read_text import read_2column_text + + +class NpyScpWriter: + """Writer class for a scp file of numpy file. + + Examples: + key1 /some/path/a.npy + key2 /some/path/b.npy + key3 /some/path/c.npy + key4 /some/path/d.npy + ... + + >>> writer = NpyScpWriter('./data/', './data/feat.scp') + >>> writer['aa'] = numpy_array + >>> writer['bb'] = numpy_array + + """ + + def __init__(self, outdir: Union[Path, str], scpfile: Union[Path, str]): + assert check_argument_types() + self.dir = Path(outdir) + self.dir.mkdir(parents=True, exist_ok=True) + scpfile = Path(scpfile) + scpfile.parent.mkdir(parents=True, exist_ok=True) + self.fscp = scpfile.open("w", encoding="utf-8") + + self.data = {} + + def get_path(self, key): + return self.data[key] + + def __setitem__(self, key, value): + assert isinstance(value, np.ndarray), type(value) + p = self.dir / f"{key}.npy" + p.parent.mkdir(parents=True, exist_ok=True) + np.save(str(p), value) + self.fscp.write(f"{key} {p}\n") + + # Store the file path + self.data[key] = str(p) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + self.fscp.close() + + +class NpyScpReader(collections.abc.Mapping): + """Reader class for a scp file of numpy file. + + Examples: + key1 /some/path/a.npy + key2 /some/path/b.npy + key3 /some/path/c.npy + key4 /some/path/d.npy + ... + + >>> reader = NpyScpReader('npy.scp') + >>> array = reader['key1'] + + """ + + def __init__(self, fname: Union[Path, str]): + assert check_argument_types() + self.fname = Path(fname) + self.data = read_2column_text(fname) + + def get_path(self, key): + return self.data[key] + + def __getitem__(self, key) -> np.ndarray: + p = self.data[key] + return np.load(p) + + def __contains__(self, item): + return item + + def __len__(self): + return len(self.data) + + def __iter__(self): + return iter(self.data) + + def keys(self): + return self.data.keys() diff --git a/funasr/fileio/rand_gen_dataset.py b/funasr/fileio/rand_gen_dataset.py new file mode 100644 index 000000000..2faef3a04 --- /dev/null +++ b/funasr/fileio/rand_gen_dataset.py @@ -0,0 +1,86 @@ +import collections +from pathlib import Path +from typing import Union + +import numpy as np +from typeguard import check_argument_types + +from funasr.fileio.read_text import load_num_sequence_text + + +class FloatRandomGenerateDataset(collections.abc.Mapping): + """Generate float array from shape.txt. + + Examples: + shape.txt + uttA 123,83 + uttB 34,83 + >>> dataset = FloatRandomGenerateDataset("shape.txt") + >>> array = dataset["uttA"] + >>> assert array.shape == (123, 83) + >>> array = dataset["uttB"] + >>> assert array.shape == (34, 83) + + """ + + def __init__( + self, + shape_file: Union[Path, str], + dtype: Union[str, np.dtype] = "float32", + loader_type: str = "csv_int", + ): + assert check_argument_types() + shape_file = Path(shape_file) + self.utt2shape = load_num_sequence_text(shape_file, loader_type) + self.dtype = np.dtype(dtype) + + def __iter__(self): + return iter(self.utt2shape) + + def __len__(self): + return len(self.utt2shape) + + def __getitem__(self, item) -> np.ndarray: + shape = self.utt2shape[item] + return np.random.randn(*shape).astype(self.dtype) + + +class IntRandomGenerateDataset(collections.abc.Mapping): + """Generate float array from shape.txt + + Examples: + shape.txt + uttA 123,83 + uttB 34,83 + >>> dataset = IntRandomGenerateDataset("shape.txt", low=0, high=10) + >>> array = dataset["uttA"] + >>> assert array.shape == (123, 83) + >>> array = dataset["uttB"] + >>> assert array.shape == (34, 83) + + """ + + def __init__( + self, + shape_file: Union[Path, str], + low: int, + high: int = None, + dtype: Union[str, np.dtype] = "int64", + loader_type: str = "csv_int", + ): + assert check_argument_types() + shape_file = Path(shape_file) + self.utt2shape = load_num_sequence_text(shape_file, loader_type) + self.dtype = np.dtype(dtype) + self.low = low + self.high = high + + def __iter__(self): + return iter(self.utt2shape) + + def __len__(self): + return len(self.utt2shape) + + def __getitem__(self, item) -> np.ndarray: + shape = self.utt2shape[item] + return np.random.randint(self.low, self.high, size=shape, dtype=self.dtype) diff --git a/funasr/fileio/read_text.py b/funasr/fileio/read_text.py new file mode 100644 index 000000000..e26e7a1c5 --- /dev/null +++ b/funasr/fileio/read_text.py @@ -0,0 +1,81 @@ +import logging +from pathlib import Path +from typing import Dict +from typing import List +from typing import Union + +from typeguard import check_argument_types + + +def read_2column_text(path: Union[Path, str]) -> Dict[str, str]: + """Read a text file having 2 column as dict object. + + Examples: + wav.scp: + key1 /some/path/a.wav + key2 /some/path/b.wav + + >>> read_2column_text('wav.scp') + {'key1': '/some/path/a.wav', 'key2': '/some/path/b.wav'} + + """ + assert check_argument_types() + + data = {} + with Path(path).open("r", encoding="utf-8") as f: + for linenum, line in enumerate(f, 1): + sps = line.rstrip().split(maxsplit=1) + if len(sps) == 1: + k, v = sps[0], "" + else: + k, v = sps + if k in data: + raise RuntimeError(f"{k} is duplicated ({path}:{linenum})") + data[k] = v + return data + + +def load_num_sequence_text( + path: Union[Path, str], loader_type: str = "csv_int" +) -> Dict[str, List[Union[float, int]]]: + """Read a text file indicating sequences of number + + Examples: + key1 1 2 3 + key2 34 5 6 + + >>> d = load_num_sequence_text('text') + >>> np.testing.assert_array_equal(d["key1"], np.array([1, 2, 3])) + """ + assert check_argument_types() + if loader_type == "text_int": + delimiter = " " + dtype = int + elif loader_type == "text_float": + delimiter = " " + dtype = float + elif loader_type == "csv_int": + delimiter = "," + dtype = int + elif loader_type == "csv_float": + delimiter = "," + dtype = float + else: + raise ValueError(f"Not supported loader_type={loader_type}") + + # path looks like: + # utta 1,0 + # uttb 3,4,5 + # -> return {'utta': np.ndarray([1, 0]), + # 'uttb': np.ndarray([3, 4, 5])} + d = read_2column_text(path) + + # Using for-loop instead of dict-comprehension for debuggability + retval = {} + for k, v in d.items(): + try: + retval[k] = [dtype(i) for i in v.split(delimiter)] + except TypeError: + logging.error(f'Error happened with path="{path}", id="{k}", value="{v}"') + raise + return retval diff --git a/funasr/fileio/sound_scp.py b/funasr/fileio/sound_scp.py new file mode 100644 index 000000000..459369efb --- /dev/null +++ b/funasr/fileio/sound_scp.py @@ -0,0 +1,131 @@ +import collections.abc +from pathlib import Path +from typing import Union + +import numpy as np +import soundfile +from typeguard import check_argument_types + +from funasr.fileio.read_text import read_2column_text + + +class SoundScpReader(collections.abc.Mapping): + """Reader class for 'wav.scp'. + + Examples: + key1 /some/path/a.wav + key2 /some/path/b.wav + key3 /some/path/c.wav + key4 /some/path/d.wav + ... + + >>> reader = SoundScpReader('wav.scp') + >>> rate, array = reader['key1'] + + """ + + def __init__( + self, + fname, + dtype=np.int16, + always_2d: bool = False, + normalize: bool = False, + ): + assert check_argument_types() + self.fname = fname + self.dtype = dtype + self.always_2d = always_2d + self.normalize = normalize + self.data = read_2column_text(fname) + + def __getitem__(self, key): + wav = self.data[key] + if self.normalize: + # soundfile.read normalizes data to [-1,1] if dtype is not given + array, rate = soundfile.read(wav, always_2d=self.always_2d) + else: + array, rate = soundfile.read( + wav, dtype=self.dtype, always_2d=self.always_2d + ) + + return rate, array + + def get_path(self, key): + return self.data[key] + + def __contains__(self, item): + return item + + def __len__(self): + return len(self.data) + + def __iter__(self): + return iter(self.data) + + def keys(self): + return self.data.keys() + + +class SoundScpWriter: + """Writer class for 'wav.scp' + + Examples: + key1 /some/path/a.wav + key2 /some/path/b.wav + key3 /some/path/c.wav + key4 /some/path/d.wav + ... + + >>> writer = SoundScpWriter('./data/', './data/feat.scp') + >>> writer['aa'] = 16000, numpy_array + >>> writer['bb'] = 16000, numpy_array + + """ + + def __init__( + self, + outdir: Union[Path, str], + scpfile: Union[Path, str], + format="wav", + dtype=None, + ): + assert check_argument_types() + self.dir = Path(outdir) + self.dir.mkdir(parents=True, exist_ok=True) + scpfile = Path(scpfile) + scpfile.parent.mkdir(parents=True, exist_ok=True) + self.fscp = scpfile.open("w", encoding="utf-8") + self.format = format + self.dtype = dtype + + self.data = {} + + def __setitem__(self, key: str, value): + rate, signal = value + assert isinstance(rate, int), type(rate) + assert isinstance(signal, np.ndarray), type(signal) + if signal.ndim not in (1, 2): + raise RuntimeError(f"Input signal must be 1 or 2 dimension: {signal.ndim}") + if signal.ndim == 1: + signal = signal[:, None] + + wav = self.dir / f"{key}.{self.format}" + wav.parent.mkdir(parents=True, exist_ok=True) + soundfile.write(str(wav), signal, rate) + + self.fscp.write(f"{key} {wav}\n") + + # Store the file path + self.data[key] = str(wav) + + def get_path(self, key): + return self.data[key] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + self.fscp.close() diff --git a/funasr/iterators/__init__.py b/funasr/iterators/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/iterators/abs_iter_factory.py b/funasr/iterators/abs_iter_factory.py new file mode 100644 index 000000000..36e4dd2c5 --- /dev/null +++ b/funasr/iterators/abs_iter_factory.py @@ -0,0 +1,9 @@ +from abc import ABC +from abc import abstractmethod +from typing import Iterator + + +class AbsIterFactory(ABC): + @abstractmethod + def build_iter(self, epoch: int, shuffle: bool = None) -> Iterator: + raise NotImplementedError diff --git a/funasr/iterators/chunk_iter_factory.py b/funasr/iterators/chunk_iter_factory.py new file mode 100644 index 000000000..cec637040 --- /dev/null +++ b/funasr/iterators/chunk_iter_factory.py @@ -0,0 +1,215 @@ +import logging +from typing import Any +from typing import Dict +from typing import Iterator +from typing import List +from typing import Sequence +from typing import Tuple +from typing import Union + +import numpy as np +import torch +from typeguard import check_argument_types + +from funasr.iterators.abs_iter_factory import AbsIterFactory +from funasr.iterators.sequence_iter_factory import SequenceIterFactory +from funasr.samplers.abs_sampler import AbsSampler + + +class ChunkIterFactory(AbsIterFactory): + """Creates chunks from a sequence + + Examples: + >>> batches = [["id1"], ["id2"], ...] + >>> batch_size = 128 + >>> chunk_length = 1000 + >>> iter_factory = ChunkIterFactory(dataset, batches, batch_size, chunk_length) + >>> it = iter_factory.build_iter(epoch) + >>> for ids, batch in it: + ... ... + + - The number of mini-batches are varied in each epochs and + we can't get the number in advance + because IterFactory doesn't be given to the length information. + - Since the first reason, "num_iters_per_epoch" can't be implemented + for this iterator. Instead of it, "num_samples_per_epoch" is implemented. + + """ + + def __init__( + self, + dataset, + batch_size: int, + batches: Union[AbsSampler, Sequence[Sequence[Any]]], + chunk_length: Union[int, str], + chunk_shift_ratio: float = 0.5, + num_cache_chunks: int = 1024, + num_samples_per_epoch: int = None, + seed: int = 0, + shuffle: bool = False, + num_workers: int = 0, + collate_fn=None, + pin_memory: bool = False, + ): + assert check_argument_types() + assert all(len(x) == 1 for x in batches), "batch-size must be 1" + + self.per_sample_iter_factory = SequenceIterFactory( + dataset=dataset, + batches=batches, + num_iters_per_epoch=num_samples_per_epoch, + seed=seed, + shuffle=shuffle, + num_workers=num_workers, + collate_fn=collate_fn, + pin_memory=pin_memory, + ) + + self.num_cache_chunks = max(num_cache_chunks, batch_size) + if isinstance(chunk_length, str): + if len(chunk_length) == 0: + raise ValueError("e.g. 5,8 or 3-5: but got empty string") + + self.chunk_lengths = [] + for x in chunk_length.split(","): + try: + sps = list(map(int, x.split("-"))) + except ValueError: + raise ValueError(f"e.g. 5,8 or 3-5: but got {chunk_length}") + + if len(sps) > 2: + raise ValueError(f"e.g. 5,8 or 3-5: but got {chunk_length}") + elif len(sps) == 2: + # Append all numbers between the range into the candidates + self.chunk_lengths += list(range(sps[0], sps[1] + 1)) + else: + self.chunk_lengths += [sps[0]] + else: + # Single candidates: Fixed chunk length + self.chunk_lengths = [chunk_length] + + self.chunk_shift_ratio = chunk_shift_ratio + self.batch_size = batch_size + self.seed = seed + self.shuffle = shuffle + + def build_iter( + self, + epoch: int, + shuffle: bool = None, + ) -> Iterator[Tuple[List[str], Dict[str, torch.Tensor]]]: + per_sample_loader = self.per_sample_iter_factory.build_iter(epoch, shuffle) + + if shuffle is None: + shuffle = self.shuffle + state = np.random.RandomState(epoch + self.seed) + + # NOTE(kamo): + # This iterator supports multiple chunk lengths and + # keep chunks for each lengths here until collecting specified numbers + cache_chunks_dict = {} + cache_id_list_dict = {} + for ids, batch in per_sample_loader: + # Must be per-sample-loader + assert len(ids) == 1, f"Must be per-sample-loader: {len(ids)}" + assert all(len(x) == 1 for x in batch.values()) + + # Get keys of sequence data + sequence_keys = [] + for key in batch: + if key + "_lengths" in batch: + sequence_keys.append(key) + # Remove lengths data and get the first sample + batch = {k: v[0] for k, v in batch.items() if not k.endswith("_lengths")} + id_ = ids[0] + + for key in sequence_keys: + if len(batch[key]) != len(batch[sequence_keys[0]]): + raise RuntimeError( + f"All sequences must has same length: " + f"{len(batch[key])} != {len(batch[sequence_keys[0]])}" + ) + + L = len(batch[sequence_keys[0]]) + # Select chunk length + chunk_lengths = [lg for lg in self.chunk_lengths if lg < L] + if len(chunk_lengths) == 0: + logging.warning( + f"The length of '{id_}' is {L}, but it is shorter than " + f"any candidates of chunk-length: {self.chunk_lengths}" + ) + continue + + W = int(state.choice(chunk_lengths, 1)) + cache_id_list = cache_id_list_dict.setdefault(W, []) + cache_chunks = cache_chunks_dict.setdefault(W, {}) + + # Shift width to the next chunk + S = int(W * self.chunk_shift_ratio) + # Number of chunks + N = (L - W) // S + 1 + if shuffle: + Z = state.randint(0, (L - W) % S + 1) + else: + Z = 0 + + # Split a sequence into chunks. + # Note that the marginal frames divided by chunk length are discarded + for k, v in batch.items(): + if k not in cache_chunks: + cache_chunks[k] = [] + if k in sequence_keys: + # Shift chunks with overlapped length for data augmentation + cache_chunks[k] += [v[Z + i * S : Z + i * S + W] for i in range(N)] + else: + # If not sequence, use whole data instead of chunk + cache_chunks[k] += [v for _ in range(N)] + cache_id_list += [id_ for _ in range(N)] + + if len(cache_id_list) > self.num_cache_chunks: + cache_id_list, cache_chunks = yield from self._generate_mini_batches( + cache_id_list, + cache_chunks, + shuffle, + state, + ) + + cache_id_list_dict[W] = cache_id_list + cache_chunks_dict[W] = cache_chunks + + else: + for W in cache_id_list_dict: + cache_id_list = cache_id_list_dict.setdefault(W, []) + cache_chunks = cache_chunks_dict.setdefault(W, {}) + + yield from self._generate_mini_batches( + cache_id_list, + cache_chunks, + shuffle, + state, + ) + + def _generate_mini_batches( + self, + id_list: List[str], + batches: Dict[str, List[torch.Tensor]], + shuffle: bool, + state: np.random.RandomState, + ): + if shuffle: + indices = np.arange(0, len(id_list)) + state.shuffle(indices) + batches = {k: [v[i] for i in indices] for k, v in batches.items()} + id_list = [id_list[i] for i in indices] + + bs = self.batch_size + while len(id_list) >= bs: + # Make mini-batch and yield + yield ( + id_list[:bs], + {k: torch.stack(v[:bs], 0) for k, v in batches.items()}, + ) + id_list = id_list[bs:] + batches = {k: v[bs:] for k, v in batches.items()} + + return id_list, batches diff --git a/funasr/iterators/multiple_iter_factory.py b/funasr/iterators/multiple_iter_factory.py new file mode 100644 index 000000000..088016cf3 --- /dev/null +++ b/funasr/iterators/multiple_iter_factory.py @@ -0,0 +1,37 @@ +import logging +from typing import Callable +from typing import Collection +from typing import Iterator + +import numpy as np +from typeguard import check_argument_types + +from funasr.iterators.abs_iter_factory import AbsIterFactory + + +class MultipleIterFactory(AbsIterFactory): + def __init__( + self, + build_funcs: Collection[Callable[[], AbsIterFactory]], + seed: int = 0, + shuffle: bool = False, + ): + assert check_argument_types() + self.build_funcs = list(build_funcs) + self.seed = seed + self.shuffle = shuffle + + def build_iter(self, epoch: int, shuffle: bool = None) -> Iterator: + if shuffle is None: + shuffle = self.shuffle + + build_funcs = list(self.build_funcs) + + if shuffle: + np.random.RandomState(epoch + self.seed).shuffle(build_funcs) + + for i, build_func in enumerate(build_funcs): + logging.info(f"Building {i}th iter-factory...") + iter_factory = build_func() + assert isinstance(iter_factory, AbsIterFactory), type(iter_factory) + yield from iter_factory.build_iter(epoch, shuffle) diff --git a/funasr/iterators/sequence_iter_factory.py b/funasr/iterators/sequence_iter_factory.py new file mode 100644 index 000000000..39d083446 --- /dev/null +++ b/funasr/iterators/sequence_iter_factory.py @@ -0,0 +1,143 @@ +from typing import Any +from typing import Sequence +from typing import Union + +import numpy as np +from torch.utils.data import DataLoader +from typeguard import check_argument_types + +from funasr.iterators.abs_iter_factory import AbsIterFactory +from funasr.samplers.abs_sampler import AbsSampler + + +class RawSampler(AbsSampler): + def __init__(self, batches): + self.batches = batches + + def __len__(self): + return len(self.batches) + + def __iter__(self): + return iter(self.batches) + + def generate(self, seed): + return list(self.batches) + + +class SequenceIterFactory(AbsIterFactory): + """Build iterator for each epoch. + + This class simply creates pytorch DataLoader except for the following points: + - The random seed is decided according to the number of epochs. This feature + guarantees reproducibility when resuming from middle of training process. + - Enable to restrict the number of samples for one epoch. This features + controls the interval number between training and evaluation. + + """ + + def __init__( + self, + dataset, + batches: Union[AbsSampler, Sequence[Sequence[Any]]], + num_iters_per_epoch: int = None, + seed: int = 0, + shuffle: bool = False, + num_workers: int = 0, + collate_fn=None, + pin_memory: bool = False, + ): + assert check_argument_types() + + if not isinstance(batches, AbsSampler): + self.sampler = RawSampler(batches) + else: + self.sampler = batches + + self.dataset = dataset + self.num_iters_per_epoch = num_iters_per_epoch + self.shuffle = shuffle + self.seed = seed + self.num_workers = num_workers + self.collate_fn = collate_fn + # https://discuss.pytorch.org/t/what-is-the-disadvantage-of-using-pin-memory/1702 + self.pin_memory = pin_memory + + def build_iter(self, epoch: int, shuffle: bool = None) -> DataLoader: + if shuffle is None: + shuffle = self.shuffle + + if self.num_iters_per_epoch is not None: + N = len(self.sampler) + # If corpus size is larger than the num_per_epoch + if self.num_iters_per_epoch < N: + N = len(self.sampler) + real_epoch, offset = divmod(self.num_iters_per_epoch * epoch, N) + + if offset >= self.num_iters_per_epoch: + current_batches = self.sampler.generate(real_epoch + self.seed) + if shuffle: + np.random.RandomState(real_epoch + self.seed).shuffle( + current_batches + ) + batches = current_batches[ + offset - self.num_iters_per_epoch : offset + ] + else: + prev_batches = self.sampler.generate(real_epoch - 1 + self.seed) + current_batches = self.sampler.generate(real_epoch + self.seed) + if shuffle: + np.random.RandomState(real_epoch - 1 + self.seed).shuffle( + prev_batches + ) + np.random.RandomState(real_epoch + self.seed).shuffle( + current_batches + ) + batches = ( + prev_batches[offset - self.num_iters_per_epoch :] + + current_batches[:offset] + ) + + # If corpus size is less than the num_per_epoch + else: + _epoch, _cursor = divmod(self.num_iters_per_epoch * (epoch - 1), N) + _remain = self.num_iters_per_epoch + batches = [] + current_batches = self.sampler.generate(_epoch + self.seed) + if shuffle: + np.random.RandomState(_epoch + self.seed).shuffle(current_batches) + while _remain > 0: + + _batches = current_batches[_cursor : _cursor + _remain] + batches += _batches + if _cursor + _remain >= N: + _epoch += 1 + _cursor = 0 + current_batches = self.sampler.generate(_epoch + self.seed) + if shuffle: + np.random.RandomState(_epoch + self.seed).shuffle( + current_batches + ) + else: + _cursor = _cursor + _remain + _remain -= len(_batches) + + assert len(batches) == self.num_iters_per_epoch + + else: + batches = self.sampler.generate(epoch + self.seed) + if shuffle: + np.random.RandomState(epoch + self.seed).shuffle(batches) + + # For backward compatibility for pytorch DataLoader + if self.collate_fn is not None: + kwargs = dict(collate_fn=self.collate_fn) + else: + kwargs = {} + + return DataLoader( + dataset=self.dataset, + batch_sampler=batches, + num_workers=self.num_workers, + pin_memory=self.pin_memory, + **kwargs, + ) diff --git a/funasr/layers/__init__.py b/funasr/layers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/layers/abs_normalize.py b/funasr/layers/abs_normalize.py new file mode 100644 index 000000000..f2be748dd --- /dev/null +++ b/funasr/layers/abs_normalize.py @@ -0,0 +1,14 @@ +from abc import ABC +from abc import abstractmethod +from typing import Tuple + +import torch + + +class AbsNormalize(torch.nn.Module, ABC): + @abstractmethod + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + # return output, output_lengths + raise NotImplementedError diff --git a/funasr/layers/complex_utils.py b/funasr/layers/complex_utils.py new file mode 100644 index 000000000..bf4799f58 --- /dev/null +++ b/funasr/layers/complex_utils.py @@ -0,0 +1,191 @@ +"""Beamformer module.""" +from distutils.version import LooseVersion +from typing import Sequence +from typing import Tuple +from typing import Union + +import torch +from torch_complex import functional as FC +from torch_complex.tensor import ComplexTensor + + +EPS = torch.finfo(torch.double).eps +is_torch_1_8_plus = LooseVersion(torch.__version__) >= LooseVersion("1.8.0") +is_torch_1_9_plus = LooseVersion(torch.__version__) >= LooseVersion("1.9.0") + + +def new_complex_like( + ref: Union[torch.Tensor, ComplexTensor], + real_imag: Tuple[torch.Tensor, torch.Tensor], +): + if isinstance(ref, ComplexTensor): + return ComplexTensor(*real_imag) + elif is_torch_complex_tensor(ref): + return torch.complex(*real_imag) + else: + raise ValueError( + "Please update your PyTorch version to 1.9+ for complex support." + ) + + +def is_torch_complex_tensor(c): + return ( + not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c) + ) + + +def is_complex(c): + return isinstance(c, ComplexTensor) or is_torch_complex_tensor(c) + + +def to_double(c): + if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c): + return c.to(dtype=torch.complex128) + else: + return c.double() + + +def to_float(c): + if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c): + return c.to(dtype=torch.complex64) + else: + return c.float() + + +def cat(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs): + if not isinstance(seq, (list, tuple)): + raise TypeError( + "cat(): argument 'tensors' (position 1) must be tuple of Tensors, " + "not Tensor" + ) + if isinstance(seq[0], ComplexTensor): + return FC.cat(seq, *args, **kwargs) + else: + return torch.cat(seq, *args, **kwargs) + + +def complex_norm( + c: Union[torch.Tensor, ComplexTensor], dim=-1, keepdim=False +) -> torch.Tensor: + if not is_complex(c): + raise TypeError("Input is not a complex tensor.") + if is_torch_complex_tensor(c): + return torch.norm(c, dim=dim, keepdim=keepdim) + else: + return torch.sqrt( + (c.real**2 + c.imag**2).sum(dim=dim, keepdim=keepdim) + EPS + ) + + +def einsum(equation, *operands): + # NOTE: Do not mix ComplexTensor and torch.complex in the input! + # NOTE (wangyou): Until PyTorch 1.9.0, torch.einsum does not support + # mixed input with complex and real tensors. + if len(operands) == 1: + if isinstance(operands[0], (tuple, list)): + operands = operands[0] + complex_module = FC if isinstance(operands[0], ComplexTensor) else torch + return complex_module.einsum(equation, *operands) + elif len(operands) != 2: + op0 = operands[0] + same_type = all(op.dtype == op0.dtype for op in operands[1:]) + if same_type: + _einsum = FC.einsum if isinstance(op0, ComplexTensor) else torch.einsum + return _einsum(equation, *operands) + else: + raise ValueError("0 or More than 2 operands are not supported.") + a, b = operands + if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): + return FC.einsum(equation, a, b) + elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): + if not torch.is_complex(a): + o_real = torch.einsum(equation, a, b.real) + o_imag = torch.einsum(equation, a, b.imag) + return torch.complex(o_real, o_imag) + elif not torch.is_complex(b): + o_real = torch.einsum(equation, a.real, b) + o_imag = torch.einsum(equation, a.imag, b) + return torch.complex(o_real, o_imag) + else: + return torch.einsum(equation, a, b) + else: + return torch.einsum(equation, a, b) + + +def inverse( + c: Union[torch.Tensor, ComplexTensor] +) -> Union[torch.Tensor, ComplexTensor]: + if isinstance(c, ComplexTensor): + return c.inverse2() + else: + return c.inverse() + + +def matmul( + a: Union[torch.Tensor, ComplexTensor], b: Union[torch.Tensor, ComplexTensor] +) -> Union[torch.Tensor, ComplexTensor]: + # NOTE: Do not mix ComplexTensor and torch.complex in the input! + # NOTE (wangyou): Until PyTorch 1.9.0, torch.matmul does not support + # multiplication between complex and real tensors. + if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): + return FC.matmul(a, b) + elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): + if not torch.is_complex(a): + o_real = torch.matmul(a, b.real) + o_imag = torch.matmul(a, b.imag) + return torch.complex(o_real, o_imag) + elif not torch.is_complex(b): + o_real = torch.matmul(a.real, b) + o_imag = torch.matmul(a.imag, b) + return torch.complex(o_real, o_imag) + else: + return torch.matmul(a, b) + else: + return torch.matmul(a, b) + + +def trace(a: Union[torch.Tensor, ComplexTensor]): + # NOTE (wangyou): until PyTorch 1.9.0, torch.trace does not + # support bacth processing. Use FC.trace() as fallback. + return FC.trace(a) + + +def reverse(a: Union[torch.Tensor, ComplexTensor], dim=0): + if isinstance(a, ComplexTensor): + return FC.reverse(a, dim=dim) + else: + return torch.flip(a, dims=(dim,)) + + +def solve(b: Union[torch.Tensor, ComplexTensor], a: Union[torch.Tensor, ComplexTensor]): + """Solve the linear equation ax = b.""" + # NOTE: Do not mix ComplexTensor and torch.complex in the input! + # NOTE (wangyou): Until PyTorch 1.9.0, torch.solve does not support + # mixed input with complex and real tensors. + if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor): + if isinstance(a, ComplexTensor) and isinstance(b, ComplexTensor): + return FC.solve(b, a, return_LU=False) + else: + return matmul(inverse(a), b) + elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)): + if torch.is_complex(a) and torch.is_complex(b): + return torch.linalg.solve(a, b) + else: + return matmul(inverse(a), b) + else: + if is_torch_1_8_plus: + return torch.linalg.solve(a, b) + else: + return torch.solve(b, a)[0] + + +def stack(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs): + if not isinstance(seq, (list, tuple)): + raise TypeError( + "stack(): argument 'tensors' (position 1) must be tuple of Tensors, " + "not Tensor" + ) + if isinstance(seq[0], ComplexTensor): + return FC.stack(seq, *args, **kwargs) + else: + return torch.stack(seq, *args, **kwargs) diff --git a/funasr/layers/global_mvn.py b/funasr/layers/global_mvn.py new file mode 100644 index 000000000..5515cdde6 --- /dev/null +++ b/funasr/layers/global_mvn.py @@ -0,0 +1,121 @@ +from pathlib import Path +from typing import Tuple +from typing import Union + +import numpy as np +import torch +from typeguard import check_argument_types + +from funasr.modules.nets_utils import make_pad_mask +from funasr.layers.abs_normalize import AbsNormalize +from funasr.layers.inversible_interface import InversibleInterface + + +class GlobalMVN(AbsNormalize, InversibleInterface): + """Apply global mean and variance normalization + + TODO(kamo): Make this class portable somehow + + Args: + stats_file: npy file + norm_means: Apply mean normalization + norm_vars: Apply var normalization + eps: + """ + + def __init__( + self, + stats_file: Union[Path, str], + norm_means: bool = True, + norm_vars: bool = True, + eps: float = 1.0e-20, + ): + assert check_argument_types() + super().__init__() + self.norm_means = norm_means + self.norm_vars = norm_vars + self.eps = eps + stats_file = Path(stats_file) + + self.stats_file = stats_file + stats = np.load(stats_file) + if isinstance(stats, np.ndarray): + # Kaldi like stats + count = stats[0].flatten()[-1] + mean = stats[0, :-1] / count + var = stats[1, :-1] / count - mean * mean + else: + # New style: Npz file + count = stats["count"] + sum_v = stats["sum"] + sum_square_v = stats["sum_square"] + mean = sum_v / count + var = sum_square_v / count - mean * mean + std = np.sqrt(np.maximum(var, eps)) + + self.register_buffer("mean", torch.from_numpy(mean)) + self.register_buffer("std", torch.from_numpy(std)) + + def extra_repr(self): + return ( + f"stats_file={self.stats_file}, " + f"norm_means={self.norm_means}, norm_vars={self.norm_vars}" + ) + + def forward( + self, x: torch.Tensor, ilens: torch.Tensor = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward function + + Args: + x: (B, L, ...) + ilens: (B,) + """ + if ilens is None: + ilens = x.new_full([x.size(0)], x.size(1)) + norm_means = self.norm_means + norm_vars = self.norm_vars + self.mean = self.mean.to(x.device, x.dtype) + self.std = self.std.to(x.device, x.dtype) + mask = make_pad_mask(ilens, x, 1) + + # feat: (B, T, D) + if norm_means: + if x.requires_grad: + x = x - self.mean + else: + x -= self.mean + if x.requires_grad: + x = x.masked_fill(mask, 0.0) + else: + x.masked_fill_(mask, 0.0) + + if norm_vars: + x /= self.std + + return x, ilens + + def inverse( + self, x: torch.Tensor, ilens: torch.Tensor = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + if ilens is None: + ilens = x.new_full([x.size(0)], x.size(1)) + norm_means = self.norm_means + norm_vars = self.norm_vars + self.mean = self.mean.to(x.device, x.dtype) + self.std = self.std.to(x.device, x.dtype) + mask = make_pad_mask(ilens, x, 1) + + if x.requires_grad: + x = x.masked_fill(mask, 0.0) + else: + x.masked_fill_(mask, 0.0) + + if norm_vars: + x *= self.std + + # feat: (B, T, D) + if norm_means: + x += self.mean + x.masked_fill_(make_pad_mask(ilens, x, 1), 0.0) + return x, ilens diff --git a/funasr/layers/inversible_interface.py b/funasr/layers/inversible_interface.py new file mode 100644 index 000000000..a1a59399a --- /dev/null +++ b/funasr/layers/inversible_interface.py @@ -0,0 +1,14 @@ +from abc import ABC +from abc import abstractmethod +from typing import Tuple + +import torch + + +class InversibleInterface(ABC): + @abstractmethod + def inverse( + self, input: torch.Tensor, input_lengths: torch.Tensor = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + # return output, output_lengths + raise NotImplementedError diff --git a/funasr/layers/label_aggregation.py b/funasr/layers/label_aggregation.py new file mode 100644 index 000000000..075e19d90 --- /dev/null +++ b/funasr/layers/label_aggregation.py @@ -0,0 +1,82 @@ +import torch +from typeguard import check_argument_types +from typing import Optional +from typing import Tuple + +from funasr.modules.nets_utils import make_pad_mask + + +class LabelAggregate(torch.nn.Module): + def __init__( + self, + win_length: int = 512, + hop_length: int = 128, + center: bool = True, + ): + assert check_argument_types() + super().__init__() + + self.win_length = win_length + self.hop_length = hop_length + self.center = center + + def extra_repr(self): + return ( + f"win_length={self.win_length}, " + f"hop_length={self.hop_length}, " + f"center={self.center}, " + ) + + def forward( + self, input: torch.Tensor, ilens: torch.Tensor = None + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """LabelAggregate forward function. + + Args: + input: (Batch, Nsamples, Label_dim) + ilens: (Batch) + Returns: + output: (Batch, Frames, Label_dim) + + """ + bs = input.size(0) + max_length = input.size(1) + label_dim = input.size(2) + + # NOTE(jiatong): + # The default behaviour of label aggregation is compatible with + # torch.stft about framing and padding. + + # Step1: center padding + if self.center: + pad = self.win_length // 2 + max_length = max_length + 2 * pad + input = torch.nn.functional.pad(input, (0, 0, pad, pad), "constant", 0) + input[:, :pad, :] = input[:, pad : (2 * pad), :] + input[:, (max_length - pad) : max_length, :] = input[ + :, (max_length - 2 * pad) : (max_length - pad), : + ] + nframe = (max_length - self.win_length) // self.hop_length + 1 + + # Step2: framing + output = input.as_strided( + (bs, nframe, self.win_length, label_dim), + (max_length * label_dim, self.hop_length * label_dim, label_dim, 1), + ) + + # Step3: aggregate label + output = torch.gt(output.sum(dim=2, keepdim=False), self.win_length // 2) + output = output.float() + + # Step4: process lengths + if ilens is not None: + if self.center: + pad = self.win_length // 2 + ilens = ilens + 2 * pad + + olens = (ilens - self.win_length) // self.hop_length + 1 + output.masked_fill_(make_pad_mask(olens, output, 1), 0.0) + else: + olens = None + + return output, olens diff --git a/funasr/layers/log_mel.py b/funasr/layers/log_mel.py new file mode 100644 index 000000000..2285f6d4f --- /dev/null +++ b/funasr/layers/log_mel.py @@ -0,0 +1,83 @@ +import librosa +import torch +from typing import Tuple + +from funasr.modules.nets_utils import make_pad_mask + + +class LogMel(torch.nn.Module): + """Convert STFT to fbank feats + + The arguments is same as librosa.filters.mel + + Args: + fs: number > 0 [scalar] sampling rate of the incoming signal + n_fft: int > 0 [scalar] number of FFT components + n_mels: int > 0 [scalar] number of Mel bands to generate + fmin: float >= 0 [scalar] lowest frequency (in Hz) + fmax: float >= 0 [scalar] highest frequency (in Hz). + If `None`, use `fmax = fs / 2.0` + htk: use HTK formula instead of Slaney + """ + + def __init__( + self, + fs: int = 16000, + n_fft: int = 512, + n_mels: int = 80, + fmin: float = None, + fmax: float = None, + htk: bool = False, + log_base: float = None, + ): + super().__init__() + + fmin = 0 if fmin is None else fmin + fmax = fs / 2 if fmax is None else fmax + _mel_options = dict( + sr=fs, + n_fft=n_fft, + n_mels=n_mels, + fmin=fmin, + fmax=fmax, + htk=htk, + ) + self.mel_options = _mel_options + self.log_base = log_base + + # Note(kamo): The mel matrix of librosa is different from kaldi. + melmat = librosa.filters.mel(**_mel_options) + # melmat: (D2, D1) -> (D1, D2) + self.register_buffer("melmat", torch.from_numpy(melmat.T).float()) + + def extra_repr(self): + return ", ".join(f"{k}={v}" for k, v in self.mel_options.items()) + + def forward( + self, + feat: torch.Tensor, + ilens: torch.Tensor = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # feat: (B, T, D1) x melmat: (D1, D2) -> mel_feat: (B, T, D2) + mel_feat = torch.matmul(feat, self.melmat) + mel_feat = torch.clamp(mel_feat, min=1e-10) + + if self.log_base is None: + logmel_feat = mel_feat.log() + elif self.log_base == 2.0: + logmel_feat = mel_feat.log2() + elif self.log_base == 10.0: + logmel_feat = mel_feat.log10() + else: + logmel_feat = mel_feat.log() / torch.log(self.log_base) + + # Zero padding + if ilens is not None: + logmel_feat = logmel_feat.masked_fill( + make_pad_mask(ilens, logmel_feat, 1), 0.0 + ) + else: + ilens = feat.new_full( + [feat.size(0)], fill_value=feat.size(1), dtype=torch.long + ) + return logmel_feat, ilens diff --git a/funasr/layers/mask_along_axis.py b/funasr/layers/mask_along_axis.py new file mode 100644 index 000000000..e49e621cc --- /dev/null +++ b/funasr/layers/mask_along_axis.py @@ -0,0 +1,340 @@ +import math +import torch +from typeguard import check_argument_types +from typing import Sequence +from typing import Union + + +def mask_along_axis( + spec: torch.Tensor, + spec_lengths: torch.Tensor, + mask_width_range: Sequence[int] = (0, 30), + dim: int = 1, + num_mask: int = 2, + replace_with_zero: bool = True, +): + """Apply mask along the specified direction. + + Args: + spec: (Batch, Length, Freq) + spec_lengths: (Length): Not using lengths in this implementation + mask_width_range: Select the width randomly between this range + """ + + org_size = spec.size() + if spec.dim() == 4: + # spec: (Batch, Channel, Length, Freq) -> (Batch * Channel, Length, Freq) + spec = spec.view(-1, spec.size(2), spec.size(3)) + + B = spec.shape[0] + # D = Length or Freq + D = spec.shape[dim] + # mask_length: (B, num_mask, 1) + mask_length = torch.randint( + mask_width_range[0], + mask_width_range[1], + (B, num_mask), + device=spec.device, + ).unsqueeze(2) + + # mask_pos: (B, num_mask, 1) + mask_pos = torch.randint( + 0, max(1, D - mask_length.max()), (B, num_mask), device=spec.device + ).unsqueeze(2) + + # aran: (1, 1, D) + aran = torch.arange(D, device=spec.device)[None, None, :] + # mask: (Batch, num_mask, D) + mask = (mask_pos <= aran) * (aran < (mask_pos + mask_length)) + # Multiply masks: (Batch, num_mask, D) -> (Batch, D) + mask = mask.any(dim=1) + if dim == 1: + # mask: (Batch, Length, 1) + mask = mask.unsqueeze(2) + elif dim == 2: + # mask: (Batch, 1, Freq) + mask = mask.unsqueeze(1) + + if replace_with_zero: + value = 0.0 + else: + value = spec.mean() + + if spec.requires_grad: + spec = spec.masked_fill(mask, value) + else: + spec = spec.masked_fill_(mask, value) + spec = spec.view(*org_size) + return spec, spec_lengths + +def mask_along_axis_lfr( + spec: torch.Tensor, + spec_lengths: torch.Tensor, + mask_width_range: Sequence[int] = (0, 30), + dim: int = 1, + num_mask: int = 2, + replace_with_zero: bool = True, + lfr_rate: int = 1, +): + """Apply mask along the specified direction. + + Args: + spec: (Batch, Length, Freq) + spec_lengths: (Length): Not using lengths in this implementation + mask_width_range: Select the width randomly between this range + lfr_rate：low frame rate + """ + + org_size = spec.size() + if spec.dim() == 4: + # spec: (Batch, Channel, Length, Freq) -> (Batch * Channel, Length, Freq) + spec = spec.view(-1, spec.size(2), spec.size(3)) + + B = spec.shape[0] + # D = Length or Freq + D = spec.shape[dim] // lfr_rate + # mask_length: (B, num_mask, 1) + mask_length = torch.randint( + mask_width_range[0], + mask_width_range[1], + (B, num_mask), + device=spec.device, + ).unsqueeze(2) + if lfr_rate > 1: + mask_length = mask_length.repeat(1, lfr_rate, 1) + # mask_pos: (B, num_mask, 1) + mask_pos = torch.randint( + 0, max(1, D - mask_length.max()), (B, num_mask), device=spec.device + ).unsqueeze(2) + if lfr_rate > 1: + mask_pos_raw = mask_pos.clone() + mask_pos = torch.zeros((B, 0, 1), device=spec.device, dtype=torch.int32) + for i in range(lfr_rate): + mask_pos_i = mask_pos_raw + D * i + mask_pos = torch.cat((mask_pos, mask_pos_i), dim=1) + # aran: (1, 1, D) + D = spec.shape[dim] + aran = torch.arange(D, device=spec.device)[None, None, :] + # mask: (Batch, num_mask, D) + mask = (mask_pos <= aran) * (aran < (mask_pos + mask_length)) + # Multiply masks: (Batch, num_mask, D) -> (Batch, D) + mask = mask.any(dim=1) + if dim == 1: + # mask: (Batch, Length, 1) + mask = mask.unsqueeze(2) + elif dim == 2: + # mask: (Batch, 1, Freq) + mask = mask.unsqueeze(1) + + if replace_with_zero: + value = 0.0 + else: + value = spec.mean() + + if spec.requires_grad: + spec = spec.masked_fill(mask, value) + else: + spec = spec.masked_fill_(mask, value) + spec = spec.view(*org_size) + return spec, spec_lengths + + +class MaskAlongAxis(torch.nn.Module): + def __init__( + self, + mask_width_range: Union[int, Sequence[int]] = (0, 30), + num_mask: int = 2, + dim: Union[int, str] = "time", + replace_with_zero: bool = True, + ): + assert check_argument_types() + if isinstance(mask_width_range, int): + mask_width_range = (0, mask_width_range) + if len(mask_width_range) != 2: + raise TypeError( + f"mask_width_range must be a tuple of int and int values: " + f"{mask_width_range}", + ) + + assert mask_width_range[1] > mask_width_range[0] + if isinstance(dim, str): + if dim == "time": + dim = 1 + elif dim == "freq": + dim = 2 + else: + raise ValueError("dim must be int, 'time' or 'freq'") + if dim == 1: + self.mask_axis = "time" + elif dim == 2: + self.mask_axis = "freq" + else: + self.mask_axis = "unknown" + + super().__init__() + self.mask_width_range = mask_width_range + self.num_mask = num_mask + self.dim = dim + self.replace_with_zero = replace_with_zero + + def extra_repr(self): + return ( + f"mask_width_range={self.mask_width_range}, " + f"num_mask={self.num_mask}, axis={self.mask_axis}" + ) + + def forward(self, spec: torch.Tensor, spec_lengths: torch.Tensor = None): + """Forward function. + + Args: + spec: (Batch, Length, Freq) + """ + + return mask_along_axis( + spec, + spec_lengths, + mask_width_range=self.mask_width_range, + dim=self.dim, + num_mask=self.num_mask, + replace_with_zero=self.replace_with_zero, + ) + + +class MaskAlongAxisVariableMaxWidth(torch.nn.Module): + """Mask input spec along a specified axis with variable maximum width. + + Formula: + max_width = max_width_ratio * seq_len + """ + + def __init__( + self, + mask_width_ratio_range: Union[float, Sequence[float]] = (0.0, 0.05), + num_mask: int = 2, + dim: Union[int, str] = "time", + replace_with_zero: bool = True, + ): + assert check_argument_types() + if isinstance(mask_width_ratio_range, float): + mask_width_ratio_range = (0.0, mask_width_ratio_range) + if len(mask_width_ratio_range) != 2: + raise TypeError( + f"mask_width_ratio_range must be a tuple of float and float values: " + f"{mask_width_ratio_range}", + ) + + assert mask_width_ratio_range[1] > mask_width_ratio_range[0] + if isinstance(dim, str): + if dim == "time": + dim = 1 + elif dim == "freq": + dim = 2 + else: + raise ValueError("dim must be int, 'time' or 'freq'") + if dim == 1: + self.mask_axis = "time" + elif dim == 2: + self.mask_axis = "freq" + else: + self.mask_axis = "unknown" + + super().__init__() + self.mask_width_ratio_range = mask_width_ratio_range + self.num_mask = num_mask + self.dim = dim + self.replace_with_zero = replace_with_zero + + def extra_repr(self): + return ( + f"mask_width_ratio_range={self.mask_width_ratio_range}, " + f"num_mask={self.num_mask}, axis={self.mask_axis}" + ) + + def forward(self, spec: torch.Tensor, spec_lengths: torch.Tensor = None): + """Forward function. + + Args: + spec: (Batch, Length, Freq) + """ + + max_seq_len = spec.shape[self.dim] + min_mask_width = math.floor(max_seq_len * self.mask_width_ratio_range[0]) + min_mask_width = max([0, min_mask_width]) + max_mask_width = math.floor(max_seq_len * self.mask_width_ratio_range[1]) + max_mask_width = min([max_seq_len, max_mask_width]) + + if max_mask_width > min_mask_width: + return mask_along_axis( + spec, + spec_lengths, + mask_width_range=(min_mask_width, max_mask_width), + dim=self.dim, + num_mask=self.num_mask, + replace_with_zero=self.replace_with_zero, + ) + return spec, spec_lengths + +class MaskAlongAxisLFR(torch.nn.Module): + def __init__( + self, + mask_width_range: Union[int, Sequence[int]] = (0, 30), + num_mask: int = 2, + dim: Union[int, str] = "time", + replace_with_zero: bool = True, + lfr_rate: int = 1, + ): + assert check_argument_types() + if isinstance(mask_width_range, int): + mask_width_range = (0, mask_width_range) + if len(mask_width_range) != 2: + raise TypeError( + f"mask_width_range must be a tuple of int and int values: " + f"{mask_width_range}", + ) + + assert mask_width_range[1] > mask_width_range[0] + if isinstance(dim, str): + if dim == "time": + dim = 1 + lfr_rate = 1 + elif dim == "freq": + dim = 2 + else: + raise ValueError("dim must be int, 'time' or 'freq'") + if dim == 1: + self.mask_axis = "time" + lfr_rate = 1 + elif dim == 2: + self.mask_axis = "freq" + else: + self.mask_axis = "unknown" + + super().__init__() + self.mask_width_range = mask_width_range + self.num_mask = num_mask + self.dim = dim + self.replace_with_zero = replace_with_zero + self.lfr_rate = lfr_rate + + def extra_repr(self): + return ( + f"mask_width_range={self.mask_width_range}, " + f"num_mask={self.num_mask}, axis={self.mask_axis}" + ) + + def forward(self, spec: torch.Tensor, spec_lengths: torch.Tensor = None): + """Forward function. + + Args: + spec: (Batch, Length, Freq) + """ + + return mask_along_axis_lfr( + spec, + spec_lengths, + mask_width_range=self.mask_width_range, + dim=self.dim, + num_mask=self.num_mask, + replace_with_zero=self.replace_with_zero, + lfr_rate=self.lfr_rate, + ) \ No newline at end of file diff --git a/funasr/layers/sinc_conv.py b/funasr/layers/sinc_conv.py new file mode 100644 index 000000000..33df97fbc --- /dev/null +++ b/funasr/layers/sinc_conv.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +# 2020, Technische Universität München; Ludwig Kürzinger +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Sinc convolutions.""" +import math +import torch +from typeguard import check_argument_types +from typing import Union + + +class LogCompression(torch.nn.Module): + """Log Compression Activation. + + Activation function `log(abs(x) + 1)`. + """ + + def __init__(self): + """Initialize.""" + super().__init__() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward. + + Applies the Log Compression function elementwise on tensor x. + """ + return torch.log(torch.abs(x) + 1) + + +class SincConv(torch.nn.Module): + """Sinc Convolution. + + This module performs a convolution using Sinc filters in time domain as kernel. + Sinc filters function as band passes in spectral domain. + The filtering is done as a convolution in time domain, and no transformation + to spectral domain is necessary. + + This implementation of the Sinc convolution is heavily inspired + by Ravanelli et al. https://github.com/mravanelli/SincNet, + and adapted for the ESpnet toolkit. + Combine Sinc convolutions with a log compression activation function, as in: + https://arxiv.org/abs/2010.07597 + + Notes: + Currently, the same filters are applied to all input channels. + The windowing function is applied on the kernel to obtained a smoother filter, + and not on the input values, which is different to traditional ASR. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int, + stride: int = 1, + padding: int = 0, + dilation: int = 1, + window_func: str = "hamming", + scale_type: str = "mel", + fs: Union[int, float] = 16000, + ): + """Initialize Sinc convolutions. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + kernel_size: Sinc filter kernel size (needs to be an odd number). + stride: See torch.nn.functional.conv1d. + padding: See torch.nn.functional.conv1d. + dilation: See torch.nn.functional.conv1d. + window_func: Window function on the filter, one of ["hamming", "none"]. + fs (str, int, float): Sample rate of the input data + """ + assert check_argument_types() + super().__init__() + window_funcs = { + "none": self.none_window, + "hamming": self.hamming_window, + } + if window_func not in window_funcs: + raise NotImplementedError( + f"Window function has to be one of {list(window_funcs.keys())}", + ) + self.window_func = window_funcs[window_func] + scale_choices = { + "mel": MelScale, + "bark": BarkScale, + } + if scale_type not in scale_choices: + raise NotImplementedError( + f"Scale has to be one of {list(scale_choices.keys())}", + ) + self.scale = scale_choices[scale_type] + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.padding = padding + self.dilation = dilation + self.stride = stride + self.fs = float(fs) + if self.kernel_size % 2 == 0: + raise ValueError("SincConv: Kernel size must be odd.") + self.f = None + N = self.kernel_size // 2 + self._x = 2 * math.pi * torch.linspace(1, N, N) + self._window = self.window_func(torch.linspace(1, N, N)) + # init may get overwritten by E2E network, + # but is still required to calculate output dim + self.init_filters() + + @staticmethod + def sinc(x: torch.Tensor) -> torch.Tensor: + """Sinc function.""" + x2 = x + 1e-6 + return torch.sin(x2) / x2 + + @staticmethod + def none_window(x: torch.Tensor) -> torch.Tensor: + """Identity-like windowing function.""" + return torch.ones_like(x) + + @staticmethod + def hamming_window(x: torch.Tensor) -> torch.Tensor: + """Hamming Windowing function.""" + L = 2 * x.size(0) + 1 + x = x.flip(0) + return 0.54 - 0.46 * torch.cos(2.0 * math.pi * x / L) + + def init_filters(self): + """Initialize filters with filterbank values.""" + f = self.scale.bank(self.out_channels, self.fs) + f = torch.div(f, self.fs) + self.f = torch.nn.Parameter(f, requires_grad=True) + + def _create_filters(self, device: str): + """Calculate coefficients. + + This function (re-)calculates the filter convolutions coefficients. + """ + f_mins = torch.abs(self.f[:, 0]) + f_maxs = torch.abs(self.f[:, 0]) + torch.abs(self.f[:, 1] - self.f[:, 0]) + + self._x = self._x.to(device) + self._window = self._window.to(device) + + f_mins_x = torch.matmul(f_mins.view(-1, 1), self._x.view(1, -1)) + f_maxs_x = torch.matmul(f_maxs.view(-1, 1), self._x.view(1, -1)) + + kernel = (torch.sin(f_maxs_x) - torch.sin(f_mins_x)) / (0.5 * self._x) + kernel = kernel * self._window + + kernel_left = kernel.flip(1) + kernel_center = (2 * f_maxs - 2 * f_mins).unsqueeze(1) + filters = torch.cat([kernel_left, kernel_center, kernel], dim=1) + + filters = filters.view(filters.size(0), 1, filters.size(1)) + self.sinc_filters = filters + + def forward(self, xs: torch.Tensor) -> torch.Tensor: + """Sinc convolution forward function. + + Args: + xs: Batch in form of torch.Tensor (B, C_in, D_in). + + Returns: + xs: Batch in form of torch.Tensor (B, C_out, D_out). + """ + self._create_filters(xs.device) + xs = torch.nn.functional.conv1d( + xs, + self.sinc_filters, + padding=self.padding, + stride=self.stride, + dilation=self.dilation, + groups=self.in_channels, + ) + return xs + + def get_odim(self, idim: int) -> int: + """Obtain the output dimension of the filter.""" + D_out = idim + 2 * self.padding - self.dilation * (self.kernel_size - 1) - 1 + D_out = (D_out // self.stride) + 1 + return D_out + + +class MelScale: + """Mel frequency scale.""" + + @staticmethod + def convert(f): + """Convert Hz to mel.""" + return 1125.0 * torch.log(torch.div(f, 700.0) + 1.0) + + @staticmethod + def invert(x): + """Convert mel to Hz.""" + return 700.0 * (torch.exp(torch.div(x, 1125.0)) - 1.0) + + @classmethod + def bank(cls, channels: int, fs: float) -> torch.Tensor: + """Obtain initialization values for the mel scale. + + Args: + channels: Number of channels. + fs: Sample rate. + + Returns: + torch.Tensor: Filter start frequencíes. + torch.Tensor: Filter stop frequencies. + """ + assert check_argument_types() + # min and max bandpass edge frequencies + min_frequency = torch.tensor(30.0) + max_frequency = torch.tensor(fs * 0.5) + frequencies = torch.linspace( + cls.convert(min_frequency), cls.convert(max_frequency), channels + 2 + ) + frequencies = cls.invert(frequencies) + f1, f2 = frequencies[:-2], frequencies[2:] + return torch.stack([f1, f2], dim=1) + + +class BarkScale: + """Bark frequency scale. + + Has wider bandwidths at lower frequencies, see: + Critical bandwidth: BARK + Zwicker and Terhardt, 1980 + """ + + @staticmethod + def convert(f): + """Convert Hz to Bark.""" + b = torch.div(f, 1000.0) + b = torch.pow(b, 2.0) * 1.4 + b = torch.pow(b + 1.0, 0.69) + return b * 75.0 + 25.0 + + @staticmethod + def invert(x): + """Convert Bark to Hz.""" + f = torch.div(x - 25.0, 75.0) + f = torch.pow(f, (1.0 / 0.69)) + f = torch.div(f - 1.0, 1.4) + f = torch.pow(f, 0.5) + return f * 1000.0 + + @classmethod + def bank(cls, channels: int, fs: float) -> torch.Tensor: + """Obtain initialization values for the Bark scale. + + Args: + channels: Number of channels. + fs: Sample rate. + + Returns: + torch.Tensor: Filter start frequencíes. + torch.Tensor: Filter stop frequencíes. + """ + assert check_argument_types() + # min and max BARK center frequencies by approximation + min_center_frequency = torch.tensor(70.0) + max_center_frequency = torch.tensor(fs * 0.45) + center_frequencies = torch.linspace( + cls.convert(min_center_frequency), + cls.convert(max_center_frequency), + channels, + ) + center_frequencies = cls.invert(center_frequencies) + + f1 = center_frequencies - torch.div(cls.convert(center_frequencies), 2) + f2 = center_frequencies + torch.div(cls.convert(center_frequencies), 2) + return torch.stack([f1, f2], dim=1) diff --git a/funasr/layers/stft.py b/funasr/layers/stft.py new file mode 100644 index 000000000..21beaae6f --- /dev/null +++ b/funasr/layers/stft.py @@ -0,0 +1,229 @@ +from distutils.version import LooseVersion +from typing import Optional +from typing import Tuple +from typing import Union + +import torch +from torch_complex.tensor import ComplexTensor +from typeguard import check_argument_types + +from funasr.modules.nets_utils import make_pad_mask +from funasr.layers.complex_utils import is_complex +from funasr.layers.inversible_interface import InversibleInterface +import librosa +import numpy as np + +is_torch_1_9_plus = LooseVersion(torch.__version__) >= LooseVersion("1.9.0") + + +is_torch_1_7_plus = LooseVersion(torch.__version__) >= LooseVersion("1.7") + + +class Stft(torch.nn.Module, InversibleInterface): + def __init__( + self, + n_fft: int = 512, + win_length: int = None, + hop_length: int = 128, + window: Optional[str] = "hann", + center: bool = True, + normalized: bool = False, + onesided: bool = True, + ): + assert check_argument_types() + super().__init__() + self.n_fft = n_fft + if win_length is None: + self.win_length = n_fft + else: + self.win_length = win_length + self.hop_length = hop_length + self.center = center + self.normalized = normalized + self.onesided = onesided + if window is not None and not hasattr(torch, f"{window}_window"): + raise ValueError(f"{window} window is not implemented") + self.window = window + + def extra_repr(self): + return ( + f"n_fft={self.n_fft}, " + f"win_length={self.win_length}, " + f"hop_length={self.hop_length}, " + f"center={self.center}, " + f"normalized={self.normalized}, " + f"onesided={self.onesided}" + ) + + def forward( + self, input: torch.Tensor, ilens: torch.Tensor = None + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """STFT forward function. + + Args: + input: (Batch, Nsamples) or (Batch, Nsample, Channels) + ilens: (Batch) + Returns: + output: (Batch, Frames, Freq, 2) or (Batch, Frames, Channels, Freq, 2) + + """ + bs = input.size(0) + if input.dim() == 3: + multi_channel = True + # input: (Batch, Nsample, Channels) -> (Batch * Channels, Nsample) + input = input.transpose(1, 2).reshape(-1, input.size(1)) + else: + multi_channel = False + + # NOTE(kamo): + # The default behaviour of torch.stft is compatible with librosa.stft + # about padding and scaling. + # Note that it's different from scipy.signal.stft + + # output: (Batch, Freq, Frames, 2=real_imag) + # or (Batch, Channel, Freq, Frames, 2=real_imag) + if self.window is not None: + window_func = getattr(torch, f"{self.window}_window") + window = window_func( + self.win_length, dtype=input.dtype, device=input.device + ) + else: + window = None + + # For the compatibility of ARM devices, which do not support + # torch.stft() due to the lake of MKL. + if input.is_cuda or torch.backends.mkl.is_available(): + stft_kwargs = dict( + n_fft=self.n_fft, + win_length=self.win_length, + hop_length=self.hop_length, + center=self.center, + window=window, + normalized=self.normalized, + onesided=self.onesided, + ) + if is_torch_1_7_plus: + stft_kwargs["return_complex"] = False + output = torch.stft(input, **stft_kwargs) + else: + if self.training: + raise NotImplementedError( + "stft is implemented with librosa on this device, which does not " + "support the training mode." + ) + + # use stft_kwargs to flexibly control different PyTorch versions' kwargs + stft_kwargs = dict( + n_fft=self.n_fft, + win_length=self.win_length, + hop_length=self.hop_length, + center=self.center, + window=window, + ) + + if window is not None: + # pad the given window to n_fft + n_pad_left = (self.n_fft - window.shape[0]) // 2 + n_pad_right = self.n_fft - window.shape[0] - n_pad_left + stft_kwargs["window"] = torch.cat( + [torch.zeros(n_pad_left), window, torch.zeros(n_pad_right)], 0 + ).numpy() + else: + win_length = ( + self.win_length if self.win_length is not None else self.n_fft + ) + stft_kwargs["window"] = torch.ones(win_length) + + output = [] + # iterate over istances in a batch + for i, instance in enumerate(input): + stft = librosa.stft(input[i].numpy(), **stft_kwargs) + output.append(torch.tensor(np.stack([stft.real, stft.imag], -1))) + output = torch.stack(output, 0) + if not self.onesided: + len_conj = self.n_fft - output.shape[1] + conj = output[:, 1 : 1 + len_conj].flip(1) + conj[:, :, :, -1].data *= -1 + output = torch.cat([output, conj], 1) + if self.normalized: + output = output * (stft_kwargs["window"].shape[0] ** (-0.5)) + + # output: (Batch, Freq, Frames, 2=real_imag) + # -> (Batch, Frames, Freq, 2=real_imag) + output = output.transpose(1, 2) + if multi_channel: + # output: (Batch * Channel, Frames, Freq, 2=real_imag) + # -> (Batch, Frame, Channel, Freq, 2=real_imag) + output = output.view(bs, -1, output.size(1), output.size(2), 2).transpose( + 1, 2 + ) + + if ilens is not None: + if self.center: + pad = self.n_fft // 2 + ilens = ilens + 2 * pad + + olens = (ilens - self.n_fft) // self.hop_length + 1 + output.masked_fill_(make_pad_mask(olens, output, 1), 0.0) + else: + olens = None + + return output, olens + + def inverse( + self, input: Union[torch.Tensor, ComplexTensor], ilens: torch.Tensor = None + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + """Inverse STFT. + + Args: + input: Tensor(batch, T, F, 2) or ComplexTensor(batch, T, F) + ilens: (batch,) + Returns: + wavs: (batch, samples) + ilens: (batch,) + """ + if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): + istft = torch.functional.istft + else: + try: + import torchaudio + except ImportError: + raise ImportError( + "Please install torchaudio>=0.3.0 or use torch>=1.6.0" + ) + + if not hasattr(torchaudio.functional, "istft"): + raise ImportError( + "Please install torchaudio>=0.3.0 or use torch>=1.6.0" + ) + istft = torchaudio.functional.istft + + if self.window is not None: + window_func = getattr(torch, f"{self.window}_window") + if is_complex(input): + datatype = input.real.dtype + else: + datatype = input.dtype + window = window_func(self.win_length, dtype=datatype, device=input.device) + else: + window = None + + if is_complex(input): + input = torch.stack([input.real, input.imag], dim=-1) + elif input.shape[-1] != 2: + raise TypeError("Invalid input type") + input = input.transpose(1, 2) + + wavs = istft( + input, + n_fft=self.n_fft, + hop_length=self.hop_length, + win_length=self.win_length, + window=window, + center=self.center, + normalized=self.normalized, + onesided=self.onesided, + length=ilens.max() if ilens is not None else ilens, + ) + + return wavs, ilens diff --git a/funasr/layers/time_warp.py b/funasr/layers/time_warp.py new file mode 100644 index 000000000..b55461872 --- /dev/null +++ b/funasr/layers/time_warp.py @@ -0,0 +1,88 @@ +"""Time warp module.""" +import torch + +from funasr.modules.nets_utils import pad_list + +DEFAULT_TIME_WARP_MODE = "bicubic" + + +def time_warp(x: torch.Tensor, window: int = 80, mode: str = DEFAULT_TIME_WARP_MODE): + """Time warping using torch.interpolate. + + Args: + x: (Batch, Time, Freq) + window: time warp parameter + mode: Interpolate mode + """ + + # bicubic supports 4D or more dimension tensor + org_size = x.size() + if x.dim() == 3: + # x: (Batch, Time, Freq) -> (Batch, 1, Time, Freq) + x = x[:, None] + + t = x.shape[2] + if t - window <= window: + return x.view(*org_size) + + center = torch.randint(window, t - window, (1,))[0] + warped = torch.randint(center - window, center + window, (1,))[0] + 1 + + # left: (Batch, Channel, warped, Freq) + # right: (Batch, Channel, time - warped, Freq) + left = torch.nn.functional.interpolate( + x[:, :, :center], (warped, x.shape[3]), mode=mode, align_corners=False + ) + right = torch.nn.functional.interpolate( + x[:, :, center:], (t - warped, x.shape[3]), mode=mode, align_corners=False + ) + + if x.requires_grad: + x = torch.cat([left, right], dim=-2) + else: + x[:, :, :warped] = left + x[:, :, warped:] = right + + return x.view(*org_size) + + +class TimeWarp(torch.nn.Module): + """Time warping using torch.interpolate. + + Args: + window: time warp parameter + mode: Interpolate mode + """ + + def __init__(self, window: int = 80, mode: str = DEFAULT_TIME_WARP_MODE): + super().__init__() + self.window = window + self.mode = mode + + def extra_repr(self): + return f"window={self.window}, mode={self.mode}" + + def forward(self, x: torch.Tensor, x_lengths: torch.Tensor = None): + """Forward function. + + Args: + x: (Batch, Time, Freq) + x_lengths: (Batch,) + """ + + if x_lengths is None or all(le == x_lengths[0] for le in x_lengths): + # Note that applying same warping for each sample + y = time_warp(x, window=self.window, mode=self.mode) + else: + # FIXME(kamo): I have no idea to batchify Timewarp + ys = [] + for i in range(x.size(0)): + _y = time_warp( + x[i][None, : x_lengths[i]], + window=self.window, + mode=self.mode, + )[0] + ys.append(_y) + y = pad_list(ys, 0.0) + + return y, x_lengths diff --git a/funasr/layers/utterance_mvn.py b/funasr/layers/utterance_mvn.py new file mode 100644 index 000000000..50f27cd55 --- /dev/null +++ b/funasr/layers/utterance_mvn.py @@ -0,0 +1,88 @@ +from typing import Tuple + +import torch +from typeguard import check_argument_types + +from funasr.modules.nets_utils import make_pad_mask +from funasr.layers.abs_normalize import AbsNormalize + + +class UtteranceMVN(AbsNormalize): + def __init__( + self, + norm_means: bool = True, + norm_vars: bool = False, + eps: float = 1.0e-20, + ): + assert check_argument_types() + super().__init__() + self.norm_means = norm_means + self.norm_vars = norm_vars + self.eps = eps + + def extra_repr(self): + return f"norm_means={self.norm_means}, norm_vars={self.norm_vars}" + + def forward( + self, x: torch.Tensor, ilens: torch.Tensor = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward function + + Args: + x: (B, L, ...) + ilens: (B,) + + """ + return utterance_mvn( + x, + ilens, + norm_means=self.norm_means, + norm_vars=self.norm_vars, + eps=self.eps, + ) + + +def utterance_mvn( + x: torch.Tensor, + ilens: torch.Tensor = None, + norm_means: bool = True, + norm_vars: bool = False, + eps: float = 1.0e-20, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Apply utterance mean and variance normalization + + Args: + x: (B, T, D), assumed zero padded + ilens: (B,) + norm_means: + norm_vars: + eps: + + """ + if ilens is None: + ilens = x.new_full([x.size(0)], x.size(1)) + ilens_ = ilens.to(x.device, x.dtype).view(-1, *[1 for _ in range(x.dim() - 1)]) + # Zero padding + if x.requires_grad: + x = x.masked_fill(make_pad_mask(ilens, x, 1), 0.0) + else: + x.masked_fill_(make_pad_mask(ilens, x, 1), 0.0) + # mean: (B, 1, D) + mean = x.sum(dim=1, keepdim=True) / ilens_ + + if norm_means: + x -= mean + + if norm_vars: + var = x.pow(2).sum(dim=1, keepdim=True) / ilens_ + std = torch.clamp(var.sqrt(), min=eps) + x = x / std.sqrt() + return x, ilens + else: + if norm_vars: + y = x - mean + y.masked_fill_(make_pad_mask(ilens, y, 1), 0.0) + var = y.pow(2).sum(dim=1, keepdim=True) / ilens_ + std = torch.clamp(var.sqrt(), min=eps) + x /= std + return x, ilens diff --git a/funasr/lm/__init__.py b/funasr/lm/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/lm/abs_model.py b/funasr/lm/abs_model.py new file mode 100644 index 000000000..0ad1e71bc --- /dev/null +++ b/funasr/lm/abs_model.py @@ -0,0 +1,29 @@ +from abc import ABC +from abc import abstractmethod +from typing import Tuple + +import torch + +from funasr.modules.scorers.scorer_interface import BatchScorerInterface + + +class AbsLM(torch.nn.Module, BatchScorerInterface, ABC): + """The abstract LM class + + To share the loss calculation way among different models, + We uses delegate pattern here: + The instance of this class should be passed to "LanguageModel" + + >>> from funasr.lm.abs_model import AbsLM + >>> lm = AbsLM() + >>> model = LanguageESPnetModel(lm=lm) + + This "model" is one of mediator objects for "Task" class. + + """ + + @abstractmethod + def forward( + self, input: torch.Tensor, hidden: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError diff --git a/funasr/lm/espnet_model.py b/funasr/lm/espnet_model.py new file mode 100644 index 000000000..4fc3b49c8 --- /dev/null +++ b/funasr/lm/espnet_model.py @@ -0,0 +1,131 @@ +from typing import Dict +from typing import Optional +from typing import Tuple + +import torch +import torch.nn.functional as F +from typeguard import check_argument_types + +from funasr.modules.nets_utils import make_pad_mask +from funasr.lm.abs_model import AbsLM +from funasr.torch_utils.device_funcs import force_gatherable +from funasr.train.abs_espnet_model import AbsESPnetModel + + +class ESPnetLanguageModel(AbsESPnetModel): + def __init__(self, lm: AbsLM, vocab_size: int, ignore_id: int = 0): + assert check_argument_types() + super().__init__() + self.lm = lm + self.sos = 1 + self.eos = 2 + + # ignore_id may be assumed as 0, shared with CTC-blank symbol for ASR. + self.ignore_id = ignore_id + + def nll( + self, + text: torch.Tensor, + text_lengths: torch.Tensor, + max_length: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute negative log likelihood(nll) + + Normally, this function is called in batchify_nll. + Args: + text: (Batch, Length) + text_lengths: (Batch,) + max_lengths: int + """ + batch_size = text.size(0) + # For data parallel + if max_length is None: + text = text[:, : text_lengths.max()] + else: + text = text[:, :max_length] + + # 1. Create a sentence pair like ' w1 w2 w3' and 'w1 w2 w3 ' + # text: (Batch, Length) -> x, y: (Batch, Length + 1) + x = F.pad(text, [1, 0], "constant", self.eos) + t = F.pad(text, [0, 1], "constant", self.ignore_id) + for i, l in enumerate(text_lengths): + t[i, l] = self.sos + x_lengths = text_lengths + 1 + + # 2. Forward Language model + # x: (Batch, Length) -> y: (Batch, Length, NVocab) + y, _ = self.lm(x, None) + + # 3. Calc negative log likelihood + # nll: (BxL,) + nll = F.cross_entropy(y.view(-1, y.shape[-1]), t.view(-1), reduction="none") + # nll: (BxL,) -> (BxL,) + if max_length is None: + nll.masked_fill_(make_pad_mask(x_lengths).to(nll.device).view(-1), 0.0) + else: + nll.masked_fill_( + make_pad_mask(x_lengths, maxlen=max_length + 1).to(nll.device).view(-1), + 0.0, + ) + # nll: (BxL,) -> (B, L) + nll = nll.view(batch_size, -1) + return nll, x_lengths + + def batchify_nll( + self, text: torch.Tensor, text_lengths: torch.Tensor, batch_size: int = 100 + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute negative log likelihood(nll) from transformer language model + + To avoid OOM, this fuction seperate the input into batches. + Then call nll for each batch and combine and return results. + Args: + text: (Batch, Length) + text_lengths: (Batch,) + batch_size: int, samples each batch contain when computing nll, + you may change this to avoid OOM or increase + + """ + total_num = text.size(0) + if total_num <= batch_size: + nll, x_lengths = self.nll(text, text_lengths) + else: + nlls = [] + x_lengths = [] + max_length = text_lengths.max() + + start_idx = 0 + while True: + end_idx = min(start_idx + batch_size, total_num) + batch_text = text[start_idx:end_idx, :] + batch_text_lengths = text_lengths[start_idx:end_idx] + # batch_nll: [B * T] + batch_nll, batch_x_lengths = self.nll( + batch_text, batch_text_lengths, max_length=max_length + ) + nlls.append(batch_nll) + x_lengths.append(batch_x_lengths) + start_idx = end_idx + if start_idx == total_num: + break + nll = torch.cat(nlls) + x_lengths = torch.cat(x_lengths) + assert nll.size(0) == total_num + assert x_lengths.size(0) == total_num + return nll, x_lengths + + def forward( + self, text: torch.Tensor, text_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + nll, y_lengths = self.nll(text, text_lengths) + ntokens = y_lengths.sum() + loss = nll.sum() / ntokens + stats = dict(loss=loss.detach()) + + # force_gatherable: to-device and to-tensor if scalar for DataParallel + loss, stats, weight = force_gatherable((loss, stats, ntokens), loss.device) + return loss, stats, weight + + def collect_feats( + self, text: torch.Tensor, text_lengths: torch.Tensor + ) -> Dict[str, torch.Tensor]: + return {} diff --git a/funasr/lm/seq_rnn_lm.py b/funasr/lm/seq_rnn_lm.py new file mode 100644 index 000000000..09d1e4ae5 --- /dev/null +++ b/funasr/lm/seq_rnn_lm.py @@ -0,0 +1,174 @@ +"""Sequential implementation of Recurrent Neural Network Language Model.""" +from typing import Tuple +from typing import Union + +import torch +import torch.nn as nn +from typeguard import check_argument_types + +from funasr.lm.abs_model import AbsLM + + +class SequentialRNNLM(AbsLM): + """Sequential RNNLM. + + See also: + https://github.com/pytorch/examples/blob/4581968193699de14b56527296262dd76ab43557/word_language_model/model.py + + """ + + def __init__( + self, + vocab_size: int, + unit: int = 650, + nhid: int = None, + nlayers: int = 2, + dropout_rate: float = 0.0, + tie_weights: bool = False, + rnn_type: str = "lstm", + ignore_id: int = 0, + ): + assert check_argument_types() + super().__init__() + + ninp = unit + if nhid is None: + nhid = unit + rnn_type = rnn_type.upper() + + self.drop = nn.Dropout(dropout_rate) + self.encoder = nn.Embedding(vocab_size, ninp, padding_idx=ignore_id) + if rnn_type in ["LSTM", "GRU"]: + rnn_class = getattr(nn, rnn_type) + self.rnn = rnn_class( + ninp, nhid, nlayers, dropout=dropout_rate, batch_first=True + ) + else: + try: + nonlinearity = {"RNN_TANH": "tanh", "RNN_RELU": "relu"}[rnn_type] + except KeyError: + raise ValueError( + """An invalid option for `--model` was supplied, + options are ['LSTM', 'GRU', 'RNN_TANH' or 'RNN_RELU']""" + ) + self.rnn = nn.RNN( + ninp, + nhid, + nlayers, + nonlinearity=nonlinearity, + dropout=dropout_rate, + batch_first=True, + ) + self.decoder = nn.Linear(nhid, vocab_size) + + # Optionally tie weights as in: + # "Using the Output Embedding to Improve Language Models" + # (Press & Wolf 2016) https://arxiv.org/abs/1608.05859 + # and + # "Tying Word Vectors and Word Classifiers: + # A Loss Framework for Language Modeling" (Inan et al. 2016) + # https://arxiv.org/abs/1611.01462 + if tie_weights: + if nhid != ninp: + raise ValueError( + "When using the tied flag, nhid must be equal to emsize" + ) + self.decoder.weight = self.encoder.weight + + self.rnn_type = rnn_type + self.nhid = nhid + self.nlayers = nlayers + + def zero_state(self): + """Initialize LM state filled with zero values.""" + if isinstance(self.rnn, torch.nn.LSTM): + h = torch.zeros((self.nlayers, self.nhid), dtype=torch.float) + c = torch.zeros((self.nlayers, self.nhid), dtype=torch.float) + state = h, c + else: + state = torch.zeros((self.nlayers, self.nhid), dtype=torch.float) + + return state + + def forward( + self, input: torch.Tensor, hidden: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + emb = self.drop(self.encoder(input)) + output, hidden = self.rnn(emb, hidden) + output = self.drop(output) + decoded = self.decoder( + output.contiguous().view(output.size(0) * output.size(1), output.size(2)) + ) + return ( + decoded.view(output.size(0), output.size(1), decoded.size(1)), + hidden, + ) + + def score( + self, + y: torch.Tensor, + state: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]], + x: torch.Tensor, + ) -> Tuple[torch.Tensor, Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]]: + """Score new token. + + Args: + y: 1D torch.int64 prefix tokens. + state: Scorer state for prefix tokens + x: 2D encoder feature that generates ys. + + Returns: + Tuple of + torch.float32 scores for next token (n_vocab) + and next state for ys + + """ + y, new_state = self(y[-1].view(1, 1), state) + logp = y.log_softmax(dim=-1).view(-1) + return logp, new_state + + def batch_score( + self, ys: torch.Tensor, states: torch.Tensor, xs: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Score new token batch. + + Args: + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). + + Returns: + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, n_vocab)` + and next state list for ys. + + """ + if states[0] is None: + states = None + elif isinstance(self.rnn, torch.nn.LSTM): + # states: Batch x 2 x (Nlayers, Dim) -> 2 x (Nlayers, Batch, Dim) + h = torch.stack([h for h, c in states], dim=1) + c = torch.stack([c for h, c in states], dim=1) + states = h, c + else: + # states: Batch x (Nlayers, Dim) -> (Nlayers, Batch, Dim) + states = torch.stack(states, dim=1) + + ys, states = self(ys[:, -1:], states) + # ys: (Batch, 1, Nvocab) -> (Batch, NVocab) + assert ys.size(1) == 1, ys.shape + ys = ys.squeeze(1) + logp = ys.log_softmax(dim=-1) + + # state: Change to batch first + if isinstance(self.rnn, torch.nn.LSTM): + # h, c: (Nlayers, Batch, Dim) + h, c = states + # states: Batch x 2 x (Nlayers, Dim) + states = [(h[:, i], c[:, i]) for i in range(h.size(1))] + else: + # states: (Nlayers, Batch, Dim) -> Batch x (Nlayers, Dim) + states = [states[:, i] for i in range(states.size(1))] + + return logp, states diff --git a/funasr/lm/transformer_lm.py b/funasr/lm/transformer_lm.py new file mode 100644 index 000000000..52af45bdc --- /dev/null +++ b/funasr/lm/transformer_lm.py @@ -0,0 +1,131 @@ +from typing import Any +from typing import List +from typing import Tuple + +import torch +import torch.nn as nn + +from funasr.modules.embedding import PositionalEncoding +from funasr.models.encoder.transformer_encoder import TransformerEncoder_s0 as Encoder +from funasr.modules.mask import subsequent_mask +from funasr.lm.abs_model import AbsLM + + +class TransformerLM(AbsLM): + def __init__( + self, + vocab_size: int, + pos_enc: str = None, + embed_unit: int = 128, + att_unit: int = 256, + head: int = 2, + unit: int = 1024, + layer: int = 4, + dropout_rate: float = 0.5, + ): + super().__init__() + if pos_enc == "sinusoidal": + pos_enc_class = PositionalEncoding + elif pos_enc is None: + + def pos_enc_class(*args, **kwargs): + return nn.Sequential() # indentity + + else: + raise ValueError(f"unknown pos-enc option: {pos_enc}") + + self.embed = nn.Embedding(vocab_size, embed_unit) + self.encoder = Encoder( + idim=embed_unit, + attention_dim=att_unit, + attention_heads=head, + linear_units=unit, + num_blocks=layer, + dropout_rate=dropout_rate, + input_layer="linear", + pos_enc_class=pos_enc_class, + ) + self.decoder = nn.Linear(att_unit, vocab_size) + + def _target_mask(self, ys_in_pad): + ys_mask = ys_in_pad != 0 + m = subsequent_mask(ys_mask.size(-1), device=ys_mask.device).unsqueeze(0) + return ys_mask.unsqueeze(-2) & m + + def forward(self, input: torch.Tensor, hidden: None) -> Tuple[torch.Tensor, None]: + """Compute LM loss value from buffer sequences. + + Args: + input (torch.Tensor): Input ids. (batch, len) + hidden (torch.Tensor): Target ids. (batch, len) + + """ + x = self.embed(input) + mask = self._target_mask(input) + h, _ = self.encoder(x, mask) + y = self.decoder(h) + return y, None + + def score( + self, y: torch.Tensor, state: Any, x: torch.Tensor + ) -> Tuple[torch.Tensor, Any]: + """Score new token. + + Args: + y (torch.Tensor): 1D torch.int64 prefix tokens. + state: Scorer state for prefix tokens + x (torch.Tensor): encoder feature that generates ys. + + Returns: + tuple[torch.Tensor, Any]: Tuple of + torch.float32 scores for next token (vocab_size) + and next state for ys + + """ + y = y.unsqueeze(0) + h, _, cache = self.encoder.forward_one_step( + self.embed(y), self._target_mask(y), cache=state + ) + h = self.decoder(h[:, -1]) + logp = h.log_softmax(dim=-1).squeeze(0) + return logp, cache + + def batch_score( + self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor + ) -> Tuple[torch.Tensor, List[Any]]: + """Score new token batch. + + Args: + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). + + Returns: + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, vocab_size)` + and next state list for ys. + + """ + # merge states + n_batch = len(ys) + n_layers = len(self.encoder.encoders) + if states[0] is None: + batch_state = None + else: + # transpose state of [batch, layer] into [layer, batch] + batch_state = [ + torch.stack([states[b][i] for b in range(n_batch)]) + for i in range(n_layers) + ] + + # batch decoding + h, _, states = self.encoder.forward_one_step( + self.embed(ys), self._target_mask(ys), cache=batch_state + ) + h = self.decoder(h[:, -1]) + logp = h.log_softmax(dim=-1) + + # transpose state of [layer, batch] into [batch, layer] + state_list = [[states[i][b] for i in range(n_layers)] for b in range(n_batch)] + return logp, state_list diff --git a/funasr/losses/label_smoothing_loss.py b/funasr/losses/label_smoothing_loss.py new file mode 100644 index 000000000..0d8b30338 --- /dev/null +++ b/funasr/losses/label_smoothing_loss.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Label smoothing module.""" + +import torch +from torch import nn + + +class LabelSmoothingLoss(nn.Module): + """Label-smoothing loss. + + :param int size: the number of class + :param int padding_idx: ignored class id + :param float smoothing: smoothing rate (0.0 means the conventional CE) + :param bool normalize_length: normalize loss by sequence length if True + :param torch.nn.Module criterion: loss function to be smoothed + """ + + def __init__( + self, + size, + padding_idx, + smoothing, + normalize_length=False, + criterion=nn.KLDivLoss(reduction="none"), + ): + """Construct an LabelSmoothingLoss object.""" + super(LabelSmoothingLoss, self).__init__() + self.criterion = criterion + self.padding_idx = padding_idx + self.confidence = 1.0 - smoothing + self.smoothing = smoothing + self.size = size + self.true_dist = None + self.normalize_length = normalize_length + + def forward(self, x, target): + """Compute loss between x and target. + + :param torch.Tensor x: prediction (batch, seqlen, class) + :param torch.Tensor target: + target signal masked with self.padding_id (batch, seqlen) + :return: scalar float value + :rtype torch.Tensor + """ + assert x.size(2) == self.size + batch_size = x.size(0) + x = x.view(-1, self.size) + target = target.view(-1) + with torch.no_grad(): + true_dist = x.clone() + true_dist.fill_(self.smoothing / (self.size - 1)) + ignore = target == self.padding_idx # (B,) + total = len(target) - ignore.sum().item() + target = target.masked_fill(ignore, 0) # avoid -1 index + true_dist.scatter_(1, target.unsqueeze(1), self.confidence) + kl = self.criterion(torch.log_softmax(x, dim=1), true_dist) + denom = total if self.normalize_length else batch_size + return kl.masked_fill(ignore.unsqueeze(1), 0).sum() / denom diff --git a/funasr/main_funcs/__init__.py b/funasr/main_funcs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/main_funcs/average_nbest_models.py b/funasr/main_funcs/average_nbest_models.py new file mode 100644 index 000000000..53f956800 --- /dev/null +++ b/funasr/main_funcs/average_nbest_models.py @@ -0,0 +1,127 @@ +import logging +from pathlib import Path +from typing import Optional +from typing import Sequence +from typing import Union +import warnings +import os +from io import BytesIO + +import torch +from typeguard import check_argument_types +from typing import Collection + +from funasr.train.reporter import Reporter + + +@torch.no_grad() +def average_nbest_models( + output_dir: Path, + reporter: Reporter, + best_model_criterion: Sequence[Sequence[str]], + nbest: Union[Collection[int], int], + suffix: Optional[str] = None, + oss_bucket=None, + pai_output_dir=None, +) -> None: + """Generate averaged model from n-best models + + Args: + output_dir: The directory contains the model file for each epoch + reporter: Reporter instance + best_model_criterion: Give criterions to decide the best model. + e.g. [("valid", "loss", "min"), ("train", "acc", "max")] + nbest: Number of best model files to be averaged + suffix: A suffix added to the averaged model file name + """ + assert check_argument_types() + if isinstance(nbest, int): + nbests = [nbest] + else: + nbests = list(nbest) + if len(nbests) == 0: + warnings.warn("At least 1 nbest values are required") + nbests = [1] + if suffix is not None: + suffix = suffix + "." + else: + suffix = "" + + # 1. Get nbests: List[Tuple[str, str, List[Tuple[epoch, value]]]] + nbest_epochs = [ + (ph, k, reporter.sort_epochs_and_values(ph, k, m)[: max(nbests)]) + for ph, k, m in best_model_criterion + if reporter.has(ph, k) + ] + + _loaded = {} + for ph, cr, epoch_and_values in nbest_epochs: + _nbests = [i for i in nbests if i <= len(epoch_and_values)] + if len(_nbests) == 0: + _nbests = [1] + + for n in _nbests: + if n == 0: + continue + elif n == 1: + # The averaged model is same as the best model + e, _ = epoch_and_values[0] + op = output_dir / f"{e}epoch.pth" + sym_op = output_dir / f"{ph}.{cr}.ave_1best.{suffix}pth" + if sym_op.is_symlink() or sym_op.exists(): + sym_op.unlink() + sym_op.symlink_to(op.name) + else: + op = output_dir / f"{ph}.{cr}.ave_{n}best.{suffix}pth" + logging.info( + f"Averaging {n}best models: " f'criterion="{ph}.{cr}": {op}' + ) + + avg = None + # 2.a. Averaging model + for e, _ in epoch_and_values[:n]: + if e not in _loaded: + if oss_bucket is None: + _loaded[e] = torch.load( + output_dir / f"{e}epoch.pth", + map_location="cpu", + ) + else: + buffer = BytesIO( + oss_bucket.get_object(os.path.join(pai_output_dir, f"{e}epoch.pth")).read()) + _loaded[e] = torch.load(buffer) + states = _loaded[e] + + if avg is None: + avg = states + else: + # Accumulated + for k in avg: + avg[k] = avg[k] + states[k] + for k in avg: + if str(avg[k].dtype).startswith("torch.int"): + # For int type, not averaged, but only accumulated. + # e.g. BatchNorm.num_batches_tracked + # (If there are any cases that requires averaging + # or the other reducing method, e.g. max/min, for integer type, + # please report.) + pass + else: + avg[k] = avg[k] / n + + # 2.b. Save the ave model and create a symlink + if oss_bucket is None: + torch.save(avg, op) + else: + buffer = BytesIO() + torch.save(avg, buffer) + oss_bucket.put_object(os.path.join(pai_output_dir, f"{ph}.{cr}.ave_{n}best.{suffix}pth"), + buffer.getvalue()) + + # 3. *.*.ave.pth is a symlink to the max ave model + if oss_bucket is None: + op = output_dir / f"{ph}.{cr}.ave_{max(_nbests)}best.{suffix}pth" + sym_op = output_dir / f"{ph}.{cr}.ave.{suffix}pth" + if sym_op.is_symlink() or sym_op.exists(): + sym_op.unlink() + sym_op.symlink_to(op.name) diff --git a/funasr/main_funcs/calculate_all_attentions.py b/funasr/main_funcs/calculate_all_attentions.py new file mode 100644 index 000000000..8f238c6bf --- /dev/null +++ b/funasr/main_funcs/calculate_all_attentions.py @@ -0,0 +1,160 @@ +from collections import defaultdict +from typing import Dict +from typing import List + +import torch + +from funasr.modules.rnn.attentions import AttAdd +from funasr.modules.rnn.attentions import AttCov +from funasr.modules.rnn.attentions import AttCovLoc +from funasr.modules.rnn.attentions import AttDot +from funasr.modules.rnn.attentions import AttForward +from funasr.modules.rnn.attentions import AttForwardTA +from funasr.modules.rnn.attentions import AttLoc +from funasr.modules.rnn.attentions import AttLoc2D +from funasr.modules.rnn.attentions import AttLocRec +from funasr.modules.rnn.attentions import AttMultiHeadAdd +from funasr.modules.rnn.attentions import AttMultiHeadDot +from funasr.modules.rnn.attentions import AttMultiHeadLoc +from funasr.modules.rnn.attentions import AttMultiHeadMultiResLoc +from funasr.modules.rnn.attentions import NoAtt +from funasr.modules.attention import MultiHeadedAttention + + +from funasr.train.abs_espnet_model import AbsESPnetModel + + +@torch.no_grad() +def calculate_all_attentions( + model: AbsESPnetModel, batch: Dict[str, torch.Tensor] +) -> Dict[str, List[torch.Tensor]]: + """Derive the outputs from the all attention layers + + Args: + model: + batch: same as forward + Returns: + return_dict: A dict of a list of tensor. + key_names x batch x (D1, D2, ...) + + """ + bs = len(next(iter(batch.values()))) + assert all(len(v) == bs for v in batch.values()), { + k: v.shape for k, v in batch.items() + } + + # 1. Register forward_hook fn to save the output from specific layers + outputs = {} + handles = {} + for name, modu in model.named_modules(): + + def hook(module, input, output, name=name): + if isinstance(module, MultiHeadedAttention): + # NOTE(kamo): MultiHeadedAttention doesn't return attention weight + # attn: (B, Head, Tout, Tin) + outputs[name] = module.attn.detach().cpu() + elif isinstance(module, AttLoc2D): + c, w = output + # w: previous concate attentions + # w: (B, nprev, Tin) + att_w = w[:, -1].detach().cpu() + outputs.setdefault(name, []).append(att_w) + elif isinstance(module, (AttCov, AttCovLoc)): + c, w = output + assert isinstance(w, list), type(w) + # w: list of previous attentions + # w: nprev x (B, Tin) + att_w = w[-1].detach().cpu() + outputs.setdefault(name, []).append(att_w) + elif isinstance(module, AttLocRec): + # w: (B, Tin) + c, (w, (att_h, att_c)) = output + att_w = w.detach().cpu() + outputs.setdefault(name, []).append(att_w) + elif isinstance( + module, + ( + AttMultiHeadDot, + AttMultiHeadAdd, + AttMultiHeadLoc, + AttMultiHeadMultiResLoc, + ), + ): + c, w = output + # w: nhead x (B, Tin) + assert isinstance(w, list), type(w) + att_w = [_w.detach().cpu() for _w in w] + outputs.setdefault(name, []).append(att_w) + elif isinstance( + module, + ( + AttAdd, + AttDot, + AttForward, + AttForwardTA, + AttLoc, + NoAtt, + ), + ): + c, w = output + att_w = w.detach().cpu() + outputs.setdefault(name, []).append(att_w) + + handle = modu.register_forward_hook(hook) + handles[name] = handle + + # 2. Just forward one by one sample. + # Batch-mode can't be used to keep requirements small for each models. + keys = [] + for k in batch: + if not k.endswith("_lengths"): + keys.append(k) + + return_dict = defaultdict(list) + for ibatch in range(bs): + # *: (B, L, ...) -> (1, L2, ...) + _sample = { + k: batch[k][ibatch, None, : batch[k + "_lengths"][ibatch]] + if k + "_lengths" in batch + else batch[k][ibatch, None] + for k in keys + } + + # *_lengths: (B,) -> (1,) + _sample.update( + { + k + "_lengths": batch[k + "_lengths"][ibatch, None] + for k in keys + if k + "_lengths" in batch + } + ) + model(**_sample) + + # Derive the attention results + for name, output in outputs.items(): + if isinstance(output, list): + if isinstance(output[0], list): + # output: nhead x (Tout, Tin) + output = torch.stack( + [ + # Tout x (1, Tin) -> (Tout, Tin) + torch.cat([o[idx] for o in output], dim=0) + for idx in range(len(output[0])) + ], + dim=0, + ) + else: + # Tout x (1, Tin) -> (Tout, Tin) + output = torch.cat(output, dim=0) + else: + # output: (1, NHead, Tout, Tin) -> (NHead, Tout, Tin) + output = output.squeeze(0) + # output: (Tout, Tin) or (NHead, Tout, Tin) + return_dict[name].append(output) + outputs.clear() + + # 3. Remove all hooks + for _, handle in handles.items(): + handle.remove() + + return dict(return_dict) diff --git a/funasr/main_funcs/collect_stats.py b/funasr/main_funcs/collect_stats.py new file mode 100644 index 000000000..bacda8f2f --- /dev/null +++ b/funasr/main_funcs/collect_stats.py @@ -0,0 +1,126 @@ +from collections import defaultdict +import logging +from pathlib import Path +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Tuple + +import numpy as np +import torch +from torch.nn.parallel import data_parallel +from torch.utils.data import DataLoader +from typeguard import check_argument_types + +from funasr.fileio.datadir_writer import DatadirWriter +from funasr.fileio.npy_scp import NpyScpWriter +from funasr.torch_utils.device_funcs import to_device +from funasr.torch_utils.forward_adaptor import ForwardAdaptor +from funasr.train.abs_espnet_model import AbsESPnetModel + + +@torch.no_grad() +def collect_stats( + model: AbsESPnetModel, + train_iter: DataLoader and Iterable[Tuple[List[str], Dict[str, torch.Tensor]]], + valid_iter: DataLoader and Iterable[Tuple[List[str], Dict[str, torch.Tensor]]], + output_dir: Path, + ngpu: Optional[int], + log_interval: Optional[int], + write_collected_feats: bool, +) -> None: + """Perform on collect_stats mode. + + Running for deriving the shape information from data + and gathering statistics. + This method is used before executing train(). + + """ + assert check_argument_types() + + npy_scp_writers = {} + for itr, mode in zip([train_iter, valid_iter], ["train", "valid"]): + if log_interval is None: + try: + log_interval = max(len(itr) // 20, 10) + except TypeError: + log_interval = 100 + + sum_dict = defaultdict(lambda: 0) + sq_dict = defaultdict(lambda: 0) + count_dict = defaultdict(lambda: 0) + + with DatadirWriter(output_dir / mode) as datadir_writer: + for iiter, (keys, batch) in enumerate(itr, 1): + batch = to_device(batch, "cuda" if ngpu > 0 else "cpu") + + # 1. Write shape file + for name in batch: + if name.endswith("_lengths"): + continue + for i, (key, data) in enumerate(zip(keys, batch[name])): + if f"{name}_lengths" in batch: + lg = int(batch[f"{name}_lengths"][i]) + data = data[:lg] + datadir_writer[f"{name}_shape"][key] = ",".join( + map(str, data.shape) + ) + + # 2. Extract feats + if ngpu <= 1: + data = model.collect_feats(**batch) + else: + # Note that data_parallel can parallelize only "forward()" + data = data_parallel( + ForwardAdaptor(model, "collect_feats"), + (), + range(ngpu), + module_kwargs=batch, + ) + + # 3. Calculate sum and square sum + for key, v in data.items(): + for i, (uttid, seq) in enumerate(zip(keys, v.cpu().numpy())): + # Truncate zero-padding region + if f"{key}_lengths" in data: + length = data[f"{key}_lengths"][i] + # seq: (Length, Dim, ...) + seq = seq[:length] + else: + # seq: (Dim, ...) -> (1, Dim, ...) + seq = seq[None] + # Accumulate value, its square, and count + sum_dict[key] += seq.sum(0) + sq_dict[key] += (seq**2).sum(0) + count_dict[key] += len(seq) + + # 4. [Option] Write derived features as npy format file. + if write_collected_feats: + # Instantiate NpyScpWriter for the first iteration + if (key, mode) not in npy_scp_writers: + p = output_dir / mode / "collect_feats" + npy_scp_writers[(key, mode)] = NpyScpWriter( + p / f"data_{key}", p / f"{key}.scp" + ) + # Save array as npy file + npy_scp_writers[(key, mode)][uttid] = seq + + if iiter % log_interval == 0: + logging.info(f"Niter: {iiter}") + + for key in sum_dict: + np.savez( + output_dir / mode / f"{key}_stats.npz", + count=count_dict[key], + sum=sum_dict[key], + sum_square=sq_dict[key], + ) + + # batch_keys and stats_keys are used by aggregate_stats_dirs.py + with (output_dir / mode / "batch_keys").open("w", encoding="utf-8") as f: + f.write( + "\n".join(filter(lambda x: not x.endswith("_lengths"), batch)) + "\n" + ) + with (output_dir / mode / "stats_keys").open("w", encoding="utf-8") as f: + f.write("\n".join(sum_dict) + "\n") diff --git a/funasr/main_funcs/pack_funcs.py b/funasr/main_funcs/pack_funcs.py new file mode 100644 index 000000000..ffa807e23 --- /dev/null +++ b/funasr/main_funcs/pack_funcs.py @@ -0,0 +1,302 @@ +from datetime import datetime +from io import BytesIO +from io import TextIOWrapper +import os +from pathlib import Path +import sys +import tarfile +from typing import Dict +from typing import Iterable +from typing import Optional +from typing import Union +import zipfile + +import yaml + + +class Archiver: + def __init__(self, file, mode="r"): + if Path(file).suffix == ".tar": + self.type = "tar" + elif Path(file).suffix == ".tgz" or Path(file).suffixes == [".tar", ".gz"]: + self.type = "tar" + if mode == "w": + mode = "w:gz" + elif Path(file).suffix == ".tbz2" or Path(file).suffixes == [".tar", ".bz2"]: + self.type = "tar" + if mode == "w": + mode = "w:bz2" + elif Path(file).suffix == ".txz" or Path(file).suffixes == [".tar", ".xz"]: + self.type = "tar" + if mode == "w": + mode = "w:xz" + elif Path(file).suffix == ".zip": + self.type = "zip" + else: + raise ValueError(f"Cannot detect archive format: type={file}") + + if self.type == "tar": + self.fopen = tarfile.open(file, mode=mode) + elif self.type == "zip": + + self.fopen = zipfile.ZipFile(file, mode=mode) + else: + raise ValueError(f"Not supported: type={type}") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.fopen.close() + + def close(self): + self.fopen.close() + + def __iter__(self): + if self.type == "tar": + return iter(self.fopen) + elif self.type == "zip": + return iter(self.fopen.infolist()) + else: + raise ValueError(f"Not supported: type={self.type}") + + def add(self, filename, arcname=None, recursive: bool = True): + if arcname is not None: + print(f"adding: {arcname}") + else: + print(f"adding: {filename}") + + if recursive and Path(filename).is_dir(): + for f in Path(filename).glob("**/*"): + if f.is_dir(): + continue + + if arcname is not None: + _arcname = Path(arcname) / f + else: + _arcname = None + + self.add(f, _arcname) + return + + if self.type == "tar": + return self.fopen.add(filename, arcname) + elif self.type == "zip": + return self.fopen.write(filename, arcname) + else: + raise ValueError(f"Not supported: type={self.type}") + + def addfile(self, info, fileobj): + print(f"adding: {self.get_name_from_info(info)}") + + if self.type == "tar": + return self.fopen.addfile(info, fileobj) + elif self.type == "zip": + return self.fopen.writestr(info, fileobj.read()) + else: + raise ValueError(f"Not supported: type={self.type}") + + def generate_info(self, name, size) -> Union[tarfile.TarInfo, zipfile.ZipInfo]: + """Generate TarInfo using system information""" + if self.type == "tar": + tarinfo = tarfile.TarInfo(str(name)) + if os.name == "posix": + tarinfo.gid = os.getgid() + tarinfo.uid = os.getuid() + tarinfo.mtime = datetime.now().timestamp() + tarinfo.size = size + # Keep mode as default + return tarinfo + elif self.type == "zip": + zipinfo = zipfile.ZipInfo(str(name), datetime.now().timetuple()[:6]) + zipinfo.file_size = size + return zipinfo + else: + raise ValueError(f"Not supported: type={self.type}") + + def get_name_from_info(self, info): + if self.type == "tar": + assert isinstance(info, tarfile.TarInfo), type(info) + return info.name + elif self.type == "zip": + assert isinstance(info, zipfile.ZipInfo), type(info) + return info.filename + else: + raise ValueError(f"Not supported: type={self.type}") + + def extract(self, info, path=None): + if self.type == "tar": + return self.fopen.extract(info, path) + elif self.type == "zip": + return self.fopen.extract(info, path) + else: + raise ValueError(f"Not supported: type={self.type}") + + def extractfile(self, info, mode="r"): + if self.type == "tar": + f = self.fopen.extractfile(info) + if mode == "r": + return TextIOWrapper(f) + else: + return f + elif self.type == "zip": + if mode == "rb": + mode = "r" + return self.fopen.open(info, mode) + else: + raise ValueError(f"Not supported: type={self.type}") + + +def find_path_and_change_it_recursive(value, src: str, tgt: str): + if isinstance(value, dict): + return { + k: find_path_and_change_it_recursive(v, src, tgt) for k, v in value.items() + } + elif isinstance(value, (list, tuple)): + return [find_path_and_change_it_recursive(v, src, tgt) for v in value] + elif isinstance(value, str) and Path(value) == Path(src): + return tgt + else: + return value + + +def get_dict_from_cache(meta: Union[Path, str]) -> Optional[Dict[str, str]]: + meta = Path(meta) + outpath = meta.parent.parent + if not meta.exists(): + return None + + with meta.open("r", encoding="utf-8") as f: + d = yaml.safe_load(f) + assert isinstance(d, dict), type(d) + yaml_files = d["yaml_files"] + files = d["files"] + assert isinstance(yaml_files, dict), type(yaml_files) + assert isinstance(files, dict), type(files) + + retval = {} + for key, value in list(yaml_files.items()) + list(files.items()): + if not (outpath / value).exists(): + return None + retval[key] = str(outpath / value) + return retval + + +def unpack( + input_archive: Union[Path, str], + outpath: Union[Path, str], + use_cache: bool = True, +) -> Dict[str, str]: + """Scan all files in the archive file and return as a dict of files. + + Examples: + tarfile: + model.pth + some1.file + some2.file + + >>> unpack("tarfile", "out") + {'asr_model_file': 'out/model.pth'} + """ + input_archive = Path(input_archive) + outpath = Path(outpath) + + with Archiver(input_archive) as archive: + for info in archive: + if Path(archive.get_name_from_info(info)).name == "meta.yaml": + if ( + use_cache + and (outpath / Path(archive.get_name_from_info(info))).exists() + ): + retval = get_dict_from_cache( + outpath / Path(archive.get_name_from_info(info)) + ) + if retval is not None: + return retval + d = yaml.safe_load(archive.extractfile(info)) + assert isinstance(d, dict), type(d) + yaml_files = d["yaml_files"] + files = d["files"] + assert isinstance(yaml_files, dict), type(yaml_files) + assert isinstance(files, dict), type(files) + break + else: + raise RuntimeError("Format error: not found meta.yaml") + + for info in archive: + fname = archive.get_name_from_info(info) + outname = outpath / fname + outname.parent.mkdir(parents=True, exist_ok=True) + if fname in set(yaml_files.values()): + d = yaml.safe_load(archive.extractfile(info)) + # Rewrite yaml + for info2 in archive: + name = archive.get_name_from_info(info2) + d = find_path_and_change_it_recursive(d, name, str(outpath / name)) + with outname.open("w", encoding="utf-8") as f: + yaml.safe_dump(d, f) + else: + archive.extract(info, path=outpath) + + retval = {} + for key, value in list(yaml_files.items()) + list(files.items()): + retval[key] = str(outpath / value) + return retval + + +def _to_relative_or_resolve(f): + # Resolve to avoid symbolic link + p = Path(f).resolve() + try: + # Change to relative if it can + p = p.relative_to(Path(".").resolve()) + except ValueError: + pass + return str(p) + + +def pack( + files: Dict[str, Union[str, Path]], + yaml_files: Dict[str, Union[str, Path]], + outpath: Union[str, Path], + option: Iterable[Union[str, Path]] = (), +): + for v in list(files.values()) + list(yaml_files.values()) + list(option): + if not Path(v).exists(): + raise FileNotFoundError(f"No such file or directory: {v}") + + files = {k: _to_relative_or_resolve(v) for k, v in files.items()} + yaml_files = {k: _to_relative_or_resolve(v) for k, v in yaml_files.items()} + option = [_to_relative_or_resolve(v) for v in option] + + meta_objs = dict( + files=files, + yaml_files=yaml_files, + timestamp=datetime.now().timestamp(), + python=sys.version, + ) + + try: + import torch + + meta_objs.update(torch=str(torch.__version__)) + except ImportError: + pass + try: + import espnet + + meta_objs.update(espnet=espnet.__version__) + except ImportError: + pass + + Path(outpath).parent.mkdir(parents=True, exist_ok=True) + with Archiver(outpath, mode="w") as archive: + # Write packed/meta.yaml + fileobj = BytesIO(yaml.safe_dump(meta_objs).encode()) + info = archive.generate_info("meta.yaml", fileobj.getbuffer().nbytes) + archive.addfile(info, fileobj=fileobj) + + for f in list(yaml_files.values()) + list(files.values()) + list(option): + archive.add(f) + + print(f"Generate: {outpath}") diff --git a/funasr/models/ctc.py b/funasr/models/ctc.py new file mode 100644 index 000000000..64b87106a --- /dev/null +++ b/funasr/models/ctc.py @@ -0,0 +1,187 @@ +import logging + +import torch +import torch.nn.functional as F +from typeguard import check_argument_types + + +class CTC(torch.nn.Module): + """CTC module. + + Args: + odim: dimension of outputs + encoder_output_size: number of encoder projection units + dropout_rate: dropout rate (0.0 ~ 1.0) + ctc_type: builtin or warpctc + reduce: reduce the CTC loss into a scalar + """ + + def __init__( + self, + odim: int, + encoder_output_size: int, + dropout_rate: float = 0.0, + ctc_type: str = "builtin", + reduce: bool = True, + ignore_nan_grad: bool = True, + ): + assert check_argument_types() + super().__init__() + eprojs = encoder_output_size + self.dropout_rate = dropout_rate + self.ctc_lo = torch.nn.Linear(eprojs, odim) + self.ctc_type = ctc_type + self.ignore_nan_grad = ignore_nan_grad + + if self.ctc_type == "builtin": + self.ctc_loss = torch.nn.CTCLoss(reduction="none") + elif self.ctc_type == "warpctc": + import warpctc_pytorch as warp_ctc + + if ignore_nan_grad: + logging.warning("ignore_nan_grad option is not supported for warp_ctc") + self.ctc_loss = warp_ctc.CTCLoss(size_average=True, reduce=reduce) + + elif self.ctc_type == "gtnctc": + from espnet.nets.pytorch_backend.gtn_ctc import GTNCTCLossFunction + + self.ctc_loss = GTNCTCLossFunction.apply + else: + raise ValueError( + f'ctc_type must be "builtin" or "warpctc": {self.ctc_type}' + ) + + self.reduce = reduce + + def loss_fn(self, th_pred, th_target, th_ilen, th_olen) -> torch.Tensor: + if self.ctc_type == "builtin": + th_pred = th_pred.log_softmax(2) + loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen) + + if loss.requires_grad and self.ignore_nan_grad: + # ctc_grad: (L, B, O) + ctc_grad = loss.grad_fn(torch.ones_like(loss)) + ctc_grad = ctc_grad.sum([0, 2]) + indices = torch.isfinite(ctc_grad) + size = indices.long().sum() + if size == 0: + # Return as is + logging.warning( + "All samples in this mini-batch got nan grad." + " Returning nan value instead of CTC loss" + ) + elif size != th_pred.size(1): + logging.warning( + f"{th_pred.size(1) - size}/{th_pred.size(1)}" + " samples got nan grad." + " These were ignored for CTC loss." + ) + + # Create mask for target + target_mask = torch.full( + [th_target.size(0)], + 1, + dtype=torch.bool, + device=th_target.device, + ) + s = 0 + for ind, le in enumerate(th_olen): + if not indices[ind]: + target_mask[s : s + le] = 0 + s += le + + # Calc loss again using maksed data + loss = self.ctc_loss( + th_pred[:, indices, :], + th_target[target_mask], + th_ilen[indices], + th_olen[indices], + ) + else: + size = th_pred.size(1) + + if self.reduce: + # Batch-size average + loss = loss.sum() / size + else: + loss = loss / size + return loss + + elif self.ctc_type == "warpctc": + # warpctc only supports float32 + th_pred = th_pred.to(dtype=torch.float32) + + th_target = th_target.cpu().int() + th_ilen = th_ilen.cpu().int() + th_olen = th_olen.cpu().int() + loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen) + if self.reduce: + # NOTE: sum() is needed to keep consistency since warpctc + # return as tensor w/ shape (1,) + # but builtin return as tensor w/o shape (scalar). + loss = loss.sum() + return loss + + elif self.ctc_type == "gtnctc": + log_probs = torch.nn.functional.log_softmax(th_pred, dim=2) + return self.ctc_loss(log_probs, th_target, th_ilen, 0, "none") + + else: + raise NotImplementedError + + def forward(self, hs_pad, hlens, ys_pad, ys_lens): + """Calculate CTC loss. + + Args: + hs_pad: batch of padded hidden state sequences (B, Tmax, D) + hlens: batch of lengths of hidden state sequences (B) + ys_pad: batch of padded character id sequence tensor (B, Lmax) + ys_lens: batch of lengths of character sequence (B) + """ + # hs_pad: (B, L, NProj) -> ys_hat: (B, L, Nvocab) + ys_hat = self.ctc_lo(F.dropout(hs_pad, p=self.dropout_rate)) + + if self.ctc_type == "gtnctc": + # gtn expects list form for ys + ys_true = [y[y != -1] for y in ys_pad] # parse padded ys + else: + # ys_hat: (B, L, D) -> (L, B, D) + ys_hat = ys_hat.transpose(0, 1) + # (B, L) -> (BxL,) + ys_true = torch.cat([ys_pad[i, :l] for i, l in enumerate(ys_lens)]) + + loss = self.loss_fn(ys_hat, ys_true, hlens, ys_lens).to( + device=hs_pad.device, dtype=hs_pad.dtype + ) + + return loss + + def softmax(self, hs_pad): + """softmax of frame activations + + Args: + Tensor hs_pad: 3d tensor (B, Tmax, eprojs) + Returns: + torch.Tensor: softmax applied 3d tensor (B, Tmax, odim) + """ + return F.softmax(self.ctc_lo(hs_pad), dim=2) + + def log_softmax(self, hs_pad): + """log_softmax of frame activations + + Args: + Tensor hs_pad: 3d tensor (B, Tmax, eprojs) + Returns: + torch.Tensor: log softmax applied 3d tensor (B, Tmax, odim) + """ + return F.log_softmax(self.ctc_lo(hs_pad), dim=2) + + def argmax(self, hs_pad): + """argmax of frame activations + + Args: + torch.Tensor hs_pad: 3d tensor (B, Tmax, eprojs) + Returns: + torch.Tensor: argmax applied 2d tensor (B, Tmax) + """ + return torch.argmax(self.ctc_lo(hs_pad), dim=2) diff --git a/funasr/models/decoder/__init__.py b/funasr/models/decoder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/models/decoder/abs_decoder.py b/funasr/models/decoder/abs_decoder.py new file mode 100644 index 000000000..bc8acf44c --- /dev/null +++ b/funasr/models/decoder/abs_decoder.py @@ -0,0 +1,19 @@ +from abc import ABC +from abc import abstractmethod +from typing import Tuple + +import torch + +from funasr.modules.scorers.scorer_interface import ScorerInterface + + +class AbsDecoder(torch.nn.Module, ScorerInterface, ABC): + @abstractmethod + def forward( + self, + hs_pad: torch.Tensor, + hlens: torch.Tensor, + ys_in_pad: torch.Tensor, + ys_in_lens: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError diff --git a/funasr/models/decoder/rnn_decoder.py b/funasr/models/decoder/rnn_decoder.py new file mode 100644 index 000000000..80709c9be --- /dev/null +++ b/funasr/models/decoder/rnn_decoder.py @@ -0,0 +1,334 @@ +import random + +import numpy as np +import torch +import torch.nn.functional as F +from typeguard import check_argument_types + +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.nets_utils import to_device +from funasr.modules.rnn.attentions import initial_att +from funasr.models.decoder.abs_decoder import AbsDecoder +from funasr.utils.get_default_kwargs import get_default_kwargs + + +def build_attention_list( + eprojs: int, + dunits: int, + atype: str = "location", + num_att: int = 1, + num_encs: int = 1, + aheads: int = 4, + adim: int = 320, + awin: int = 5, + aconv_chans: int = 10, + aconv_filts: int = 100, + han_mode: bool = False, + han_type=None, + han_heads: int = 4, + han_dim: int = 320, + han_conv_chans: int = -1, + han_conv_filts: int = 100, + han_win: int = 5, +): + + att_list = torch.nn.ModuleList() + if num_encs == 1: + for i in range(num_att): + att = initial_att( + atype, + eprojs, + dunits, + aheads, + adim, + awin, + aconv_chans, + aconv_filts, + ) + att_list.append(att) + elif num_encs > 1: # no multi-speaker mode + if han_mode: + att = initial_att( + han_type, + eprojs, + dunits, + han_heads, + han_dim, + han_win, + han_conv_chans, + han_conv_filts, + han_mode=True, + ) + return att + else: + att_list = torch.nn.ModuleList() + for idx in range(num_encs): + att = initial_att( + atype[idx], + eprojs, + dunits, + aheads[idx], + adim[idx], + awin[idx], + aconv_chans[idx], + aconv_filts[idx], + ) + att_list.append(att) + else: + raise ValueError( + "Number of encoders needs to be more than one. {}".format(num_encs) + ) + return att_list + + +class RNNDecoder(AbsDecoder): + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + rnn_type: str = "lstm", + num_layers: int = 1, + hidden_size: int = 320, + sampling_probability: float = 0.0, + dropout: float = 0.0, + context_residual: bool = False, + replace_sos: bool = False, + num_encs: int = 1, + att_conf: dict = get_default_kwargs(build_attention_list), + ): + # FIXME(kamo): The parts of num_spk should be refactored more more more + assert check_argument_types() + if rnn_type not in {"lstm", "gru"}: + raise ValueError(f"Not supported: rnn_type={rnn_type}") + + super().__init__() + eprojs = encoder_output_size + self.dtype = rnn_type + self.dunits = hidden_size + self.dlayers = num_layers + self.context_residual = context_residual + self.sos = vocab_size - 1 + self.eos = vocab_size - 1 + self.odim = vocab_size + self.sampling_probability = sampling_probability + self.dropout = dropout + self.num_encs = num_encs + + # for multilingual translation + self.replace_sos = replace_sos + + self.embed = torch.nn.Embedding(vocab_size, hidden_size) + self.dropout_emb = torch.nn.Dropout(p=dropout) + + self.decoder = torch.nn.ModuleList() + self.dropout_dec = torch.nn.ModuleList() + self.decoder += [ + torch.nn.LSTMCell(hidden_size + eprojs, hidden_size) + if self.dtype == "lstm" + else torch.nn.GRUCell(hidden_size + eprojs, hidden_size) + ] + self.dropout_dec += [torch.nn.Dropout(p=dropout)] + for _ in range(1, self.dlayers): + self.decoder += [ + torch.nn.LSTMCell(hidden_size, hidden_size) + if self.dtype == "lstm" + else torch.nn.GRUCell(hidden_size, hidden_size) + ] + self.dropout_dec += [torch.nn.Dropout(p=dropout)] + # NOTE: dropout is applied only for the vertical connections + # see https://arxiv.org/pdf/1409.2329.pdf + + if context_residual: + self.output = torch.nn.Linear(hidden_size + eprojs, vocab_size) + else: + self.output = torch.nn.Linear(hidden_size, vocab_size) + + self.att_list = build_attention_list( + eprojs=eprojs, dunits=hidden_size, **att_conf + ) + + def zero_state(self, hs_pad): + return hs_pad.new_zeros(hs_pad.size(0), self.dunits) + + def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev): + if self.dtype == "lstm": + z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0])) + for i in range(1, self.dlayers): + z_list[i], c_list[i] = self.decoder[i]( + self.dropout_dec[i - 1](z_list[i - 1]), + (z_prev[i], c_prev[i]), + ) + else: + z_list[0] = self.decoder[0](ey, z_prev[0]) + for i in range(1, self.dlayers): + z_list[i] = self.decoder[i]( + self.dropout_dec[i - 1](z_list[i - 1]), z_prev[i] + ) + return z_list, c_list + + def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens, strm_idx=0): + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + hs_pad = [hs_pad] + hlens = [hlens] + + # attention index for the attention module + # in SPA (speaker parallel attention), + # att_idx is used to select attention module. In other cases, it is 0. + att_idx = min(strm_idx, len(self.att_list) - 1) + + # hlens should be list of list of integer + hlens = [list(map(int, hlens[idx])) for idx in range(self.num_encs)] + + # get dim, length info + olength = ys_in_pad.size(1) + + # initialization + c_list = [self.zero_state(hs_pad[0])] + z_list = [self.zero_state(hs_pad[0])] + for _ in range(1, self.dlayers): + c_list.append(self.zero_state(hs_pad[0])) + z_list.append(self.zero_state(hs_pad[0])) + z_all = [] + if self.num_encs == 1: + att_w = None + self.att_list[att_idx].reset() # reset pre-computation of h + else: + att_w_list = [None] * (self.num_encs + 1) # atts + han + att_c_list = [None] * self.num_encs # atts + for idx in range(self.num_encs + 1): + # reset pre-computation of h in atts and han + self.att_list[idx].reset() + + # pre-computation of embedding + eys = self.dropout_emb(self.embed(ys_in_pad)) # utt x olen x zdim + + # loop for an output sequence + for i in range(olength): + if self.num_encs == 1: + att_c, att_w = self.att_list[att_idx]( + hs_pad[0], hlens[0], self.dropout_dec[0](z_list[0]), att_w + ) + else: + for idx in range(self.num_encs): + att_c_list[idx], att_w_list[idx] = self.att_list[idx]( + hs_pad[idx], + hlens[idx], + self.dropout_dec[0](z_list[0]), + att_w_list[idx], + ) + hs_pad_han = torch.stack(att_c_list, dim=1) + hlens_han = [self.num_encs] * len(ys_in_pad) + att_c, att_w_list[self.num_encs] = self.att_list[self.num_encs]( + hs_pad_han, + hlens_han, + self.dropout_dec[0](z_list[0]), + att_w_list[self.num_encs], + ) + if i > 0 and random.random() < self.sampling_probability: + z_out = self.output(z_all[-1]) + z_out = np.argmax(z_out.detach().cpu(), axis=1) + z_out = self.dropout_emb(self.embed(to_device(self, z_out))) + ey = torch.cat((z_out, att_c), dim=1) # utt x (zdim + hdim) + else: + # utt x (zdim + hdim) + ey = torch.cat((eys[:, i, :], att_c), dim=1) + z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_list, c_list) + if self.context_residual: + z_all.append( + torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) + ) # utt x (zdim + hdim) + else: + z_all.append(self.dropout_dec[-1](z_list[-1])) # utt x (zdim) + + z_all = torch.stack(z_all, dim=1) + z_all = self.output(z_all) + z_all.masked_fill_( + make_pad_mask(ys_in_lens, z_all, 1), + 0, + ) + return z_all, ys_in_lens + + def init_state(self, x): + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + x = [x] + + c_list = [self.zero_state(x[0].unsqueeze(0))] + z_list = [self.zero_state(x[0].unsqueeze(0))] + for _ in range(1, self.dlayers): + c_list.append(self.zero_state(x[0].unsqueeze(0))) + z_list.append(self.zero_state(x[0].unsqueeze(0))) + # TODO(karita): support strm_index for `asr_mix` + strm_index = 0 + att_idx = min(strm_index, len(self.att_list) - 1) + if self.num_encs == 1: + a = None + self.att_list[att_idx].reset() # reset pre-computation of h + else: + a = [None] * (self.num_encs + 1) # atts + han + for idx in range(self.num_encs + 1): + # reset pre-computation of h in atts and han + self.att_list[idx].reset() + return dict( + c_prev=c_list[:], + z_prev=z_list[:], + a_prev=a, + workspace=(att_idx, z_list, c_list), + ) + + def score(self, yseq, state, x): + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + x = [x] + + att_idx, z_list, c_list = state["workspace"] + vy = yseq[-1].unsqueeze(0) + ey = self.dropout_emb(self.embed(vy)) # utt list (1) x zdim + if self.num_encs == 1: + att_c, att_w = self.att_list[att_idx]( + x[0].unsqueeze(0), + [x[0].size(0)], + self.dropout_dec[0](state["z_prev"][0]), + state["a_prev"], + ) + else: + att_w = [None] * (self.num_encs + 1) # atts + han + att_c_list = [None] * self.num_encs # atts + for idx in range(self.num_encs): + att_c_list[idx], att_w[idx] = self.att_list[idx]( + x[idx].unsqueeze(0), + [x[idx].size(0)], + self.dropout_dec[0](state["z_prev"][0]), + state["a_prev"][idx], + ) + h_han = torch.stack(att_c_list, dim=1) + att_c, att_w[self.num_encs] = self.att_list[self.num_encs]( + h_han, + [self.num_encs], + self.dropout_dec[0](state["z_prev"][0]), + state["a_prev"][self.num_encs], + ) + ey = torch.cat((ey, att_c), dim=1) # utt(1) x (zdim + hdim) + z_list, c_list = self.rnn_forward( + ey, z_list, c_list, state["z_prev"], state["c_prev"] + ) + if self.context_residual: + logits = self.output( + torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) + ) + else: + logits = self.output(self.dropout_dec[-1](z_list[-1])) + logp = F.log_softmax(logits, dim=1).squeeze(0) + return ( + logp, + dict( + c_prev=c_list[:], + z_prev=z_list[:], + a_prev=att_w, + workspace=(att_idx, z_list, c_list), + ), + ) diff --git a/funasr/models/decoder/sanm_decoder.py b/funasr/models/decoder/sanm_decoder.py new file mode 100644 index 000000000..a5db353ba --- /dev/null +++ b/funasr/models/decoder/sanm_decoder.py @@ -0,0 +1,616 @@ +from typing import List +from typing import Tuple + +import torch +import torch.nn as nn +from funasr.modules.streaming_utils import utils as myutils +from funasr.models.decoder.transformer_decoder import BaseTransformerDecoder +from typeguard import check_argument_types + +from funasr.modules.attention import MultiHeadedAttentionSANMDecoder, MultiHeadedAttentionCrossAtt +from funasr.modules.embedding import PositionalEncoding +from funasr.modules.layer_norm import LayerNorm +from funasr.modules.positionwise_feed_forward import PositionwiseFeedForwardDecoderSANM +from funasr.modules.repeat import repeat + + +class DecoderLayerSANM(nn.Module): + """Single decoder layer module. + + Args: + size (int): Input dimension. + self_attn (torch.nn.Module): Self-attention module instance. + `MultiHeadedAttention` instance can be used as the argument. + src_attn (torch.nn.Module): Self-attention module instance. + `MultiHeadedAttention` instance can be used as the argument. + feed_forward (torch.nn.Module): Feed-forward module instance. + `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance + can be used as the argument. + dropout_rate (float): Dropout rate. + normalize_before (bool): Whether to use layer_norm before the first block. + concat_after (bool): Whether to concat attention layer's input and output. + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. i.e. x -> x + att(x) + + + """ + + def __init__( + self, + size, + self_attn, + src_attn, + feed_forward, + dropout_rate, + normalize_before=True, + concat_after=False, + ): + """Construct an DecoderLayer object.""" + super(DecoderLayerSANM, self).__init__() + self.size = size + self.self_attn = self_attn + self.src_attn = src_attn + self.feed_forward = feed_forward + self.norm1 = LayerNorm(size) + if self_attn is not None: + self.norm2 = LayerNorm(size) + if src_attn is not None: + self.norm3 = LayerNorm(size) + self.dropout = nn.Dropout(dropout_rate) + self.normalize_before = normalize_before + self.concat_after = concat_after + if self.concat_after: + self.concat_linear1 = nn.Linear(size + size, size) + self.concat_linear2 = nn.Linear(size + size, size) + + def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None): + """Compute decoded features. + + Args: + tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). + tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). + memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). + memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). + cache (List[torch.Tensor]): List of cached tensors. + Each tensor shape should be (#batch, maxlen_out - 1, size). + + Returns: + torch.Tensor: Output tensor(#batch, maxlen_out, size). + torch.Tensor: Mask for output tensor (#batch, maxlen_out). + torch.Tensor: Encoded memory (#batch, maxlen_in, size). + torch.Tensor: Encoded memory mask (#batch, maxlen_in). + + """ + # tgt = self.dropout(tgt) + residual = tgt + if self.normalize_before: + tgt = self.norm1(tgt) + tgt = self.feed_forward(tgt) + + x = tgt + if self.self_attn: + if self.normalize_before: + tgt = self.norm2(tgt) + if self.training: + cache = None + x, cache = self.self_attn(tgt, tgt_mask, cache=cache) + x = residual + self.dropout(x) + + if self.src_attn is not None: + residual = x + if self.normalize_before: + x = self.norm3(x) + + x = residual + self.dropout(self.src_attn(x, memory, memory_mask)) + + + return x, tgt_mask, memory, memory_mask, cache + + +class FsmnDecoderSCAMAOpt(BaseTransformerDecoder): + """ + author: Speech Lab, Alibaba Group, China + SCAMA: Streaming chunk-aware multihead attention for online end-to-end speech recognition + https://arxiv.org/abs/2006.01713 + + """ + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + self_attention_dropout_rate: float = 0.0, + src_attention_dropout_rate: float = 0.0, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + att_layer_num: int = 6, + kernel_size: int = 21, + sanm_shfit: int = None, + concat_embeds: bool = False, + attention_dim: int = None, + ): + assert check_argument_types() + super().__init__( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + dropout_rate=dropout_rate, + positional_dropout_rate=positional_dropout_rate, + input_layer=input_layer, + use_output_layer=use_output_layer, + pos_enc_class=pos_enc_class, + normalize_before=normalize_before, + ) + if attention_dim is None: + attention_dim = encoder_output_size + + if input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(vocab_size, attention_dim), + ) + elif input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(vocab_size, attention_dim), + torch.nn.LayerNorm(attention_dim), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + pos_enc_class(attention_dim, positional_dropout_rate), + ) + else: + raise ValueError(f"only 'embed' or 'linear' is supported: {input_layer}") + + self.normalize_before = normalize_before + if self.normalize_before: + self.after_norm = LayerNorm(attention_dim) + if use_output_layer: + self.output_layer = torch.nn.Linear(attention_dim, vocab_size) + else: + self.output_layer = None + + self.att_layer_num = att_layer_num + self.num_blocks = num_blocks + if sanm_shfit is None: + sanm_shfit = (kernel_size - 1) // 2 + self.decoders = repeat( + att_layer_num, + lambda lnum: DecoderLayerSANM( + attention_dim, + MultiHeadedAttentionSANMDecoder( + attention_dim, self_attention_dropout_rate, kernel_size, sanm_shfit=sanm_shfit + ), + MultiHeadedAttentionCrossAtt( + attention_heads, attention_dim, src_attention_dropout_rate, encoder_output_size=encoder_output_size + ), + PositionwiseFeedForwardDecoderSANM(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + if num_blocks - att_layer_num <= 0: + self.decoders2 = None + else: + self.decoders2 = repeat( + num_blocks - att_layer_num, + lambda lnum: DecoderLayerSANM( + attention_dim, + MultiHeadedAttentionSANMDecoder( + attention_dim, self_attention_dropout_rate, kernel_size, sanm_shfit=sanm_shfit + ), + None, + PositionwiseFeedForwardDecoderSANM(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + self.decoders3 = repeat( + 1, + lambda lnum: DecoderLayerSANM( + attention_dim, + None, + None, + PositionwiseFeedForwardDecoderSANM(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + if concat_embeds: + self.embed_concat_ffn = repeat( + 1, + lambda lnum: DecoderLayerSANM( + attention_dim + encoder_output_size, + None, + None, + PositionwiseFeedForwardDecoderSANM(attention_dim + encoder_output_size, linear_units, dropout_rate, + adim=attention_dim), + dropout_rate, + normalize_before, + concat_after, + ), + ) + else: + self.embed_concat_ffn = None + self.concat_embeds = concat_embeds + + def forward( + self, + hs_pad: torch.Tensor, + hlens: torch.Tensor, + ys_in_pad: torch.Tensor, + ys_in_lens: torch.Tensor, + chunk_mask: torch.Tensor = None, + pre_acoustic_embeds: torch.Tensor = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward decoder. + + Args: + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) + Returns: + (tuple): tuple containing: + + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) + """ + tgt = ys_in_pad + tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None] + + memory = hs_pad + memory_mask = myutils.sequence_mask(hlens, device=memory.device)[:, None, :] + if chunk_mask is not None: + memory_mask = memory_mask * chunk_mask + if tgt_mask.size(1) != memory_mask.size(1): + memory_mask = torch.cat((memory_mask, memory_mask[:, -2:-1, :]), dim=1) + + x = self.embed(tgt) + + if pre_acoustic_embeds is not None and self.concat_embeds: + x = torch.cat((x, pre_acoustic_embeds), dim=-1) + x, _, _, _, _ = self.embed_concat_ffn(x, None, None, None, None) + + x, tgt_mask, memory, memory_mask, _ = self.decoders( + x, tgt_mask, memory, memory_mask + ) + if self.decoders2 is not None: + x, tgt_mask, memory, memory_mask, _ = self.decoders2( + x, tgt_mask, memory, memory_mask + ) + x, tgt_mask, memory, memory_mask, _ = self.decoders3( + x, tgt_mask, memory, memory_mask + ) + if self.normalize_before: + x = self.after_norm(x) + if self.output_layer is not None: + x = self.output_layer(x) + + olens = tgt_mask.sum(1) + return x, olens + + def score(self, ys, state, x, x_mask=None, pre_acoustic_embeds: torch.Tensor = None, ): + """Score.""" + ys_mask = myutils.sequence_mask(torch.tensor([len(ys)], dtype=torch.int32), device=x.device)[:, :, None] + logp, state = self.forward_one_step( + ys.unsqueeze(0), ys_mask, x.unsqueeze(0), memory_mask=x_mask, pre_acoustic_embeds=pre_acoustic_embeds, + cache=state + ) + return logp.squeeze(0), state + + def forward_one_step( + self, + tgt: torch.Tensor, + tgt_mask: torch.Tensor, + memory: torch.Tensor, + memory_mask: torch.Tensor = None, + pre_acoustic_embeds: torch.Tensor = None, + cache: List[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Forward one step. + + Args: + tgt: input token ids, int64 (batch, maxlen_out) + tgt_mask: input token mask, (batch, maxlen_out) + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (include 1.2) + memory: encoded memory, float32 (batch, maxlen_in, feat) + cache: cached output list of (batch, max_time_out-1, size) + Returns: + y, cache: NN output value and cache per `self.decoders`. + y.shape` is (batch, maxlen_out, token) + """ + + x = tgt[:, -1:] + tgt_mask = None + x = self.embed(x) + + if pre_acoustic_embeds is not None and self.concat_embeds: + x = torch.cat((x, pre_acoustic_embeds), dim=-1) + x, _, _, _, _ = self.embed_concat_ffn(x, None, None, None, None) + + if cache is None: + cache_layer_num = len(self.decoders) + if self.decoders2 is not None: + cache_layer_num += len(self.decoders2) + cache = [None] * cache_layer_num + new_cache = [] + # for c, decoder in zip(cache, self.decoders): + for i in range(self.att_layer_num): + decoder = self.decoders[i] + c = cache[i] + x, tgt_mask, memory, memory_mask, c_ret = decoder( + x, tgt_mask, memory, memory_mask, cache=c + ) + new_cache.append(c_ret) + + if self.num_blocks - self.att_layer_num >= 1: + for i in range(self.num_blocks - self.att_layer_num): + j = i + self.att_layer_num + decoder = self.decoders2[i] + c = cache[j] + x, tgt_mask, memory, memory_mask, c_ret = decoder( + x, tgt_mask, memory, memory_mask, cache=c + ) + new_cache.append(c_ret) + + for decoder in self.decoders3: + x, tgt_mask, memory, memory_mask, _ = decoder( + x, tgt_mask, memory, None, cache=None + ) + + if self.normalize_before: + y = self.after_norm(x[:, -1]) + else: + y = x[:, -1] + if self.output_layer is not None: + y = self.output_layer(y) + y = torch.log_softmax(y, dim=-1) + + return y, new_cache + +class ParaformerSANMDecoder(BaseTransformerDecoder): + """ + author: Speech Lab, Alibaba Group, China + Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition + https://arxiv.org/abs/2006.01713 + """ + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + self_attention_dropout_rate: float = 0.0, + src_attention_dropout_rate: float = 0.0, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + att_layer_num: int = 6, + kernel_size: int = 21, + sanm_shfit: int = 0, + ): + assert check_argument_types() + super().__init__( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + dropout_rate=dropout_rate, + positional_dropout_rate=positional_dropout_rate, + input_layer=input_layer, + use_output_layer=use_output_layer, + pos_enc_class=pos_enc_class, + normalize_before=normalize_before, + ) + + attention_dim = encoder_output_size + + if input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(vocab_size, attention_dim), + # pos_enc_class(attention_dim, positional_dropout_rate), + ) + elif input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(vocab_size, attention_dim), + torch.nn.LayerNorm(attention_dim), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + pos_enc_class(attention_dim, positional_dropout_rate), + ) + else: + raise ValueError(f"only 'embed' or 'linear' is supported: {input_layer}") + + self.normalize_before = normalize_before + if self.normalize_before: + self.after_norm = LayerNorm(attention_dim) + if use_output_layer: + self.output_layer = torch.nn.Linear(attention_dim, vocab_size) + else: + self.output_layer = None + + self.att_layer_num = att_layer_num + self.num_blocks = num_blocks + if sanm_shfit is None: + sanm_shfit = (kernel_size - 1) // 2 + self.decoders = repeat( + att_layer_num, + lambda lnum: DecoderLayerSANM( + attention_dim, + MultiHeadedAttentionSANMDecoder( + attention_dim, self_attention_dropout_rate, kernel_size, sanm_shfit=sanm_shfit + ), + MultiHeadedAttentionCrossAtt( + attention_heads, attention_dim, src_attention_dropout_rate + ), + PositionwiseFeedForwardDecoderSANM(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + if num_blocks - att_layer_num <= 0: + self.decoders2 = None + else: + self.decoders2 = repeat( + num_blocks - att_layer_num, + lambda lnum: DecoderLayerSANM( + attention_dim, + MultiHeadedAttentionSANMDecoder( + attention_dim, self_attention_dropout_rate, kernel_size, sanm_shfit=0 + ), + None, + PositionwiseFeedForwardDecoderSANM(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + self.decoders3 = repeat( + 1, + lambda lnum: DecoderLayerSANM( + attention_dim, + None, + None, + PositionwiseFeedForwardDecoderSANM(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + def forward( + self, + hs_pad: torch.Tensor, + hlens: torch.Tensor, + ys_in_pad: torch.Tensor, + ys_in_lens: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward decoder. + + Args: + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) + Returns: + (tuple): tuple containing: + + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) + """ + tgt = ys_in_pad + tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None] + + memory = hs_pad + memory_mask = myutils.sequence_mask(hlens, device=memory.device)[:, None, :] + + x = tgt + x, tgt_mask, memory, memory_mask, _ = self.decoders( + x, tgt_mask, memory, memory_mask + ) + if self.decoders2 is not None: + x, tgt_mask, memory, memory_mask, _ = self.decoders2( + x, tgt_mask, memory, memory_mask + ) + x, tgt_mask, memory, memory_mask, _ = self.decoders3( + x, tgt_mask, memory, memory_mask + ) + if self.normalize_before: + x = self.after_norm(x) + if self.output_layer is not None: + x = self.output_layer(x) + + olens = tgt_mask.sum(1) + return x, olens + + def score(self, ys, state, x): + """Score.""" + ys_mask = myutils.sequence_mask(torch.tensor([len(ys)], dtype=torch.int32), device=x.device)[:, :, None] + logp, state = self.forward_one_step( + ys.unsqueeze(0), ys_mask, x.unsqueeze(0), cache=state + ) + return logp.squeeze(0), state + + def forward_one_step( + self, + tgt: torch.Tensor, + tgt_mask: torch.Tensor, + memory: torch.Tensor, + cache: List[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Forward one step. + + Args: + tgt: input token ids, int64 (batch, maxlen_out) + tgt_mask: input token mask, (batch, maxlen_out) + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (include 1.2) + memory: encoded memory, float32 (batch, maxlen_in, feat) + cache: cached output list of (batch, max_time_out-1, size) + Returns: + y, cache: NN output value and cache per `self.decoders`. + y.shape` is (batch, maxlen_out, token) + """ + x = self.embed(tgt) + if cache is None: + cache_layer_num = len(self.decoders) + if self.decoders2 is not None: + cache_layer_num += len(self.decoders2) + cache = [None] * cache_layer_num + new_cache = [] + # for c, decoder in zip(cache, self.decoders): + for i in range(self.att_layer_num): + decoder = self.decoders[i] + c = cache[i] + x, tgt_mask, memory, memory_mask, c_ret = decoder( + x, tgt_mask, memory, None, cache=c + ) + new_cache.append(c_ret) + + if self.num_blocks - self.att_layer_num > 1: + for i in range(self.num_blocks - self.att_layer_num): + j = i + self.att_layer_num + decoder = self.decoders2[i] + c = cache[j] + x, tgt_mask, memory, memory_mask, c_ret = decoder( + x, tgt_mask, memory, None, cache=c + ) + new_cache.append(c_ret) + + for decoder in self.decoders3: + + x, tgt_mask, memory, memory_mask, _ = decoder( + x, tgt_mask, memory, None, cache=None + ) + + if self.normalize_before: + y = self.after_norm(x[:, -1]) + else: + y = x[:, -1] + if self.output_layer is not None: + y = torch.log_softmax(self.output_layer(y), dim=-1) + + return y, new_cache \ No newline at end of file diff --git a/funasr/models/decoder/transformer_decoder.py b/funasr/models/decoder/transformer_decoder.py new file mode 100644 index 000000000..5f1bb2436 --- /dev/null +++ b/funasr/models/decoder/transformer_decoder.py @@ -0,0 +1,766 @@ +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Decoder definition.""" +from typing import Any +from typing import List +from typing import Sequence +from typing import Tuple + +import torch +from torch import nn +from typeguard import check_argument_types + +from funasr.models.decoder.abs_decoder import AbsDecoder +from funasr.modules.attention import MultiHeadedAttention +from funasr.modules.dynamic_conv import DynamicConvolution +from funasr.modules.dynamic_conv2d import DynamicConvolution2D +from funasr.modules.embedding import PositionalEncoding +from funasr.modules.layer_norm import LayerNorm +from funasr.modules.lightconv import LightweightConvolution +from funasr.modules.lightconv2d import LightweightConvolution2D +from funasr.modules.mask import subsequent_mask +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.positionwise_feed_forward import ( + PositionwiseFeedForward, # noqa: H301 +) +from funasr.modules.repeat import repeat +from funasr.modules.scorers.scorer_interface import BatchScorerInterface + + +class DecoderLayer(nn.Module): + """Single decoder layer module. + + Args: + size (int): Input dimension. + self_attn (torch.nn.Module): Self-attention module instance. + `MultiHeadedAttention` instance can be used as the argument. + src_attn (torch.nn.Module): Self-attention module instance. + `MultiHeadedAttention` instance can be used as the argument. + feed_forward (torch.nn.Module): Feed-forward module instance. + `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance + can be used as the argument. + dropout_rate (float): Dropout rate. + normalize_before (bool): Whether to use layer_norm before the first block. + concat_after (bool): Whether to concat attention layer's input and output. + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. i.e. x -> x + att(x) + + + """ + + def __init__( + self, + size, + self_attn, + src_attn, + feed_forward, + dropout_rate, + normalize_before=True, + concat_after=False, + ): + """Construct an DecoderLayer object.""" + super(DecoderLayer, self).__init__() + self.size = size + self.self_attn = self_attn + self.src_attn = src_attn + self.feed_forward = feed_forward + self.norm1 = LayerNorm(size) + self.norm2 = LayerNorm(size) + self.norm3 = LayerNorm(size) + self.dropout = nn.Dropout(dropout_rate) + self.normalize_before = normalize_before + self.concat_after = concat_after + if self.concat_after: + self.concat_linear1 = nn.Linear(size + size, size) + self.concat_linear2 = nn.Linear(size + size, size) + + def forward(self, tgt, tgt_mask, memory, memory_mask, cache=None): + """Compute decoded features. + + Args: + tgt (torch.Tensor): Input tensor (#batch, maxlen_out, size). + tgt_mask (torch.Tensor): Mask for input tensor (#batch, maxlen_out). + memory (torch.Tensor): Encoded memory, float32 (#batch, maxlen_in, size). + memory_mask (torch.Tensor): Encoded memory mask (#batch, maxlen_in). + cache (List[torch.Tensor]): List of cached tensors. + Each tensor shape should be (#batch, maxlen_out - 1, size). + + Returns: + torch.Tensor: Output tensor(#batch, maxlen_out, size). + torch.Tensor: Mask for output tensor (#batch, maxlen_out). + torch.Tensor: Encoded memory (#batch, maxlen_in, size). + torch.Tensor: Encoded memory mask (#batch, maxlen_in). + + """ + residual = tgt + if self.normalize_before: + tgt = self.norm1(tgt) + + if cache is None: + tgt_q = tgt + tgt_q_mask = tgt_mask + else: + # compute only the last frame query keeping dim: max_time_out -> 1 + assert cache.shape == ( + tgt.shape[0], + tgt.shape[1] - 1, + self.size, + ), f"{cache.shape} == {(tgt.shape[0], tgt.shape[1] - 1, self.size)}" + tgt_q = tgt[:, -1:, :] + residual = residual[:, -1:, :] + tgt_q_mask = None + if tgt_mask is not None: + tgt_q_mask = tgt_mask[:, -1:, :] + + if self.concat_after: + tgt_concat = torch.cat( + (tgt_q, self.self_attn(tgt_q, tgt, tgt, tgt_q_mask)), dim=-1 + ) + x = residual + self.concat_linear1(tgt_concat) + else: + x = residual + self.dropout(self.self_attn(tgt_q, tgt, tgt, tgt_q_mask)) + if not self.normalize_before: + x = self.norm1(x) + + residual = x + if self.normalize_before: + x = self.norm2(x) + if self.concat_after: + x_concat = torch.cat( + (x, self.src_attn(x, memory, memory, memory_mask)), dim=-1 + ) + x = residual + self.concat_linear2(x_concat) + else: + x = residual + self.dropout(self.src_attn(x, memory, memory, memory_mask)) + if not self.normalize_before: + x = self.norm2(x) + + residual = x + if self.normalize_before: + x = self.norm3(x) + x = residual + self.dropout(self.feed_forward(x)) + if not self.normalize_before: + x = self.norm3(x) + + if cache is not None: + x = torch.cat([cache, x], dim=1) + + return x, tgt_mask, memory, memory_mask + + +class BaseTransformerDecoder(AbsDecoder, BatchScorerInterface): + """Base class of Transfomer decoder module. + + Args: + vocab_size: output dim + encoder_output_size: dimension of attention + attention_heads: the number of heads of multi head attention + linear_units: the number of units of position-wise feed forward + num_blocks: the number of decoder blocks + dropout_rate: dropout rate + self_attention_dropout_rate: dropout rate for attention + input_layer: input layer type + use_output_layer: whether to use output layer + pos_enc_class: PositionalEncoding or ScaledPositionalEncoding + normalize_before: whether to use layer_norm before the first block + concat_after: whether to concat attention layer's input and output + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. + i.e. x -> x + att(x) + """ + + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + ): + assert check_argument_types() + super().__init__() + attention_dim = encoder_output_size + + if input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(vocab_size, attention_dim), + pos_enc_class(attention_dim, positional_dropout_rate), + ) + elif input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(vocab_size, attention_dim), + torch.nn.LayerNorm(attention_dim), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + pos_enc_class(attention_dim, positional_dropout_rate), + ) + else: + raise ValueError(f"only 'embed' or 'linear' is supported: {input_layer}") + + self.normalize_before = normalize_before + if self.normalize_before: + self.after_norm = LayerNorm(attention_dim) + if use_output_layer: + self.output_layer = torch.nn.Linear(attention_dim, vocab_size) + else: + self.output_layer = None + + # Must set by the inheritance + self.decoders = None + + def forward( + self, + hs_pad: torch.Tensor, + hlens: torch.Tensor, + ys_in_pad: torch.Tensor, + ys_in_lens: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward decoder. + + Args: + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) + Returns: + (tuple): tuple containing: + + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) + """ + tgt = ys_in_pad + # tgt_mask: (B, 1, L) + tgt_mask = (~make_pad_mask(ys_in_lens)[:, None, :]).to(tgt.device) + # m: (1, L, L) + m = subsequent_mask(tgt_mask.size(-1), device=tgt_mask.device).unsqueeze(0) + # tgt_mask: (B, L, L) + tgt_mask = tgt_mask & m + + memory = hs_pad + memory_mask = (~make_pad_mask(hlens, maxlen=memory.size(1)))[:, None, :].to( + memory.device + ) + # Padding for Longformer + if memory_mask.shape[-1] != memory.shape[1]: + padlen = memory.shape[1] - memory_mask.shape[-1] + memory_mask = torch.nn.functional.pad( + memory_mask, (0, padlen), "constant", False + ) + + x = self.embed(tgt) + x, tgt_mask, memory, memory_mask = self.decoders( + x, tgt_mask, memory, memory_mask + ) + if self.normalize_before: + x = self.after_norm(x) + if self.output_layer is not None: + x = self.output_layer(x) + + olens = tgt_mask.sum(1) + return x, olens + + def forward_one_step( + self, + tgt: torch.Tensor, + tgt_mask: torch.Tensor, + memory: torch.Tensor, + cache: List[torch.Tensor] = None, + ) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """Forward one step. + + Args: + tgt: input token ids, int64 (batch, maxlen_out) + tgt_mask: input token mask, (batch, maxlen_out) + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (include 1.2) + memory: encoded memory, float32 (batch, maxlen_in, feat) + cache: cached output list of (batch, max_time_out-1, size) + Returns: + y, cache: NN output value and cache per `self.decoders`. + y.shape` is (batch, maxlen_out, token) + """ + x = self.embed(tgt) + if cache is None: + cache = [None] * len(self.decoders) + new_cache = [] + for c, decoder in zip(cache, self.decoders): + x, tgt_mask, memory, memory_mask = decoder( + x, tgt_mask, memory, None, cache=c + ) + new_cache.append(x) + + if self.normalize_before: + y = self.after_norm(x[:, -1]) + else: + y = x[:, -1] + if self.output_layer is not None: + y = torch.log_softmax(self.output_layer(y), dim=-1) + + return y, new_cache + + def score(self, ys, state, x): + """Score.""" + ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0) + logp, state = self.forward_one_step( + ys.unsqueeze(0), ys_mask, x.unsqueeze(0), cache=state + ) + return logp.squeeze(0), state + + def batch_score( + self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor + ) -> Tuple[torch.Tensor, List[Any]]: + """Score new token batch. + + Args: + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). + + Returns: + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, n_vocab)` + and next state list for ys. + + """ + # merge states + n_batch = len(ys) + n_layers = len(self.decoders) + if states[0] is None: + batch_state = None + else: + # transpose state of [batch, layer] into [layer, batch] + batch_state = [ + torch.stack([states[b][i] for b in range(n_batch)]) + for i in range(n_layers) + ] + + # batch decoding + ys_mask = subsequent_mask(ys.size(-1), device=xs.device).unsqueeze(0) + logp, states = self.forward_one_step(ys, ys_mask, xs, cache=batch_state) + + # transpose state of [layer, batch] into [batch, layer] + state_list = [[states[i][b] for i in range(n_layers)] for b in range(n_batch)] + return logp, state_list + + +class TransformerDecoder(BaseTransformerDecoder): + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + self_attention_dropout_rate: float = 0.0, + src_attention_dropout_rate: float = 0.0, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + ): + assert check_argument_types() + super().__init__( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + dropout_rate=dropout_rate, + positional_dropout_rate=positional_dropout_rate, + input_layer=input_layer, + use_output_layer=use_output_layer, + pos_enc_class=pos_enc_class, + normalize_before=normalize_before, + ) + + attention_dim = encoder_output_size + self.decoders = repeat( + num_blocks, + lambda lnum: DecoderLayer( + attention_dim, + MultiHeadedAttention( + attention_heads, attention_dim, self_attention_dropout_rate + ), + MultiHeadedAttention( + attention_heads, attention_dim, src_attention_dropout_rate + ), + PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + +class ParaformerDecoderSAN(BaseTransformerDecoder): + """ + author: Speech Lab, Alibaba Group, China + Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition + https://arxiv.org/abs/2006.01713 + """ + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + self_attention_dropout_rate: float = 0.0, + src_attention_dropout_rate: float = 0.0, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + embeds_id: int = -1, + ): + assert check_argument_types() + super().__init__( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + dropout_rate=dropout_rate, + positional_dropout_rate=positional_dropout_rate, + input_layer=input_layer, + use_output_layer=use_output_layer, + pos_enc_class=pos_enc_class, + normalize_before=normalize_before, + ) + + attention_dim = encoder_output_size + self.decoders = repeat( + num_blocks, + lambda lnum: DecoderLayer( + attention_dim, + MultiHeadedAttention( + attention_heads, attention_dim, self_attention_dropout_rate + ), + MultiHeadedAttention( + attention_heads, attention_dim, src_attention_dropout_rate + ), + PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + self.embeds_id = embeds_id + self.attention_dim = attention_dim + + def forward( + self, + hs_pad: torch.Tensor, + hlens: torch.Tensor, + ys_in_pad: torch.Tensor, + ys_in_lens: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward decoder. + + Args: + hs_pad: encoded memory, float32 (batch, maxlen_in, feat) + hlens: (batch) + ys_in_pad: + input token ids, int64 (batch, maxlen_out) + if input_layer == "embed" + input tensor (batch, maxlen_out, #mels) in the other cases + ys_in_lens: (batch) + Returns: + (tuple): tuple containing: + + x: decoded token score before softmax (batch, maxlen_out, token) + if use_output_layer is True, + olens: (batch, ) + """ + tgt = ys_in_pad + tgt_mask = (~make_pad_mask(ys_in_lens)[:, None, :]).to(tgt.device) + + memory = hs_pad + memory_mask = (~make_pad_mask(hlens, maxlen=memory.size(1)))[:, None, :].to( + memory.device + ) + # Padding for Longformer + if memory_mask.shape[-1] != memory.shape[1]: + padlen = memory.shape[1] - memory_mask.shape[-1] + memory_mask = torch.nn.functional.pad( + memory_mask, (0, padlen), "constant", False + ) + + # x = self.embed(tgt) + x = tgt + embeds_outputs = None + for layer_id, decoder in enumerate(self.decoders): + x, tgt_mask, memory, memory_mask = decoder( + x, tgt_mask, memory, memory_mask + ) + if layer_id == self.embeds_id: + embeds_outputs = x + if self.normalize_before: + x = self.after_norm(x) + if self.output_layer is not None: + x = self.output_layer(x) + + olens = tgt_mask.sum(1) + if embeds_outputs is not None: + return x, olens, embeds_outputs + else: + return x, olens + + +class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder): + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + self_attention_dropout_rate: float = 0.0, + src_attention_dropout_rate: float = 0.0, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + conv_wshare: int = 4, + conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), + conv_usebias: int = False, + ): + assert check_argument_types() + if len(conv_kernel_length) != num_blocks: + raise ValueError( + "conv_kernel_length must have equal number of values to num_blocks: " + f"{len(conv_kernel_length)} != {num_blocks}" + ) + super().__init__( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + dropout_rate=dropout_rate, + positional_dropout_rate=positional_dropout_rate, + input_layer=input_layer, + use_output_layer=use_output_layer, + pos_enc_class=pos_enc_class, + normalize_before=normalize_before, + ) + + attention_dim = encoder_output_size + self.decoders = repeat( + num_blocks, + lambda lnum: DecoderLayer( + attention_dim, + LightweightConvolution( + wshare=conv_wshare, + n_feat=attention_dim, + dropout_rate=self_attention_dropout_rate, + kernel_size=conv_kernel_length[lnum], + use_kernel_mask=True, + use_bias=conv_usebias, + ), + MultiHeadedAttention( + attention_heads, attention_dim, src_attention_dropout_rate + ), + PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + +class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder): + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + self_attention_dropout_rate: float = 0.0, + src_attention_dropout_rate: float = 0.0, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + conv_wshare: int = 4, + conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), + conv_usebias: int = False, + ): + assert check_argument_types() + if len(conv_kernel_length) != num_blocks: + raise ValueError( + "conv_kernel_length must have equal number of values to num_blocks: " + f"{len(conv_kernel_length)} != {num_blocks}" + ) + super().__init__( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + dropout_rate=dropout_rate, + positional_dropout_rate=positional_dropout_rate, + input_layer=input_layer, + use_output_layer=use_output_layer, + pos_enc_class=pos_enc_class, + normalize_before=normalize_before, + ) + + attention_dim = encoder_output_size + self.decoders = repeat( + num_blocks, + lambda lnum: DecoderLayer( + attention_dim, + LightweightConvolution2D( + wshare=conv_wshare, + n_feat=attention_dim, + dropout_rate=self_attention_dropout_rate, + kernel_size=conv_kernel_length[lnum], + use_kernel_mask=True, + use_bias=conv_usebias, + ), + MultiHeadedAttention( + attention_heads, attention_dim, src_attention_dropout_rate + ), + PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + +class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder): + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + self_attention_dropout_rate: float = 0.0, + src_attention_dropout_rate: float = 0.0, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + conv_wshare: int = 4, + conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), + conv_usebias: int = False, + ): + assert check_argument_types() + if len(conv_kernel_length) != num_blocks: + raise ValueError( + "conv_kernel_length must have equal number of values to num_blocks: " + f"{len(conv_kernel_length)} != {num_blocks}" + ) + super().__init__( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + dropout_rate=dropout_rate, + positional_dropout_rate=positional_dropout_rate, + input_layer=input_layer, + use_output_layer=use_output_layer, + pos_enc_class=pos_enc_class, + normalize_before=normalize_before, + ) + attention_dim = encoder_output_size + + self.decoders = repeat( + num_blocks, + lambda lnum: DecoderLayer( + attention_dim, + DynamicConvolution( + wshare=conv_wshare, + n_feat=attention_dim, + dropout_rate=self_attention_dropout_rate, + kernel_size=conv_kernel_length[lnum], + use_kernel_mask=True, + use_bias=conv_usebias, + ), + MultiHeadedAttention( + attention_heads, attention_dim, src_attention_dropout_rate + ), + PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + +class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder): + def __init__( + self, + vocab_size: int, + encoder_output_size: int, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + self_attention_dropout_rate: float = 0.0, + src_attention_dropout_rate: float = 0.0, + input_layer: str = "embed", + use_output_layer: bool = True, + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + conv_wshare: int = 4, + conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11), + conv_usebias: int = False, + ): + assert check_argument_types() + if len(conv_kernel_length) != num_blocks: + raise ValueError( + "conv_kernel_length must have equal number of values to num_blocks: " + f"{len(conv_kernel_length)} != {num_blocks}" + ) + super().__init__( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + dropout_rate=dropout_rate, + positional_dropout_rate=positional_dropout_rate, + input_layer=input_layer, + use_output_layer=use_output_layer, + pos_enc_class=pos_enc_class, + normalize_before=normalize_before, + ) + attention_dim = encoder_output_size + + self.decoders = repeat( + num_blocks, + lambda lnum: DecoderLayer( + attention_dim, + DynamicConvolution2D( + wshare=conv_wshare, + n_feat=attention_dim, + dropout_rate=self_attention_dropout_rate, + kernel_size=conv_kernel_length[lnum], + use_kernel_mask=True, + use_bias=conv_usebias, + ), + MultiHeadedAttention( + attention_heads, attention_dim, src_attention_dropout_rate + ), + PositionwiseFeedForward(attention_dim, linear_units, dropout_rate), + dropout_rate, + normalize_before, + concat_after, + ), + ) \ No newline at end of file diff --git a/funasr/models/e2e_asr.py b/funasr/models/e2e_asr.py new file mode 100644 index 000000000..f64ea3dbe --- /dev/null +++ b/funasr/models/e2e_asr.py @@ -0,0 +1,458 @@ +# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import logging +from contextlib import contextmanager +from distutils.version import LooseVersion +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + +import torch +from typeguard import check_argument_types + +from funasr.layers.abs_normalize import AbsNormalize +from funasr.losses.label_smoothing_loss import ( + LabelSmoothingLoss, # noqa: H301 +) +from funasr.models.ctc import CTC +from funasr.models.decoder.abs_decoder import AbsDecoder +from funasr.models.encoder.abs_encoder import AbsEncoder +from funasr.models.frontend.abs_frontend import AbsFrontend +from funasr.models.postencoder.abs_postencoder import AbsPostEncoder +from funasr.models.preencoder.abs_preencoder import AbsPreEncoder +from funasr.models.specaug.abs_specaug import AbsSpecAug +from funasr.modules.add_sos_eos import add_sos_eos +from funasr.modules.e2e_asr_common import ErrorCalculator +from funasr.modules.nets_utils import th_accuracy +from funasr.torch_utils.device_funcs import force_gatherable +from funasr.train.abs_espnet_model import AbsESPnetModel + +if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): + from torch.cuda.amp import autocast +else: + # Nothing to do if torch<1.6.0 + @contextmanager + def autocast(enabled=True): + yield + + +class ESPnetASRModel(AbsESPnetModel): + """CTC-attention hybrid Encoder-Decoder model""" + + def __init__( + self, + vocab_size: int, + token_list: Union[Tuple[str, ...], List[str]], + frontend: Optional[AbsFrontend], + specaug: Optional[AbsSpecAug], + normalize: Optional[AbsNormalize], + preencoder: Optional[AbsPreEncoder], + encoder: AbsEncoder, + postencoder: Optional[AbsPostEncoder], + decoder: AbsDecoder, + ctc: CTC, + ctc_weight: float = 0.5, + interctc_weight: float = 0.0, + ignore_id: int = -1, + lsm_weight: float = 0.0, + length_normalized_loss: bool = False, + report_cer: bool = True, + report_wer: bool = True, + sym_space: str = "", + sym_blank: str = "", + extract_feats_in_collect_stats: bool = True, + ): + assert check_argument_types() + assert 0.0 <= ctc_weight <= 1.0, ctc_weight + assert 0.0 <= interctc_weight < 1.0, interctc_weight + + super().__init__() + # note that eos is the same as sos (equivalent ID) + self.blank_id = 0 + self.sos = 1 + self.eos = 2 + self.vocab_size = vocab_size + self.ignore_id = ignore_id + self.ctc_weight = ctc_weight + self.interctc_weight = interctc_weight + self.token_list = token_list.copy() + + self.frontend = frontend + self.specaug = specaug + self.normalize = normalize + self.preencoder = preencoder + self.postencoder = postencoder + self.encoder = encoder + + if not hasattr(self.encoder, "interctc_use_conditioning"): + self.encoder.interctc_use_conditioning = False + if self.encoder.interctc_use_conditioning: + self.encoder.conditioning_layer = torch.nn.Linear( + vocab_size, self.encoder.output_size() + ) + + self.error_calculator = None + + + # we set self.decoder = None in the CTC mode since + # self.decoder parameters were never used and PyTorch complained + # and threw an Exception in the multi-GPU experiment. + # thanks Jeff Farris for pointing out the issue. + if ctc_weight == 1.0: + self.decoder = None + else: + self.decoder = decoder + + self.criterion_att = LabelSmoothingLoss( + size=vocab_size, + padding_idx=ignore_id, + smoothing=lsm_weight, + normalize_length=length_normalized_loss, + ) + + if report_cer or report_wer: + self.error_calculator = ErrorCalculator( + token_list, sym_space, sym_blank, report_cer, report_wer + ) + + if ctc_weight == 0.0: + self.ctc = None + else: + self.ctc = ctc + + self.extract_feats_in_collect_stats = extract_feats_in_collect_stats + + def forward( + self, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + text: torch.Tensor, + text_lengths: torch.Tensor, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + """Frontend + Encoder + Decoder + Calc loss + + Args: + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + text: (Batch, Length) + text_lengths: (Batch,) + """ + assert text_lengths.dim() == 1, text_lengths.shape + # Check that batch_size is unified + assert ( + speech.shape[0] + == speech_lengths.shape[0] + == text.shape[0] + == text_lengths.shape[0] + ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) + batch_size = speech.shape[0] + + # for data-parallel + text = text[:, : text_lengths.max()] + + # 1. Encoder + encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + + loss_att, acc_att, cer_att, wer_att = None, None, None, None + loss_ctc, cer_ctc = None, None + stats = dict() + + # 1. CTC branch + if self.ctc_weight != 0.0: + loss_ctc, cer_ctc = self._calc_ctc_loss( + encoder_out, encoder_out_lens, text, text_lengths + ) + + # Collect CTC branch stats + stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None + stats["cer_ctc"] = cer_ctc + + # Intermediate CTC (optional) + loss_interctc = 0.0 + if self.interctc_weight != 0.0 and intermediate_outs is not None: + for layer_idx, intermediate_out in intermediate_outs: + # we assume intermediate_out has the same length & padding + # as those of encoder_out + loss_ic, cer_ic = self._calc_ctc_loss( + intermediate_out, encoder_out_lens, text, text_lengths + ) + loss_interctc = loss_interctc + loss_ic + + # Collect Intermedaite CTC stats + stats["loss_interctc_layer{}".format(layer_idx)] = ( + loss_ic.detach() if loss_ic is not None else None + ) + stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic + + loss_interctc = loss_interctc / len(intermediate_outs) + + # calculate whole encoder loss + loss_ctc = ( + 1 - self.interctc_weight + ) * loss_ctc + self.interctc_weight * loss_interctc + + + # 2b. Attention decoder branch + if self.ctc_weight != 1.0: + loss_att, acc_att, cer_att, wer_att = self._calc_att_loss( + encoder_out, encoder_out_lens, text, text_lengths + ) + + # 3. CTC-Att loss definition + if self.ctc_weight == 0.0: + loss = loss_att + elif self.ctc_weight == 1.0: + loss = loss_ctc + else: + loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + + # Collect Attn branch stats + stats["loss_att"] = loss_att.detach() if loss_att is not None else None + stats["acc"] = acc_att + stats["cer"] = cer_att + stats["wer"] = wer_att + + # Collect total loss stats + stats["loss"] = torch.clone(loss.detach()) + + # force_gatherable: to-device and to-tensor if scalar for DataParallel + loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) + return loss, stats, weight + + def collect_feats( + self, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + text: torch.Tensor, + text_lengths: torch.Tensor, + ) -> Dict[str, torch.Tensor]: + if self.extract_feats_in_collect_stats: + feats, feats_lengths = self._extract_feats(speech, speech_lengths) + else: + # Generate dummy stats if extract_feats_in_collect_stats is False + logging.warning( + "Generating dummy stats for feats and feats_lengths, " + "because encoder_conf.extract_feats_in_collect_stats is " + f"{self.extract_feats_in_collect_stats}" + ) + feats, feats_lengths = speech, speech_lengths + return {"feats": feats, "feats_lengths": feats_lengths} + + def encode( + self, speech: torch.Tensor, speech_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Frontend + Encoder. Note that this method is used by asr_inference.py + + Args: + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + """ + with autocast(False): + # 1. Extract feats + feats, feats_lengths = self._extract_feats(speech, speech_lengths) + + # 2. Data augmentation + if self.specaug is not None and self.training: + feats, feats_lengths = self.specaug(feats, feats_lengths) + + # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN + if self.normalize is not None: + feats, feats_lengths = self.normalize(feats, feats_lengths) + + # Pre-encoder, e.g. used for raw input data + if self.preencoder is not None: + feats, feats_lengths = self.preencoder(feats, feats_lengths) + + # 4. Forward encoder + # feats: (Batch, Length, Dim) + # -> encoder_out: (Batch, Length2, Dim2) + if self.encoder.interctc_use_conditioning: + encoder_out, encoder_out_lens, _ = self.encoder( + feats, feats_lengths, ctc=self.ctc + ) + else: + encoder_out, encoder_out_lens, _ = self.encoder(feats, feats_lengths) + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + + # Post-encoder, e.g. NLU + if self.postencoder is not None: + encoder_out, encoder_out_lens = self.postencoder( + encoder_out, encoder_out_lens + ) + + assert encoder_out.size(0) == speech.size(0), ( + encoder_out.size(), + speech.size(0), + ) + assert encoder_out.size(1) <= encoder_out_lens.max(), ( + encoder_out.size(), + encoder_out_lens.max(), + ) + + if intermediate_outs is not None: + return (encoder_out, intermediate_outs), encoder_out_lens + + return encoder_out, encoder_out_lens + + def _extract_feats( + self, speech: torch.Tensor, speech_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + assert speech_lengths.dim() == 1, speech_lengths.shape + + # for data-parallel + speech = speech[:, : speech_lengths.max()] + + if self.frontend is not None: + # Frontend + # e.g. STFT and Feature extract + # data_loader may send time-domain signal in this case + # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim) + feats, feats_lengths = self.frontend(speech, speech_lengths) + else: + # No frontend and no feature extract + feats, feats_lengths = speech, speech_lengths + return feats, feats_lengths + + def nll( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ) -> torch.Tensor: + """Compute negative log likelihood(nll) from transformer-decoder + + Normally, this function is called in batchify_nll. + + Args: + encoder_out: (Batch, Length, Dim) + encoder_out_lens: (Batch,) + ys_pad: (Batch, Length) + ys_pad_lens: (Batch,) + """ + ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_in_lens = ys_pad_lens + 1 + + # 1. Forward decoder + decoder_out, _ = self.decoder( + encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens + ) # [batch, seqlen, dim] + batch_size = decoder_out.size(0) + decoder_num_class = decoder_out.size(2) + # nll: negative log-likelihood + nll = torch.nn.functional.cross_entropy( + decoder_out.view(-1, decoder_num_class), + ys_out_pad.view(-1), + ignore_index=self.ignore_id, + reduction="none", + ) + nll = nll.view(batch_size, -1) + nll = nll.sum(dim=1) + assert nll.size(0) == batch_size + return nll + + def batchify_nll( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + batch_size: int = 100, + ): + """Compute negative log likelihood(nll) from transformer-decoder + + To avoid OOM, this fuction seperate the input into batches. + Then call nll for each batch and combine and return results. + Args: + encoder_out: (Batch, Length, Dim) + encoder_out_lens: (Batch,) + ys_pad: (Batch, Length) + ys_pad_lens: (Batch,) + batch_size: int, samples each batch contain when computing nll, + you may change this to avoid OOM or increase + GPU memory usage + """ + total_num = encoder_out.size(0) + if total_num <= batch_size: + nll = self.nll(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) + else: + nll = [] + start_idx = 0 + while True: + end_idx = min(start_idx + batch_size, total_num) + batch_encoder_out = encoder_out[start_idx:end_idx, :, :] + batch_encoder_out_lens = encoder_out_lens[start_idx:end_idx] + batch_ys_pad = ys_pad[start_idx:end_idx, :] + batch_ys_pad_lens = ys_pad_lens[start_idx:end_idx] + batch_nll = self.nll( + batch_encoder_out, + batch_encoder_out_lens, + batch_ys_pad, + batch_ys_pad_lens, + ) + nll.append(batch_nll) + start_idx = end_idx + if start_idx == total_num: + break + nll = torch.cat(nll) + assert nll.size(0) == total_num + return nll + + def _calc_att_loss( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_in_lens = ys_pad_lens + 1 + + # 1. Forward decoder + decoder_out, _ = self.decoder( + encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens + ) + + # 2. Compute attention loss + loss_att = self.criterion_att(decoder_out, ys_out_pad) + acc_att = th_accuracy( + decoder_out.view(-1, self.vocab_size), + ys_out_pad, + ignore_label=self.ignore_id, + ) + + # Compute cer/wer using attention-decoder + if self.training or self.error_calculator is None: + cer_att, wer_att = None, None + else: + ys_hat = decoder_out.argmax(dim=-1) + cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) + + return loss_att, acc_att, cer_att, wer_att + + def _calc_ctc_loss( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + # Calc CTC loss + loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) + + # Calc CER using CTC + cer_ctc = None + if not self.training and self.error_calculator is not None: + ys_hat = self.ctc.argmax(encoder_out).data + cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True) + return loss_ctc, cer_ctc diff --git a/funasr/models/e2e_asr_common.py b/funasr/models/e2e_asr_common.py new file mode 100644 index 000000000..92f90796a --- /dev/null +++ b/funasr/models/e2e_asr_common.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Common functions for ASR.""" + +import json +import logging +import sys + +from itertools import groupby +import numpy as np +import six + + +def end_detect(ended_hyps, i, M=3, D_end=np.log(1 * np.exp(-10))): + """End detection. + + described in Eq. (50) of S. Watanabe et al + "Hybrid CTC/Attention Architecture for End-to-End Speech Recognition" + + :param ended_hyps: + :param i: + :param M: + :param D_end: + :return: + """ + if len(ended_hyps) == 0: + return False + count = 0 + best_hyp = sorted(ended_hyps, key=lambda x: x["score"], reverse=True)[0] + for m in six.moves.range(M): + # get ended_hyps with their length is i - m + hyp_length = i - m + hyps_same_length = [x for x in ended_hyps if len(x["yseq"]) == hyp_length] + if len(hyps_same_length) > 0: + best_hyp_same_length = sorted( + hyps_same_length, key=lambda x: x["score"], reverse=True + )[0] + if best_hyp_same_length["score"] - best_hyp["score"] < D_end: + count += 1 + + if count == M: + return True + else: + return False + + +# TODO(takaaki-hori): add different smoothing methods +def label_smoothing_dist(odim, lsm_type, transcript=None, blank=0): + """Obtain label distribution for loss smoothing. + + :param odim: + :param lsm_type: + :param blank: + :param transcript: + :return: + """ + if transcript is not None: + with open(transcript, "rb") as f: + trans_json = json.load(f)["utts"] + + if lsm_type == "unigram": + assert transcript is not None, ( + "transcript is required for %s label smoothing" % lsm_type + ) + labelcount = np.zeros(odim) + for k, v in trans_json.items(): + ids = np.array([int(n) for n in v["output"][0]["tokenid"].split()]) + # to avoid an error when there is no text in an uttrance + if len(ids) > 0: + labelcount[ids] += 1 + labelcount[odim - 1] = len(transcript) # count + labelcount[labelcount == 0] = 1 # flooring + labelcount[blank] = 0 # remove counts for blank + labeldist = labelcount.astype(np.float32) / np.sum(labelcount) + else: + logging.error("Error: unexpected label smoothing type: %s" % lsm_type) + sys.exit() + + return labeldist + + +def get_vgg2l_odim(idim, in_channel=3, out_channel=128): + """Return the output size of the VGG frontend. + + :param in_channel: input channel size + :param out_channel: output channel size + :return: output size + :rtype int + """ + idim = idim / in_channel + idim = np.ceil(np.array(idim, dtype=np.float32) / 2) # 1st max pooling + idim = np.ceil(np.array(idim, dtype=np.float32) / 2) # 2nd max pooling + return int(idim) * out_channel # numer of channels + + +class ErrorCalculator(object): + """Calculate CER and WER for E2E_ASR and CTC models during training. + + :param y_hats: numpy array with predicted text + :param y_pads: numpy array with true (target) text + :param char_list: + :param sym_space: + :param sym_blank: + :return: + """ + + def __init__( + self, char_list, sym_space, sym_blank, report_cer=False, report_wer=False + ): + """Construct an ErrorCalculator object.""" + super(ErrorCalculator, self).__init__() + + self.report_cer = report_cer + self.report_wer = report_wer + + self.char_list = char_list + self.space = sym_space + self.blank = sym_blank + self.idx_blank = self.char_list.index(self.blank) + if self.space in self.char_list: + self.idx_space = self.char_list.index(self.space) + else: + self.idx_space = None + + def __call__(self, ys_hat, ys_pad, is_ctc=False): + """Calculate sentence-level WER/CER score. + + :param torch.Tensor ys_hat: prediction (batch, seqlen) + :param torch.Tensor ys_pad: reference (batch, seqlen) + :param bool is_ctc: calculate CER score for CTC + :return: sentence-level WER score + :rtype float + :return: sentence-level CER score + :rtype float + """ + cer, wer = None, None + if is_ctc: + return self.calculate_cer_ctc(ys_hat, ys_pad) + elif not self.report_cer and not self.report_wer: + return cer, wer + + seqs_hat, seqs_true = self.convert_to_char(ys_hat, ys_pad) + if self.report_cer: + cer = self.calculate_cer(seqs_hat, seqs_true) + + if self.report_wer: + wer = self.calculate_wer(seqs_hat, seqs_true) + return cer, wer + + def calculate_cer_ctc(self, ys_hat, ys_pad): + """Calculate sentence-level CER score for CTC. + + :param torch.Tensor ys_hat: prediction (batch, seqlen) + :param torch.Tensor ys_pad: reference (batch, seqlen) + :return: average sentence-level CER score + :rtype float + """ + import editdistance + + cers, char_ref_lens = [], [] + for i, y in enumerate(ys_hat): + y_hat = [x[0] for x in groupby(y)] + y_true = ys_pad[i] + seq_hat, seq_true = [], [] + for idx in y_hat: + idx = int(idx) + if idx != -1 and idx != self.idx_blank and idx != self.idx_space: + seq_hat.append(self.char_list[int(idx)]) + + for idx in y_true: + idx = int(idx) + if idx != -1 and idx != self.idx_blank and idx != self.idx_space: + seq_true.append(self.char_list[int(idx)]) + + hyp_chars = "".join(seq_hat) + ref_chars = "".join(seq_true) + if len(ref_chars) > 0: + cers.append(editdistance.eval(hyp_chars, ref_chars)) + char_ref_lens.append(len(ref_chars)) + + cer_ctc = float(sum(cers)) / sum(char_ref_lens) if cers else None + return cer_ctc + + def convert_to_char(self, ys_hat, ys_pad): + """Convert index to character. + + :param torch.Tensor seqs_hat: prediction (batch, seqlen) + :param torch.Tensor seqs_true: reference (batch, seqlen) + :return: token list of prediction + :rtype list + :return: token list of reference + :rtype list + """ + seqs_hat, seqs_true = [], [] + for i, y_hat in enumerate(ys_hat): + y_true = ys_pad[i] + eos_true = np.where(y_true == -1)[0] + ymax = eos_true[0] if len(eos_true) > 0 else len(y_true) + # NOTE: padding index (-1) in y_true is used to pad y_hat + seq_hat = [self.char_list[int(idx)] for idx in y_hat[:ymax]] + seq_true = [self.char_list[int(idx)] for idx in y_true if int(idx) != -1] + seq_hat_text = "".join(seq_hat).replace(self.space, " ") + seq_hat_text = seq_hat_text.replace(self.blank, "") + seq_true_text = "".join(seq_true).replace(self.space, " ") + seqs_hat.append(seq_hat_text) + seqs_true.append(seq_true_text) + return seqs_hat, seqs_true + + def calculate_cer(self, seqs_hat, seqs_true): + """Calculate sentence-level CER score. + + :param list seqs_hat: prediction + :param list seqs_true: reference + :return: average sentence-level CER score + :rtype float + """ + import editdistance + + char_eds, char_ref_lens = [], [] + for i, seq_hat_text in enumerate(seqs_hat): + seq_true_text = seqs_true[i] + hyp_chars = seq_hat_text.replace(" ", "") + ref_chars = seq_true_text.replace(" ", "") + char_eds.append(editdistance.eval(hyp_chars, ref_chars)) + char_ref_lens.append(len(ref_chars)) + return float(sum(char_eds)) / sum(char_ref_lens) + + def calculate_wer(self, seqs_hat, seqs_true): + """Calculate sentence-level WER score. + + :param list seqs_hat: prediction + :param list seqs_true: reference + :return: average sentence-level WER score + :rtype float + """ + import editdistance + + word_eds, word_ref_lens = [], [] + for i, seq_hat_text in enumerate(seqs_hat): + seq_true_text = seqs_true[i] + hyp_words = seq_hat_text.split() + ref_words = seq_true_text.split() + word_eds.append(editdistance.eval(hyp_words, ref_words)) + word_ref_lens.append(len(ref_words)) + return float(sum(word_eds)) / sum(word_ref_lens) diff --git a/funasr/models/e2e_asr_paraformer.py b/funasr/models/e2e_asr_paraformer.py new file mode 100644 index 000000000..5ea28f31a --- /dev/null +++ b/funasr/models/e2e_asr_paraformer.py @@ -0,0 +1,820 @@ +import logging +from contextlib import contextmanager +from distutils.version import LooseVersion +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + +import torch +from typeguard import check_argument_types + +from funasr.layers.abs_normalize import AbsNormalize +from funasr.losses.label_smoothing_loss import ( + LabelSmoothingLoss, # noqa: H301 +) +from funasr.models.ctc import CTC +from funasr.models.decoder.abs_decoder import AbsDecoder +from funasr.models.e2e_asr_common import ErrorCalculator +from funasr.models.encoder.abs_encoder import AbsEncoder +from funasr.models.frontend.abs_frontend import AbsFrontend +from funasr.models.postencoder.abs_postencoder import AbsPostEncoder +from funasr.models.preencoder.abs_preencoder import AbsPreEncoder +from funasr.models.specaug.abs_specaug import AbsSpecAug +from funasr.modules.add_sos_eos import add_sos_eos +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.nets_utils import th_accuracy +from funasr.models.predictor.cif import mae_loss +from funasr.torch_utils.device_funcs import force_gatherable +from funasr.train.abs_espnet_model import AbsESPnetModel + +if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): + from torch.cuda.amp import autocast +else: + # Nothing to do if torch<1.6.0 + @contextmanager + def autocast(enabled=True): + yield + +class Paraformer(AbsESPnetModel): + """ + Author: Speech Lab, Alibaba Group, China + Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition + https://arxiv.org/abs/2206.08317 + """ + + def __init__( + self, + vocab_size: int, + token_list: Union[Tuple[str, ...], List[str]], + frontend: Optional[AbsFrontend], + specaug: Optional[AbsSpecAug], + normalize: Optional[AbsNormalize], + preencoder: Optional[AbsPreEncoder], + encoder: AbsEncoder, + postencoder: Optional[AbsPostEncoder], + decoder: AbsDecoder, + ctc: CTC, + ctc_weight: float = 0.5, + interctc_weight: float = 0.0, + ignore_id: int = -1, + blank_id: int = 0, + sos: int = 1, + eos: int = 2, + lsm_weight: float = 0.0, + length_normalized_loss: bool = False, + report_cer: bool = True, + report_wer: bool = True, + sym_space: str = "", + sym_blank: str = "", + extract_feats_in_collect_stats: bool = True, + predictor = None, + predictor_weight: float = 0.0, + predictor_bias: int = 0, + sampling_ratio: float = 0.2, + + ): + assert check_argument_types() + assert 0.0 <= ctc_weight <= 1.0, ctc_weight + assert 0.0 <= interctc_weight < 1.0, interctc_weight + + super().__init__() + # note that eos is the same as sos (equivalent ID) + self.blank_id = blank_id + self.sos = vocab_size - 1 if sos is None else sos + self.eos = vocab_size - 1 if eos is None else eos + self.vocab_size = vocab_size + self.ignore_id = ignore_id + self.ctc_weight = ctc_weight + self.interctc_weight = interctc_weight + self.token_list = token_list.copy() + + self.frontend = frontend + self.specaug = specaug + self.normalize = normalize + self.preencoder = preencoder + self.postencoder = postencoder + self.encoder = encoder + + if not hasattr(self.encoder, "interctc_use_conditioning"): + self.encoder.interctc_use_conditioning = False + if self.encoder.interctc_use_conditioning: + self.encoder.conditioning_layer = torch.nn.Linear( + vocab_size, self.encoder.output_size() + ) + + self.error_calculator = None + + + if ctc_weight == 1.0: + self.decoder = None + else: + self.decoder = decoder + + self.criterion_att = LabelSmoothingLoss( + size=vocab_size, + padding_idx=ignore_id, + smoothing=lsm_weight, + normalize_length=length_normalized_loss, + ) + + if report_cer or report_wer: + self.error_calculator = ErrorCalculator( + token_list, sym_space, sym_blank, report_cer, report_wer + ) + + if ctc_weight == 0.0: + self.ctc = None + else: + self.ctc = ctc + + self.extract_feats_in_collect_stats = extract_feats_in_collect_stats + self.predictor = predictor + self.predictor_weight = predictor_weight + self.predictor_bias = predictor_bias + self.sampling_ratio = sampling_ratio + self.criterion_pre = mae_loss(normalize_length=length_normalized_loss) + self.step_cur = 0 + + + def forward( + self, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + text: torch.Tensor, + text_lengths: torch.Tensor, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + """Frontend + Encoder + Decoder + Calc loss + + Args: + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + text: (Batch, Length) + text_lengths: (Batch,) + """ + assert text_lengths.dim() == 1, text_lengths.shape + # Check that batch_size is unified + assert ( + speech.shape[0] + == speech_lengths.shape[0] + == text.shape[0] + == text_lengths.shape[0] + ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) + batch_size = speech.shape[0] + self.step_cur += 1 + # for data-parallel + text = text[:, : text_lengths.max()] + speech = speech[:, :speech_lengths.max(), :] + + # 1. Encoder + encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + + + loss_att, acc_att, cer_att, wer_att = None, None, None, None + loss_ctc, cer_ctc = None, None + loss_pre = None + stats = dict() + + # 1. CTC branch + if self.ctc_weight != 0.0: + loss_ctc, cer_ctc = self._calc_ctc_loss( + encoder_out, encoder_out_lens, text, text_lengths + ) + + # Collect CTC branch stats + stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None + stats["cer_ctc"] = cer_ctc + + # Intermediate CTC (optional) + loss_interctc = 0.0 + if self.interctc_weight != 0.0 and intermediate_outs is not None: + for layer_idx, intermediate_out in intermediate_outs: + # we assume intermediate_out has the same length & padding + # as those of encoder_out + loss_ic, cer_ic = self._calc_ctc_loss( + intermediate_out, encoder_out_lens, text, text_lengths + ) + loss_interctc = loss_interctc + loss_ic + + # Collect Intermedaite CTC stats + stats["loss_interctc_layer{}".format(layer_idx)] = ( + loss_ic.detach() if loss_ic is not None else None + ) + stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic + + loss_interctc = loss_interctc / len(intermediate_outs) + + # calculate whole encoder loss + loss_ctc = ( + 1 - self.interctc_weight + ) * loss_ctc + self.interctc_weight * loss_interctc + + + # 2b. Attention decoder branch + if self.ctc_weight != 1.0: + + loss_att, acc_att, cer_att, wer_att, loss_pre = self._calc_att_loss( + encoder_out, encoder_out_lens, text, text_lengths + ) + + # 3. CTC-Att loss definition + if self.ctc_weight == 0.0: + loss = loss_att + loss_pre * self.predictor_weight + elif self.ctc_weight == 1.0: + loss = loss_ctc + else: + loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight + + # Collect Attn branch stats + stats["loss_att"] = loss_att.detach() if loss_att is not None else None + stats["acc"] = acc_att + stats["cer"] = cer_att + stats["wer"] = wer_att + stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None + + stats["loss"] =torch.clone(loss.detach()) + + # force_gatherable: to-device and to-tensor if scalar for DataParallel + loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) + return loss, stats, weight + + def collect_feats( + self, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + text: torch.Tensor, + text_lengths: torch.Tensor, + ) -> Dict[str, torch.Tensor]: + if self.extract_feats_in_collect_stats: + feats, feats_lengths = self._extract_feats(speech, speech_lengths) + else: + # Generate dummy stats if extract_feats_in_collect_stats is False + logging.warning( + "Generating dummy stats for feats and feats_lengths, " + "because encoder_conf.extract_feats_in_collect_stats is " + f"{self.extract_feats_in_collect_stats}" + ) + feats, feats_lengths = speech, speech_lengths + return {"feats": feats, "feats_lengths": feats_lengths} + + def encode( + self, speech: torch.Tensor, speech_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Frontend + Encoder. Note that this method is used by asr_inference.py + + Args: + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + """ + with autocast(False): + # 1. Extract feats + feats, feats_lengths = self._extract_feats(speech, speech_lengths) + + # 2. Data augmentation + if self.specaug is not None and self.training: + feats, feats_lengths = self.specaug(feats, feats_lengths) + + # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN + if self.normalize is not None: + feats, feats_lengths = self.normalize(feats, feats_lengths) + + # Pre-encoder, e.g. used for raw input data + if self.preencoder is not None: + feats, feats_lengths = self.preencoder(feats, feats_lengths) + + # 4. Forward encoder + # feats: (Batch, Length, Dim) + # -> encoder_out: (Batch, Length2, Dim2) + if self.encoder.interctc_use_conditioning: + encoder_out, encoder_out_lens, _ = self.encoder( + feats, feats_lengths, ctc=self.ctc + ) + else: + encoder_out, encoder_out_lens, _ = self.encoder(feats, feats_lengths) + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + + # Post-encoder, e.g. NLU + if self.postencoder is not None: + encoder_out, encoder_out_lens = self.postencoder( + encoder_out, encoder_out_lens + ) + + assert encoder_out.size(0) == speech.size(0), ( + encoder_out.size(), + speech.size(0), + ) + assert encoder_out.size(1) <= encoder_out_lens.max(), ( + encoder_out.size(), + encoder_out_lens.max(), + ) + + if intermediate_outs is not None: + return (encoder_out, intermediate_outs), encoder_out_lens + + return encoder_out, encoder_out_lens + + def calc_predictor(self, encoder_out, encoder_out_lens): + + encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device) + pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor(encoder_out, None, encoder_out_mask, ignore_id=self.ignore_id) + return pre_acoustic_embeds, pre_token_length + + + def cal_decoder_with_predictor(self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens): + + decoder_out, _ = self.decoder( + encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens + ) + decoder_out = torch.log_softmax(decoder_out, dim=-1) + return decoder_out, ys_pad_lens + + def _extract_feats( + self, speech: torch.Tensor, speech_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + assert speech_lengths.dim() == 1, speech_lengths.shape + + # for data-parallel + speech = speech[:, : speech_lengths.max()] + if self.frontend is not None: + # Frontend + # e.g. STFT and Feature extract + # data_loader may send time-domain signal in this case + # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim) + feats, feats_lengths = self.frontend(speech, speech_lengths) + else: + # No frontend and no feature extract + feats, feats_lengths = speech, speech_lengths + return feats, feats_lengths + + def nll( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ) -> torch.Tensor: + """Compute negative log likelihood(nll) from transformer-decoder + + Normally, this function is called in batchify_nll. + + Args: + encoder_out: (Batch, Length, Dim) + encoder_out_lens: (Batch,) + ys_pad: (Batch, Length) + ys_pad_lens: (Batch,) + """ + ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_in_lens = ys_pad_lens + 1 + + # 1. Forward decoder + decoder_out, _ = self.decoder( + encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens + ) # [batch, seqlen, dim] + batch_size = decoder_out.size(0) + decoder_num_class = decoder_out.size(2) + # nll: negative log-likelihood + nll = torch.nn.functional.cross_entropy( + decoder_out.view(-1, decoder_num_class), + ys_out_pad.view(-1), + ignore_index=self.ignore_id, + reduction="none", + ) + nll = nll.view(batch_size, -1) + nll = nll.sum(dim=1) + assert nll.size(0) == batch_size + return nll + + def batchify_nll( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + batch_size: int = 100, + ): + """Compute negative log likelihood(nll) from transformer-decoder + + To avoid OOM, this fuction seperate the input into batches. + Then call nll for each batch and combine and return results. + Args: + encoder_out: (Batch, Length, Dim) + encoder_out_lens: (Batch,) + ys_pad: (Batch, Length) + ys_pad_lens: (Batch,) + batch_size: int, samples each batch contain when computing nll, + you may change this to avoid OOM or increase + GPU memory usage + """ + total_num = encoder_out.size(0) + if total_num <= batch_size: + nll = self.nll(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) + else: + nll = [] + start_idx = 0 + while True: + end_idx = min(start_idx + batch_size, total_num) + batch_encoder_out = encoder_out[start_idx:end_idx, :, :] + batch_encoder_out_lens = encoder_out_lens[start_idx:end_idx] + batch_ys_pad = ys_pad[start_idx:end_idx, :] + batch_ys_pad_lens = ys_pad_lens[start_idx:end_idx] + batch_nll = self.nll( + batch_encoder_out, + batch_encoder_out_lens, + batch_ys_pad, + batch_ys_pad_lens, + ) + nll.append(batch_nll) + start_idx = end_idx + if start_idx == total_num: + break + nll = torch.cat(nll) + assert nll.size(0) == total_num + return nll + + def _calc_att_loss( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device) + if self.predictor_bias == 1: + _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_pad_lens = ys_pad_lens + self.predictor_bias + pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor(encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id) + + # 0. sampler + decoder_out_1st = None + if self.sampling_ratio > 0.0: + if self.step_cur < 2: + logging.info("enable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio)) + sematic_embeds, decoder_out_1st = self.sampler(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds) + else: + if self.step_cur < 2: + logging.info("disable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio)) + sematic_embeds = pre_acoustic_embeds + + # 1. Forward decoder + decoder_outs = self.decoder( + encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens + ) + decoder_out, _ = decoder_outs[0], decoder_outs[1] + + if decoder_out_1st is None: + decoder_out_1st = decoder_out + # 2. Compute attention loss + loss_att = self.criterion_att(decoder_out, ys_pad) + acc_att = th_accuracy( + decoder_out_1st.view(-1, self.vocab_size), + ys_pad, + ignore_label=self.ignore_id, + ) + loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length) + + # Compute cer/wer using attention-decoder + if self.training or self.error_calculator is None: + cer_att, wer_att = None, None + else: + ys_hat = decoder_out_1st.argmax(dim=-1) + cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) + + return loss_att, acc_att, cer_att, wer_att, loss_pre + + def sampler(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds): + + tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device) + ys_pad *= tgt_mask[:, :, 0] + ys_pad_embed = self.decoder.embed(ys_pad) + with torch.no_grad(): + decoder_outs = self.decoder( + encoder_out, encoder_out_lens, pre_acoustic_embeds, ys_pad_lens + ) + decoder_out, _ = decoder_outs[0], decoder_outs[1] + pred_tokens = decoder_out.argmax(-1) + nonpad_positions = ys_pad.ne(self.ignore_id) + seq_lens = (nonpad_positions).sum(1) + same_num = ((pred_tokens == ys_pad) & nonpad_positions).sum(1) + input_mask = torch.ones_like(nonpad_positions) + bsz, seq_len = ys_pad.size() + for li in range(bsz): + target_num = (((seq_lens[li] - same_num[li].sum()).float()) * self.sampling_ratio).long() + if target_num > 0: + input_mask[li].scatter_(dim=0, index=torch.randperm(seq_lens[li])[:target_num].cuda(), value=0) + input_mask = input_mask.eq(1) + input_mask = input_mask.masked_fill(~nonpad_positions, False) + input_mask_expand_dim = input_mask.unsqueeze(2).to(pre_acoustic_embeds.device) + + sematic_embeds = pre_acoustic_embeds.masked_fill(~input_mask_expand_dim, 0) + ys_pad_embed.masked_fill( + input_mask_expand_dim, 0) + return sematic_embeds * tgt_mask, decoder_out * tgt_mask + + + def _calc_ctc_loss( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + # Calc CTC loss + loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) + + # Calc CER using CTC + cer_ctc = None + if not self.training and self.error_calculator is not None: + ys_hat = self.ctc.argmax(encoder_out).data + cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True) + return loss_ctc, cer_ctc + +class ParaformerBert(Paraformer): + """ + Author: Speech Lab, Alibaba Group, China + Paraformer2: advanced paraformer with LFMMI and bert for non-autoregressive end-to-end speech recognition + """ + + def __init__( + self, + vocab_size: int, + token_list: Union[Tuple[str, ...], List[str]], + frontend: Optional[AbsFrontend], + specaug: Optional[AbsSpecAug], + normalize: Optional[AbsNormalize], + preencoder: Optional[AbsPreEncoder], + encoder: AbsEncoder, + postencoder: Optional[AbsPostEncoder], + decoder: AbsDecoder, + ctc: CTC, + joint_network: Optional[torch.nn.Module], + ctc_weight: float = 0.5, + interctc_weight: float = 0.0, + ignore_id: int = -1, + blank_id: int = 0, + sos: int = 1, + eos: int = 2, + lsm_weight: float = 0.0, + length_normalized_loss: bool = False, + report_cer: bool = True, + report_wer: bool = True, + sym_space: str = "", + sym_blank: str = "", + extract_feats_in_collect_stats: bool = True, + predictor = None, + predictor_weight: float = 0.0, + predictor_bias: int = 0, + sampling_ratio: float = 0.2, + embeds_id: int = 2, + embeds_loss_weight: float = 0.0, + embed_dims: int = 768, + ): + assert check_argument_types() + assert 0.0 <= ctc_weight <= 1.0, ctc_weight + assert 0.0 <= interctc_weight < 1.0, interctc_weight + + super().__init__( + vocab_size=vocab_size, + token_list=token_list, + frontend=frontend, + specaug=specaug, + normalize=normalize, + preencoder=preencoder, + encoder=encoder, + postencoder=postencoder, + decoder=decoder, + ctc=ctc, + joint_network=joint_network, + ctc_weight=ctc_weight, + interctc_weight=interctc_weight, + ignore_id=ignore_id, + blank_id=blank_id, + sos=sos, + eos=eos, + lsm_weight=lsm_weight, + length_normalized_loss=length_normalized_loss, + report_cer=report_cer, + report_wer=report_wer, + sym_space=sym_space, + sym_blank=sym_blank, + extract_feats_in_collect_stats=extract_feats_in_collect_stats, + predictor=predictor, + predictor_weight=predictor_weight, + predictor_bias=predictor_bias, + sampling_ratio=sampling_ratio, + ) + self.decoder.embeds_id = embeds_id + decoder_attention_dim = self.decoder.attention_dim + self.pro_nn = torch.nn.Linear(decoder_attention_dim, embed_dims) + self.cos = torch.nn.CosineSimilarity(dim=-1, eps=1e-6) + self.embeds_loss_weight = embeds_loss_weight + self.length_normalized_loss = length_normalized_loss + + def _calc_embed_loss(self, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + embed: torch.Tensor = None, + embed_lengths: torch.Tensor = None, + embeds_outputs: torch.Tensor = None, + ): + embeds_outputs = self.pro_nn(embeds_outputs) + tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device) + embeds_outputs *= tgt_mask # b x l x d + embed *= tgt_mask # b x l x d + cos_loss = 1.0 - self.cos(embeds_outputs, embed) + cos_loss *= tgt_mask.squeeze(2) + if self.length_normalized_loss: + token_num_total = torch.sum(tgt_mask) + else: + token_num_total = tgt_mask.size()[0] + cos_loss_total = torch.sum(cos_loss) + cos_loss = cos_loss_total / token_num_total + # print("cos_loss: {}".format(cos_loss)) + return cos_loss + + + def _calc_att_loss( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device) + if self.predictor_bias == 1: + _, ys_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_pad_lens = ys_pad_lens + self.predictor_bias + pre_acoustic_embeds, pre_token_length, _, pre_peak_index = self.predictor(encoder_out, ys_pad, encoder_out_mask, ignore_id=self.ignore_id) + + # 0. sampler + decoder_out_1st = None + if self.sampling_ratio > 0.0: + if self.step_cur < 2: + logging.info( + "enable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio)) + sematic_embeds, decoder_out_1st = self.sampler(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds) + else: + if self.step_cur < 2: + logging.info( + "disable sampler in paraformer, sampling_ratio: {}".format(self.sampling_ratio)) + sematic_embeds = pre_acoustic_embeds + + # 1. Forward decoder + decoder_outs = self.decoder( + encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens + ) + decoder_out, _ = decoder_outs[0], decoder_outs[1] + embeds_outputs = None + if len(decoder_outs) > 2: + embeds_outputs = decoder_outs[2] + + if decoder_out_1st is None: + decoder_out_1st = decoder_out + # 2. Compute attention loss + loss_att = self.criterion_att(decoder_out, ys_pad) + acc_att = th_accuracy( + decoder_out_1st.view(-1, self.vocab_size), + ys_pad, + ignore_label=self.ignore_id, + ) + loss_pre = self.criterion_pre(ys_pad_lens.type_as(pre_token_length), pre_token_length) + + # Compute cer/wer using attention-decoder + if self.training or self.error_calculator is None: + cer_att, wer_att = None, None + else: + ys_hat = decoder_out_1st.argmax(dim=-1) + cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) + + return loss_att, acc_att, cer_att, wer_att, loss_pre, embeds_outputs + + + def forward( + self, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + text: torch.Tensor, + text_lengths: torch.Tensor, + embed: torch.Tensor = None, + embed_lengths: torch.Tensor = None, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + """Frontend + Encoder + Decoder + Calc loss + + Args: + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + text: (Batch, Length) + text_lengths: (Batch,) + """ + assert text_lengths.dim() == 1, text_lengths.shape + # Check that batch_size is unified + assert ( + speech.shape[0] + == speech_lengths.shape[0] + == text.shape[0] + == text_lengths.shape[0] + ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) + batch_size = speech.shape[0] + self.step_cur += 1 + # for data-parallel + text = text[:, : text_lengths.max()] + speech = speech[:, :speech_lengths.max(), :] + if embed is not None: + embed = embed[:, :embed_lengths.max(), :] + + # 1. Encoder + encoder_out, encoder_out_lens = self.encode(speech, speech_lengths) + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + + + loss_att, acc_att, cer_att, wer_att = None, None, None, None + loss_ctc, cer_ctc = None, None + loss_pre = 0.0 + cos_loss = 0.0 + stats = dict() + + # 1. CTC branch + if self.ctc_weight != 0.0: + loss_ctc, cer_ctc = self._calc_ctc_loss( + encoder_out, encoder_out_lens, text, text_lengths + ) + + # Collect CTC branch stats + stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None + stats["cer_ctc"] = cer_ctc + + # Intermediate CTC (optional) + loss_interctc = 0.0 + if self.interctc_weight != 0.0 and intermediate_outs is not None: + for layer_idx, intermediate_out in intermediate_outs: + # we assume intermediate_out has the same length & padding + # as those of encoder_out + loss_ic, cer_ic = self._calc_ctc_loss( + intermediate_out, encoder_out_lens, text, text_lengths + ) + loss_interctc = loss_interctc + loss_ic + + # Collect Intermedaite CTC stats + stats["loss_interctc_layer{}".format(layer_idx)] = ( + loss_ic.detach() if loss_ic is not None else None + ) + stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic + + loss_interctc = loss_interctc / len(intermediate_outs) + + # calculate whole encoder loss + loss_ctc = ( + 1 - self.interctc_weight + ) * loss_ctc + self.interctc_weight * loss_interctc + + + # 2b. Attention decoder branch + if self.ctc_weight != 1.0: + + loss_ret = self._calc_att_loss( + encoder_out, encoder_out_lens, text, text_lengths + ) + loss_att, acc_att, cer_att, wer_att, loss_pre = loss_ret[0], loss_ret[1], loss_ret[2], loss_ret[3], loss_ret[4] + embeds_outputs = None + if len(loss_ret) > 5: + embeds_outputs = loss_ret[5] + if embeds_outputs is not None: + cos_loss = self._calc_embed_loss(text, text_lengths, embed, embed_lengths, embeds_outputs) + + # 3. CTC-Att loss definition + if self.ctc_weight == 0.0: + loss = loss_att + loss_pre * self.predictor_weight + cos_loss * self.embeds_loss_weight + elif self.ctc_weight == 1.0: + loss = loss_ctc + else: + loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight + cos_loss * self.embeds_loss_weight + + # Collect Attn branch stats + stats["loss_att"] = loss_att.detach() if loss_att is not None else None + stats["acc"] = acc_att + stats["cer"] = cer_att + stats["wer"] = wer_att + stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre > 0.0 else None + stats["cos_loss"] = cos_loss.detach().cpu() if cos_loss > 0.0 else None + + stats["loss"] =torch.clone(loss.detach()) + + # force_gatherable: to-device and to-tensor if scalar for DataParallel + loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) + return loss, stats, weight + + + + + + + diff --git a/funasr/models/e2e_uni_asr.py b/funasr/models/e2e_uni_asr.py new file mode 100644 index 000000000..03fbca9af --- /dev/null +++ b/funasr/models/e2e_uni_asr.py @@ -0,0 +1,1076 @@ +import logging +from contextlib import contextmanager +from distutils.version import LooseVersion +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + +import torch +from typeguard import check_argument_types + +from funasr.models.e2e_asr_common import ErrorCalculator +from funasr.modules.nets_utils import th_accuracy +from funasr.modules.add_sos_eos import add_sos_eos +from funasr.losses.label_smoothing_loss import ( + LabelSmoothingLoss, # noqa: H301 +) +from funasr.models.ctc import CTC +from funasr.models.decoder.abs_decoder import AbsDecoder +from funasr.models.encoder.abs_encoder import AbsEncoder +from funasr.models.frontend.abs_frontend import AbsFrontend +from funasr.models.postencoder.abs_postencoder import AbsPostEncoder +from funasr.models.preencoder.abs_preencoder import AbsPreEncoder +from funasr.models.specaug.abs_specaug import AbsSpecAug +from funasr.layers.abs_normalize import AbsNormalize +from funasr.torch_utils.device_funcs import force_gatherable +from funasr.train.abs_espnet_model import AbsESPnetModel +from funasr.modules.streaming_utils.chunk_utilis import sequence_mask +from funasr.models.predictor.cif import mae_loss + +if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): + from torch.cuda.amp import autocast +else: + # Nothing to do if torch<1.6.0 + @contextmanager + def autocast(enabled=True): + yield + + +class UniASR(AbsESPnetModel): + """ + Author: Speech Lab, Alibaba Group, China + """ + + def __init__( + self, + vocab_size: int, + token_list: Union[Tuple[str, ...], List[str]], + frontend: Optional[AbsFrontend], + specaug: Optional[AbsSpecAug], + normalize: Optional[AbsNormalize], + preencoder: Optional[AbsPreEncoder], + encoder: AbsEncoder, + postencoder: Optional[AbsPostEncoder], + decoder: AbsDecoder, + ctc: CTC, + ctc_weight: float = 0.5, + interctc_weight: float = 0.0, + ignore_id: int = -1, + lsm_weight: float = 0.0, + length_normalized_loss: bool = False, + report_cer: bool = True, + report_wer: bool = True, + sym_space: str = "", + sym_blank: str = "", + extract_feats_in_collect_stats: bool = True, + predictor=None, + predictor_weight: float = 0.0, + decoder_attention_chunk_type: str = 'chunk', + encoder2: AbsEncoder = None, + decoder2: AbsDecoder = None, + ctc2: CTC = None, + ctc_weight2: float = 0.5, + interctc_weight2: float = 0.0, + predictor2=None, + predictor_weight2: float = 0.0, + decoder_attention_chunk_type2: str = 'chunk', + stride_conv=None, + loss_weight_model1: float = 0.5, + enable_maas_finetune: bool = False, + freeze_encoder2: bool = False, + encoder1_encoder2_joint_training: bool = True, + ): + assert check_argument_types() + assert 0.0 <= ctc_weight <= 1.0, ctc_weight + assert 0.0 <= interctc_weight < 1.0, interctc_weight + + super().__init__() + self.blank_id = 0 + self.sos = 1 + self.eos = 2 + self.vocab_size = vocab_size + self.ignore_id = ignore_id + self.ctc_weight = ctc_weight + self.interctc_weight = interctc_weight + self.token_list = token_list.copy() + + self.frontend = frontend + self.specaug = specaug + self.normalize = normalize + self.preencoder = preencoder + self.postencoder = postencoder + self.encoder = encoder + + if not hasattr(self.encoder, "interctc_use_conditioning"): + self.encoder.interctc_use_conditioning = False + if self.encoder.interctc_use_conditioning: + self.encoder.conditioning_layer = torch.nn.Linear( + vocab_size, self.encoder.output_size() + ) + + self.error_calculator = None + + # we set self.decoder = None in the CTC mode since + # self.decoder parameters were never used and PyTorch complained + # and threw an Exception in the multi-GPU experiment. + # thanks Jeff Farris for pointing out the issue. + if ctc_weight == 1.0: + self.decoder = None + else: + self.decoder = decoder + + self.criterion_att = LabelSmoothingLoss( + size=vocab_size, + padding_idx=ignore_id, + smoothing=lsm_weight, + normalize_length=length_normalized_loss, + ) + + if report_cer or report_wer: + self.error_calculator = ErrorCalculator( + token_list, sym_space, sym_blank, report_cer, report_wer + ) + + if ctc_weight == 0.0: + self.ctc = None + else: + self.ctc = ctc + + self.extract_feats_in_collect_stats = extract_feats_in_collect_stats + self.predictor = predictor + self.predictor_weight = predictor_weight + self.criterion_pre = mae_loss(normalize_length=length_normalized_loss) + self.step_cur = 0 + if self.encoder.overlap_chunk_cls is not None: + from funasr.modules.streaming_utils.chunk_utilis import build_scama_mask_for_cross_attention_decoder + self.build_scama_mask_for_cross_attention_decoder_fn = build_scama_mask_for_cross_attention_decoder + self.decoder_attention_chunk_type = decoder_attention_chunk_type + + self.encoder2 = encoder2 + self.decoder2 = decoder2 + self.ctc_weight2 = ctc_weight2 + if ctc_weight2 == 0.0: + self.ctc2 = None + else: + self.ctc2 = ctc2 + self.interctc_weight2 = interctc_weight2 + self.predictor2 = predictor2 + self.predictor_weight2 = predictor_weight2 + self.decoder_attention_chunk_type2 = decoder_attention_chunk_type2 + self.stride_conv = stride_conv + self.loss_weight_model1 = loss_weight_model1 + if self.encoder2.overlap_chunk_cls is not None: + from funasr.modules.streaming_utils.chunk_utilis import build_scama_mask_for_cross_attention_decoder + self.build_scama_mask_for_cross_attention_decoder_fn2 = build_scama_mask_for_cross_attention_decoder + self.decoder_attention_chunk_type2 = decoder_attention_chunk_type2 + + self.enable_maas_finetune = enable_maas_finetune + self.freeze_encoder2 = freeze_encoder2 + self.encoder1_encoder2_joint_training = encoder1_encoder2_joint_training + + def forward( + self, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + text: torch.Tensor, + text_lengths: torch.Tensor, + decoding_ind: int = None, + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + """Frontend + Encoder + Decoder + Calc loss + + Args: + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + text: (Batch, Length) + text_lengths: (Batch,) + """ + assert text_lengths.dim() == 1, text_lengths.shape + # Check that batch_size is unified + assert ( + speech.shape[0] + == speech_lengths.shape[0] + == text.shape[0] + == text_lengths.shape[0] + ), (speech.shape, speech_lengths.shape, text.shape, text_lengths.shape) + batch_size = speech.shape[0] + + # for data-parallel + text = text[:, : text_lengths.max()] + speech = speech[:, :speech_lengths.max(), :] + + ind = self.encoder.overlap_chunk_cls.random_choice(self.training, decoding_ind) + speech_raw = speech.clone().to(speech.device) + # 1. Encoder + if self.enable_maas_finetune: + with torch.no_grad(): + encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind) + else: + encoder_out, encoder_out_lens = self.encode(speech, speech_lengths, ind=ind) + + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + + loss_att, acc_att, cer_att, wer_att = None, None, None, None + loss_ctc, cer_ctc = None, None + stats = dict() + loss_pre = None + loss, loss1, loss2 = 0.0, 0.0, 0.0 + + if self.loss_weight_model1 > 0.0: + ## model1 + # 1. CTC branch + if self.enable_maas_finetune: + with torch.no_grad(): + if self.ctc_weight != 0.0: + if self.encoder.overlap_chunk_cls is not None: + encoder_out_ctc, encoder_out_lens_ctc = self.encoder.overlap_chunk_cls.remove_chunk(encoder_out, + encoder_out_lens, + chunk_outs=None) + loss_ctc, cer_ctc = self._calc_ctc_loss( + encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths + ) + + # Collect CTC branch stats + stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None + stats["cer_ctc"] = cer_ctc + + # Intermediate CTC (optional) + loss_interctc = 0.0 + if self.interctc_weight != 0.0 and intermediate_outs is not None: + for layer_idx, intermediate_out in intermediate_outs: + # we assume intermediate_out has the same length & padding + # as those of encoder_out + if self.encoder.overlap_chunk_cls is not None: + encoder_out_ctc, encoder_out_lens_ctc = \ + self.encoder.overlap_chunk_cls.remove_chunk( + intermediate_out, + encoder_out_lens, + chunk_outs=None) + loss_ic, cer_ic = self._calc_ctc_loss( + encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths + ) + loss_interctc = loss_interctc + loss_ic + + # Collect Intermedaite CTC stats + stats["loss_interctc_layer{}".format(layer_idx)] = ( + loss_ic.detach() if loss_ic is not None else None + ) + stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic + + loss_interctc = loss_interctc / len(intermediate_outs) + + # calculate whole encoder loss + loss_ctc = ( + 1 - self.interctc_weight + ) * loss_ctc + self.interctc_weight * loss_interctc + + # 2b. Attention decoder branch + if self.ctc_weight != 1.0: + loss_att, acc_att, cer_att, wer_att, loss_pre = self._calc_att_predictor_loss( + encoder_out, encoder_out_lens, text, text_lengths + ) + + # 3. CTC-Att loss definition + if self.ctc_weight == 0.0: + loss = loss_att + loss_pre * self.predictor_weight + elif self.ctc_weight == 1.0: + loss = loss_ctc + else: + loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight + + # Collect Attn branch stats + stats["loss_att"] = loss_att.detach() if loss_att is not None else None + stats["acc"] = acc_att + stats["cer"] = cer_att + stats["wer"] = wer_att + stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None + else: + if self.ctc_weight != 0.0: + if self.encoder.overlap_chunk_cls is not None: + encoder_out_ctc, encoder_out_lens_ctc = self.encoder.overlap_chunk_cls.remove_chunk(encoder_out, + encoder_out_lens, + chunk_outs=None) + loss_ctc, cer_ctc = self._calc_ctc_loss( + encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths + ) + + # Collect CTC branch stats + stats["loss_ctc"] = loss_ctc.detach() if loss_ctc is not None else None + stats["cer_ctc"] = cer_ctc + + # Intermediate CTC (optional) + loss_interctc = 0.0 + if self.interctc_weight != 0.0 and intermediate_outs is not None: + for layer_idx, intermediate_out in intermediate_outs: + # we assume intermediate_out has the same length & padding + # as those of encoder_out + if self.encoder.overlap_chunk_cls is not None: + encoder_out_ctc, encoder_out_lens_ctc = \ + self.encoder.overlap_chunk_cls.remove_chunk( + intermediate_out, + encoder_out_lens, + chunk_outs=None) + loss_ic, cer_ic = self._calc_ctc_loss( + encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths + ) + loss_interctc = loss_interctc + loss_ic + + # Collect Intermedaite CTC stats + stats["loss_interctc_layer{}".format(layer_idx)] = ( + loss_ic.detach() if loss_ic is not None else None + ) + stats["cer_interctc_layer{}".format(layer_idx)] = cer_ic + + loss_interctc = loss_interctc / len(intermediate_outs) + + # calculate whole encoder loss + loss_ctc = ( + 1 - self.interctc_weight + ) * loss_ctc + self.interctc_weight * loss_interctc + + # 2b. Attention decoder branch + if self.ctc_weight != 1.0: + loss_att, acc_att, cer_att, wer_att, loss_pre = self._calc_att_predictor_loss( + encoder_out, encoder_out_lens, text, text_lengths + ) + + # 3. CTC-Att loss definition + if self.ctc_weight == 0.0: + loss = loss_att + loss_pre * self.predictor_weight + elif self.ctc_weight == 1.0: + loss = loss_ctc + else: + loss = self.ctc_weight * loss_ctc + (1 - self.ctc_weight) * loss_att + loss_pre * self.predictor_weight + + # Collect Attn branch stats + stats["loss_att"] = loss_att.detach() if loss_att is not None else None + stats["acc"] = acc_att + stats["cer"] = cer_att + stats["wer"] = wer_att + stats["loss_pre"] = loss_pre.detach().cpu() if loss_pre is not None else None + + loss1 = loss + + if self.loss_weight_model1 < 1.0: + ## model2 + + # encoder2 + if self.freeze_encoder2: + with torch.no_grad(): + encoder_out, encoder_out_lens = self.encode2(encoder_out, encoder_out_lens, speech_raw, speech_lengths, ind=ind) + else: + encoder_out, encoder_out_lens = self.encode2(encoder_out, encoder_out_lens, speech_raw, speech_lengths, ind=ind) + + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + # CTC2 + if self.ctc_weight2 != 0.0: + if self.encoder2.overlap_chunk_cls is not None: + encoder_out_ctc, encoder_out_lens_ctc = \ + self.encoder2.overlap_chunk_cls.remove_chunk( + encoder_out, + encoder_out_lens, + chunk_outs=None, + ) + loss_ctc, cer_ctc = self._calc_ctc_loss2( + encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths + ) + + # Collect CTC branch stats + stats["loss_ctc2"] = loss_ctc.detach() if loss_ctc is not None else None + stats["cer_ctc2"] = cer_ctc + + # Intermediate CTC (optional) + loss_interctc = 0.0 + if self.interctc_weight2 != 0.0 and intermediate_outs is not None: + for layer_idx, intermediate_out in intermediate_outs: + # we assume intermediate_out has the same length & padding + # as those of encoder_out + if self.encoder2.overlap_chunk_cls is not None: + encoder_out_ctc, encoder_out_lens_ctc = \ + self.encoder2.overlap_chunk_cls.remove_chunk( + intermediate_out, + encoder_out_lens, + chunk_outs=None) + loss_ic, cer_ic = self._calc_ctc_loss2( + encoder_out_ctc, encoder_out_lens_ctc, text, text_lengths + ) + loss_interctc = loss_interctc + loss_ic + + # Collect Intermedaite CTC stats + stats["loss_interctc_layer{}2".format(layer_idx)] = ( + loss_ic.detach() if loss_ic is not None else None + ) + stats["cer_interctc_layer{}2".format(layer_idx)] = cer_ic + + loss_interctc = loss_interctc / len(intermediate_outs) + + # calculate whole encoder loss + loss_ctc = ( + 1 - self.interctc_weight2 + ) * loss_ctc + self.interctc_weight2 * loss_interctc + + # 2b. Attention decoder branch + if self.ctc_weight2 != 1.0: + loss_att, acc_att, cer_att, wer_att, loss_pre = self._calc_att_predictor_loss2( + encoder_out, encoder_out_lens, text, text_lengths + ) + + # 3. CTC-Att loss definition + if self.ctc_weight2 == 0.0: + loss = loss_att + loss_pre * self.predictor_weight2 + elif self.ctc_weight2 == 1.0: + loss = loss_ctc + else: + loss = self.ctc_weight2 * loss_ctc + ( + 1 - self.ctc_weight2) * loss_att + loss_pre * self.predictor_weight2 + + # Collect Attn branch stats + stats["loss_att2"] = loss_att.detach() if loss_att is not None else None + stats["acc2"] = acc_att + stats["cer2"] = cer_att + stats["wer2"] = wer_att + stats["loss_pre2"] = loss_pre.detach().cpu() if loss_pre is not None else None + loss2 = loss + + loss = loss1 * self.loss_weight_model1 + loss2 * (1 - self.loss_weight_model1) + stats["loss1"] = torch.clone(loss1.detach()) + stats["loss2"] = torch.clone(loss2.detach()) + stats["loss"] = torch.clone(loss.detach()) + # force_gatherable: to-device and to-tensor if scalar for DataParallel + loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device) + return loss, stats, weight + + def collect_feats( + self, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + text: torch.Tensor, + text_lengths: torch.Tensor, + ) -> Dict[str, torch.Tensor]: + if self.extract_feats_in_collect_stats: + feats, feats_lengths = self._extract_feats(speech, speech_lengths) + else: + # Generate dummy stats if extract_feats_in_collect_stats is False + logging.warning( + "Generating dummy stats for feats and feats_lengths, " + "because encoder_conf.extract_feats_in_collect_stats is " + f"{self.extract_feats_in_collect_stats}" + ) + feats, feats_lengths = speech, speech_lengths + return {"feats": feats, "feats_lengths": feats_lengths} + + def encode( + self, speech: torch.Tensor, speech_lengths: torch.Tensor, ind: int = 0, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Frontend + Encoder. Note that this method is used by asr_inference.py + + Args: + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + """ + with autocast(False): + # 1. Extract feats + feats, feats_lengths = self._extract_feats(speech, speech_lengths) + + # 2. Data augmentation + if self.specaug is not None and self.training: + feats, feats_lengths = self.specaug(feats, feats_lengths) + + # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN + if self.normalize is not None: + feats, feats_lengths = self.normalize(feats, feats_lengths) + + # Pre-encoder, e.g. used for raw input data + if self.preencoder is not None: + feats, feats_lengths = self.preencoder(feats, feats_lengths) + + # 4. Forward encoder + # feats: (Batch, Length, Dim) + # -> encoder_out: (Batch, Length2, Dim2) + if self.encoder.interctc_use_conditioning: + encoder_out, encoder_out_lens, _ = self.encoder( + feats, feats_lengths, ctc=self.ctc, ind=ind + ) + else: + encoder_out, encoder_out_lens, _ = self.encoder(feats, feats_lengths, ind=ind) + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + + # Post-encoder, e.g. NLU + if self.postencoder is not None: + encoder_out, encoder_out_lens = self.postencoder( + encoder_out, encoder_out_lens + ) + + assert encoder_out.size(0) == speech.size(0), ( + encoder_out.size(), + speech.size(0), + ) + assert encoder_out.size(1) <= encoder_out_lens.max(), ( + encoder_out.size(), + encoder_out_lens.max(), + ) + + if intermediate_outs is not None: + return (encoder_out, intermediate_outs), encoder_out_lens + + return encoder_out, encoder_out_lens + + def encode2( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + ind: int = 0, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Frontend + Encoder. Note that this method is used by asr_inference.py + + Args: + speech: (Batch, Length, ...) + speech_lengths: (Batch, ) + """ + # with autocast(False): + # # 1. Extract feats + # feats, feats_lengths = self._extract_feats(speech, speech_lengths) + # + # # 2. Data augmentation + # if self.specaug is not None and self.training: + # feats, feats_lengths = self.specaug(feats, feats_lengths) + # + # # 3. Normalization for feature: e.g. Global-CMVN, Utterance-CMVN + # if self.normalize is not None: + # feats, feats_lengths = self.normalize(feats, feats_lengths) + + # Pre-encoder, e.g. used for raw input data + # if self.preencoder is not None: + # feats, feats_lengths = self.preencoder(feats, feats_lengths) + encoder_out_rm, encoder_out_lens_rm = self.encoder.overlap_chunk_cls.remove_chunk( + encoder_out, + encoder_out_lens, + chunk_outs=None, + ) + # residual_input + encoder_out = torch.cat((speech, encoder_out_rm), dim=-1) + encoder_out_lens = encoder_out_lens_rm + if self.stride_conv is not None: + speech, speech_lengths = self.stride_conv(encoder_out, encoder_out_lens) + if not self.encoder1_encoder2_joint_training: + speech = speech.detach() + speech_lengths = speech_lengths.detach() + # 4. Forward encoder + # feats: (Batch, Length, Dim) + # -> encoder_out: (Batch, Length2, Dim2) + if self.encoder2.interctc_use_conditioning: + encoder_out, encoder_out_lens, _ = self.encoder2( + speech, speech_lengths, ctc=self.ctc2, ind=ind + ) + else: + encoder_out, encoder_out_lens, _ = self.encoder2(speech, speech_lengths, ind=ind) + intermediate_outs = None + if isinstance(encoder_out, tuple): + intermediate_outs = encoder_out[1] + encoder_out = encoder_out[0] + + # # Post-encoder, e.g. NLU + # if self.postencoder is not None: + # encoder_out, encoder_out_lens = self.postencoder( + # encoder_out, encoder_out_lens + # ) + + assert encoder_out.size(0) == speech.size(0), ( + encoder_out.size(), + speech.size(0), + ) + assert encoder_out.size(1) <= encoder_out_lens.max(), ( + encoder_out.size(), + encoder_out_lens.max(), + ) + + if intermediate_outs is not None: + return (encoder_out, intermediate_outs), encoder_out_lens + + return encoder_out, encoder_out_lens + + def _extract_feats( + self, speech: torch.Tensor, speech_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + assert speech_lengths.dim() == 1, speech_lengths.shape + + # for data-parallel + speech = speech[:, : speech_lengths.max()] + + if self.frontend is not None: + # Frontend + # e.g. STFT and Feature extract + # data_loader may send time-domain signal in this case + # speech (Batch, NSamples) -> feats: (Batch, NFrames, Dim) + feats, feats_lengths = self.frontend(speech, speech_lengths) + else: + # No frontend and no feature extract + feats, feats_lengths = speech, speech_lengths + return feats, feats_lengths + + def nll( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ) -> torch.Tensor: + """Compute negative log likelihood(nll) from transformer-decoder + + Normally, this function is called in batchify_nll. + + Args: + encoder_out: (Batch, Length, Dim) + encoder_out_lens: (Batch,) + ys_pad: (Batch, Length) + ys_pad_lens: (Batch,) + """ + ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_in_lens = ys_pad_lens + 1 + + # 1. Forward decoder + decoder_out, _ = self.decoder( + encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens + ) # [batch, seqlen, dim] + batch_size = decoder_out.size(0) + decoder_num_class = decoder_out.size(2) + # nll: negative log-likelihood + nll = torch.nn.functional.cross_entropy( + decoder_out.view(-1, decoder_num_class), + ys_out_pad.view(-1), + ignore_index=self.ignore_id, + reduction="none", + ) + nll = nll.view(batch_size, -1) + nll = nll.sum(dim=1) + assert nll.size(0) == batch_size + return nll + + def batchify_nll( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + batch_size: int = 100, + ): + """Compute negative log likelihood(nll) from transformer-decoder + + To avoid OOM, this fuction seperate the input into batches. + Then call nll for each batch and combine and return results. + Args: + encoder_out: (Batch, Length, Dim) + encoder_out_lens: (Batch,) + ys_pad: (Batch, Length) + ys_pad_lens: (Batch,) + batch_size: int, samples each batch contain when computing nll, + you may change this to avoid OOM or increase + GPU memory usage + """ + total_num = encoder_out.size(0) + if total_num <= batch_size: + nll = self.nll(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) + else: + nll = [] + start_idx = 0 + while True: + end_idx = min(start_idx + batch_size, total_num) + batch_encoder_out = encoder_out[start_idx:end_idx, :, :] + batch_encoder_out_lens = encoder_out_lens[start_idx:end_idx] + batch_ys_pad = ys_pad[start_idx:end_idx, :] + batch_ys_pad_lens = ys_pad_lens[start_idx:end_idx] + batch_nll = self.nll( + batch_encoder_out, + batch_encoder_out_lens, + batch_ys_pad, + batch_ys_pad_lens, + ) + nll.append(batch_nll) + start_idx = end_idx + if start_idx == total_num: + break + nll = torch.cat(nll) + assert nll.size(0) == total_num + return nll + + def _calc_att_loss( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_in_lens = ys_pad_lens + 1 + + # 1. Forward decoder + decoder_out, _ = self.decoder( + encoder_out, encoder_out_lens, ys_in_pad, ys_in_lens + ) + + # 2. Compute attention loss + loss_att = self.criterion_att(decoder_out, ys_out_pad) + acc_att = th_accuracy( + decoder_out.view(-1, self.vocab_size), + ys_out_pad, + ignore_label=self.ignore_id, + ) + + # Compute cer/wer using attention-decoder + if self.training or self.error_calculator is None: + cer_att, wer_att = None, None + else: + ys_hat = decoder_out.argmax(dim=-1) + cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) + + return loss_att, acc_att, cer_att, wer_att + + def _calc_att_predictor_loss( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_in_lens = ys_pad_lens + 1 + + encoder_out_mask = sequence_mask(encoder_out_lens, maxlen=encoder_out.size(1), dtype=encoder_out.dtype, + device=encoder_out.device)[:, None, :] + mask_chunk_predictor = None + if self.encoder.overlap_chunk_cls is not None: + mask_chunk_predictor = self.encoder.overlap_chunk_cls.get_mask_chunk_predictor(None, + device=encoder_out.device, + batch_size=encoder_out.size( + 0)) + mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk(None, device=encoder_out.device, + batch_size=encoder_out.size(0)) + encoder_out = encoder_out * mask_shfit_chunk + pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor(encoder_out, + ys_out_pad, + encoder_out_mask, + ignore_id=self.ignore_id, + mask_chunk_predictor=mask_chunk_predictor, + target_label_length=ys_in_lens, + ) + predictor_alignments, predictor_alignments_len = self.predictor.gen_frame_alignments(pre_alphas, + encoder_out_lens) + + scama_mask = None + if self.encoder.overlap_chunk_cls is not None and self.decoder_attention_chunk_type == 'chunk': + encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur + attention_chunk_center_bias = 0 + attention_chunk_size = encoder_chunk_size + decoder_att_look_back_factor = self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur + mask_shift_att_chunk_decoder = self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder(None, + device=encoder_out.device, + batch_size=encoder_out.size( + 0)) + scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn( + predictor_alignments=predictor_alignments, + encoder_sequence_length=encoder_out_lens, + chunk_size=1, + encoder_chunk_size=encoder_chunk_size, + attention_chunk_center_bias=attention_chunk_center_bias, + attention_chunk_size=attention_chunk_size, + attention_chunk_type=self.decoder_attention_chunk_type, + step=None, + predictor_mask_chunk_hopping=mask_chunk_predictor, + decoder_att_look_back_factor=decoder_att_look_back_factor, + mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, + target_length=ys_in_lens, + is_training=self.training, + ) + elif self.encoder.overlap_chunk_cls is not None: + encoder_out, encoder_out_lens = self.encoder.overlap_chunk_cls.remove_chunk(encoder_out, encoder_out_lens, + chunk_outs=None) + # try: + # 1. Forward decoder + decoder_out, _ = self.decoder( + encoder_out, + encoder_out_lens, + ys_in_pad, + ys_in_lens, + chunk_mask=scama_mask, + pre_acoustic_embeds=pre_acoustic_embeds, + + ) + + # 2. Compute attention loss + loss_att = self.criterion_att(decoder_out, ys_out_pad) + acc_att = th_accuracy( + decoder_out.view(-1, self.vocab_size), + ys_out_pad, + ignore_label=self.ignore_id, + ) + # predictor loss + loss_pre = self.criterion_pre(ys_in_lens.type_as(pre_token_length), pre_token_length) + # Compute cer/wer using attention-decoder + if self.training or self.error_calculator is None: + cer_att, wer_att = None, None + else: + ys_hat = decoder_out.argmax(dim=-1) + cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) + + return loss_att, acc_att, cer_att, wer_att, loss_pre + + def _calc_att_predictor_loss2( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + ys_in_lens = ys_pad_lens + 1 + + encoder_out_mask = sequence_mask(encoder_out_lens, maxlen=encoder_out.size(1), dtype=encoder_out.dtype, + device=encoder_out.device)[:, None, :] + mask_chunk_predictor = None + if self.encoder2.overlap_chunk_cls is not None: + mask_chunk_predictor = self.encoder2.overlap_chunk_cls.get_mask_chunk_predictor(None, + device=encoder_out.device, + batch_size=encoder_out.size( + 0)) + mask_shfit_chunk = self.encoder2.overlap_chunk_cls.get_mask_shfit_chunk(None, device=encoder_out.device, + batch_size=encoder_out.size(0)) + encoder_out = encoder_out * mask_shfit_chunk + pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor2(encoder_out, + ys_out_pad, + encoder_out_mask, + ignore_id=self.ignore_id, + mask_chunk_predictor=mask_chunk_predictor, + target_label_length=ys_in_lens, + ) + predictor_alignments, predictor_alignments_len = self.predictor2.gen_frame_alignments(pre_alphas, + encoder_out_lens) + + scama_mask = None + if self.encoder2.overlap_chunk_cls is not None and self.decoder_attention_chunk_type2 == 'chunk': + encoder_chunk_size = self.encoder2.overlap_chunk_cls.chunk_size_pad_shift_cur + attention_chunk_center_bias = 0 + attention_chunk_size = encoder_chunk_size + decoder_att_look_back_factor = self.encoder2.overlap_chunk_cls.decoder_att_look_back_factor_cur + mask_shift_att_chunk_decoder = self.encoder2.overlap_chunk_cls.get_mask_shift_att_chunk_decoder(None, + device=encoder_out.device, + batch_size=encoder_out.size( + 0)) + scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn2( + predictor_alignments=predictor_alignments, + encoder_sequence_length=encoder_out_lens, + chunk_size=1, + encoder_chunk_size=encoder_chunk_size, + attention_chunk_center_bias=attention_chunk_center_bias, + attention_chunk_size=attention_chunk_size, + attention_chunk_type=self.decoder_attention_chunk_type2, + step=None, + predictor_mask_chunk_hopping=mask_chunk_predictor, + decoder_att_look_back_factor=decoder_att_look_back_factor, + mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, + target_length=ys_in_lens, + is_training=self.training, + ) + elif self.encoder2.overlap_chunk_cls is not None: + encoder_out, encoder_out_lens = self.encoder2.overlap_chunk_cls.remove_chunk(encoder_out, encoder_out_lens, + chunk_outs=None) + # try: + # 1. Forward decoder + decoder_out, _ = self.decoder2( + encoder_out, + encoder_out_lens, + ys_in_pad, + ys_in_lens, + chunk_mask=scama_mask, + pre_acoustic_embeds=pre_acoustic_embeds, + ) + + # 2. Compute attention loss + loss_att = self.criterion_att(decoder_out, ys_out_pad) + acc_att = th_accuracy( + decoder_out.view(-1, self.vocab_size), + ys_out_pad, + ignore_label=self.ignore_id, + ) + # predictor loss + loss_pre = self.criterion_pre(ys_in_lens.type_as(pre_token_length), pre_token_length) + # Compute cer/wer using attention-decoder + if self.training or self.error_calculator is None: + cer_att, wer_att = None, None + else: + ys_hat = decoder_out.argmax(dim=-1) + cer_att, wer_att = self.error_calculator(ys_hat.cpu(), ys_pad.cpu()) + + return loss_att, acc_att, cer_att, wer_att, loss_pre + + def calc_predictor_mask( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor = None, + ys_pad_lens: torch.Tensor = None, + ): + # ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + # ys_in_lens = ys_pad_lens + 1 + ys_out_pad, ys_in_lens = None, None + + encoder_out_mask = sequence_mask(encoder_out_lens, maxlen=encoder_out.size(1), dtype=encoder_out.dtype, + device=encoder_out.device)[:, None, :] + mask_chunk_predictor = None + if self.encoder.overlap_chunk_cls is not None: + mask_chunk_predictor = self.encoder.overlap_chunk_cls.get_mask_chunk_predictor(None, + device=encoder_out.device, + batch_size=encoder_out.size( + 0)) + mask_shfit_chunk = self.encoder.overlap_chunk_cls.get_mask_shfit_chunk(None, device=encoder_out.device, + batch_size=encoder_out.size(0)) + encoder_out = encoder_out * mask_shfit_chunk + pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor(encoder_out, + ys_out_pad, + encoder_out_mask, + ignore_id=self.ignore_id, + mask_chunk_predictor=mask_chunk_predictor, + target_label_length=ys_in_lens, + ) + predictor_alignments, predictor_alignments_len = self.predictor.gen_frame_alignments(pre_alphas, + encoder_out_lens) + + scama_mask = None + if self.encoder.overlap_chunk_cls is not None and self.decoder_attention_chunk_type == 'chunk': + encoder_chunk_size = self.encoder.overlap_chunk_cls.chunk_size_pad_shift_cur + attention_chunk_center_bias = 0 + attention_chunk_size = encoder_chunk_size + decoder_att_look_back_factor = self.encoder.overlap_chunk_cls.decoder_att_look_back_factor_cur + mask_shift_att_chunk_decoder = self.encoder.overlap_chunk_cls.get_mask_shift_att_chunk_decoder(None, + device=encoder_out.device, + batch_size=encoder_out.size( + 0)) + scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn( + predictor_alignments=predictor_alignments, + encoder_sequence_length=encoder_out_lens, + chunk_size=1, + encoder_chunk_size=encoder_chunk_size, + attention_chunk_center_bias=attention_chunk_center_bias, + attention_chunk_size=attention_chunk_size, + attention_chunk_type=self.decoder_attention_chunk_type, + step=None, + predictor_mask_chunk_hopping=mask_chunk_predictor, + decoder_att_look_back_factor=decoder_att_look_back_factor, + mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, + target_length=ys_in_lens, + is_training=self.training, + ) + elif self.encoder.overlap_chunk_cls is not None: + encoder_out, encoder_out_lens = self.encoder.overlap_chunk_cls.remove_chunk(encoder_out, encoder_out_lens, + chunk_outs=None) + + return pre_acoustic_embeds, pre_token_length, predictor_alignments, predictor_alignments_len, scama_mask + + def calc_predictor_mask2( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor = None, + ys_pad_lens: torch.Tensor = None, + ): + # ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id) + # ys_in_lens = ys_pad_lens + 1 + ys_out_pad, ys_in_lens = None, None + + encoder_out_mask = sequence_mask(encoder_out_lens, maxlen=encoder_out.size(1), dtype=encoder_out.dtype, + device=encoder_out.device)[:, None, :] + mask_chunk_predictor = None + if self.encoder2.overlap_chunk_cls is not None: + mask_chunk_predictor = self.encoder2.overlap_chunk_cls.get_mask_chunk_predictor(None, + device=encoder_out.device, + batch_size=encoder_out.size( + 0)) + mask_shfit_chunk = self.encoder2.overlap_chunk_cls.get_mask_shfit_chunk(None, device=encoder_out.device, + batch_size=encoder_out.size(0)) + encoder_out = encoder_out * mask_shfit_chunk + pre_acoustic_embeds, pre_token_length, pre_alphas, _ = self.predictor2(encoder_out, + ys_out_pad, + encoder_out_mask, + ignore_id=self.ignore_id, + mask_chunk_predictor=mask_chunk_predictor, + target_label_length=ys_in_lens, + ) + predictor_alignments, predictor_alignments_len = self.predictor2.gen_frame_alignments(pre_alphas, + encoder_out_lens) + + scama_mask = None + if self.encoder2.overlap_chunk_cls is not None and self.decoder_attention_chunk_type2 == 'chunk': + encoder_chunk_size = self.encoder2.overlap_chunk_cls.chunk_size_pad_shift_cur + attention_chunk_center_bias = 0 + attention_chunk_size = encoder_chunk_size + decoder_att_look_back_factor = self.encoder2.overlap_chunk_cls.decoder_att_look_back_factor_cur + mask_shift_att_chunk_decoder = self.encoder2.overlap_chunk_cls.get_mask_shift_att_chunk_decoder(None, + device=encoder_out.device, + batch_size=encoder_out.size( + 0)) + scama_mask = self.build_scama_mask_for_cross_attention_decoder_fn2( + predictor_alignments=predictor_alignments, + encoder_sequence_length=encoder_out_lens, + chunk_size=1, + encoder_chunk_size=encoder_chunk_size, + attention_chunk_center_bias=attention_chunk_center_bias, + attention_chunk_size=attention_chunk_size, + attention_chunk_type=self.decoder_attention_chunk_type2, + step=None, + predictor_mask_chunk_hopping=mask_chunk_predictor, + decoder_att_look_back_factor=decoder_att_look_back_factor, + mask_shift_att_chunk_decoder=mask_shift_att_chunk_decoder, + target_length=ys_in_lens, + is_training=self.training, + ) + elif self.encoder2.overlap_chunk_cls is not None: + encoder_out, encoder_out_lens = self.encoder2.overlap_chunk_cls.remove_chunk(encoder_out, encoder_out_lens, + chunk_outs=None) + + return pre_acoustic_embeds, pre_token_length, predictor_alignments, predictor_alignments_len, scama_mask + + def _calc_ctc_loss( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + # Calc CTC loss + loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) + + # Calc CER using CTC + cer_ctc = None + if not self.training and self.error_calculator is not None: + ys_hat = self.ctc.argmax(encoder_out).data + cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True) + return loss_ctc, cer_ctc + + def _calc_ctc_loss2( + self, + encoder_out: torch.Tensor, + encoder_out_lens: torch.Tensor, + ys_pad: torch.Tensor, + ys_pad_lens: torch.Tensor, + ): + # Calc CTC loss + loss_ctc = self.ctc2(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens) + + # Calc CER using CTC + cer_ctc = None + if not self.training and self.error_calculator is not None: + ys_hat = self.ctc2.argmax(encoder_out).data + cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True) + return loss_ctc, cer_ctc + diff --git a/funasr/models/encoder/__init__.py b/funasr/models/encoder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/models/encoder/abs_encoder.py b/funasr/models/encoder/abs_encoder.py new file mode 100644 index 000000000..1fb7c97c3 --- /dev/null +++ b/funasr/models/encoder/abs_encoder.py @@ -0,0 +1,21 @@ +from abc import ABC +from abc import abstractmethod +from typing import Optional +from typing import Tuple + +import torch + + +class AbsEncoder(torch.nn.Module, ABC): + @abstractmethod + def output_size(self) -> int: + raise NotImplementedError + + @abstractmethod + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + raise NotImplementedError diff --git a/funasr/models/encoder/conformer_encoder.py b/funasr/models/encoder/conformer_encoder.py new file mode 100644 index 000000000..2df2ba608 --- /dev/null +++ b/funasr/models/encoder/conformer_encoder.py @@ -0,0 +1,598 @@ +# Copyright 2020 Tomoki Hayashi +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Conformer encoder definition.""" + +import logging +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + +import torch +from torch import nn +from typeguard import check_argument_types + +from funasr.models.ctc import CTC +from funasr.models.encoder.abs_encoder import AbsEncoder +from funasr.modules.attention import ( + MultiHeadedAttention, # noqa: H301 + RelPositionMultiHeadedAttention, # noqa: H301 + LegacyRelPositionMultiHeadedAttention, # noqa: H301 +) +from funasr.modules.embedding import ( + PositionalEncoding, # noqa: H301 + ScaledPositionalEncoding, # noqa: H301 + RelPositionalEncoding, # noqa: H301 + LegacyRelPositionalEncoding, # noqa: H301 +) +from funasr.modules.layer_norm import LayerNorm +from funasr.modules.multi_layer_conv import Conv1dLinear +from funasr.modules.multi_layer_conv import MultiLayeredConv1d +from funasr.modules.nets_utils import get_activation +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.positionwise_feed_forward import ( + PositionwiseFeedForward, # noqa: H301 +) +from funasr.modules.repeat import repeat +from funasr.modules.subsampling import Conv2dSubsampling +from funasr.modules.subsampling import Conv2dSubsampling2 +from funasr.modules.subsampling import Conv2dSubsampling6 +from funasr.modules.subsampling import Conv2dSubsampling8 +from funasr.modules.subsampling import TooShortUttError +from funasr.modules.subsampling import check_short_utt + +class ConvolutionModule(nn.Module): + """ConvolutionModule in Conformer model. + + Args: + channels (int): The number of channels of conv layers. + kernel_size (int): Kernerl size of conv layers. + + """ + + def __init__(self, channels, kernel_size, activation=nn.ReLU(), bias=True): + """Construct an ConvolutionModule object.""" + super(ConvolutionModule, self).__init__() + # kernerl_size should be a odd number for 'SAME' padding + assert (kernel_size - 1) % 2 == 0 + + self.pointwise_conv1 = nn.Conv1d( + channels, + 2 * channels, + kernel_size=1, + stride=1, + padding=0, + bias=bias, + ) + self.depthwise_conv = nn.Conv1d( + channels, + channels, + kernel_size, + stride=1, + padding=(kernel_size - 1) // 2, + groups=channels, + bias=bias, + ) + self.norm = nn.BatchNorm1d(channels) + self.pointwise_conv2 = nn.Conv1d( + channels, + channels, + kernel_size=1, + stride=1, + padding=0, + bias=bias, + ) + self.activation = activation + + def forward(self, x): + """Compute convolution module. + + Args: + x (torch.Tensor): Input tensor (#batch, time, channels). + + Returns: + torch.Tensor: Output tensor (#batch, time, channels). + + """ + # exchange the temporal dimension and the feature dimension + x = x.transpose(1, 2) + + # GLU mechanism + x = self.pointwise_conv1(x) # (batch, 2*channel, dim) + x = nn.functional.glu(x, dim=1) # (batch, channel, dim) + + # 1D Depthwise Conv + x = self.depthwise_conv(x) + x = self.activation(self.norm(x)) + + x = self.pointwise_conv2(x) + + return x.transpose(1, 2) + + +class EncoderLayer(nn.Module): + """Encoder layer module. + + Args: + size (int): Input dimension. + self_attn (torch.nn.Module): Self-attention module instance. + `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` instance + can be used as the argument. + feed_forward (torch.nn.Module): Feed-forward module instance. + `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance + can be used as the argument. + feed_forward_macaron (torch.nn.Module): Additional feed-forward module instance. + `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance + can be used as the argument. + conv_module (torch.nn.Module): Convolution module instance. + `ConvlutionModule` instance can be used as the argument. + dropout_rate (float): Dropout rate. + normalize_before (bool): Whether to use layer_norm before the first block. + concat_after (bool): Whether to concat attention layer's input and output. + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. i.e. x -> x + att(x) + stochastic_depth_rate (float): Proability to skip this layer. + During training, the layer may skip residual computation and return input + as-is with given probability. + """ + + def __init__( + self, + size, + self_attn, + feed_forward, + feed_forward_macaron, + conv_module, + dropout_rate, + normalize_before=True, + concat_after=False, + stochastic_depth_rate=0.0, + ): + """Construct an EncoderLayer object.""" + super(EncoderLayer, self).__init__() + self.self_attn = self_attn + self.feed_forward = feed_forward + self.feed_forward_macaron = feed_forward_macaron + self.conv_module = conv_module + self.norm_ff = LayerNorm(size) # for the FNN module + self.norm_mha = LayerNorm(size) # for the MHA module + if feed_forward_macaron is not None: + self.norm_ff_macaron = LayerNorm(size) + self.ff_scale = 0.5 + else: + self.ff_scale = 1.0 + if self.conv_module is not None: + self.norm_conv = LayerNorm(size) # for the CNN module + self.norm_final = LayerNorm(size) # for the final output of the block + self.dropout = nn.Dropout(dropout_rate) + self.size = size + self.normalize_before = normalize_before + self.concat_after = concat_after + if self.concat_after: + self.concat_linear = nn.Linear(size + size, size) + self.stochastic_depth_rate = stochastic_depth_rate + + def forward(self, x_input, mask, cache=None): + """Compute encoded features. + + Args: + x_input (Union[Tuple, torch.Tensor]): Input tensor w/ or w/o pos emb. + - w/ pos emb: Tuple of tensors [(#batch, time, size), (1, time, size)]. + - w/o pos emb: Tensor (#batch, time, size). + mask (torch.Tensor): Mask tensor for the input (#batch, time). + cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). + + Returns: + torch.Tensor: Output tensor (#batch, time, size). + torch.Tensor: Mask tensor (#batch, time). + + """ + if isinstance(x_input, tuple): + x, pos_emb = x_input[0], x_input[1] + else: + x, pos_emb = x_input, None + + skip_layer = False + # with stochastic depth, residual connection `x + f(x)` becomes + # `x <- x + 1 / (1 - p) * f(x)` at training time. + stoch_layer_coeff = 1.0 + if self.training and self.stochastic_depth_rate > 0: + skip_layer = torch.rand(1).item() < self.stochastic_depth_rate + stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) + + if skip_layer: + if cache is not None: + x = torch.cat([cache, x], dim=1) + if pos_emb is not None: + return (x, pos_emb), mask + return x, mask + + # whether to use macaron style + if self.feed_forward_macaron is not None: + residual = x + if self.normalize_before: + x = self.norm_ff_macaron(x) + x = residual + stoch_layer_coeff * self.ff_scale * self.dropout( + self.feed_forward_macaron(x) + ) + if not self.normalize_before: + x = self.norm_ff_macaron(x) + + # multi-headed self-attention module + residual = x + if self.normalize_before: + x = self.norm_mha(x) + + if cache is None: + x_q = x + else: + assert cache.shape == (x.shape[0], x.shape[1] - 1, self.size) + x_q = x[:, -1:, :] + residual = residual[:, -1:, :] + mask = None if mask is None else mask[:, -1:, :] + + if pos_emb is not None: + x_att = self.self_attn(x_q, x, x, pos_emb, mask) + else: + x_att = self.self_attn(x_q, x, x, mask) + + if self.concat_after: + x_concat = torch.cat((x, x_att), dim=-1) + x = residual + stoch_layer_coeff * self.concat_linear(x_concat) + else: + x = residual + stoch_layer_coeff * self.dropout(x_att) + if not self.normalize_before: + x = self.norm_mha(x) + + # convolution module + if self.conv_module is not None: + residual = x + if self.normalize_before: + x = self.norm_conv(x) + x = residual + stoch_layer_coeff * self.dropout(self.conv_module(x)) + if not self.normalize_before: + x = self.norm_conv(x) + + # feed forward module + residual = x + if self.normalize_before: + x = self.norm_ff(x) + x = residual + stoch_layer_coeff * self.ff_scale * self.dropout( + self.feed_forward(x) + ) + if not self.normalize_before: + x = self.norm_ff(x) + + if self.conv_module is not None: + x = self.norm_final(x) + + if cache is not None: + x = torch.cat([cache, x], dim=1) + + if pos_emb is not None: + return (x, pos_emb), mask + + return x, mask + + +class ConformerEncoder(AbsEncoder): + """Conformer encoder module. + + Args: + input_size (int): Input dimension. + output_size (int): Dimension of attention. + attention_heads (int): The number of heads of multi head attention. + linear_units (int): The number of units of position-wise feed forward. + num_blocks (int): The number of decoder blocks. + dropout_rate (float): Dropout rate. + attention_dropout_rate (float): Dropout rate in attention. + positional_dropout_rate (float): Dropout rate after adding positional encoding. + input_layer (Union[str, torch.nn.Module]): Input layer type. + normalize_before (bool): Whether to use layer_norm before the first block. + concat_after (bool): Whether to concat attention layer's input and output. + If True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + If False, no additional linear will be applied. i.e. x -> x + att(x) + positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". + positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. + rel_pos_type (str): Whether to use the latest relative positional encoding or + the legacy one. The legacy relative positional encoding will be deprecated + in the future. More Details can be found in + https://github.com/espnet/espnet/pull/2816. + encoder_pos_enc_layer_type (str): Encoder positional encoding layer type. + encoder_attn_layer_type (str): Encoder attention layer type. + activation_type (str): Encoder activation function type. + macaron_style (bool): Whether to use macaron style for positionwise layer. + use_cnn_module (bool): Whether to use convolution module. + zero_triu (bool): Whether to zero the upper triangular part of attention matrix. + cnn_module_kernel (int): Kernerl size of convolution module. + padding_idx (int): Padding idx for input_layer=embed. + + """ + + def __init__( + self, + input_size: int, + output_size: int = 256, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + attention_dropout_rate: float = 0.0, + input_layer: str = "conv2d", + normalize_before: bool = True, + concat_after: bool = False, + positionwise_layer_type: str = "linear", + positionwise_conv_kernel_size: int = 3, + macaron_style: bool = False, + rel_pos_type: str = "legacy", + pos_enc_layer_type: str = "rel_pos", + selfattention_layer_type: str = "rel_selfattn", + activation_type: str = "swish", + use_cnn_module: bool = True, + zero_triu: bool = False, + cnn_module_kernel: int = 31, + padding_idx: int = -1, + interctc_layer_idx: List[int] = [], + interctc_use_conditioning: bool = False, + stochastic_depth_rate: Union[float, List[float]] = 0.0, + ): + assert check_argument_types() + super().__init__() + self._output_size = output_size + + if rel_pos_type == "legacy": + if pos_enc_layer_type == "rel_pos": + pos_enc_layer_type = "legacy_rel_pos" + if selfattention_layer_type == "rel_selfattn": + selfattention_layer_type = "legacy_rel_selfattn" + elif rel_pos_type == "latest": + assert selfattention_layer_type != "legacy_rel_selfattn" + assert pos_enc_layer_type != "legacy_rel_pos" + else: + raise ValueError("unknown rel_pos_type: " + rel_pos_type) + + activation = get_activation(activation_type) + if pos_enc_layer_type == "abs_pos": + pos_enc_class = PositionalEncoding + elif pos_enc_layer_type == "scaled_abs_pos": + pos_enc_class = ScaledPositionalEncoding + elif pos_enc_layer_type == "rel_pos": + assert selfattention_layer_type == "rel_selfattn" + pos_enc_class = RelPositionalEncoding + elif pos_enc_layer_type == "legacy_rel_pos": + assert selfattention_layer_type == "legacy_rel_selfattn" + pos_enc_class = LegacyRelPositionalEncoding + logging.warning( + "Using legacy_rel_pos and it will be deprecated in the future." + ) + else: + raise ValueError("unknown pos_enc_layer: " + pos_enc_layer_type) + + if input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(input_size, output_size), + torch.nn.LayerNorm(output_size), + torch.nn.Dropout(dropout_rate), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "conv2d": + self.embed = Conv2dSubsampling( + input_size, + output_size, + dropout_rate, + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "conv2d2": + self.embed = Conv2dSubsampling2( + input_size, + output_size, + dropout_rate, + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "conv2d6": + self.embed = Conv2dSubsampling6( + input_size, + output_size, + dropout_rate, + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "conv2d8": + self.embed = Conv2dSubsampling8( + input_size, + output_size, + dropout_rate, + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif isinstance(input_layer, torch.nn.Module): + self.embed = torch.nn.Sequential( + input_layer, + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer is None: + self.embed = torch.nn.Sequential( + pos_enc_class(output_size, positional_dropout_rate) + ) + else: + raise ValueError("unknown input_layer: " + input_layer) + self.normalize_before = normalize_before + if positionwise_layer_type == "linear": + positionwise_layer = PositionwiseFeedForward + positionwise_layer_args = ( + output_size, + linear_units, + dropout_rate, + activation, + ) + elif positionwise_layer_type == "conv1d": + positionwise_layer = MultiLayeredConv1d + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d-linear": + positionwise_layer = Conv1dLinear + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + else: + raise NotImplementedError("Support only linear or conv1d.") + + if selfattention_layer_type == "selfattn": + encoder_selfattn_layer = MultiHeadedAttention + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + attention_dropout_rate, + ) + elif selfattention_layer_type == "legacy_rel_selfattn": + assert pos_enc_layer_type == "legacy_rel_pos" + encoder_selfattn_layer = LegacyRelPositionMultiHeadedAttention + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + attention_dropout_rate, + ) + logging.warning( + "Using legacy_rel_selfattn and it will be deprecated in the future." + ) + elif selfattention_layer_type == "rel_selfattn": + assert pos_enc_layer_type == "rel_pos" + encoder_selfattn_layer = RelPositionMultiHeadedAttention + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + attention_dropout_rate, + zero_triu, + ) + else: + raise ValueError("unknown encoder_attn_layer: " + selfattention_layer_type) + + convolution_layer = ConvolutionModule + convolution_layer_args = (output_size, cnn_module_kernel, activation) + + if isinstance(stochastic_depth_rate, float): + stochastic_depth_rate = [stochastic_depth_rate] * num_blocks + + if len(stochastic_depth_rate) != num_blocks: + raise ValueError( + f"Length of stochastic_depth_rate ({len(stochastic_depth_rate)}) " + f"should be equal to num_blocks ({num_blocks})" + ) + + self.encoders = repeat( + num_blocks, + lambda lnum: EncoderLayer( + output_size, + encoder_selfattn_layer(*encoder_selfattn_layer_args), + positionwise_layer(*positionwise_layer_args), + positionwise_layer(*positionwise_layer_args) if macaron_style else None, + convolution_layer(*convolution_layer_args) if use_cnn_module else None, + dropout_rate, + normalize_before, + concat_after, + stochastic_depth_rate[lnum], + ), + ) + if self.normalize_before: + self.after_norm = LayerNorm(output_size) + + self.interctc_layer_idx = interctc_layer_idx + if len(interctc_layer_idx) > 0: + assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks + self.interctc_use_conditioning = interctc_use_conditioning + self.conditioning_layer = None + + def output_size(self) -> int: + return self._output_size + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ctc: CTC = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Calculate forward propagation. + + Args: + xs_pad (torch.Tensor): Input tensor (#batch, L, input_size). + ilens (torch.Tensor): Input length (#batch). + prev_states (torch.Tensor): Not to be used now. + + Returns: + torch.Tensor: Output tensor (#batch, L, output_size). + torch.Tensor: Output length (#batch). + torch.Tensor: Not to be used now. + + """ + masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) + + if ( + isinstance(self.embed, Conv2dSubsampling) + or isinstance(self.embed, Conv2dSubsampling2) + or isinstance(self.embed, Conv2dSubsampling6) + or isinstance(self.embed, Conv2dSubsampling8) + ): + short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) + if short_status: + raise TooShortUttError( + f"has {xs_pad.size(1)} frames and is too short for subsampling " + + f"(it needs more than {limit_size} frames), return empty results", + xs_pad.size(1), + limit_size, + ) + xs_pad, masks = self.embed(xs_pad, masks) + else: + xs_pad = self.embed(xs_pad) + + intermediate_outs = [] + if len(self.interctc_layer_idx) == 0: + xs_pad, masks = self.encoders(xs_pad, masks) + else: + for layer_idx, encoder_layer in enumerate(self.encoders): + xs_pad, masks = encoder_layer(xs_pad, masks) + + if layer_idx + 1 in self.interctc_layer_idx: + encoder_out = xs_pad + if isinstance(encoder_out, tuple): + encoder_out = encoder_out[0] + + # intermediate outputs are also normalized + if self.normalize_before: + encoder_out = self.after_norm(encoder_out) + + intermediate_outs.append((layer_idx + 1, encoder_out)) + + if self.interctc_use_conditioning: + ctc_out = ctc.softmax(encoder_out) + + if isinstance(xs_pad, tuple): + x, pos_emb = xs_pad + x = x + self.conditioning_layer(ctc_out) + xs_pad = (x, pos_emb) + else: + xs_pad = xs_pad + self.conditioning_layer(ctc_out) + + if isinstance(xs_pad, tuple): + xs_pad = xs_pad[0] + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + + olens = masks.squeeze(1).sum(1) + if len(intermediate_outs) > 0: + return (xs_pad, intermediate_outs), olens, None + return xs_pad, olens, None diff --git a/funasr/models/encoder/rnn_encoder.py b/funasr/models/encoder/rnn_encoder.py new file mode 100644 index 000000000..7a3b05399 --- /dev/null +++ b/funasr/models/encoder/rnn_encoder.py @@ -0,0 +1,115 @@ +from typing import Optional +from typing import Sequence +from typing import Tuple + +import numpy as np +import torch +from typeguard import check_argument_types + +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.rnn.encoders import RNN +from funasr.modules.rnn.encoders import RNNP +from funasr.models.encoder.abs_encoder import AbsEncoder + + +class RNNEncoder(AbsEncoder): + """RNNEncoder class. + + Args: + input_size: The number of expected features in the input + output_size: The number of output features + hidden_size: The number of hidden features + bidirectional: If ``True`` becomes a bidirectional LSTM + use_projection: Use projection layer or not + num_layers: Number of recurrent layers + dropout: dropout probability + + """ + + def __init__( + self, + input_size: int, + rnn_type: str = "lstm", + bidirectional: bool = True, + use_projection: bool = True, + num_layers: int = 4, + hidden_size: int = 320, + output_size: int = 320, + dropout: float = 0.0, + subsample: Optional[Sequence[int]] = (2, 2, 1, 1), + ): + assert check_argument_types() + super().__init__() + self._output_size = output_size + self.rnn_type = rnn_type + self.bidirectional = bidirectional + self.use_projection = use_projection + + if rnn_type not in {"lstm", "gru"}: + raise ValueError(f"Not supported rnn_type={rnn_type}") + + if subsample is None: + subsample = np.ones(num_layers + 1, dtype=np.int) + else: + subsample = subsample[:num_layers] + # Append 1 at the beginning because the second or later is used + subsample = np.pad( + np.array(subsample, dtype=np.int), + [1, num_layers - len(subsample)], + mode="constant", + constant_values=1, + ) + + rnn_type = ("b" if bidirectional else "") + rnn_type + if use_projection: + self.enc = torch.nn.ModuleList( + [ + RNNP( + input_size, + num_layers, + hidden_size, + output_size, + subsample, + dropout, + typ=rnn_type, + ) + ] + ) + + else: + self.enc = torch.nn.ModuleList( + [ + RNN( + input_size, + num_layers, + hidden_size, + output_size, + dropout, + typ=rnn_type, + ) + ] + ) + + def output_size(self) -> int: + return self._output_size + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if prev_states is None: + prev_states = [None] * len(self.enc) + assert len(prev_states) == len(self.enc) + + current_states = [] + for module, prev_state in zip(self.enc, prev_states): + xs_pad, ilens, states = module(xs_pad, ilens, prev_state=prev_state) + current_states.append(states) + + if self.use_projection: + xs_pad.masked_fill_(make_pad_mask(ilens, xs_pad, 1), 0.0) + else: + xs_pad = xs_pad.masked_fill(make_pad_mask(ilens, xs_pad, 1), 0.0) + return xs_pad, ilens, current_states diff --git a/funasr/models/encoder/sanm_encoder.py b/funasr/models/encoder/sanm_encoder.py new file mode 100644 index 000000000..3d8079dd3 --- /dev/null +++ b/funasr/models/encoder/sanm_encoder.py @@ -0,0 +1,595 @@ +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import torch +import torch.nn as nn +from funasr.modules.streaming_utils.chunk_utilis import overlap_chunk +from typeguard import check_argument_types + +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.attention import MultiHeadedAttention, MultiHeadedAttentionSANM +from funasr.modules.embedding import SinusoidalPositionEncoder +from funasr.modules.layer_norm import LayerNorm +from funasr.modules.multi_layer_conv import Conv1dLinear +from funasr.modules.multi_layer_conv import MultiLayeredConv1d +from funasr.modules.positionwise_feed_forward import ( + PositionwiseFeedForward, # noqa: H301 +) +from funasr.modules.repeat import repeat +from funasr.modules.subsampling import Conv2dSubsampling +from funasr.modules.subsampling import Conv2dSubsampling2 +from funasr.modules.subsampling import Conv2dSubsampling6 +from funasr.modules.subsampling import Conv2dSubsampling8 +from funasr.modules.subsampling import TooShortUttError +from funasr.modules.subsampling import check_short_utt +from funasr.models.ctc import CTC +from funasr.models.encoder.abs_encoder import AbsEncoder + +class EncoderLayerSANM(nn.Module): + def __init__( + self, + in_size, + size, + self_attn, + feed_forward, + dropout_rate, + normalize_before=True, + concat_after=False, + stochastic_depth_rate=0.0, + ): + """Construct an EncoderLayer object.""" + super(EncoderLayerSANM, self).__init__() + self.self_attn = self_attn + self.feed_forward = feed_forward + self.norm1 = LayerNorm(in_size) + self.norm2 = LayerNorm(size) + self.dropout = nn.Dropout(dropout_rate) + self.in_size = in_size + self.size = size + self.normalize_before = normalize_before + self.concat_after = concat_after + if self.concat_after: + self.concat_linear = nn.Linear(size + size, size) + self.stochastic_depth_rate = stochastic_depth_rate + self.dropout_rate = dropout_rate + + def forward(self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None): + """Compute encoded features. + + Args: + x_input (torch.Tensor): Input tensor (#batch, time, size). + mask (torch.Tensor): Mask tensor for the input (#batch, time). + cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). + + Returns: + torch.Tensor: Output tensor (#batch, time, size). + torch.Tensor: Mask tensor (#batch, time). + + """ + skip_layer = False + # with stochastic depth, residual connection `x + f(x)` becomes + # `x <- x + 1 / (1 - p) * f(x)` at training time. + stoch_layer_coeff = 1.0 + if self.training and self.stochastic_depth_rate > 0: + skip_layer = torch.rand(1).item() < self.stochastic_depth_rate + stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) + + if skip_layer: + if cache is not None: + x = torch.cat([cache, x], dim=1) + return x, mask + + residual = x + if self.normalize_before: + x = self.norm1(x) + + if self.concat_after: + x_concat = torch.cat((x, self.self_attn(x, mask, mask_shfit_chunk=mask_shfit_chunk, mask_att_chunk_encoder=mask_att_chunk_encoder)), dim=-1) + if self.in_size == self.size: + x = residual + stoch_layer_coeff * self.concat_linear(x_concat) + else: + x = stoch_layer_coeff * self.concat_linear(x_concat) + else: + if self.in_size == self.size: + x = residual + stoch_layer_coeff * self.dropout( + self.self_attn(x, mask, mask_shfit_chunk=mask_shfit_chunk, mask_att_chunk_encoder=mask_att_chunk_encoder) + ) + else: + x = stoch_layer_coeff * self.dropout( + self.self_attn(x, mask, mask_shfit_chunk=mask_shfit_chunk, mask_att_chunk_encoder=mask_att_chunk_encoder) + ) + if not self.normalize_before: + x = self.norm1(x) + + residual = x + if self.normalize_before: + x = self.norm2(x) + x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) + if not self.normalize_before: + x = self.norm2(x) + + + return x, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder + +class SANMEncoder(AbsEncoder): + """ + author: Speech Lab, Alibaba Group, China + San-m: Memory equipped self-attention for end-to-end speech recognition + https://arxiv.org/abs/2006.01713 + + """ + + def __init__( + self, + input_size: int, + output_size: int = 256, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + attention_dropout_rate: float = 0.0, + input_layer: Optional[str] = "conv2d", + pos_enc_class=SinusoidalPositionEncoder, + normalize_before: bool = True, + concat_after: bool = False, + positionwise_layer_type: str = "linear", + positionwise_conv_kernel_size: int = 1, + padding_idx: int = -1, + interctc_layer_idx: List[int] = [], + interctc_use_conditioning: bool = False, + kernel_size : int = 11, + sanm_shfit : int = 0, + selfattention_layer_type: str = "sanm", + ): + assert check_argument_types() + super().__init__() + self._output_size = output_size + + if input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(input_size, output_size), + torch.nn.LayerNorm(output_size), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "conv2d": + self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) + elif input_layer == "conv2d2": + self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) + elif input_layer == "conv2d6": + self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) + elif input_layer == "conv2d8": + self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) + elif input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer is None: + if input_size == output_size: + self.embed = None + else: + self.embed = torch.nn.Linear(input_size, output_size) + elif input_layer == "pe": + self.embed = SinusoidalPositionEncoder() + else: + raise ValueError("unknown input_layer: " + input_layer) + self.normalize_before = normalize_before + if positionwise_layer_type == "linear": + positionwise_layer = PositionwiseFeedForward + positionwise_layer_args = ( + output_size, + linear_units, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d": + positionwise_layer = MultiLayeredConv1d + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d-linear": + positionwise_layer = Conv1dLinear + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + else: + raise NotImplementedError("Support only linear or conv1d.") + + if selfattention_layer_type == "selfattn": + encoder_selfattn_layer = MultiHeadedAttention + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + attention_dropout_rate, + ) + + elif selfattention_layer_type == "sanm": + encoder_selfattn_layer = MultiHeadedAttentionSANM + encoder_selfattn_layer_args0 = ( + attention_heads, + input_size, + output_size, + attention_dropout_rate, + kernel_size, + sanm_shfit, + ) + + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + output_size, + attention_dropout_rate, + kernel_size, + sanm_shfit, + ) + self.encoders0 = repeat( + 1, + lambda lnum: EncoderLayerSANM( + input_size, + output_size, + encoder_selfattn_layer(*encoder_selfattn_layer_args0), + positionwise_layer(*positionwise_layer_args), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + self.encoders = repeat( + num_blocks-1, + lambda lnum: EncoderLayerSANM( + output_size, + output_size, + encoder_selfattn_layer(*encoder_selfattn_layer_args), + positionwise_layer(*positionwise_layer_args), + dropout_rate, + normalize_before, + concat_after, + ), + ) + if self.normalize_before: + self.after_norm = LayerNorm(output_size) + + self.interctc_layer_idx = interctc_layer_idx + if len(interctc_layer_idx) > 0: + assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks + self.interctc_use_conditioning = interctc_use_conditioning + self.conditioning_layer = None + self.dropout = nn.Dropout(dropout_rate) + + def output_size(self) -> int: + return self._output_size + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ctc: CTC = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Embed positions in tensor. + + Args: + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + Returns: + position embedded tensor and mask + """ + masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) + xs_pad *= self.output_size()**0.5 + if self.embed is None: + xs_pad = xs_pad + elif ( + isinstance(self.embed, Conv2dSubsampling) + or isinstance(self.embed, Conv2dSubsampling2) + or isinstance(self.embed, Conv2dSubsampling6) + or isinstance(self.embed, Conv2dSubsampling8) + ): + short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) + if short_status: + raise TooShortUttError( + f"has {xs_pad.size(1)} frames and is too short for subsampling " + + f"(it needs more than {limit_size} frames), return empty results", + xs_pad.size(1), + limit_size, + ) + xs_pad, masks = self.embed(xs_pad, masks) + else: + xs_pad = self.embed(xs_pad) + + # xs_pad = self.dropout(xs_pad) + encoder_outs = self.encoders0(xs_pad, masks) + xs_pad, masks = encoder_outs[0], encoder_outs[1] + intermediate_outs = [] + if len(self.interctc_layer_idx) == 0: + encoder_outs = self.encoders(xs_pad, masks) + xs_pad, masks = encoder_outs[0], encoder_outs[1] + else: + for layer_idx, encoder_layer in enumerate(self.encoders): + encoder_outs = encoder_layer(xs_pad, masks) + xs_pad, masks = encoder_outs[0], encoder_outs[1] + + if layer_idx + 1 in self.interctc_layer_idx: + encoder_out = xs_pad + + # intermediate outputs are also normalized + if self.normalize_before: + encoder_out = self.after_norm(encoder_out) + + intermediate_outs.append((layer_idx + 1, encoder_out)) + + if self.interctc_use_conditioning: + ctc_out = ctc.softmax(encoder_out) + xs_pad = xs_pad + self.conditioning_layer(ctc_out) + + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + + olens = masks.squeeze(1).sum(1) + if len(intermediate_outs) > 0: + return (xs_pad, intermediate_outs), olens, None + return xs_pad, olens, None + + +class SANMEncoderChunkOpt(AbsEncoder): + """ + author: Speech Lab, Alibaba Group, China + SCAMA: Streaming chunk-aware multihead attention for online end-to-end speech recognition + https://arxiv.org/abs/2006.01713 + + """ + + def __init__( + self, + input_size: int, + output_size: int = 256, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + attention_dropout_rate: float = 0.0, + input_layer: Optional[str] = "conv2d", + pos_enc_class=SinusoidalPositionEncoder, + normalize_before: bool = True, + concat_after: bool = False, + positionwise_layer_type: str = "linear", + positionwise_conv_kernel_size: int = 1, + padding_idx: int = -1, + interctc_layer_idx: List[int] = [], + interctc_use_conditioning: bool = False, + kernel_size: int = 11, + sanm_shfit: int = 0, + selfattention_layer_type: str = "sanm", + chunk_size: Union[int, Sequence[int]] = (16,), + stride: Union[int, Sequence[int]] = (10,), + pad_left: Union[int, Sequence[int]] = (0,), + encoder_att_look_back_factor: Union[int, Sequence[int]] = (1,), + decoder_att_look_back_factor: Union[int, Sequence[int]] = (1,), + ): + assert check_argument_types() + super().__init__() + self._output_size = output_size + + if input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(input_size, output_size), + torch.nn.LayerNorm(output_size), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "conv2d": + self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) + elif input_layer == "conv2d2": + self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) + elif input_layer == "conv2d6": + self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) + elif input_layer == "conv2d8": + self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) + elif input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer is None: + if input_size == output_size: + self.embed = None + else: + self.embed = torch.nn.Linear(input_size, output_size) + elif input_layer == "pe": + self.embed = SinusoidalPositionEncoder() + else: + raise ValueError("unknown input_layer: " + input_layer) + self.normalize_before = normalize_before + if positionwise_layer_type == "linear": + positionwise_layer = PositionwiseFeedForward + positionwise_layer_args = ( + output_size, + linear_units, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d": + positionwise_layer = MultiLayeredConv1d + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d-linear": + positionwise_layer = Conv1dLinear + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + else: + raise NotImplementedError("Support only linear or conv1d.") + + if selfattention_layer_type == "selfattn": + encoder_selfattn_layer = MultiHeadedAttention + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + attention_dropout_rate, + ) + elif selfattention_layer_type == "sanm": + encoder_selfattn_layer = MultiHeadedAttentionSANM + encoder_selfattn_layer_args0 = ( + attention_heads, + input_size, + output_size, + attention_dropout_rate, + kernel_size, + sanm_shfit, + ) + + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + output_size, + attention_dropout_rate, + kernel_size, + sanm_shfit, + ) + self.encoders0 = repeat( + 1, + lambda lnum: EncoderLayerSANM( + input_size, + output_size, + encoder_selfattn_layer(*encoder_selfattn_layer_args0), + positionwise_layer(*positionwise_layer_args), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + self.encoders = repeat( + num_blocks - 1, + lambda lnum: EncoderLayerSANM( + output_size, + output_size, + encoder_selfattn_layer(*encoder_selfattn_layer_args), + positionwise_layer(*positionwise_layer_args), + dropout_rate, + normalize_before, + concat_after, + ), + ) + if self.normalize_before: + self.after_norm = LayerNorm(output_size) + + self.interctc_layer_idx = interctc_layer_idx + if len(interctc_layer_idx) > 0: + assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks + self.interctc_use_conditioning = interctc_use_conditioning + self.conditioning_layer = None + shfit_fsmn = (kernel_size - 1) // 2 + self.overlap_chunk_cls = overlap_chunk( + chunk_size=chunk_size, + stride=stride, + pad_left=pad_left, + shfit_fsmn=shfit_fsmn, + encoder_att_look_back_factor=encoder_att_look_back_factor, + decoder_att_look_back_factor=decoder_att_look_back_factor, + ) + + def output_size(self) -> int: + return self._output_size + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ctc: CTC = None, + ind: int = 0, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Embed positions in tensor. + + Args: + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + Returns: + position embedded tensor and mask + """ + masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) + xs_pad *= self.output_size() ** 0.5 + if self.embed is None: + xs_pad = xs_pad + elif ( + isinstance(self.embed, Conv2dSubsampling) + or isinstance(self.embed, Conv2dSubsampling2) + or isinstance(self.embed, Conv2dSubsampling6) + or isinstance(self.embed, Conv2dSubsampling8) + ): + short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) + if short_status: + raise TooShortUttError( + f"has {xs_pad.size(1)} frames and is too short for subsampling " + + f"(it needs more than {limit_size} frames), return empty results", + xs_pad.size(1), + limit_size, + ) + xs_pad, masks = self.embed(xs_pad, masks) + else: + xs_pad = self.embed(xs_pad) + + mask_shfit_chunk, mask_att_chunk_encoder = None, None + if self.overlap_chunk_cls is not None: + ilens = masks.squeeze(1).sum(1) + chunk_outs = self.overlap_chunk_cls.gen_chunk_mask(ilens, ind) + xs_pad, ilens = self.overlap_chunk_cls.split_chunk(xs_pad, ilens, chunk_outs=chunk_outs) + masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) + mask_shfit_chunk = self.overlap_chunk_cls.get_mask_shfit_chunk(chunk_outs, xs_pad.device, xs_pad.size(0), + dtype=xs_pad.dtype) + mask_att_chunk_encoder = self.overlap_chunk_cls.get_mask_att_chunk_encoder(chunk_outs, xs_pad.device, + xs_pad.size(0), + dtype=xs_pad.dtype) + + encoder_outs = self.encoders0(xs_pad, masks, None, mask_shfit_chunk, mask_att_chunk_encoder) + xs_pad, masks = encoder_outs[0], encoder_outs[1] + intermediate_outs = [] + if len(self.interctc_layer_idx) == 0: + encoder_outs = self.encoders(xs_pad, masks, None, mask_shfit_chunk, mask_att_chunk_encoder) + xs_pad, masks = encoder_outs[0], encoder_outs[1] + else: + for layer_idx, encoder_layer in enumerate(self.encoders): + encoder_outs = encoder_layer(xs_pad, masks, None, mask_shfit_chunk, mask_att_chunk_encoder) + xs_pad, masks = encoder_outs[0], encoder_outs[1] + if layer_idx + 1 in self.interctc_layer_idx: + encoder_out = xs_pad + + # intermediate outputs are also normalized + if self.normalize_before: + encoder_out = self.after_norm(encoder_out) + + intermediate_outs.append((layer_idx + 1, encoder_out)) + + if self.interctc_use_conditioning: + ctc_out = ctc.softmax(encoder_out) + xs_pad = xs_pad + self.conditioning_layer(ctc_out) + + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + + olens = masks.squeeze(1).sum(1) + if len(intermediate_outs) > 0: + return (xs_pad, intermediate_outs), olens, None + return xs_pad, olens, None diff --git a/funasr/models/encoder/transformer_encoder.py b/funasr/models/encoder/transformer_encoder.py new file mode 100644 index 000000000..ff9c3db51 --- /dev/null +++ b/funasr/models/encoder/transformer_encoder.py @@ -0,0 +1,684 @@ +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Transformer encoder definition.""" + +from typing import List +from typing import Optional +from typing import Tuple + +import torch +from torch import nn +from typeguard import check_argument_types +import logging + +from funasr.models.ctc import CTC +from funasr.models.encoder.abs_encoder import AbsEncoder +from funasr.modules.attention import MultiHeadedAttention +from funasr.modules.embedding import PositionalEncoding +from funasr.modules.layer_norm import LayerNorm +from funasr.modules.multi_layer_conv import Conv1dLinear +from funasr.modules.multi_layer_conv import MultiLayeredConv1d +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.positionwise_feed_forward import ( + PositionwiseFeedForward, # noqa: H301 +) +from funasr.modules.repeat import repeat +from funasr.modules.nets_utils import rename_state_dict +from funasr.modules.dynamic_conv import DynamicConvolution +from funasr.modules.dynamic_conv2d import DynamicConvolution2D +from funasr.modules.lightconv import LightweightConvolution +from funasr.modules.lightconv2d import LightweightConvolution2D +from funasr.modules.subsampling import Conv2dSubsampling +from funasr.modules.subsampling import Conv2dSubsampling2 +from funasr.modules.subsampling import Conv2dSubsampling6 +from funasr.modules.subsampling import Conv2dSubsampling8 +from funasr.modules.subsampling import TooShortUttError +from funasr.modules.subsampling import check_short_utt + + +class EncoderLayer(nn.Module): + """Encoder layer module. + + Args: + size (int): Input dimension. + self_attn (torch.nn.Module): Self-attention module instance. + `MultiHeadedAttention` or `RelPositionMultiHeadedAttention` instance + can be used as the argument. + feed_forward (torch.nn.Module): Feed-forward module instance. + `PositionwiseFeedForward`, `MultiLayeredConv1d`, or `Conv1dLinear` instance + can be used as the argument. + dropout_rate (float): Dropout rate. + normalize_before (bool): Whether to use layer_norm before the first block. + concat_after (bool): Whether to concat attention layer's input and output. + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. i.e. x -> x + att(x) + stochastic_depth_rate (float): Proability to skip this layer. + During training, the layer may skip residual computation and return input + as-is with given probability. + """ + + def __init__( + self, + size, + self_attn, + feed_forward, + dropout_rate, + normalize_before=True, + concat_after=False, + stochastic_depth_rate=0.0, + ): + """Construct an EncoderLayer object.""" + super(EncoderLayer, self).__init__() + self.self_attn = self_attn + self.feed_forward = feed_forward + self.norm1 = LayerNorm(size) + self.norm2 = LayerNorm(size) + self.dropout = nn.Dropout(dropout_rate) + self.size = size + self.normalize_before = normalize_before + self.concat_after = concat_after + if self.concat_after: + self.concat_linear = nn.Linear(size + size, size) + self.stochastic_depth_rate = stochastic_depth_rate + + def forward(self, x, mask, cache=None): + """Compute encoded features. + + Args: + x_input (torch.Tensor): Input tensor (#batch, time, size). + mask (torch.Tensor): Mask tensor for the input (#batch, time). + cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). + + Returns: + torch.Tensor: Output tensor (#batch, time, size). + torch.Tensor: Mask tensor (#batch, time). + + """ + skip_layer = False + # with stochastic depth, residual connection `x + f(x)` becomes + # `x <- x + 1 / (1 - p) * f(x)` at training time. + stoch_layer_coeff = 1.0 + if self.training and self.stochastic_depth_rate > 0: + skip_layer = torch.rand(1).item() < self.stochastic_depth_rate + stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) + + if skip_layer: + if cache is not None: + x = torch.cat([cache, x], dim=1) + return x, mask + + residual = x + if self.normalize_before: + x = self.norm1(x) + + if cache is None: + x_q = x + else: + assert cache.shape == (x.shape[0], x.shape[1] - 1, self.size) + x_q = x[:, -1:, :] + residual = residual[:, -1:, :] + mask = None if mask is None else mask[:, -1:, :] + + if self.concat_after: + x_concat = torch.cat((x, self.self_attn(x_q, x, x, mask)), dim=-1) + x = residual + stoch_layer_coeff * self.concat_linear(x_concat) + else: + x = residual + stoch_layer_coeff * self.dropout( + self.self_attn(x_q, x, x, mask) + ) + if not self.normalize_before: + x = self.norm1(x) + + residual = x + if self.normalize_before: + x = self.norm2(x) + x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) + if not self.normalize_before: + x = self.norm2(x) + + if cache is not None: + x = torch.cat([cache, x], dim=1) + + return x, mask + + +class TransformerEncoder(AbsEncoder): + """Transformer encoder module. + + Args: + input_size: input dim + output_size: dimension of attention + attention_heads: the number of heads of multi head attention + linear_units: the number of units of position-wise feed forward + num_blocks: the number of decoder blocks + dropout_rate: dropout rate + attention_dropout_rate: dropout rate in attention + positional_dropout_rate: dropout rate after adding positional encoding + input_layer: input layer type + pos_enc_class: PositionalEncoding or ScaledPositionalEncoding + normalize_before: whether to use layer_norm before the first block + concat_after: whether to concat attention layer's input and output + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. + i.e. x -> x + att(x) + positionwise_layer_type: linear of conv1d + positionwise_conv_kernel_size: kernel size of positionwise conv1d layer + padding_idx: padding_idx for input_layer=embed + """ + + def __init__( + self, + input_size: int, + output_size: int = 256, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + attention_dropout_rate: float = 0.0, + input_layer: Optional[str] = "conv2d", + pos_enc_class=PositionalEncoding, + normalize_before: bool = True, + concat_after: bool = False, + positionwise_layer_type: str = "linear", + positionwise_conv_kernel_size: int = 1, + padding_idx: int = -1, + interctc_layer_idx: List[int] = [], + interctc_use_conditioning: bool = False, + ): + assert check_argument_types() + super().__init__() + self._output_size = output_size + + if input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(input_size, output_size), + torch.nn.LayerNorm(output_size), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "conv2d": + self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) + elif input_layer == "conv2d2": + self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) + elif input_layer == "conv2d6": + self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) + elif input_layer == "conv2d8": + self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) + elif input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer is None: + if input_size == output_size: + self.embed = None + else: + self.embed = torch.nn.Linear(input_size, output_size) + else: + raise ValueError("unknown input_layer: " + input_layer) + self.normalize_before = normalize_before + if positionwise_layer_type == "linear": + positionwise_layer = PositionwiseFeedForward + positionwise_layer_args = ( + output_size, + linear_units, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d": + positionwise_layer = MultiLayeredConv1d + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d-linear": + positionwise_layer = Conv1dLinear + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + else: + raise NotImplementedError("Support only linear or conv1d.") + self.encoders = repeat( + num_blocks, + lambda lnum: EncoderLayer( + output_size, + MultiHeadedAttention( + attention_heads, output_size, attention_dropout_rate + ), + positionwise_layer(*positionwise_layer_args), + dropout_rate, + normalize_before, + concat_after, + ), + ) + if self.normalize_before: + self.after_norm = LayerNorm(output_size) + + self.interctc_layer_idx = interctc_layer_idx + if len(interctc_layer_idx) > 0: + assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks + self.interctc_use_conditioning = interctc_use_conditioning + self.conditioning_layer = None + + def output_size(self) -> int: + return self._output_size + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + prev_states: torch.Tensor = None, + ctc: CTC = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Embed positions in tensor. + + Args: + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + Returns: + position embedded tensor and mask + """ + masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) + + if self.embed is None: + xs_pad = xs_pad + elif ( + isinstance(self.embed, Conv2dSubsampling) + or isinstance(self.embed, Conv2dSubsampling2) + or isinstance(self.embed, Conv2dSubsampling6) + or isinstance(self.embed, Conv2dSubsampling8) + ): + short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) + if short_status: + raise TooShortUttError( + f"has {xs_pad.size(1)} frames and is too short for subsampling " + + f"(it needs more than {limit_size} frames), return empty results", + xs_pad.size(1), + limit_size, + ) + xs_pad, masks = self.embed(xs_pad, masks) + else: + xs_pad = self.embed(xs_pad) + + intermediate_outs = [] + if len(self.interctc_layer_idx) == 0: + xs_pad, masks = self.encoders(xs_pad, masks) + else: + for layer_idx, encoder_layer in enumerate(self.encoders): + xs_pad, masks = encoder_layer(xs_pad, masks) + + if layer_idx + 1 in self.interctc_layer_idx: + encoder_out = xs_pad + + # intermediate outputs are also normalized + if self.normalize_before: + encoder_out = self.after_norm(encoder_out) + + intermediate_outs.append((layer_idx + 1, encoder_out)) + + if self.interctc_use_conditioning: + ctc_out = ctc.softmax(encoder_out) + xs_pad = xs_pad + self.conditioning_layer(ctc_out) + + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + + olens = masks.squeeze(1).sum(1) + if len(intermediate_outs) > 0: + return (xs_pad, intermediate_outs), olens, None + return xs_pad, olens, None + + +def _pre_hook( + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, +): + # https://github.com/espnet/espnet/commit/21d70286c354c66c0350e65dc098d2ee236faccc#diff-bffb1396f038b317b2b64dd96e6d3563 + rename_state_dict(prefix + "input_layer.", prefix + "embed.", state_dict) + # https://github.com/espnet/espnet/commit/3d422f6de8d4f03673b89e1caef698745ec749ea#diff-bffb1396f038b317b2b64dd96e6d3563 + rename_state_dict(prefix + "norm.", prefix + "after_norm.", state_dict) + + +class TransformerEncoder_s0(torch.nn.Module): + """Transformer encoder module. + + Args: + idim (int): Input dimension. + attention_dim (int): Dimension of attention. + attention_heads (int): The number of heads of multi head attention. + conv_wshare (int): The number of kernel of convolution. Only used in + selfattention_layer_type == "lightconv*" or "dynamiconv*". + conv_kernel_length (Union[int, str]): Kernel size str of convolution + (e.g. 71_71_71_71_71_71). Only used in selfattention_layer_type + == "lightconv*" or "dynamiconv*". + conv_usebias (bool): Whether to use bias in convolution. Only used in + selfattention_layer_type == "lightconv*" or "dynamiconv*". + linear_units (int): The number of units of position-wise feed forward. + num_blocks (int): The number of decoder blocks. + dropout_rate (float): Dropout rate. + positional_dropout_rate (float): Dropout rate after adding positional encoding. + attention_dropout_rate (float): Dropout rate in attention. + input_layer (Union[str, torch.nn.Module]): Input layer type. + pos_enc_class (torch.nn.Module): Positional encoding module class. + `PositionalEncoding `or `ScaledPositionalEncoding` + normalize_before (bool): Whether to use layer_norm before the first block. + concat_after (bool): Whether to concat attention layer's input and output. + if True, additional linear will be applied. + i.e. x -> x + linear(concat(x, att(x))) + if False, no additional linear will be applied. i.e. x -> x + att(x) + positionwise_layer_type (str): "linear", "conv1d", or "conv1d-linear". + positionwise_conv_kernel_size (int): Kernel size of positionwise conv1d layer. + selfattention_layer_type (str): Encoder attention layer type. + padding_idx (int): Padding idx for input_layer=embed. + stochastic_depth_rate (float): Maximum probability to skip the encoder layer. + intermediate_layers (Union[List[int], None]): indices of intermediate CTC layer. + indices start from 1. + if not None, intermediate outputs are returned (which changes return type + signature.) + + """ + + def __init__( + self, + idim, + attention_dim=256, + attention_heads=4, + conv_wshare=4, + conv_kernel_length="11", + conv_usebias=False, + linear_units=2048, + num_blocks=6, + dropout_rate=0.1, + positional_dropout_rate=0.1, + attention_dropout_rate=0.0, + input_layer="conv2d", + pos_enc_class=PositionalEncoding, + normalize_before=True, + concat_after=False, + positionwise_layer_type="linear", + positionwise_conv_kernel_size=1, + selfattention_layer_type="selfattn", + padding_idx=-1, + stochastic_depth_rate=0.0, + intermediate_layers=None, + ctc_softmax=None, + conditioning_layer_dim=None, + ): + """Construct an Encoder object.""" + super(TransformerEncoder_s0, self).__init__() + self._register_load_state_dict_pre_hook(_pre_hook) + + self.conv_subsampling_factor = 1 + if input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(idim, attention_dim), + torch.nn.LayerNorm(attention_dim), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + pos_enc_class(attention_dim, positional_dropout_rate), + ) + elif input_layer == "conv2d": + self.embed = Conv2dSubsampling(idim, attention_dim, dropout_rate) + self.conv_subsampling_factor = 4 + elif input_layer == "conv2d-scaled-pos-enc": + self.embed = Conv2dSubsampling( + idim, + attention_dim, + dropout_rate, + pos_enc_class(attention_dim, positional_dropout_rate), + ) + self.conv_subsampling_factor = 4 + elif input_layer == "conv2d6": + self.embed = Conv2dSubsampling6(idim, attention_dim, dropout_rate) + self.conv_subsampling_factor = 6 + elif input_layer == "conv2d8": + self.embed = Conv2dSubsampling8(idim, attention_dim, dropout_rate) + self.conv_subsampling_factor = 8 + elif input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(idim, attention_dim, padding_idx=padding_idx), + pos_enc_class(attention_dim, positional_dropout_rate), + ) + elif isinstance(input_layer, torch.nn.Module): + self.embed = torch.nn.Sequential( + input_layer, + pos_enc_class(attention_dim, positional_dropout_rate), + ) + elif input_layer is None: + self.embed = torch.nn.Sequential( + pos_enc_class(attention_dim, positional_dropout_rate) + ) + else: + raise ValueError("unknown input_layer: " + input_layer) + self.normalize_before = normalize_before + positionwise_layer, positionwise_layer_args = self.get_positionwise_layer( + positionwise_layer_type, + attention_dim, + linear_units, + dropout_rate, + positionwise_conv_kernel_size, + ) + if selfattention_layer_type in [ + "selfattn", + "rel_selfattn", + "legacy_rel_selfattn", + ]: + logging.info("encoder self-attention layer type = self-attention") + encoder_selfattn_layer = MultiHeadedAttention + encoder_selfattn_layer_args = [ + ( + attention_heads, + attention_dim, + attention_dropout_rate, + ) + ] * num_blocks + elif selfattention_layer_type == "lightconv": + logging.info("encoder self-attention layer type = lightweight convolution") + encoder_selfattn_layer = LightweightConvolution + encoder_selfattn_layer_args = [ + ( + conv_wshare, + attention_dim, + attention_dropout_rate, + int(conv_kernel_length.split("_")[lnum]), + False, + conv_usebias, + ) + for lnum in range(num_blocks) + ] + elif selfattention_layer_type == "lightconv2d": + logging.info( + "encoder self-attention layer " + "type = lightweight convolution 2-dimensional" + ) + encoder_selfattn_layer = LightweightConvolution2D + encoder_selfattn_layer_args = [ + ( + conv_wshare, + attention_dim, + attention_dropout_rate, + int(conv_kernel_length.split("_")[lnum]), + False, + conv_usebias, + ) + for lnum in range(num_blocks) + ] + elif selfattention_layer_type == "dynamicconv": + logging.info("encoder self-attention layer type = dynamic convolution") + encoder_selfattn_layer = DynamicConvolution + encoder_selfattn_layer_args = [ + ( + conv_wshare, + attention_dim, + attention_dropout_rate, + int(conv_kernel_length.split("_")[lnum]), + False, + conv_usebias, + ) + for lnum in range(num_blocks) + ] + elif selfattention_layer_type == "dynamicconv2d": + logging.info( + "encoder self-attention layer type = dynamic convolution 2-dimensional" + ) + encoder_selfattn_layer = DynamicConvolution2D + encoder_selfattn_layer_args = [ + ( + conv_wshare, + attention_dim, + attention_dropout_rate, + int(conv_kernel_length.split("_")[lnum]), + False, + conv_usebias, + ) + for lnum in range(num_blocks) + ] + else: + raise NotImplementedError(selfattention_layer_type) + + self.encoders = repeat( + num_blocks, + lambda lnum: EncoderLayer( + attention_dim, + encoder_selfattn_layer(*encoder_selfattn_layer_args[lnum]), + positionwise_layer(*positionwise_layer_args), + dropout_rate, + normalize_before, + concat_after, + stochastic_depth_rate * float(1 + lnum) / num_blocks, + ), + ) + if self.normalize_before: + self.after_norm = LayerNorm(attention_dim) + + self.intermediate_layers = intermediate_layers + self.use_conditioning = True if ctc_softmax is not None else False + if self.use_conditioning: + self.ctc_softmax = ctc_softmax + self.conditioning_layer = torch.nn.Linear( + conditioning_layer_dim, attention_dim + ) + + def get_positionwise_layer( + self, + positionwise_layer_type="linear", + attention_dim=256, + linear_units=2048, + dropout_rate=0.1, + positionwise_conv_kernel_size=1, + ): + """Define positionwise layer.""" + if positionwise_layer_type == "linear": + positionwise_layer = PositionwiseFeedForward + positionwise_layer_args = (attention_dim, linear_units, dropout_rate) + elif positionwise_layer_type == "conv1d": + positionwise_layer = MultiLayeredConv1d + positionwise_layer_args = ( + attention_dim, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d-linear": + positionwise_layer = Conv1dLinear + positionwise_layer_args = ( + attention_dim, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + else: + raise NotImplementedError("Support only linear or conv1d.") + return positionwise_layer, positionwise_layer_args + + def forward(self, xs, masks): + """Encode input sequence. + + Args: + xs (torch.Tensor): Input tensor (#batch, time, idim). + masks (torch.Tensor): Mask tensor (#batch, time). + + Returns: + torch.Tensor: Output tensor (#batch, time, attention_dim). + torch.Tensor: Mask tensor (#batch, time). + + """ + if isinstance( + self.embed, + (Conv2dSubsampling, Conv2dSubsampling6, Conv2dSubsampling8), + ): + xs, masks = self.embed(xs, masks) + else: + xs = self.embed(xs) + + if self.intermediate_layers is None: + xs, masks = self.encoders(xs, masks) + else: + intermediate_outputs = [] + for layer_idx, encoder_layer in enumerate(self.encoders): + xs, masks = encoder_layer(xs, masks) + + if ( + self.intermediate_layers is not None + and layer_idx + 1 in self.intermediate_layers + ): + encoder_output = xs + # intermediate branches also require normalization. + if self.normalize_before: + encoder_output = self.after_norm(encoder_output) + intermediate_outputs.append(encoder_output) + + if self.use_conditioning: + intermediate_result = self.ctc_softmax(encoder_output) + xs = xs + self.conditioning_layer(intermediate_result) + + if self.normalize_before: + xs = self.after_norm(xs) + + if self.intermediate_layers is not None: + return xs, masks, intermediate_outputs + return xs, masks + + def forward_one_step(self, xs, masks, cache=None): + """Encode input frame. + + Args: + xs (torch.Tensor): Input tensor. + masks (torch.Tensor): Mask tensor. + cache (List[torch.Tensor]): List of cache tensors. + + Returns: + torch.Tensor: Output tensor. + torch.Tensor: Mask tensor. + List[torch.Tensor]: List of new cache tensors. + + """ + if isinstance(self.embed, Conv2dSubsampling): + xs, masks = self.embed(xs, masks) + else: + xs = self.embed(xs) + if cache is None: + cache = [None for _ in range(len(self.encoders))] + new_cache = [] + for c, e in zip(cache, self.encoders): + xs, masks = e(xs, masks, cache=c) + new_cache.append(xs) + if self.normalize_before: + xs = self.after_norm(xs) + return xs, masks, new_cache + diff --git a/funasr/models/frontend/__init__.py b/funasr/models/frontend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/models/frontend/abs_frontend.py b/funasr/models/frontend/abs_frontend.py new file mode 100644 index 000000000..538236fe9 --- /dev/null +++ b/funasr/models/frontend/abs_frontend.py @@ -0,0 +1,17 @@ +from abc import ABC +from abc import abstractmethod +from typing import Tuple + +import torch + + +class AbsFrontend(torch.nn.Module, ABC): + @abstractmethod + def output_size(self) -> int: + raise NotImplementedError + + @abstractmethod + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError diff --git a/funasr/models/frontend/default.py b/funasr/models/frontend/default.py new file mode 100644 index 000000000..fad6b70f2 --- /dev/null +++ b/funasr/models/frontend/default.py @@ -0,0 +1,133 @@ +import copy +from typing import Optional +from typing import Tuple +from typing import Union + +import humanfriendly +import numpy as np +import torch +from torch_complex.tensor import ComplexTensor +from typeguard import check_argument_types + +from funasr.layers.log_mel import LogMel +from funasr.layers.stft import Stft +from funasr.models.frontend.abs_frontend import AbsFrontend +from funasr.modules.frontends.frontend import Frontend +from funasr.utils.get_default_kwargs import get_default_kwargs + + +class DefaultFrontend(AbsFrontend): + """Conventional frontend structure for ASR. + + Stft -> WPE -> MVDR-Beamformer -> Power-spec -> Mel-Fbank -> CMVN + """ + + def __init__( + self, + fs: Union[int, str] = 16000, + n_fft: int = 512, + win_length: int = None, + hop_length: int = 128, + window: Optional[str] = "hann", + center: bool = True, + normalized: bool = False, + onesided: bool = True, + n_mels: int = 80, + fmin: int = None, + fmax: int = None, + htk: bool = False, + frontend_conf: Optional[dict] = get_default_kwargs(Frontend), + apply_stft: bool = True, + ): + assert check_argument_types() + super().__init__() + if isinstance(fs, str): + fs = humanfriendly.parse_size(fs) + + # Deepcopy (In general, dict shouldn't be used as default arg) + frontend_conf = copy.deepcopy(frontend_conf) + self.hop_length = hop_length + + if apply_stft: + self.stft = Stft( + n_fft=n_fft, + win_length=win_length, + hop_length=hop_length, + center=center, + window=window, + normalized=normalized, + onesided=onesided, + ) + else: + self.stft = None + self.apply_stft = apply_stft + + if frontend_conf is not None: + self.frontend = Frontend(idim=n_fft // 2 + 1, **frontend_conf) + else: + self.frontend = None + + self.logmel = LogMel( + fs=fs, + n_fft=n_fft, + n_mels=n_mels, + fmin=fmin, + fmax=fmax, + htk=htk, + ) + self.n_mels = n_mels + self.frontend_type = "default" + + def output_size(self) -> int: + return self.n_mels + + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + # 1. Domain-conversion: e.g. Stft: time -> time-freq + if self.stft is not None: + input_stft, feats_lens = self._compute_stft(input, input_lengths) + else: + input_stft = ComplexTensor(input[..., 0], input[..., 1]) + feats_lens = input_lengths + # 2. [Option] Speech enhancement + if self.frontend is not None: + assert isinstance(input_stft, ComplexTensor), type(input_stft) + # input_stft: (Batch, Length, [Channel], Freq) + input_stft, _, mask = self.frontend(input_stft, feats_lens) + + # 3. [Multi channel case]: Select a channel + if input_stft.dim() == 4: + # h: (B, T, C, F) -> h: (B, T, F) + if self.training: + # Select 1ch randomly + ch = np.random.randint(input_stft.size(2)) + input_stft = input_stft[:, :, ch, :] + else: + # Use the first channel + input_stft = input_stft[:, :, 0, :] + + # 4. STFT -> Power spectrum + # h: ComplexTensor(B, T, F) -> torch.Tensor(B, T, F) + input_power = input_stft.real ** 2 + input_stft.imag ** 2 + + # 5. Feature transform e.g. Stft -> Log-Mel-Fbank + # input_power: (Batch, [Channel,] Length, Freq) + # -> input_feats: (Batch, Length, Dim) + input_feats, _ = self.logmel(input_power, feats_lens) + + return input_feats, feats_lens + + def _compute_stft( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> torch.Tensor: + input_stft, feats_lens = self.stft(input, input_lengths) + + assert input_stft.dim() >= 4, input_stft.shape + # "2" refers to the real/imag parts of Complex + assert input_stft.shape[-1] == 2, input_stft.shape + + # Change torch.Tensor to ComplexTensor + # input_stft: (..., F, 2) -> (..., F) + input_stft = ComplexTensor(input_stft[..., 0], input_stft[..., 1]) + return input_stft, feats_lens diff --git a/funasr/models/frontend/fused.py b/funasr/models/frontend/fused.py new file mode 100644 index 000000000..8b5e56ebf --- /dev/null +++ b/funasr/models/frontend/fused.py @@ -0,0 +1,146 @@ +from funasr.models.frontend.abs_frontend import AbsFrontend +from funasr.models.frontend.default import DefaultFrontend +from funasr.models.frontend.s3prl import S3prlFrontend +import numpy as np +import torch +from typeguard import check_argument_types +from typing import Tuple + + +class FusedFrontends(AbsFrontend): + def __init__( + self, frontends=None, align_method="linear_projection", proj_dim=100, fs=16000 + ): + + assert check_argument_types() + super().__init__() + self.align_method = ( + align_method # fusing method : linear_projection only for now + ) + self.proj_dim = proj_dim # dim of the projection done on each frontend + self.frontends = [] # list of the frontends to combine + + for i, frontend in enumerate(frontends): + frontend_type = frontend["frontend_type"] + if frontend_type == "default": + n_mels, fs, n_fft, win_length, hop_length = ( + frontend.get("n_mels", 80), + fs, + frontend.get("n_fft", 512), + frontend.get("win_length"), + frontend.get("hop_length", 128), + ) + window, center, normalized, onesided = ( + frontend.get("window", "hann"), + frontend.get("center", True), + frontend.get("normalized", False), + frontend.get("onesided", True), + ) + fmin, fmax, htk, apply_stft = ( + frontend.get("fmin", None), + frontend.get("fmax", None), + frontend.get("htk", False), + frontend.get("apply_stft", True), + ) + + self.frontends.append( + DefaultFrontend( + n_mels=n_mels, + n_fft=n_fft, + fs=fs, + win_length=win_length, + hop_length=hop_length, + window=window, + center=center, + normalized=normalized, + onesided=onesided, + fmin=fmin, + fmax=fmax, + htk=htk, + apply_stft=apply_stft, + ) + ) + elif frontend_type == "s3prl": + frontend_conf, download_dir, multilayer_feature = ( + frontend.get("frontend_conf"), + frontend.get("download_dir"), + frontend.get("multilayer_feature"), + ) + self.frontends.append( + S3prlFrontend( + fs=fs, + frontend_conf=frontend_conf, + download_dir=download_dir, + multilayer_feature=multilayer_feature, + ) + ) + + else: + raise NotImplementedError # frontends are only default or s3prl + + self.frontends = torch.nn.ModuleList(self.frontends) + + self.gcd = np.gcd.reduce([frontend.hop_length for frontend in self.frontends]) + self.factors = [frontend.hop_length // self.gcd for frontend in self.frontends] + if torch.cuda.is_available(): + dev = "cuda" + else: + dev = "cpu" + if self.align_method == "linear_projection": + self.projection_layers = [ + torch.nn.Linear( + in_features=frontend.output_size(), + out_features=self.factors[i] * self.proj_dim, + ) + for i, frontend in enumerate(self.frontends) + ] + self.projection_layers = torch.nn.ModuleList(self.projection_layers) + self.projection_layers = self.projection_layers.to(torch.device(dev)) + + def output_size(self) -> int: + return len(self.frontends) * self.proj_dim + + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + + # step 0 : get all frontends features + self.feats = [] + for frontend in self.frontends: + with torch.no_grad(): + input_feats, feats_lens = frontend.forward(input, input_lengths) + self.feats.append([input_feats, feats_lens]) + + if ( + self.align_method == "linear_projection" + ): # TODO(Dan): to add other align methods + + # first step : projections + self.feats_proj = [] + for i, frontend in enumerate(self.frontends): + input_feats = self.feats[i][0] + self.feats_proj.append(self.projection_layers[i](input_feats)) + + # 2nd step : reshape + self.feats_reshaped = [] + for i, frontend in enumerate(self.frontends): + input_feats_proj = self.feats_proj[i] + bs, nf, dim = input_feats_proj.shape + input_feats_reshaped = torch.reshape( + input_feats_proj, (bs, nf * self.factors[i], dim // self.factors[i]) + ) + self.feats_reshaped.append(input_feats_reshaped) + + # 3rd step : drop the few last frames + m = min([x.shape[1] for x in self.feats_reshaped]) + self.feats_final = [x[:, :m, :] for x in self.feats_reshaped] + + input_feats = torch.cat( + self.feats_final, dim=-1 + ) # change the input size of the preencoder : proj_dim * n_frontends + feats_lens = torch.ones_like(self.feats[0][1]) * (m) + + else: + raise NotImplementedError + + return input_feats, feats_lens diff --git a/funasr/models/frontend/s3prl.py b/funasr/models/frontend/s3prl.py new file mode 100644 index 000000000..f2b6107d9 --- /dev/null +++ b/funasr/models/frontend/s3prl.py @@ -0,0 +1,143 @@ +import copy +import logging +import os +from argparse import Namespace +from typing import Optional +from typing import Tuple +from typing import Union + +import humanfriendly +import torch +from typeguard import check_argument_types + +from funasr.models.frontend.abs_frontend import AbsFrontend +from funasr.modules.frontends.frontend import Frontend +from funasr.modules.nets_utils import pad_list +from funasr.utils.get_default_kwargs import get_default_kwargs + + +def base_s3prl_setup(args): + args.upstream_feature_selection = getattr(args, "upstream_feature_selection", None) + args.upstream_model_config = getattr(args, "upstream_model_config", None) + args.upstream_refresh = getattr(args, "upstream_refresh", False) + args.upstream_ckpt = getattr(args, "upstream_ckpt", None) + args.init_ckpt = getattr(args, "init_ckpt", None) + args.verbose = getattr(args, "verbose", False) + args.tile_factor = getattr(args, "tile_factor", 1) + return args + + +class S3prlFrontend(AbsFrontend): + """Speech Pretrained Representation frontend structure for ASR.""" + + def __init__( + self, + fs: Union[int, str] = 16000, + frontend_conf: Optional[dict] = get_default_kwargs(Frontend), + download_dir: str = None, + multilayer_feature: bool = False, + ): + assert check_argument_types() + super().__init__() + if isinstance(fs, str): + fs = humanfriendly.parse_size(fs) + + if download_dir is not None: + torch.hub.set_dir(download_dir) + + self.multilayer_feature = multilayer_feature + self.upstream, self.featurizer = self._get_upstream(frontend_conf) + self.pretrained_params = copy.deepcopy(self.upstream.state_dict()) + self.output_dim = self.featurizer.output_dim + self.frontend_type = "s3prl" + self.hop_length = self.upstream.get_downsample_rates("key") + + def _get_upstream(self, frontend_conf): + """Get S3PRL upstream model.""" + s3prl_args = base_s3prl_setup( + Namespace(**frontend_conf, device="cpu"), + ) + self.args = s3prl_args + + s3prl_path = None + python_path_list = os.environ.get("PYTHONPATH", "(None)").split(":") + for p in python_path_list: + if p.endswith("s3prl"): + s3prl_path = p + break + assert s3prl_path is not None + + s3prl_upstream = torch.hub.load( + s3prl_path, + s3prl_args.upstream, + ckpt=s3prl_args.upstream_ckpt, + model_config=s3prl_args.upstream_model_config, + refresh=s3prl_args.upstream_refresh, + source="local", + ).to("cpu") + + if getattr( + s3prl_upstream, "model", None + ) is not None and s3prl_upstream.model.__class__.__name__ in [ + "Wav2Vec2Model", + "HubertModel", + ]: + s3prl_upstream.model.encoder.layerdrop = 0.0 + + from s3prl.upstream.interfaces import Featurizer + + if self.multilayer_feature is None: + feature_selection = "last_hidden_state" + else: + feature_selection = "hidden_states" + s3prl_featurizer = Featurizer( + upstream=s3prl_upstream, + feature_selection=feature_selection, + upstream_device="cpu", + ) + + return s3prl_upstream, s3prl_featurizer + + def _tile_representations(self, feature): + """Tile up the representations by `tile_factor`. + + Input - sequence of representations + shape: (batch_size, seq_len, feature_dim) + Output - sequence of tiled representations + shape: (batch_size, seq_len * factor, feature_dim) + """ + assert ( + len(feature.shape) == 3 + ), "Input argument `feature` has invalid shape: {}".format(feature.shape) + tiled_feature = feature.repeat(1, 1, self.args.tile_factor) + tiled_feature = tiled_feature.reshape( + feature.size(0), feature.size(1) * self.args.tile_factor, feature.size(2) + ) + return tiled_feature + + def output_size(self) -> int: + return self.output_dim + + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + wavs = [wav[: input_lengths[i]] for i, wav in enumerate(input)] + self.upstream.eval() + with torch.no_grad(): + feats = self.upstream(wavs) + feats = self.featurizer(wavs, feats) + + if self.args.tile_factor != 1: + feats = self._tile_representations(feats) + + input_feats = pad_list(feats, 0.0) + feats_lens = torch.tensor([f.shape[0] for f in feats], dtype=torch.long) + + # Saving CUDA Memory + del feats + + return input_feats, feats_lens + + def reload_pretrained_parameters(self): + self.upstream.load_state_dict(self.pretrained_params) + logging.info("Pretrained S3PRL frontend model parameters reloaded!") diff --git a/funasr/models/frontend/windowing.py b/funasr/models/frontend/windowing.py new file mode 100644 index 000000000..7c4c56853 --- /dev/null +++ b/funasr/models/frontend/windowing.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +# 2020, Technische Universität München; Ludwig Kürzinger +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Sliding Window for raw audio input data.""" + +from funasr.models.frontend.abs_frontend import AbsFrontend +import torch +from typeguard import check_argument_types +from typing import Tuple + + +class SlidingWindow(AbsFrontend): + """Sliding Window. + + Provides a sliding window over a batched continuous raw audio tensor. + Optionally, provides padding (Currently not implemented). + Combine this module with a pre-encoder compatible with raw audio data, + for example Sinc convolutions. + + Known issues: + Output length is calculated incorrectly if audio shorter than win_length. + WARNING: trailing values are discarded - padding not implemented yet. + There is currently no additional window function applied to input values. + """ + + def __init__( + self, + win_length: int = 400, + hop_length: int = 160, + channels: int = 1, + padding: int = None, + fs=None, + ): + """Initialize. + + Args: + win_length: Length of frame. + hop_length: Relative starting point of next frame. + channels: Number of input channels. + padding: Padding (placeholder, currently not implemented). + fs: Sampling rate (placeholder for compatibility, not used). + """ + assert check_argument_types() + super().__init__() + self.fs = fs + self.win_length = win_length + self.hop_length = hop_length + self.channels = channels + self.padding = padding + + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Apply a sliding window on the input. + + Args: + input: Input (B, T, C*D) or (B, T*C*D), with D=C=1. + input_lengths: Input lengths within batch. + + Returns: + Tensor: Output with dimensions (B, T, C, D), with D=win_length. + Tensor: Output lengths within batch. + """ + input_size = input.size() + B = input_size[0] + T = input_size[1] + C = self.channels + D = self.win_length + # (B, T, C) --> (T, B, C) + continuous = input.view(B, T, C).permute(1, 0, 2) + windowed = continuous.unfold(0, D, self.hop_length) + # (T, B, C, D) --> (B, T, C, D) + output = windowed.permute(1, 0, 2, 3).contiguous() + # After unfold(), windowed lengths change: + output_lengths = (input_lengths - self.win_length) // self.hop_length + 1 + return output, output_lengths + + def output_size(self) -> int: + """Return output length of feature dimension D, i.e. the window length.""" + return self.win_length diff --git a/funasr/models/postencoder/__init__.py b/funasr/models/postencoder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/models/postencoder/abs_postencoder.py b/funasr/models/postencoder/abs_postencoder.py new file mode 100644 index 000000000..f5ac03be2 --- /dev/null +++ b/funasr/models/postencoder/abs_postencoder.py @@ -0,0 +1,17 @@ +from abc import ABC +from abc import abstractmethod +from typing import Tuple + +import torch + + +class AbsPostEncoder(torch.nn.Module, ABC): + @abstractmethod + def output_size(self) -> int: + raise NotImplementedError + + @abstractmethod + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError diff --git a/funasr/models/postencoder/hugging_face_transformers_postencoder.py b/funasr/models/postencoder/hugging_face_transformers_postencoder.py new file mode 100644 index 000000000..1aad15d79 --- /dev/null +++ b/funasr/models/postencoder/hugging_face_transformers_postencoder.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# 2021, University of Stuttgart; Pavel Denisov +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Hugging Face Transformers PostEncoder.""" + +from funasr.modules.nets_utils import make_pad_mask +from funasr.models.postencoder.abs_postencoder import AbsPostEncoder +from typeguard import check_argument_types +from typing import Tuple + +import copy +import logging +import torch + +try: + from transformers import AutoModel + + is_transformers_available = True +except ImportError: + is_transformers_available = False + + +class HuggingFaceTransformersPostEncoder(AbsPostEncoder): + """Hugging Face Transformers PostEncoder.""" + + def __init__( + self, + input_size: int, + model_name_or_path: str, + ): + """Initialize the module.""" + assert check_argument_types() + super().__init__() + + if not is_transformers_available: + raise ImportError( + "`transformers` is not available. Please install it via `pip install" + " transformers` or `cd /path/to/espnet/tools && . ./activate_python.sh" + " && ./installers/install_transformers.sh`." + ) + + model = AutoModel.from_pretrained(model_name_or_path) + + if hasattr(model, "encoder"): + self.transformer = model.encoder + else: + self.transformer = model + + if hasattr(self.transformer, "embed_tokens"): + del self.transformer.embed_tokens + if hasattr(self.transformer, "wte"): + del self.transformer.wte + if hasattr(self.transformer, "word_embedding"): + del self.transformer.word_embedding + + self.pretrained_params = copy.deepcopy(self.transformer.state_dict()) + + if ( + self.transformer.config.is_encoder_decoder + or self.transformer.config.model_type in ["xlnet", "t5"] + ): + self.use_inputs_embeds = True + self.extend_attention_mask = False + elif self.transformer.config.model_type == "gpt2": + self.use_inputs_embeds = True + self.extend_attention_mask = True + else: + self.use_inputs_embeds = False + self.extend_attention_mask = True + + self.linear_in = torch.nn.Linear( + input_size, self.transformer.config.hidden_size + ) + + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward.""" + input = self.linear_in(input) + + args = {"return_dict": True} + + mask = (~make_pad_mask(input_lengths)).to(input.device).float() + + if self.extend_attention_mask: + args["attention_mask"] = _extend_attention_mask(mask) + else: + args["attention_mask"] = mask + + if self.use_inputs_embeds: + args["inputs_embeds"] = input + else: + args["hidden_states"] = input + + if self.transformer.config.model_type == "mpnet": + args["head_mask"] = [None for _ in self.transformer.layer] + + output = self.transformer(**args).last_hidden_state + + return output, input_lengths + + def reload_pretrained_parameters(self): + self.transformer.load_state_dict(self.pretrained_params) + logging.info("Pretrained Transformers model parameters reloaded!") + + def output_size(self) -> int: + """Get the output size.""" + return self.transformer.config.hidden_size + + +def _extend_attention_mask(mask: torch.Tensor) -> torch.Tensor: + mask = mask[:, None, None, :] + mask = (1.0 - mask) * -10000.0 + return mask diff --git a/funasr/models/predictor/__init__.py b/funasr/models/predictor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/models/predictor/cif.py b/funasr/models/predictor/cif.py new file mode 100644 index 000000000..819970870 --- /dev/null +++ b/funasr/models/predictor/cif.py @@ -0,0 +1,266 @@ +import torch +from torch import nn + +from funasr.modules.nets_utils import make_pad_mask + +class CifPredictor(nn.Module): + def __init__(self, idim, l_order, r_order, threshold=1.0, dropout=0.1, smooth_factor=1.0, noise_threshold=0): + super(CifPredictor, self).__init__() + + self.pad = nn.ConstantPad1d((l_order, r_order), 0) + self.cif_conv1d = nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim) + self.cif_output = nn.Linear(idim, 1) + self.dropout = torch.nn.Dropout(p=dropout) + self.threshold = threshold + self.smooth_factor = smooth_factor + self.noise_threshold = noise_threshold + + def forward(self, hidden, target_label=None, mask=None, ignore_id=-1, mask_chunk_predictor=None, + target_label_length=None): + h = hidden + context = h.transpose(1, 2) + queries = self.pad(context) + memory = self.cif_conv1d(queries) + output = memory + context + output = self.dropout(output) + output = output.transpose(1, 2) + output = torch.relu(output) + output = self.cif_output(output) + alphas = torch.sigmoid(output) + alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold) + if mask is not None: + alphas = alphas * mask.transpose(-1, -2).float() + if mask_chunk_predictor is not None: + alphas = alphas * mask_chunk_predictor + alphas = alphas.squeeze(-1) + if target_label_length is not None: + target_length = target_label_length + elif target_label is not None: + target_length = (target_label != ignore_id).float().sum(-1) + else: + target_length = None + token_num = alphas.sum(-1) + if target_length is not None: + alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1)) + acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold) + return acoustic_embeds, token_num, alphas, cif_peak + + def gen_frame_alignments(self, + alphas: torch.Tensor = None, + encoder_sequence_length: torch.Tensor = None): + batch_size, maximum_length = alphas.size() + int_type = torch.int32 + + is_training = self.training + if is_training: + token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type) + else: + token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type) + + max_token_num = torch.max(token_num).item() + + alphas_cumsum = torch.cumsum(alphas, dim=1) + alphas_cumsum = torch.floor(alphas_cumsum).type(int_type) + alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1) + + index = torch.ones([batch_size, max_token_num], dtype=int_type) + index = torch.cumsum(index, dim=1) + index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device) + + index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type) + index_div_bool_zeros = index_div.eq(0) + index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1 + index_div_bool_zeros_count = torch.clamp(index_div_bool_zeros_count, 0, encoder_sequence_length.max()) + token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device) + index_div_bool_zeros_count *= token_num_mask + + index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(1, 1, maximum_length) + ones = torch.ones_like(index_div_bool_zeros_count_tile) + zeros = torch.zeros_like(index_div_bool_zeros_count_tile) + ones = torch.cumsum(ones, dim=2) + cond = index_div_bool_zeros_count_tile == ones + index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones) + + index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool) + index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type) + index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1) + index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type) + predictor_mask = (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max())).type( + int_type).to(encoder_sequence_length.device) + index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask + + predictor_alignments = index_div_bool_zeros_count_tile_out + predictor_alignments_length = predictor_alignments.sum(-1).type(encoder_sequence_length.dtype) + return predictor_alignments.detach(), predictor_alignments_length.detach() + + +class CifPredictorV2(nn.Module): + def __init__(self, idim, l_order, r_order, threshold=1.0, dropout=0.1, smooth_factor=1.0, noise_threshold=0, + tail_threshold=0.0): + super(CifPredictorV2, self).__init__() + + self.pad = nn.ConstantPad1d((l_order, r_order), 0) + self.cif_conv1d = nn.Conv1d(idim, idim, l_order + r_order + 1) + self.cif_output = nn.Linear(idim, 1) + self.dropout = torch.nn.Dropout(p=dropout) + self.threshold = threshold + self.smooth_factor = smooth_factor + self.noise_threshold = noise_threshold + self.tail_threshold = tail_threshold + + def forward(self, hidden, target_label=None, mask=None, ignore_id=-1, mask_chunk_predictor=None, + target_label_length=None): + h = hidden + context = h.transpose(1, 2) + queries = self.pad(context) + output = torch.relu(self.cif_conv1d(queries)) + output = output.transpose(1, 2) + + output = self.cif_output(output) + alphas = torch.sigmoid(output) + alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold) + if mask is not None: + alphas = alphas * mask.transpose(-1, -2).float() + if mask_chunk_predictor is not None: + alphas = alphas * mask_chunk_predictor + alphas = alphas.squeeze(-1) + if target_label_length is not None: + target_length = target_label_length + elif target_label is not None: + target_length = (target_label != ignore_id).float().sum(-1) + else: + target_length = None + token_num = alphas.sum(-1) + if target_length is not None: + alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1)) + elif self.tail_threshold > 0.0: + hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num) + + acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold) + if target_length is None and self.tail_threshold > 0.0: + token_num_int = torch.max(token_num).type(torch.int32).item() + acoustic_embeds = acoustic_embeds[:, :token_num_int, :] + + return acoustic_embeds, token_num, alphas, cif_peak + + def tail_process_fn(self, hidden, alphas, token_num=None): + b, t, d = hidden.size() + tail_threshold = self.tail_threshold + tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device) + tail_threshold = torch.reshape(tail_threshold, (1, 1)) + alphas = torch.cat([alphas, tail_threshold], dim=1) + zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device) + hidden = torch.cat([hidden, zeros], dim=1) + token_num = alphas.sum(dim=-1) + token_num_floor = torch.floor(token_num) + + return hidden, alphas, token_num_floor + + def gen_frame_alignments(self, + alphas: torch.Tensor = None, + encoder_sequence_length: torch.Tensor = None): + batch_size, maximum_length = alphas.size() + int_type = torch.int32 + + is_training = self.training + if is_training: + token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type) + else: + token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type) + + max_token_num = torch.max(token_num).item() + + alphas_cumsum = torch.cumsum(alphas, dim=1) + alphas_cumsum = torch.floor(alphas_cumsum).type(int_type) + alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1) + + index = torch.ones([batch_size, max_token_num], dtype=int_type) + index = torch.cumsum(index, dim=1) + index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device) + + index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type) + index_div_bool_zeros = index_div.eq(0) + index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1 + index_div_bool_zeros_count = torch.clamp(index_div_bool_zeros_count, 0, encoder_sequence_length.max()) + token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device) + index_div_bool_zeros_count *= token_num_mask + + index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(1, 1, maximum_length) + ones = torch.ones_like(index_div_bool_zeros_count_tile) + zeros = torch.zeros_like(index_div_bool_zeros_count_tile) + ones = torch.cumsum(ones, dim=2) + cond = index_div_bool_zeros_count_tile == ones + index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones) + + index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool) + index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type) + index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1) + index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type) + predictor_mask = (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max())).type( + int_type).to(encoder_sequence_length.device) + index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask + + predictor_alignments = index_div_bool_zeros_count_tile_out + predictor_alignments_length = predictor_alignments.sum(-1).type(encoder_sequence_length.dtype) + return predictor_alignments.detach(), predictor_alignments_length.detach() + + +class mae_loss(nn.Module): + + def __init__(self, normalize_length=False): + super(mae_loss, self).__init__() + self.normalize_length = normalize_length + self.criterion = torch.nn.L1Loss(reduction='sum') + + def forward(self, token_length, pre_token_length): + loss_token_normalizer = token_length.size(0) + if self.normalize_length: + loss_token_normalizer = token_length.sum().type(torch.float32) + loss = self.criterion(token_length, pre_token_length) + loss = loss / loss_token_normalizer + return loss + + +def cif(hidden, alphas, threshold): + batch_size, len_time, hidden_size = hidden.size() + + # loop varss + integrate = torch.zeros([batch_size], device=hidden.device) + frame = torch.zeros([batch_size, hidden_size], device=hidden.device) + # intermediate vars along time + list_fires = [] + list_frames = [] + + for t in range(len_time): + alpha = alphas[:, t] + distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate + + integrate += alpha + list_fires.append(integrate) + + fire_place = integrate >= threshold + integrate = torch.where(fire_place, + integrate - torch.ones([batch_size], device=hidden.device), + integrate) + cur = torch.where(fire_place, + distribution_completion, + alpha) + remainds = alpha - cur + + frame += cur[:, None] * hidden[:, t, :] + list_frames.append(frame) + frame = torch.where(fire_place[:, None].repeat(1, hidden_size), + remainds[:, None] * hidden[:, t, :], + frame) + + fires = torch.stack(list_fires, 1) + frames = torch.stack(list_frames, 1) + list_ls = [] + len_labels = torch.round(alphas.sum(-1)).int() + max_label_len = len_labels.max() + for b in range(batch_size): + fire = fires[b, :] + l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze()) + pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device) + list_ls.append(torch.cat([l, pad_l], 0)) + return torch.stack(list_ls, 0), fires diff --git a/funasr/models/preencoder/__init__.py b/funasr/models/preencoder/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/models/preencoder/abs_preencoder.py b/funasr/models/preencoder/abs_preencoder.py new file mode 100644 index 000000000..3ecdc6b91 --- /dev/null +++ b/funasr/models/preencoder/abs_preencoder.py @@ -0,0 +1,17 @@ +from abc import ABC +from abc import abstractmethod +from typing import Tuple + +import torch + + +class AbsPreEncoder(torch.nn.Module, ABC): + @abstractmethod + def output_size(self) -> int: + raise NotImplementedError + + @abstractmethod + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError diff --git a/funasr/models/preencoder/linear.py b/funasr/models/preencoder/linear.py new file mode 100644 index 000000000..c69b6ce92 --- /dev/null +++ b/funasr/models/preencoder/linear.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# 2021, Carnegie Mellon University; Xuankai Chang +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Linear Projection.""" + +from funasr.models.preencoder.abs_preencoder import AbsPreEncoder +from typeguard import check_argument_types +from typing import Tuple + +import torch + + +class LinearProjection(AbsPreEncoder): + """Linear Projection Preencoder.""" + + def __init__( + self, + input_size: int, + output_size: int, + ): + """Initialize the module.""" + assert check_argument_types() + super().__init__() + + self.output_dim = output_size + self.linear_out = torch.nn.Linear(input_size, output_size) + + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Forward.""" + output = self.linear_out(input) + return output, input_lengths # no state in this layer + + def output_size(self) -> int: + """Get the output size.""" + return self.output_dim diff --git a/funasr/models/preencoder/sinc.py b/funasr/models/preencoder/sinc.py new file mode 100644 index 000000000..fe6d2af1b --- /dev/null +++ b/funasr/models/preencoder/sinc.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +# 2020, Technische Universität München; Ludwig Kürzinger +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Sinc convolutions for raw audio input.""" + +from collections import OrderedDict +from funasr.models.preencoder.abs_preencoder import AbsPreEncoder +from funasr.layers.sinc_conv import LogCompression +from funasr.layers.sinc_conv import SincConv +import humanfriendly +import torch +from typeguard import check_argument_types +from typing import Optional +from typing import Tuple +from typing import Union + + +class LightweightSincConvs(AbsPreEncoder): + """Lightweight Sinc Convolutions. + + Instead of using precomputed features, end-to-end speech recognition + can also be done directly from raw audio using sinc convolutions, as + described in "Lightweight End-to-End Speech Recognition from Raw Audio + Data Using Sinc-Convolutions" by Kürzinger et al. + https://arxiv.org/abs/2010.07597 + + To use Sinc convolutions in your model instead of the default f-bank + frontend, set this module as your pre-encoder with `preencoder: sinc` + and use the input of the sliding window frontend with + `frontend: sliding_window` in your yaml configuration file. + So that the process flow is: + + Frontend (SlidingWindow) -> SpecAug -> Normalization -> + Pre-encoder (LightweightSincConvs) -> Encoder -> Decoder + + Note that this method also performs data augmentation in time domain + (vs. in spectral domain in the default frontend). + Use `plot_sinc_filters.py` to visualize the learned Sinc filters. + """ + + def __init__( + self, + fs: Union[int, str, float] = 16000, + in_channels: int = 1, + out_channels: int = 256, + activation_type: str = "leakyrelu", + dropout_type: str = "dropout", + windowing_type: str = "hamming", + scale_type: str = "mel", + ): + """Initialize the module. + + Args: + fs: Sample rate. + in_channels: Number of input channels. + out_channels: Number of output channels (for each input channel). + activation_type: Choice of activation function. + dropout_type: Choice of dropout function. + windowing_type: Choice of windowing function. + scale_type: Choice of filter-bank initialization scale. + """ + assert check_argument_types() + super().__init__() + if isinstance(fs, str): + fs = humanfriendly.parse_size(fs) + self.fs = fs + self.in_channels = in_channels + self.out_channels = out_channels + self.activation_type = activation_type + self.dropout_type = dropout_type + self.windowing_type = windowing_type + self.scale_type = scale_type + + self.choices_dropout = { + "dropout": torch.nn.Dropout, + "spatial": SpatialDropout, + "dropout2d": torch.nn.Dropout2d, + } + if dropout_type not in self.choices_dropout: + raise NotImplementedError( + f"Dropout type has to be one of " + f"{list(self.choices_dropout.keys())}", + ) + + self.choices_activation = { + "leakyrelu": torch.nn.LeakyReLU, + "relu": torch.nn.ReLU, + } + if activation_type not in self.choices_activation: + raise NotImplementedError( + f"Activation type has to be one of " + f"{list(self.choices_activation.keys())}", + ) + + # initialization + self._create_sinc_convs() + # Sinc filters require custom initialization + self.espnet_initialization_fn() + + def _create_sinc_convs(self): + blocks = OrderedDict() + + # SincConvBlock + out_channels = 128 + self.filters = SincConv( + self.in_channels, + out_channels, + kernel_size=101, + stride=1, + fs=self.fs, + window_func=self.windowing_type, + scale_type=self.scale_type, + ) + block = OrderedDict( + [ + ("Filters", self.filters), + ("LogCompression", LogCompression()), + ("BatchNorm", torch.nn.BatchNorm1d(out_channels, affine=True)), + ("AvgPool", torch.nn.AvgPool1d(2)), + ] + ) + blocks["SincConvBlock"] = torch.nn.Sequential(block) + in_channels = out_channels + + # First convolutional block, connects the sinc output to the front-end "body" + out_channels = 128 + blocks["DConvBlock1"] = self.gen_lsc_block( + in_channels, + out_channels, + depthwise_kernel_size=25, + depthwise_stride=2, + pointwise_groups=0, + avgpool=True, + dropout_probability=0.1, + ) + in_channels = out_channels + + # Second convolutional block, multiple convolutional layers + out_channels = self.out_channels + for layer in [2, 3, 4]: + blocks[f"DConvBlock{layer}"] = self.gen_lsc_block( + in_channels, out_channels, depthwise_kernel_size=9, depthwise_stride=1 + ) + in_channels = out_channels + + # Third Convolutional block, acts as coupling to encoder + out_channels = self.out_channels + blocks["DConvBlock5"] = self.gen_lsc_block( + in_channels, + out_channels, + depthwise_kernel_size=7, + depthwise_stride=1, + pointwise_groups=0, + ) + + self.blocks = torch.nn.Sequential(blocks) + + def gen_lsc_block( + self, + in_channels: int, + out_channels: int, + depthwise_kernel_size: int = 9, + depthwise_stride: int = 1, + depthwise_groups=None, + pointwise_groups=0, + dropout_probability: float = 0.15, + avgpool=False, + ): + """Generate a convolutional block for Lightweight Sinc convolutions. + + Each block consists of either a depthwise or a depthwise-separable + convolutions together with dropout, (batch-)normalization layer, and + an optional average-pooling layer. + + Args: + in_channels: Number of input channels. + out_channels: Number of output channels. + depthwise_kernel_size: Kernel size of the depthwise convolution. + depthwise_stride: Stride of the depthwise convolution. + depthwise_groups: Number of groups of the depthwise convolution. + pointwise_groups: Number of groups of the pointwise convolution. + dropout_probability: Dropout probability in the block. + avgpool: If True, an AvgPool layer is inserted. + + Returns: + torch.nn.Sequential: Neural network building block. + """ + block = OrderedDict() + if not depthwise_groups: + # GCD(in_channels, out_channels) to prevent size mismatches + depthwise_groups, r = in_channels, out_channels + while r != 0: + depthwise_groups, r = depthwise_groups, depthwise_groups % r + block["depthwise"] = torch.nn.Conv1d( + in_channels, + out_channels, + depthwise_kernel_size, + depthwise_stride, + groups=depthwise_groups, + ) + if pointwise_groups: + block["pointwise"] = torch.nn.Conv1d( + out_channels, out_channels, 1, 1, groups=pointwise_groups + ) + block["activation"] = self.choices_activation[self.activation_type]() + block["batchnorm"] = torch.nn.BatchNorm1d(out_channels, affine=True) + if avgpool: + block["avgpool"] = torch.nn.AvgPool1d(2) + block["dropout"] = self.choices_dropout[self.dropout_type](dropout_probability) + return torch.nn.Sequential(block) + + def espnet_initialization_fn(self): + """Initialize sinc filters with filterbank values.""" + self.filters.init_filters() + for block in self.blocks: + for layer in block: + if type(layer) == torch.nn.BatchNorm1d and layer.affine: + layer.weight.data[:] = 1.0 + layer.bias.data[:] = 0.0 + + def forward( + self, input: torch.Tensor, input_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Apply Lightweight Sinc Convolutions. + + The input shall be formatted as (B, T, C_in, D_in) + with B as batch size, T as time dimension, C_in as channels, + and D_in as feature dimension. + + The output will then be (B, T, C_out*D_out) + with C_out and D_out as output dimensions. + + The current module structure only handles D_in=400, so that D_out=1. + Remark for the multichannel case: C_out is the number of out_channels + given at initialization multiplied with C_in. + """ + # Transform input data: + # (B, T, C_in, D_in) -> (B*T, C_in, D_in) + B, T, C_in, D_in = input.size() + input_frames = input.view(B * T, C_in, D_in) + output_frames = self.blocks.forward(input_frames) + + # ---TRANSFORM: (B*T, C_out, D_out) -> (B, T, C_out*D_out) + _, C_out, D_out = output_frames.size() + output_frames = output_frames.view(B, T, C_out * D_out) + return output_frames, input_lengths # no state in this layer + + def output_size(self) -> int: + """Get the output size.""" + return self.out_channels * self.in_channels + + +class SpatialDropout(torch.nn.Module): + """Spatial dropout module. + + Apply dropout to full channels on tensors of input (B, C, D) + """ + + def __init__( + self, + dropout_probability: float = 0.15, + shape: Optional[Union[tuple, list]] = None, + ): + """Initialize. + + Args: + dropout_probability: Dropout probability. + shape (tuple, list): Shape of input tensors. + """ + assert check_argument_types() + super().__init__() + if shape is None: + shape = (0, 2, 1) + self.dropout = torch.nn.Dropout2d(dropout_probability) + self.shape = (shape,) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward of spatial dropout module.""" + y = x.permute(*self.shape) + y = self.dropout(y) + return y.permute(*self.shape) diff --git a/funasr/models/specaug/__init__.py b/funasr/models/specaug/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/models/specaug/abs_specaug.py b/funasr/models/specaug/abs_specaug.py new file mode 100644 index 000000000..3cbac418f --- /dev/null +++ b/funasr/models/specaug/abs_specaug.py @@ -0,0 +1,18 @@ +from typing import Optional +from typing import Tuple + +import torch + + +class AbsSpecAug(torch.nn.Module): + """Abstract class for the augmentation of spectrogram + + The process-flow: + + Frontend -> SpecAug -> Normalization -> Encoder -> Decoder + """ + + def forward( + self, x: torch.Tensor, x_lengths: torch.Tensor = None + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: + raise NotImplementedError diff --git a/funasr/models/specaug/specaug.py b/funasr/models/specaug/specaug.py new file mode 100644 index 000000000..6074f86fd --- /dev/null +++ b/funasr/models/specaug/specaug.py @@ -0,0 +1,184 @@ +"""SpecAugment module.""" +from typing import Optional +from typing import Sequence +from typing import Union + +from funasr.models.specaug.abs_specaug import AbsSpecAug +from funasr.layers.mask_along_axis import MaskAlongAxis +from funasr.layers.mask_along_axis import MaskAlongAxisVariableMaxWidth +from funasr.layers.mask_along_axis import MaskAlongAxisLFR +from funasr.layers.time_warp import TimeWarp + + +class SpecAug(AbsSpecAug): + """Implementation of SpecAug. + + Reference: + Daniel S. Park et al. + "SpecAugment: A Simple Data + Augmentation Method for Automatic Speech Recognition" + + .. warning:: + When using cuda mode, time_warp doesn't have reproducibility + due to `torch.nn.functional.interpolate`. + + """ + + def __init__( + self, + apply_time_warp: bool = True, + time_warp_window: int = 5, + time_warp_mode: str = "bicubic", + apply_freq_mask: bool = True, + freq_mask_width_range: Union[int, Sequence[int]] = (0, 20), + num_freq_mask: int = 2, + apply_time_mask: bool = True, + time_mask_width_range: Optional[Union[int, Sequence[int]]] = None, + time_mask_width_ratio_range: Optional[Union[float, Sequence[float]]] = None, + num_time_mask: int = 2, + ): + if not apply_time_warp and not apply_time_mask and not apply_freq_mask: + raise ValueError( + "Either one of time_warp, time_mask, or freq_mask should be applied" + ) + if ( + apply_time_mask + and (time_mask_width_range is not None) + and (time_mask_width_ratio_range is not None) + ): + raise ValueError( + 'Either one of "time_mask_width_range" or ' + '"time_mask_width_ratio_range" can be used' + ) + super().__init__() + self.apply_time_warp = apply_time_warp + self.apply_freq_mask = apply_freq_mask + self.apply_time_mask = apply_time_mask + + if apply_time_warp: + self.time_warp = TimeWarp(window=time_warp_window, mode=time_warp_mode) + else: + self.time_warp = None + + if apply_freq_mask: + self.freq_mask = MaskAlongAxis( + dim="freq", + mask_width_range=freq_mask_width_range, + num_mask=num_freq_mask, + ) + else: + self.freq_mask = None + + if apply_time_mask: + if time_mask_width_range is not None: + self.time_mask = MaskAlongAxis( + dim="time", + mask_width_range=time_mask_width_range, + num_mask=num_time_mask, + ) + elif time_mask_width_ratio_range is not None: + self.time_mask = MaskAlongAxisVariableMaxWidth( + dim="time", + mask_width_ratio_range=time_mask_width_ratio_range, + num_mask=num_time_mask, + ) + else: + raise ValueError( + 'Either one of "time_mask_width_range" or ' + '"time_mask_width_ratio_range" should be used.' + ) + else: + self.time_mask = None + + def forward(self, x, x_lengths=None): + if self.time_warp is not None: + x, x_lengths = self.time_warp(x, x_lengths) + if self.freq_mask is not None: + x, x_lengths = self.freq_mask(x, x_lengths) + if self.time_mask is not None: + x, x_lengths = self.time_mask(x, x_lengths) + return x, x_lengths + +class SpecAugLFR(AbsSpecAug): + """Implementation of SpecAug. + lfr_rate：low frame rate + """ + + def __init__( + self, + apply_time_warp: bool = True, + time_warp_window: int = 5, + time_warp_mode: str = "bicubic", + apply_freq_mask: bool = True, + freq_mask_width_range: Union[int, Sequence[int]] = (0, 20), + num_freq_mask: int = 2, + lfr_rate: int = 0, + apply_time_mask: bool = True, + time_mask_width_range: Optional[Union[int, Sequence[int]]] = None, + time_mask_width_ratio_range: Optional[Union[float, Sequence[float]]] = None, + num_time_mask: int = 2, + ): + if not apply_time_warp and not apply_time_mask and not apply_freq_mask: + raise ValueError( + "Either one of time_warp, time_mask, or freq_mask should be applied" + ) + if ( + apply_time_mask + and (time_mask_width_range is not None) + and (time_mask_width_ratio_range is not None) + ): + raise ValueError( + 'Either one of "time_mask_width_range" or ' + '"time_mask_width_ratio_range" can be used' + ) + super().__init__() + self.apply_time_warp = apply_time_warp + self.apply_freq_mask = apply_freq_mask + self.apply_time_mask = apply_time_mask + + if apply_time_warp: + self.time_warp = TimeWarp(window=time_warp_window, mode=time_warp_mode) + else: + self.time_warp = None + + if apply_freq_mask: + self.freq_mask = MaskAlongAxisLFR( + dim="freq", + mask_width_range=freq_mask_width_range, + num_mask=num_freq_mask, + lfr_rate=lfr_rate+1, + ) + + else: + self.freq_mask = None + + if apply_time_mask: + if time_mask_width_range is not None: + self.time_mask = MaskAlongAxisLFR( + dim="time", + mask_width_range=time_mask_width_range, + num_mask=num_time_mask, + lfr_rate=lfr_rate + 1, + ) + elif time_mask_width_ratio_range is not None: + self.time_mask = MaskAlongAxisVariableMaxWidth( + dim="time", + mask_width_ratio_range=time_mask_width_ratio_range, + num_mask=num_time_mask, + ) + else: + raise ValueError( + 'Either one of "time_mask_width_range" or ' + '"time_mask_width_ratio_range" should be used.' + ) + else: + self.time_mask = None + + def forward(self, x, x_lengths=None): + if self.time_warp is not None: + x, x_lengths = self.time_warp(x, x_lengths) + if self.freq_mask is not None: + x, x_lengths = self.freq_mask(x, x_lengths) + if self.time_mask is not None: + x, x_lengths = self.time_mask(x, x_lengths) + return x, x_lengths \ No newline at end of file diff --git a/funasr/modules/add_sos_eos.py b/funasr/modules/add_sos_eos.py new file mode 100644 index 000000000..ada1c2e01 --- /dev/null +++ b/funasr/modules/add_sos_eos.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Unility functions for Transformer.""" + +import torch +from funasr.modules.nets_utils import pad_list + + +def add_sos_eos(ys_pad, sos, eos, ignore_id): + """Add and labels. + + :param torch.Tensor ys_pad: batch of padded target sequences (B, Lmax) + :param int sos: index of + :param int eos: index of + :param int ignore_id: index of padding + :return: padded tensor (B, Lmax) + :rtype: torch.Tensor + :return: padded tensor (B, Lmax) + :rtype: torch.Tensor + """ + + _sos = ys_pad.new([sos]) + _eos = ys_pad.new([eos]) + ys = [y[y != ignore_id] for y in ys_pad] # parse padded ys + ys_in = [torch.cat([_sos, y], dim=0) for y in ys] + ys_out = [torch.cat([y, _eos], dim=0) for y in ys] + return pad_list(ys_in, eos), pad_list(ys_out, ignore_id) diff --git a/funasr/modules/attention.py b/funasr/modules/attention.py new file mode 100644 index 000000000..e3ad56a5a --- /dev/null +++ b/funasr/modules/attention.py @@ -0,0 +1,625 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Multi-Head Attention layer definition.""" + +import math + +import numpy +import torch +from torch import nn + + +class MultiHeadedAttention(nn.Module): + """Multi-Head Attention layer. + + Args: + n_head (int): The number of heads. + n_feat (int): The number of features. + dropout_rate (float): Dropout rate. + + """ + + def __init__(self, n_head, n_feat, dropout_rate): + """Construct an MultiHeadedAttention object.""" + super(MultiHeadedAttention, self).__init__() + assert n_feat % n_head == 0 + # We assume d_v always equals d_k + self.d_k = n_feat // n_head + self.h = n_head + self.linear_q = nn.Linear(n_feat, n_feat) + self.linear_k = nn.Linear(n_feat, n_feat) + self.linear_v = nn.Linear(n_feat, n_feat) + self.linear_out = nn.Linear(n_feat, n_feat) + self.attn = None + self.dropout = nn.Dropout(p=dropout_rate) + + def forward_qkv(self, query, key, value): + """Transform query, key and value. + + Args: + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + + Returns: + torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). + torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). + torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). + + """ + n_batch = query.size(0) + q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k) + k = self.linear_k(key).view(n_batch, -1, self.h, self.d_k) + v = self.linear_v(value).view(n_batch, -1, self.h, self.d_k) + q = q.transpose(1, 2) # (batch, head, time1, d_k) + k = k.transpose(1, 2) # (batch, head, time2, d_k) + v = v.transpose(1, 2) # (batch, head, time2, d_k) + + return q, k, v + + def forward_attention(self, value, scores, mask): + """Compute attention context vector. + + Args: + value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). + scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). + mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). + + Returns: + torch.Tensor: Transformed value (#batch, time1, d_model) + weighted by the attention score (#batch, time1, time2). + + """ + n_batch = value.size(0) + if mask is not None: + mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) + min_value = float( + numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min + ) + scores = scores.masked_fill(mask, min_value) + self.attn = torch.softmax(scores, dim=-1).masked_fill( + mask, 0.0 + ) # (batch, head, time1, time2) + else: + self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) + + p_attn = self.dropout(self.attn) + x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) + x = ( + x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) + ) # (batch, time1, d_model) + + return self.linear_out(x) # (batch, time1, d_model) + + def forward(self, query, key, value, mask): + """Compute scaled dot product attention. + + Args: + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + mask (torch.Tensor): Mask tensor (#batch, 1, time2) or + (#batch, time1, time2). + + Returns: + torch.Tensor: Output tensor (#batch, time1, d_model). + + """ + q, k, v = self.forward_qkv(query, key, value) + scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k) + return self.forward_attention(v, scores, mask) + + +class LegacyRelPositionMultiHeadedAttention(MultiHeadedAttention): + """Multi-Head Attention layer with relative position encoding (old version). + + Details can be found in https://github.com/espnet/espnet/pull/2816. + + Paper: https://arxiv.org/abs/1901.02860 + + Args: + n_head (int): The number of heads. + n_feat (int): The number of features. + dropout_rate (float): Dropout rate. + zero_triu (bool): Whether to zero the upper triangular part of attention matrix. + + """ + + def __init__(self, n_head, n_feat, dropout_rate, zero_triu=False): + """Construct an RelPositionMultiHeadedAttention object.""" + super().__init__(n_head, n_feat, dropout_rate) + self.zero_triu = zero_triu + # linear transformation for positional encoding + self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) + # these two learnable bias are used in matrix c and matrix d + # as described in https://arxiv.org/abs/1901.02860 Section 3.3 + self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k)) + self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k)) + torch.nn.init.xavier_uniform_(self.pos_bias_u) + torch.nn.init.xavier_uniform_(self.pos_bias_v) + + def rel_shift(self, x): + """Compute relative positional encoding. + + Args: + x (torch.Tensor): Input tensor (batch, head, time1, time2). + + Returns: + torch.Tensor: Output tensor. + + """ + zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype) + x_padded = torch.cat([zero_pad, x], dim=-1) + + x_padded = x_padded.view(*x.size()[:2], x.size(3) + 1, x.size(2)) + x = x_padded[:, :, 1:].view_as(x) + + if self.zero_triu: + ones = torch.ones((x.size(2), x.size(3))) + x = x * torch.tril(ones, x.size(3) - x.size(2))[None, None, :, :] + + return x + + def forward(self, query, key, value, pos_emb, mask): + """Compute 'Scaled Dot Product Attention' with rel. positional encoding. + + Args: + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + pos_emb (torch.Tensor): Positional embedding tensor (#batch, time1, size). + mask (torch.Tensor): Mask tensor (#batch, 1, time2) or + (#batch, time1, time2). + + Returns: + torch.Tensor: Output tensor (#batch, time1, d_model). + + """ + q, k, v = self.forward_qkv(query, key, value) + q = q.transpose(1, 2) # (batch, time1, head, d_k) + + n_batch_pos = pos_emb.size(0) + p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) + p = p.transpose(1, 2) # (batch, head, time1, d_k) + + # (batch, head, time1, d_k) + q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) + # (batch, head, time1, d_k) + q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) + + # compute attention score + # first compute matrix a and matrix c + # as described in https://arxiv.org/abs/1901.02860 Section 3.3 + # (batch, head, time1, time2) + matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) + + # compute matrix b and matrix d + # (batch, head, time1, time1) + matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) + matrix_bd = self.rel_shift(matrix_bd) + + scores = (matrix_ac + matrix_bd) / math.sqrt( + self.d_k + ) # (batch, head, time1, time2) + + return self.forward_attention(v, scores, mask) + + +class RelPositionMultiHeadedAttention(MultiHeadedAttention): + """Multi-Head Attention layer with relative position encoding (new implementation). + + Details can be found in https://github.com/espnet/espnet/pull/2816. + + Paper: https://arxiv.org/abs/1901.02860 + + Args: + n_head (int): The number of heads. + n_feat (int): The number of features. + dropout_rate (float): Dropout rate. + zero_triu (bool): Whether to zero the upper triangular part of attention matrix. + + """ + + def __init__(self, n_head, n_feat, dropout_rate, zero_triu=False): + """Construct an RelPositionMultiHeadedAttention object.""" + super().__init__(n_head, n_feat, dropout_rate) + self.zero_triu = zero_triu + # linear transformation for positional encoding + self.linear_pos = nn.Linear(n_feat, n_feat, bias=False) + # these two learnable bias are used in matrix c and matrix d + # as described in https://arxiv.org/abs/1901.02860 Section 3.3 + self.pos_bias_u = nn.Parameter(torch.Tensor(self.h, self.d_k)) + self.pos_bias_v = nn.Parameter(torch.Tensor(self.h, self.d_k)) + torch.nn.init.xavier_uniform_(self.pos_bias_u) + torch.nn.init.xavier_uniform_(self.pos_bias_v) + + def rel_shift(self, x): + """Compute relative positional encoding. + + Args: + x (torch.Tensor): Input tensor (batch, head, time1, 2*time1-1). + time1 means the length of query vector. + + Returns: + torch.Tensor: Output tensor. + + """ + zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype) + x_padded = torch.cat([zero_pad, x], dim=-1) + + x_padded = x_padded.view(*x.size()[:2], x.size(3) + 1, x.size(2)) + x = x_padded[:, :, 1:].view_as(x)[ + :, :, :, : x.size(-1) // 2 + 1 + ] # only keep the positions from 0 to time2 + + if self.zero_triu: + ones = torch.ones((x.size(2), x.size(3)), device=x.device) + x = x * torch.tril(ones, x.size(3) - x.size(2))[None, None, :, :] + + return x + + def forward(self, query, key, value, pos_emb, mask): + """Compute 'Scaled Dot Product Attention' with rel. positional encoding. + + Args: + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + pos_emb (torch.Tensor): Positional embedding tensor + (#batch, 2*time1-1, size). + mask (torch.Tensor): Mask tensor (#batch, 1, time2) or + (#batch, time1, time2). + + Returns: + torch.Tensor: Output tensor (#batch, time1, d_model). + + """ + q, k, v = self.forward_qkv(query, key, value) + q = q.transpose(1, 2) # (batch, time1, head, d_k) + + n_batch_pos = pos_emb.size(0) + p = self.linear_pos(pos_emb).view(n_batch_pos, -1, self.h, self.d_k) + p = p.transpose(1, 2) # (batch, head, 2*time1-1, d_k) + + # (batch, head, time1, d_k) + q_with_bias_u = (q + self.pos_bias_u).transpose(1, 2) + # (batch, head, time1, d_k) + q_with_bias_v = (q + self.pos_bias_v).transpose(1, 2) + + # compute attention score + # first compute matrix a and matrix c + # as described in https://arxiv.org/abs/1901.02860 Section 3.3 + # (batch, head, time1, time2) + matrix_ac = torch.matmul(q_with_bias_u, k.transpose(-2, -1)) + + # compute matrix b and matrix d + # (batch, head, time1, 2*time1-1) + matrix_bd = torch.matmul(q_with_bias_v, p.transpose(-2, -1)) + matrix_bd = self.rel_shift(matrix_bd) + + scores = (matrix_ac + matrix_bd) / math.sqrt( + self.d_k + ) # (batch, head, time1, time2) + + return self.forward_attention(v, scores, mask) + + +class MultiHeadedAttentionSANM(nn.Module): + """Multi-Head Attention layer. + + Args: + n_head (int): The number of heads. + n_feat (int): The number of features. + dropout_rate (float): Dropout rate. + + """ + + def __init__(self, n_head, in_feat, n_feat, dropout_rate, kernel_size, sanm_shfit=0): + """Construct an MultiHeadedAttention object.""" + super(MultiHeadedAttentionSANM, self).__init__() + assert n_feat % n_head == 0 + # We assume d_v always equals d_k + self.d_k = n_feat // n_head + self.h = n_head + # self.linear_q = nn.Linear(n_feat, n_feat) + # self.linear_k = nn.Linear(n_feat, n_feat) + # self.linear_v = nn.Linear(n_feat, n_feat) + self.linear_out = nn.Linear(n_feat, n_feat) + self.linear_q_k_v = nn.Linear(in_feat, n_feat * 3) + self.attn = None + self.dropout = nn.Dropout(p=dropout_rate) + + self.fsmn_block = nn.Conv1d(n_feat, n_feat, kernel_size, stride=1, padding=0, groups=n_feat, bias=False) + # padding + left_padding = (kernel_size - 1) // 2 + if sanm_shfit > 0: + left_padding = left_padding + sanm_shfit + right_padding = kernel_size - 1 - left_padding + self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0) + + def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None): + b, t, d = inputs.size() + if mask is not None: + mask = torch.reshape(mask, (b, -1, 1)) + if mask_shfit_chunk is not None: + mask = mask * mask_shfit_chunk + + inputs = inputs * mask + x = inputs.transpose(1, 2) + x = self.pad_fn(x) + x = self.fsmn_block(x) + x = x.transpose(1, 2) + x += inputs + x = self.dropout(x) + return x * mask + + def forward_qkv(self, x): + """Transform query, key and value. + + Args: + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + + Returns: + torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). + torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). + torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). + + """ + b, t, d = x.size() + q_k_v = self.linear_q_k_v(x) + q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1) + q_h = torch.reshape(q, (b, t, self.h, self.d_k)).transpose(1, 2) # (batch, head, time1, d_k) + k_h = torch.reshape(k, (b, t, self.h, self.d_k)).transpose(1, 2) # (batch, head, time2, d_k) + v_h = torch.reshape(v, (b, t, self.h, self.d_k)).transpose(1, 2) # (batch, head, time2, d_k) + + return q_h, k_h, v_h, v + + def forward_attention(self, value, scores, mask, mask_att_chunk_encoder=None): + """Compute attention context vector. + + Args: + value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). + scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). + mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). + + Returns: + torch.Tensor: Transformed value (#batch, time1, d_model) + weighted by the attention score (#batch, time1, time2). + + """ + n_batch = value.size(0) + if mask is not None: + if mask_att_chunk_encoder is not None: + mask = mask * mask_att_chunk_encoder + + mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) + + min_value = float( + numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min + ) + scores = scores.masked_fill(mask, min_value) + self.attn = torch.softmax(scores, dim=-1).masked_fill( + mask, 0.0 + ) # (batch, head, time1, time2) + else: + self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) + + p_attn = self.dropout(self.attn) + x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) + x = ( + x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) + ) # (batch, time1, d_model) + + return self.linear_out(x) # (batch, time1, d_model) + + def forward(self, x, mask, mask_shfit_chunk=None, mask_att_chunk_encoder=None): + """Compute scaled dot product attention. + + Args: + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + mask (torch.Tensor): Mask tensor (#batch, 1, time2) or + (#batch, time1, time2). + + Returns: + torch.Tensor: Output tensor (#batch, time1, d_model). + + """ + q_h, k_h, v_h, v = self.forward_qkv(x) + fsmn_memory = self.forward_fsmn(v, mask, mask_shfit_chunk) + q_h = q_h * self.d_k ** (-0.5) + scores = torch.matmul(q_h, k_h.transpose(-2, -1)) + att_outs = self.forward_attention(v_h, scores, mask, mask_att_chunk_encoder) + return att_outs + fsmn_memory + +class MultiHeadedAttentionSANMDecoder(nn.Module): + """Multi-Head Attention layer. + + Args: + n_head (int): The number of heads. + n_feat (int): The number of features. + dropout_rate (float): Dropout rate. + + """ + + def __init__(self, n_feat, dropout_rate, kernel_size, sanm_shfit=0): + """Construct an MultiHeadedAttention object.""" + super(MultiHeadedAttentionSANMDecoder, self).__init__() + + self.dropout = nn.Dropout(p=dropout_rate) + + self.fsmn_block = nn.Conv1d(n_feat, n_feat, + kernel_size, stride=1, padding=0, groups=n_feat, bias=False) + # padding + # padding + left_padding = (kernel_size - 1) // 2 + if sanm_shfit > 0: + left_padding = left_padding + sanm_shfit + right_padding = kernel_size - 1 - left_padding + self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0) + self.kernel_size = kernel_size + + def forward(self, inputs, mask, cache=None, mask_shfit_chunk=None): + ''' + :param x: (#batch, time1, size). + :param mask: Mask tensor (#batch, 1, time) + :return: + ''' + # print("in fsmn, inputs", inputs.size()) + b, t, d = inputs.size() + # logging.info( + # "mask: {}".format(mask.size())) + if mask is not None: + mask = torch.reshape(mask, (b ,-1, 1)) + # logging.info("in fsmn, mask: {}, {}".format(mask.size(), mask[0:100:50, :, :])) + if mask_shfit_chunk is not None: + # logging.info("in fsmn, mask_fsmn: {}, {}".format(mask_shfit_chunk.size(), mask_shfit_chunk[0:100:50, :, :])) + mask = mask * mask_shfit_chunk + # logging.info("in fsmn, mask_after_fsmn: {}, {}".format(mask.size(), mask[0:100:50, :, :])) + # print("in fsmn, mask", mask.size()) + # print("in fsmn, inputs", inputs.size()) + inputs = inputs * mask + + x = inputs.transpose(1, 2) + b, d, t = x.size() + if cache is None: + # print("in fsmn, cache is None, x", x.size()) + + x = self.pad_fn(x) + if not self.training and t <= 1: + cache = x + else: + # print("in fsmn, cache is not None, x", x.size()) + # x = torch.cat((x, cache), dim=2)[:, :, :-1] + # if t < self.kernel_size: + # x = self.pad_fn(x) + x = torch.cat((cache[:, :, 1:], x), dim=2) + x = x[:, :, -self.kernel_size:] + # print("in fsmn, cache is not None, x_cat", x.size()) + cache = x + x = self.fsmn_block(x) + x = x.transpose(1, 2) + # print("in fsmn, fsmn_out", x.size()) + if x.size(1) != inputs.size(1): + inputs = inputs[:, -1, :] + + x = x + inputs + x = self.dropout(x) + if mask is not None: + x = x * mask + return x, cache + +class MultiHeadedAttentionCrossAtt(nn.Module): + """Multi-Head Attention layer. + + Args: + n_head (int): The number of heads. + n_feat (int): The number of features. + dropout_rate (float): Dropout rate. + + """ + + def __init__(self, n_head, n_feat, dropout_rate, encoder_output_size=None): + """Construct an MultiHeadedAttention object.""" + super(MultiHeadedAttentionCrossAtt, self).__init__() + assert n_feat % n_head == 0 + # We assume d_v always equals d_k + self.d_k = n_feat // n_head + self.h = n_head + self.linear_q = nn.Linear(n_feat, n_feat) + # self.linear_k = nn.Linear(n_feat, n_feat) + # self.linear_v = nn.Linear(n_feat, n_feat) + self.linear_k_v = nn.Linear(n_feat if encoder_output_size is None else encoder_output_size, n_feat*2) + self.linear_out = nn.Linear(n_feat, n_feat) + self.attn = None + self.dropout = nn.Dropout(p=dropout_rate) + + def forward_qkv(self, x, memory): + """Transform query, key and value. + + Args: + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + + Returns: + torch.Tensor: Transformed query tensor (#batch, n_head, time1, d_k). + torch.Tensor: Transformed key tensor (#batch, n_head, time2, d_k). + torch.Tensor: Transformed value tensor (#batch, n_head, time2, d_k). + + """ + + # print("in forward_qkv, x", x.size()) + b = x.size(0) + q = self.linear_q(x) + q_h = torch.reshape(q, (b, -1, self.h, self.d_k)).transpose(1, 2) # (batch, head, time1, d_k) + + k_v = self.linear_k_v(memory) + k, v = torch.split(k_v, int(self.h*self.d_k), dim=-1) + k_h = torch.reshape(k, (b, -1, self.h, self.d_k)).transpose(1, 2) # (batch, head, time2, d_k) + v_h = torch.reshape(v, (b, -1, self.h, self.d_k)).transpose(1, 2) # (batch, head, time2, d_k) + + + return q_h, k_h, v_h + + def forward_attention(self, value, scores, mask): + """Compute attention context vector. + + Args: + value (torch.Tensor): Transformed value (#batch, n_head, time2, d_k). + scores (torch.Tensor): Attention score (#batch, n_head, time1, time2). + mask (torch.Tensor): Mask (#batch, 1, time2) or (#batch, time1, time2). + + Returns: + torch.Tensor: Transformed value (#batch, time1, d_model) + weighted by the attention score (#batch, time1, time2). + + """ + n_batch = value.size(0) + if mask is not None: + mask = mask.unsqueeze(1).eq(0) # (batch, 1, *, time2) + min_value = float( + numpy.finfo(torch.tensor(0, dtype=scores.dtype).numpy().dtype).min + ) + # logging.info( + # "scores: {}, mask_size: {}".format(scores.size(), mask.size())) + scores = scores.masked_fill(mask, min_value) + self.attn = torch.softmax(scores, dim=-1).masked_fill( + mask, 0.0 + ) # (batch, head, time1, time2) + else: + self.attn = torch.softmax(scores, dim=-1) # (batch, head, time1, time2) + + p_attn = self.dropout(self.attn) + x = torch.matmul(p_attn, value) # (batch, head, time1, d_k) + x = ( + x.transpose(1, 2).contiguous().view(n_batch, -1, self.h * self.d_k) + ) # (batch, time1, d_model) + + return self.linear_out(x) # (batch, time1, d_model) + + def forward(self, x, memory, memory_mask): + """Compute scaled dot product attention. + + Args: + query (torch.Tensor): Query tensor (#batch, time1, size). + key (torch.Tensor): Key tensor (#batch, time2, size). + value (torch.Tensor): Value tensor (#batch, time2, size). + mask (torch.Tensor): Mask tensor (#batch, 1, time2) or + (#batch, time1, time2). + + Returns: + torch.Tensor: Output tensor (#batch, time1, d_model). + + """ + q_h, k_h, v_h = self.forward_qkv(x, memory) + q_h = q_h * self.d_k ** (-0.5) + scores = torch.matmul(q_h, k_h.transpose(-2, -1)) + return self.forward_attention(v_h, scores, memory_mask) \ No newline at end of file diff --git a/funasr/modules/beam_search/__init__.py b/funasr/modules/beam_search/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/modules/beam_search/batch_beam_search.py b/funasr/modules/beam_search/batch_beam_search.py new file mode 100644 index 000000000..6d2da8d9a --- /dev/null +++ b/funasr/modules/beam_search/batch_beam_search.py @@ -0,0 +1,348 @@ +"""Parallel beam search module.""" + +import logging +from typing import Any +from typing import Dict +from typing import List +from typing import NamedTuple +from typing import Tuple + +import torch +from torch.nn.utils.rnn import pad_sequence + +from funasr.modules.beam_search.beam_search import BeamSearch +from funasr.modules.beam_search.beam_search import Hypothesis + + +class BatchHypothesis(NamedTuple): + """Batchfied/Vectorized hypothesis data type.""" + + yseq: torch.Tensor = torch.tensor([]) # (batch, maxlen) + score: torch.Tensor = torch.tensor([]) # (batch,) + length: torch.Tensor = torch.tensor([]) # (batch,) + scores: Dict[str, torch.Tensor] = dict() # values: (batch,) + states: Dict[str, Dict] = dict() + + def __len__(self) -> int: + """Return a batch size.""" + return len(self.length) + + +class BatchBeamSearch(BeamSearch): + """Batch beam search implementation.""" + + def batchfy(self, hyps: List[Hypothesis]) -> BatchHypothesis: + """Convert list to batch.""" + if len(hyps) == 0: + return BatchHypothesis() + return BatchHypothesis( + yseq=pad_sequence( + [h.yseq for h in hyps], batch_first=True, padding_value=self.eos + ), + length=torch.tensor([len(h.yseq) for h in hyps], dtype=torch.int64), + score=torch.tensor([h.score for h in hyps]), + scores={k: torch.tensor([h.scores[k] for h in hyps]) for k in self.scorers}, + states={k: [h.states[k] for h in hyps] for k in self.scorers}, + ) + + def _batch_select(self, hyps: BatchHypothesis, ids: List[int]) -> BatchHypothesis: + return BatchHypothesis( + yseq=hyps.yseq[ids], + score=hyps.score[ids], + length=hyps.length[ids], + scores={k: v[ids] for k, v in hyps.scores.items()}, + states={ + k: [self.scorers[k].select_state(v, i) for i in ids] + for k, v in hyps.states.items() + }, + ) + + def _select(self, hyps: BatchHypothesis, i: int) -> Hypothesis: + return Hypothesis( + yseq=hyps.yseq[i, : hyps.length[i]], + score=hyps.score[i], + scores={k: v[i] for k, v in hyps.scores.items()}, + states={ + k: self.scorers[k].select_state(v, i) for k, v in hyps.states.items() + }, + ) + + def unbatchfy(self, batch_hyps: BatchHypothesis) -> List[Hypothesis]: + """Revert batch to list.""" + return [ + Hypothesis( + yseq=batch_hyps.yseq[i][: batch_hyps.length[i]], + score=batch_hyps.score[i], + scores={k: batch_hyps.scores[k][i] for k in self.scorers}, + states={ + k: v.select_state(batch_hyps.states[k], i) + for k, v in self.scorers.items() + }, + ) + for i in range(len(batch_hyps.length)) + ] + + def batch_beam( + self, weighted_scores: torch.Tensor, ids: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Batch-compute topk full token ids and partial token ids. + + Args: + weighted_scores (torch.Tensor): The weighted sum scores for each tokens. + Its shape is `(n_beam, self.vocab_size)`. + ids (torch.Tensor): The partial token ids to compute topk. + Its shape is `(n_beam, self.pre_beam_size)`. + + Returns: + Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + The topk full (prev_hyp, new_token) ids + and partial (prev_hyp, new_token) ids. + Their shapes are all `(self.beam_size,)` + + """ + top_ids = weighted_scores.view(-1).topk(self.beam_size)[1] + # Because of the flatten above, `top_ids` is organized as: + # [hyp1 * V + token1, hyp2 * V + token2, ..., hypK * V + tokenK], + # where V is `self.n_vocab` and K is `self.beam_size` + prev_hyp_ids = top_ids // self.n_vocab + new_token_ids = top_ids % self.n_vocab + return prev_hyp_ids, new_token_ids, prev_hyp_ids, new_token_ids + + def init_hyp(self, x: torch.Tensor) -> BatchHypothesis: + """Get an initial hypothesis data. + + Args: + x (torch.Tensor): The encoder output feature + + Returns: + Hypothesis: The initial hypothesis. + + """ + init_states = dict() + init_scores = dict() + for k, d in self.scorers.items(): + init_states[k] = d.batch_init_state(x) + init_scores[k] = 0.0 + return self.batchfy( + [ + Hypothesis( + score=0.0, + scores=init_scores, + states=init_states, + yseq=torch.tensor([self.sos], device=x.device), + ) + ] + ) + + def score_full( + self, hyp: BatchHypothesis, x: torch.Tensor + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + """Score new hypothesis by `self.full_scorers`. + + Args: + hyp (Hypothesis): Hypothesis with prefix tokens to score + x (torch.Tensor): Corresponding input feature + + Returns: + Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of + score dict of `hyp` that has string keys of `self.full_scorers` + and tensor score values of shape: `(self.n_vocab,)`, + and state dict that has string keys + and state values of `self.full_scorers` + + """ + scores = dict() + states = dict() + for k, d in self.full_scorers.items(): + scores[k], states[k] = d.batch_score(hyp.yseq, hyp.states[k], x) + return scores, states + + def score_partial( + self, hyp: BatchHypothesis, ids: torch.Tensor, x: torch.Tensor + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + """Score new hypothesis by `self.full_scorers`. + + Args: + hyp (Hypothesis): Hypothesis with prefix tokens to score + ids (torch.Tensor): 2D tensor of new partial tokens to score + x (torch.Tensor): Corresponding input feature + + Returns: + Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of + score dict of `hyp` that has string keys of `self.full_scorers` + and tensor score values of shape: `(self.n_vocab,)`, + and state dict that has string keys + and state values of `self.full_scorers` + + """ + scores = dict() + states = dict() + for k, d in self.part_scorers.items(): + scores[k], states[k] = d.batch_score_partial( + hyp.yseq, ids, hyp.states[k], x + ) + return scores, states + + def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: + """Merge states for new hypothesis. + + Args: + states: states of `self.full_scorers` + part_states: states of `self.part_scorers` + part_idx (int): The new token id for `part_scores` + + Returns: + Dict[str, torch.Tensor]: The new score dict. + Its keys are names of `self.full_scorers` and `self.part_scorers`. + Its values are states of the scorers. + + """ + new_states = dict() + for k, v in states.items(): + new_states[k] = v + for k, v in part_states.items(): + new_states[k] = v + return new_states + + def search(self, running_hyps: BatchHypothesis, x: torch.Tensor) -> BatchHypothesis: + """Search new tokens for running hypotheses and encoded speech x. + + Args: + running_hyps (BatchHypothesis): Running hypotheses on beam + x (torch.Tensor): Encoded speech feature (T, D) + + Returns: + BatchHypothesis: Best sorted hypotheses + + """ + n_batch = len(running_hyps) + part_ids = None # no pre-beam + # batch scoring + weighted_scores = torch.zeros( + n_batch, self.n_vocab, dtype=x.dtype, device=x.device + ) + scores, states = self.score_full(running_hyps, x.expand(n_batch, *x.shape)) + for k in self.full_scorers: + weighted_scores += self.weights[k] * scores[k] + # partial scoring + if self.do_pre_beam: + pre_beam_scores = ( + weighted_scores + if self.pre_beam_score_key == "full" + else scores[self.pre_beam_score_key] + ) + part_ids = torch.topk(pre_beam_scores, self.pre_beam_size, dim=-1)[1] + # NOTE(takaaki-hori): Unlike BeamSearch, we assume that score_partial returns + # full-size score matrices, which has non-zero scores for part_ids and zeros + # for others. + part_scores, part_states = self.score_partial(running_hyps, part_ids, x) + for k in self.part_scorers: + weighted_scores += self.weights[k] * part_scores[k] + # add previous hyp scores + weighted_scores += running_hyps.score.to( + dtype=x.dtype, device=x.device + ).unsqueeze(1) + + # TODO(karita): do not use list. use batch instead + # see also https://github.com/espnet/espnet/pull/1402#discussion_r354561029 + # update hyps + best_hyps = [] + prev_hyps = self.unbatchfy(running_hyps) + for ( + full_prev_hyp_id, + full_new_token_id, + part_prev_hyp_id, + part_new_token_id, + ) in zip(*self.batch_beam(weighted_scores, part_ids)): + prev_hyp = prev_hyps[full_prev_hyp_id] + best_hyps.append( + Hypothesis( + score=weighted_scores[full_prev_hyp_id, full_new_token_id], + yseq=self.append_token(prev_hyp.yseq, full_new_token_id), + scores=self.merge_scores( + prev_hyp.scores, + {k: v[full_prev_hyp_id] for k, v in scores.items()}, + full_new_token_id, + {k: v[part_prev_hyp_id] for k, v in part_scores.items()}, + part_new_token_id, + ), + states=self.merge_states( + { + k: self.full_scorers[k].select_state(v, full_prev_hyp_id) + for k, v in states.items() + }, + { + k: self.part_scorers[k].select_state( + v, part_prev_hyp_id, part_new_token_id + ) + for k, v in part_states.items() + }, + part_new_token_id, + ), + ) + ) + return self.batchfy(best_hyps) + + def post_process( + self, + i: int, + maxlen: int, + maxlenratio: float, + running_hyps: BatchHypothesis, + ended_hyps: List[Hypothesis], + ) -> BatchHypothesis: + """Perform post-processing of beam search iterations. + + Args: + i (int): The length of hypothesis tokens. + maxlen (int): The maximum length of tokens in beam search. + maxlenratio (int): The maximum length ratio in beam search. + running_hyps (BatchHypothesis): The running hypotheses in beam search. + ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. + + Returns: + BatchHypothesis: The new running hypotheses. + + """ + n_batch = running_hyps.yseq.shape[0] + logging.debug(f"the number of running hypothes: {n_batch}") + if self.token_list is not None: + logging.debug( + "best hypo: " + + "".join( + [ + self.token_list[x] + for x in running_hyps.yseq[0, 1 : running_hyps.length[0]] + ] + ) + ) + # add eos in the final loop to avoid that there are no ended hyps + if i == maxlen - 1: + logging.info("adding in the last position in the loop") + yseq_eos = torch.cat( + ( + running_hyps.yseq, + torch.full( + (n_batch, 1), + self.eos, + device=running_hyps.yseq.device, + dtype=torch.int64, + ), + ), + 1, + ) + running_hyps.yseq.resize_as_(yseq_eos) + running_hyps.yseq[:] = yseq_eos + running_hyps.length[:] = yseq_eos.shape[1] + + # add ended hypotheses to a final list, and removed them from current hypotheses + # (this will be a probmlem, number of hyps < beam) + is_eos = ( + running_hyps.yseq[torch.arange(n_batch), running_hyps.length - 1] + == self.eos + ) + for b in torch.nonzero(is_eos, as_tuple=False).view(-1): + hyp = self._select(running_hyps, b) + ended_hyps.append(hyp) + remained_ids = torch.nonzero(is_eos == 0, as_tuple=False).view(-1) + return self._batch_select(running_hyps, remained_ids) diff --git a/funasr/modules/beam_search/batch_beam_search_online_sim.py b/funasr/modules/beam_search/batch_beam_search_online_sim.py new file mode 100644 index 000000000..4d3debd23 --- /dev/null +++ b/funasr/modules/beam_search/batch_beam_search_online_sim.py @@ -0,0 +1,270 @@ +"""Parallel beam search module for online simulation.""" + +import logging +from pathlib import Path +from typing import List + +import yaml + +import torch + +from funasr.modules.beam_search.batch_beam_search import BatchBeamSearch +from funasr.modules.beam_search.beam_search import Hypothesis +from funasr.models.e2e_asr_common import end_detect + + +class BatchBeamSearchOnlineSim(BatchBeamSearch): + """Online beam search implementation. + + This simulates streaming decoding. + It requires encoded features of entire utterance and + extracts block by block from it as it shoud be done + in streaming processing. + This is based on Tsunoo et al, "STREAMING TRANSFORMER ASR + WITH BLOCKWISE SYNCHRONOUS BEAM SEARCH" + (https://arxiv.org/abs/2006.14941). + """ + + def set_streaming_config(self, asr_config: str): + """Set config file for streaming decoding. + + Args: + asr_config (str): The config file for asr training + + """ + train_config_file = Path(asr_config) + self.block_size = None + self.hop_size = None + self.look_ahead = None + config = None + with train_config_file.open("r", encoding="utf-8") as f: + args = yaml.safe_load(f) + if "encoder_conf" in args.keys(): + if "block_size" in args["encoder_conf"].keys(): + self.block_size = args["encoder_conf"]["block_size"] + if "hop_size" in args["encoder_conf"].keys(): + self.hop_size = args["encoder_conf"]["hop_size"] + if "look_ahead" in args["encoder_conf"].keys(): + self.look_ahead = args["encoder_conf"]["look_ahead"] + elif "config" in args.keys(): + config = args["config"] + if config is None: + logging.info( + "Cannot find config file for streaming decoding: " + + "apply batch beam search instead." + ) + return + if ( + self.block_size is None or self.hop_size is None or self.look_ahead is None + ) and config is not None: + config_file = Path(config) + with config_file.open("r", encoding="utf-8") as f: + args = yaml.safe_load(f) + if "encoder_conf" in args.keys(): + enc_args = args["encoder_conf"] + if enc_args and "block_size" in enc_args: + self.block_size = enc_args["block_size"] + if enc_args and "hop_size" in enc_args: + self.hop_size = enc_args["hop_size"] + if enc_args and "look_ahead" in enc_args: + self.look_ahead = enc_args["look_ahead"] + + def set_block_size(self, block_size: int): + """Set block size for streaming decoding. + + Args: + block_size (int): The block size of encoder + """ + self.block_size = block_size + + def set_hop_size(self, hop_size: int): + """Set hop size for streaming decoding. + + Args: + hop_size (int): The hop size of encoder + """ + self.hop_size = hop_size + + def set_look_ahead(self, look_ahead: int): + """Set look ahead size for streaming decoding. + + Args: + look_ahead (int): The look ahead size of encoder + """ + self.look_ahead = look_ahead + + def forward( + self, x: torch.Tensor, maxlenratio: float = 0.0, minlenratio: float = 0.0 + ) -> List[Hypothesis]: + """Perform beam search. + + Args: + x (torch.Tensor): Encoded speech feature (T, D) + maxlenratio (float): Input length ratio to obtain max output length. + If maxlenratio=0.0 (default), it uses a end-detect function + to automatically find maximum hypothesis lengths + minlenratio (float): Input length ratio to obtain min output length. + + Returns: + list[Hypothesis]: N-best decoding results + + """ + self.conservative = True # always true + + if self.block_size and self.hop_size and self.look_ahead: + cur_end_frame = int(self.block_size - self.look_ahead) + else: + cur_end_frame = x.shape[0] + process_idx = 0 + if cur_end_frame < x.shape[0]: + h = x.narrow(0, 0, cur_end_frame) + else: + h = x + + # set length bounds + if maxlenratio == 0: + maxlen = x.shape[0] + else: + maxlen = max(1, int(maxlenratio * x.size(0))) + minlen = int(minlenratio * x.size(0)) + logging.info("decoder input length: " + str(x.shape[0])) + logging.info("max output length: " + str(maxlen)) + logging.info("min output length: " + str(minlen)) + + # main loop of prefix search + running_hyps = self.init_hyp(h) + prev_hyps = [] + ended_hyps = [] + prev_repeat = False + + continue_decode = True + + while continue_decode: + move_to_next_block = False + if cur_end_frame < x.shape[0]: + h = x.narrow(0, 0, cur_end_frame) + else: + h = x + + # extend states for ctc + self.extend(h, running_hyps) + + while process_idx < maxlen: + logging.debug("position " + str(process_idx)) + best = self.search(running_hyps, h) + + if process_idx == maxlen - 1: + # end decoding + running_hyps = self.post_process( + process_idx, maxlen, maxlenratio, best, ended_hyps + ) + n_batch = best.yseq.shape[0] + local_ended_hyps = [] + is_local_eos = ( + best.yseq[torch.arange(n_batch), best.length - 1] == self.eos + ) + for i in range(is_local_eos.shape[0]): + if is_local_eos[i]: + hyp = self._select(best, i) + local_ended_hyps.append(hyp) + # NOTE(tsunoo): check repetitions here + # This is a implicit implementation of + # Eq (11) in https://arxiv.org/abs/2006.14941 + # A flag prev_repeat is used instead of using set + elif ( + not prev_repeat + and best.yseq[i, -1] in best.yseq[i, :-1] + and cur_end_frame < x.shape[0] + ): + move_to_next_block = True + prev_repeat = True + if maxlenratio == 0.0 and end_detect( + [lh.asdict() for lh in local_ended_hyps], process_idx + ): + logging.info(f"end detected at {process_idx}") + continue_decode = False + break + if len(local_ended_hyps) > 0 and cur_end_frame < x.shape[0]: + move_to_next_block = True + + if move_to_next_block: + if ( + self.hop_size + and cur_end_frame + int(self.hop_size) + int(self.look_ahead) + < x.shape[0] + ): + cur_end_frame += int(self.hop_size) + else: + cur_end_frame = x.shape[0] + logging.debug("Going to next block: %d", cur_end_frame) + if process_idx > 1 and len(prev_hyps) > 0 and self.conservative: + running_hyps = prev_hyps + process_idx -= 1 + prev_hyps = [] + break + + prev_repeat = False + prev_hyps = running_hyps + running_hyps = self.post_process( + process_idx, maxlen, maxlenratio, best, ended_hyps + ) + + if cur_end_frame >= x.shape[0]: + for hyp in local_ended_hyps: + ended_hyps.append(hyp) + + if len(running_hyps) == 0: + logging.info("no hypothesis. Finish decoding.") + continue_decode = False + break + else: + logging.debug(f"remained hypotheses: {len(running_hyps)}") + # increment number + process_idx += 1 + + nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) + # check the number of hypotheses reaching to eos + if len(nbest_hyps) == 0: + logging.warning( + "there is no N-best results, perform recognition " + "again with smaller minlenratio." + ) + return ( + [] + if minlenratio < 0.1 + else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) + ) + + # report the best result + best = nbest_hyps[0] + for k, v in best.scores.items(): + logging.info( + f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" + ) + logging.info(f"total log probability: {best.score:.2f}") + logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") + logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") + if self.token_list is not None: + logging.info( + "best hypo: " + + "".join([self.token_list[x] for x in best.yseq[1:-1]]) + + "\n" + ) + return nbest_hyps + + def extend(self, x: torch.Tensor, hyps: Hypothesis) -> List[Hypothesis]: + """Extend probabilities and states with more encoded chunks. + + Args: + x (torch.Tensor): The extended encoder output feature + hyps (Hypothesis): Current list of hypothesis + + Returns: + Hypothesis: The extended hypothesis + + """ + for k, d in self.scorers.items(): + if hasattr(d, "extend_prob"): + d.extend_prob(x) + if hasattr(d, "extend_state"): + hyps.states[k] = d.extend_state(hyps.states[k]) diff --git a/funasr/modules/beam_search/beam_search.py b/funasr/modules/beam_search/beam_search.py new file mode 100644 index 000000000..51fa60100 --- /dev/null +++ b/funasr/modules/beam_search/beam_search.py @@ -0,0 +1,1400 @@ +"""Beam search module.""" + +from itertools import chain +import logging +from typing import Any +from typing import Dict +from typing import List +from typing import NamedTuple +from typing import Tuple +from typing import Union + +import torch + +from funasr.modules.e2e_asr_common import end_detect +from funasr.modules.scorers.scorer_interface import PartialScorerInterface +from funasr.modules.scorers.scorer_interface import ScorerInterface + + +class Hypothesis(NamedTuple): + """Hypothesis data type.""" + + yseq: torch.Tensor + score: Union[float, torch.Tensor] = 0 + scores: Dict[str, Union[float, torch.Tensor]] = dict() + states: Dict[str, Any] = dict() + + def asdict(self) -> dict: + """Convert data to JSON-friendly dict.""" + return self._replace( + yseq=self.yseq.tolist(), + score=float(self.score), + scores={k: float(v) for k, v in self.scores.items()}, + )._asdict() + + +class BeamSearch(torch.nn.Module): + """Beam search implementation.""" + + def __init__( + self, + scorers: Dict[str, ScorerInterface], + weights: Dict[str, float], + beam_size: int, + vocab_size: int, + sos: int, + eos: int, + token_list: List[str] = None, + pre_beam_ratio: float = 1.5, + pre_beam_score_key: str = None, + ): + """Initialize beam search. + + Args: + scorers (dict[str, ScorerInterface]): Dict of decoder modules + e.g., Decoder, CTCPrefixScorer, LM + The scorer will be ignored if it is `None` + weights (dict[str, float]): Dict of weights for each scorers + The scorer will be ignored if its weight is 0 + beam_size (int): The number of hypotheses kept during search + vocab_size (int): The number of vocabulary + sos (int): Start of sequence id + eos (int): End of sequence id + token_list (list[str]): List of tokens for debug log + pre_beam_score_key (str): key of scores to perform pre-beam search + pre_beam_ratio (float): beam size in the pre-beam search + will be `int(pre_beam_ratio * beam_size)` + + """ + super().__init__() + # set scorers + self.weights = weights + self.scorers = dict() + self.full_scorers = dict() + self.part_scorers = dict() + # this module dict is required for recursive cast + # `self.to(device, dtype)` in `recog.py` + self.nn_dict = torch.nn.ModuleDict() + for k, v in scorers.items(): + w = weights.get(k, 0) + if w == 0 or v is None: + continue + assert isinstance( + v, ScorerInterface + ), f"{k} ({type(v)}) does not implement ScorerInterface" + self.scorers[k] = v + if isinstance(v, PartialScorerInterface): + self.part_scorers[k] = v + else: + self.full_scorers[k] = v + if isinstance(v, torch.nn.Module): + self.nn_dict[k] = v + + # set configurations + self.sos = sos + self.eos = eos + self.token_list = token_list + self.pre_beam_size = int(pre_beam_ratio * beam_size) + self.beam_size = beam_size + self.n_vocab = vocab_size + if ( + pre_beam_score_key is not None + and pre_beam_score_key != "full" + and pre_beam_score_key not in self.full_scorers + ): + raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") + self.pre_beam_score_key = pre_beam_score_key + self.do_pre_beam = ( + self.pre_beam_score_key is not None + and self.pre_beam_size < self.n_vocab + and len(self.part_scorers) > 0 + ) + + def init_hyp(self, x: torch.Tensor) -> List[Hypothesis]: + """Get an initial hypothesis data. + + Args: + x (torch.Tensor): The encoder output feature + + Returns: + Hypothesis: The initial hypothesis. + + """ + init_states = dict() + init_scores = dict() + for k, d in self.scorers.items(): + init_states[k] = d.init_state(x) + init_scores[k] = 0.0 + return [ + Hypothesis( + score=0.0, + scores=init_scores, + states=init_states, + yseq=torch.tensor([self.sos], device=x.device), + ) + ] + + @staticmethod + def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: + """Append new token to prefix tokens. + + Args: + xs (torch.Tensor): The prefix token + x (int): The new token to append + + Returns: + torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device + + """ + x = torch.tensor([x], dtype=xs.dtype, device=xs.device) + return torch.cat((xs, x)) + + def score_full( + self, hyp: Hypothesis, x: torch.Tensor + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + """Score new hypothesis by `self.full_scorers`. + + Args: + hyp (Hypothesis): Hypothesis with prefix tokens to score + x (torch.Tensor): Corresponding input feature + + Returns: + Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of + score dict of `hyp` that has string keys of `self.full_scorers` + and tensor score values of shape: `(self.n_vocab,)`, + and state dict that has string keys + and state values of `self.full_scorers` + + """ + scores = dict() + states = dict() + for k, d in self.full_scorers.items(): + scores[k], states[k] = d.score(hyp.yseq, hyp.states[k], x) + return scores, states + + def score_partial( + self, hyp: Hypothesis, ids: torch.Tensor, x: torch.Tensor + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + """Score new hypothesis by `self.part_scorers`. + + Args: + hyp (Hypothesis): Hypothesis with prefix tokens to score + ids (torch.Tensor): 1D tensor of new partial tokens to score + x (torch.Tensor): Corresponding input feature + + Returns: + Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of + score dict of `hyp` that has string keys of `self.part_scorers` + and tensor score values of shape: `(len(ids),)`, + and state dict that has string keys + and state values of `self.part_scorers` + + """ + scores = dict() + states = dict() + for k, d in self.part_scorers.items(): + scores[k], states[k] = d.score_partial(hyp.yseq, ids, hyp.states[k], x) + return scores, states + + def beam( + self, weighted_scores: torch.Tensor, ids: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute topk full token ids and partial token ids. + + Args: + weighted_scores (torch.Tensor): The weighted sum scores for each tokens. + Its shape is `(self.n_vocab,)`. + ids (torch.Tensor): The partial token ids to compute topk + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + The topk full token ids and partial token ids. + Their shapes are `(self.beam_size,)` + + """ + # no pre beam performed + if weighted_scores.size(0) == ids.size(0): + top_ids = weighted_scores.topk(self.beam_size)[1] + return top_ids, top_ids + + # mask pruned in pre-beam not to select in topk + tmp = weighted_scores[ids] + weighted_scores[:] = -float("inf") + weighted_scores[ids] = tmp + top_ids = weighted_scores.topk(self.beam_size)[1] + local_ids = weighted_scores[ids].topk(self.beam_size)[1] + return top_ids, local_ids + + @staticmethod + def merge_scores( + prev_scores: Dict[str, float], + next_full_scores: Dict[str, torch.Tensor], + full_idx: int, + next_part_scores: Dict[str, torch.Tensor], + part_idx: int, + ) -> Dict[str, torch.Tensor]: + """Merge scores for new hypothesis. + + Args: + prev_scores (Dict[str, float]): + The previous hypothesis scores by `self.scorers` + next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` + full_idx (int): The next token id for `next_full_scores` + next_part_scores (Dict[str, torch.Tensor]): + scores of partial tokens by `self.part_scorers` + part_idx (int): The new token id for `next_part_scores` + + Returns: + Dict[str, torch.Tensor]: The new score dict. + Its keys are names of `self.full_scorers` and `self.part_scorers`. + Its values are scalar tensors by the scorers. + + """ + new_scores = dict() + for k, v in next_full_scores.items(): + new_scores[k] = prev_scores[k] + v[full_idx] + for k, v in next_part_scores.items(): + new_scores[k] = prev_scores[k] + v[part_idx] + return new_scores + + def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: + """Merge states for new hypothesis. + + Args: + states: states of `self.full_scorers` + part_states: states of `self.part_scorers` + part_idx (int): The new token id for `part_scores` + + Returns: + Dict[str, torch.Tensor]: The new score dict. + Its keys are names of `self.full_scorers` and `self.part_scorers`. + Its values are states of the scorers. + + """ + new_states = dict() + for k, v in states.items(): + new_states[k] = v + for k, d in self.part_scorers.items(): + new_states[k] = d.select_state(part_states[k], part_idx) + return new_states + + def search( + self, running_hyps: List[Hypothesis], x: torch.Tensor + ) -> List[Hypothesis]: + """Search new tokens for running hypotheses and encoded speech x. + + Args: + running_hyps (List[Hypothesis]): Running hypotheses on beam + x (torch.Tensor): Encoded speech feature (T, D) + + Returns: + List[Hypotheses]: Best sorted hypotheses + + """ + best_hyps = [] + part_ids = torch.arange(self.n_vocab, device=x.device) # no pre-beam + for hyp in running_hyps: + # scoring + weighted_scores = torch.zeros(self.n_vocab, dtype=x.dtype, device=x.device) + scores, states = self.score_full(hyp, x) + for k in self.full_scorers: + weighted_scores += self.weights[k] * scores[k] + # partial scoring + if self.do_pre_beam: + pre_beam_scores = ( + weighted_scores + if self.pre_beam_score_key == "full" + else scores[self.pre_beam_score_key] + ) + part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] + part_scores, part_states = self.score_partial(hyp, part_ids, x) + for k in self.part_scorers: + weighted_scores[part_ids] += self.weights[k] * part_scores[k] + # add previous hyp score + weighted_scores += hyp.score + + # update hyps + for j, part_j in zip(*self.beam(weighted_scores, part_ids)): + # will be (2 x beam at most) + best_hyps.append( + Hypothesis( + score=weighted_scores[j], + yseq=self.append_token(hyp.yseq, j), + scores=self.merge_scores( + hyp.scores, scores, j, part_scores, part_j + ), + states=self.merge_states(states, part_states, part_j), + ) + ) + + # sort and prune 2 x beam -> beam + best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ + : min(len(best_hyps), self.beam_size) + ] + return best_hyps + + def forward( + self, x: torch.Tensor, maxlenratio: float = 0.0, minlenratio: float = 0.0 + ) -> List[Hypothesis]: + """Perform beam search. + + Args: + x (torch.Tensor): Encoded speech feature (T, D) + maxlenratio (float): Input length ratio to obtain max output length. + If maxlenratio=0.0 (default), it uses a end-detect function + to automatically find maximum hypothesis lengths + If maxlenratio<0.0, its absolute value is interpreted + as a constant max output length. + minlenratio (float): Input length ratio to obtain min output length. + + Returns: + list[Hypothesis]: N-best decoding results + + """ + # set length bounds + if maxlenratio == 0: + maxlen = x.shape[0] + elif maxlenratio < 0: + maxlen = -1 * int(maxlenratio) + else: + maxlen = max(1, int(maxlenratio * x.size(0))) + minlen = int(minlenratio * x.size(0)) + logging.info("decoder input length: " + str(x.shape[0])) + logging.info("max output length: " + str(maxlen)) + logging.info("min output length: " + str(minlen)) + + # main loop of prefix search + running_hyps = self.init_hyp(x) + ended_hyps = [] + for i in range(maxlen): + logging.debug("position " + str(i)) + best = self.search(running_hyps, x) + # post process of one iteration + running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) + # end detection + if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): + logging.info(f"end detected at {i}") + break + if len(running_hyps) == 0: + logging.info("no hypothesis. Finish decoding.") + break + else: + logging.debug(f"remained hypotheses: {len(running_hyps)}") + + nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) + # check the number of hypotheses reaching to eos + if len(nbest_hyps) == 0: + logging.warning( + "there is no N-best results, perform recognition " + "again with smaller minlenratio." + ) + return ( + [] + if minlenratio < 0.1 + else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) + ) + + # report the best result + best = nbest_hyps[0] + for k, v in best.scores.items(): + logging.info( + f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" + ) + logging.info(f"total log probability: {best.score:.2f}") + logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") + logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") + if self.token_list is not None: + logging.info( + "best hypo: " + + "".join([self.token_list[x] for x in best.yseq[1:-1]]) + + "\n" + ) + return nbest_hyps + + def post_process( + self, + i: int, + maxlen: int, + maxlenratio: float, + running_hyps: List[Hypothesis], + ended_hyps: List[Hypothesis], + ) -> List[Hypothesis]: + """Perform post-processing of beam search iterations. + + Args: + i (int): The length of hypothesis tokens. + maxlen (int): The maximum length of tokens in beam search. + maxlenratio (int): The maximum length ratio in beam search. + running_hyps (List[Hypothesis]): The running hypotheses in beam search. + ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. + + Returns: + List[Hypothesis]: The new running hypotheses. + + """ + logging.debug(f"the number of running hypotheses: {len(running_hyps)}") + if self.token_list is not None: + logging.debug( + "best hypo: " + + "".join([self.token_list[x] for x in running_hyps[0].yseq[1:]]) + ) + # add eos in the final loop to avoid that there are no ended hyps + if i == maxlen - 1: + logging.info("adding in the last position in the loop") + running_hyps = [ + h._replace(yseq=self.append_token(h.yseq, self.eos)) + for h in running_hyps + ] + + # add ended hypotheses to a final list, and removed them from current hypotheses + # (this will be a problem, number of hyps < beam) + remained_hyps = [] + for hyp in running_hyps: + if hyp.yseq[-1] == self.eos: + # e.g., Word LM needs to add final score + for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): + s = d.final_score(hyp.states[k]) + hyp.scores[k] += s + hyp = hyp._replace(score=hyp.score + self.weights[k] * s) + ended_hyps.append(hyp) + else: + remained_hyps.append(hyp) + return remained_hyps + + +def beam_search( + x: torch.Tensor, + sos: int, + eos: int, + beam_size: int, + vocab_size: int, + scorers: Dict[str, ScorerInterface], + weights: Dict[str, float], + token_list: List[str] = None, + maxlenratio: float = 0.0, + minlenratio: float = 0.0, + pre_beam_ratio: float = 1.5, + pre_beam_score_key: str = "full", +) -> list: + """Perform beam search with scorers. + + Args: + x (torch.Tensor): Encoded speech feature (T, D) + sos (int): Start of sequence id + eos (int): End of sequence id + beam_size (int): The number of hypotheses kept during search + vocab_size (int): The number of vocabulary + scorers (dict[str, ScorerInterface]): Dict of decoder modules + e.g., Decoder, CTCPrefixScorer, LM + The scorer will be ignored if it is `None` + weights (dict[str, float]): Dict of weights for each scorers + The scorer will be ignored if its weight is 0 + token_list (list[str]): List of tokens for debug log + maxlenratio (float): Input length ratio to obtain max output length. + If maxlenratio=0.0 (default), it uses a end-detect function + to automatically find maximum hypothesis lengths + minlenratio (float): Input length ratio to obtain min output length. + pre_beam_score_key (str): key of scores to perform pre-beam search + pre_beam_ratio (float): beam size in the pre-beam search + will be `int(pre_beam_ratio * beam_size)` + + Returns: + list: N-best decoding results + + """ + ret = BeamSearch( + scorers, + weights, + beam_size=beam_size, + vocab_size=vocab_size, + pre_beam_ratio=pre_beam_ratio, + pre_beam_score_key=pre_beam_score_key, + sos=sos, + eos=eos, + token_list=token_list, + ).forward(x=x, maxlenratio=maxlenratio, minlenratio=minlenratio) + return [h.asdict() for h in ret] + +class BeamSearchScama(torch.nn.Module): + """Beam search implementation.""" + + def __init__( + self, + scorers: Dict[str, ScorerInterface], + weights: Dict[str, float], + beam_size: int, + vocab_size: int, + sos: int, + eos: int, + token_list: List[str] = None, + pre_beam_ratio: float = 1.5, + pre_beam_score_key: str = None, + ): + """Initialize beam search. + + Args: + scorers (dict[str, ScorerInterface]): Dict of decoder modules + e.g., Decoder, CTCPrefixScorer, LM + The scorer will be ignored if it is `None` + weights (dict[str, float]): Dict of weights for each scorers + The scorer will be ignored if its weight is 0 + beam_size (int): The number of hypotheses kept during search + vocab_size (int): The number of vocabulary + sos (int): Start of sequence id + eos (int): End of sequence id + token_list (list[str]): List of tokens for debug log + pre_beam_score_key (str): key of scores to perform pre-beam search + pre_beam_ratio (float): beam size in the pre-beam search + will be `int(pre_beam_ratio * beam_size)` + + """ + super().__init__() + # set scorers + self.weights = weights + self.scorers = dict() + self.full_scorers = dict() + self.part_scorers = dict() + # this module dict is required for recursive cast + # `self.to(device, dtype)` in `recog.py` + self.nn_dict = torch.nn.ModuleDict() + for k, v in scorers.items(): + w = weights.get(k, 0) + if w == 0 or v is None: + continue + assert isinstance( + v, ScorerInterface + ), f"{k} ({type(v)}) does not implement ScorerInterface" + self.scorers[k] = v + if isinstance(v, PartialScorerInterface): + self.part_scorers[k] = v + else: + self.full_scorers[k] = v + if isinstance(v, torch.nn.Module): + self.nn_dict[k] = v + + # set configurations + self.sos = sos + self.eos = eos + self.token_list = token_list + self.pre_beam_size = int(pre_beam_ratio * beam_size) + self.beam_size = beam_size + self.n_vocab = vocab_size + if ( + pre_beam_score_key is not None + and pre_beam_score_key != "full" + and pre_beam_score_key not in self.full_scorers + ): + raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") + self.pre_beam_score_key = pre_beam_score_key + self.do_pre_beam = ( + self.pre_beam_score_key is not None + and self.pre_beam_size < self.n_vocab + and len(self.part_scorers) > 0 + ) + + def init_hyp(self, x: torch.Tensor) -> List[Hypothesis]: + """Get an initial hypothesis data. + + Args: + x (torch.Tensor): The encoder output feature + + Returns: + Hypothesis: The initial hypothesis. + + """ + init_states = dict() + init_scores = dict() + for k, d in self.scorers.items(): + init_states[k] = d.init_state(x) + init_scores[k] = 0.0 + return [ + Hypothesis( + score=0.0, + scores=init_scores, + states=init_states, + yseq=torch.tensor([self.sos], device=x.device), + ) + ] + + @staticmethod + def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: + """Append new token to prefix tokens. + + Args: + xs (torch.Tensor): The prefix token + x (int): The new token to append + + Returns: + torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device + + """ + x = torch.tensor([x], dtype=xs.dtype, device=xs.device) + return torch.cat((xs, x)) + + def score_full( + self, hyp: Hypothesis, + x: torch.Tensor, + x_mask: torch.Tensor = None, + pre_acoustic_embeds: torch.Tensor = None, + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + """Score new hypothesis by `self.full_scorers`. + + Args: + hyp (Hypothesis): Hypothesis with prefix tokens to score + x (torch.Tensor): Corresponding input feature + + Returns: + Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of + score dict of `hyp` that has string keys of `self.full_scorers` + and tensor score values of shape: `(self.n_vocab,)`, + and state dict that has string keys + and state values of `self.full_scorers` + + """ + scores = dict() + states = dict() + for k, d in self.full_scorers.items(): + scores[k], states[k] = d.score(hyp.yseq, hyp.states[k], x, x_mask=x_mask, pre_acoustic_embeds=pre_acoustic_embeds) + return scores, states + + def score_partial( + self, hyp: Hypothesis, ids: torch.Tensor, x: torch.Tensor + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + """Score new hypothesis by `self.part_scorers`. + + Args: + hyp (Hypothesis): Hypothesis with prefix tokens to score + ids (torch.Tensor): 1D tensor of new partial tokens to score + x (torch.Tensor): Corresponding input feature + + Returns: + Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of + score dict of `hyp` that has string keys of `self.part_scorers` + and tensor score values of shape: `(len(ids),)`, + and state dict that has string keys + and state values of `self.part_scorers` + + """ + scores = dict() + states = dict() + for k, d in self.part_scorers.items(): + scores[k], states[k] = d.score_partial(hyp.yseq, ids, hyp.states[k], x) + return scores, states + + def beam( + self, weighted_scores: torch.Tensor, ids: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute topk full token ids and partial token ids. + + Args: + weighted_scores (torch.Tensor): The weighted sum scores for each tokens. + Its shape is `(self.n_vocab,)`. + ids (torch.Tensor): The partial token ids to compute topk + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + The topk full token ids and partial token ids. + Their shapes are `(self.beam_size,)` + + """ + # no pre beam performed + if weighted_scores.size(0) == ids.size(0): + top_ids = weighted_scores.topk(self.beam_size)[1] + return top_ids, top_ids + + # mask pruned in pre-beam not to select in topk + tmp = weighted_scores[ids] + weighted_scores[:] = -float("inf") + weighted_scores[ids] = tmp + top_ids = weighted_scores.topk(self.beam_size)[1] + local_ids = weighted_scores[ids].topk(self.beam_size)[1] + return top_ids, local_ids + + @staticmethod + def merge_scores( + prev_scores: Dict[str, float], + next_full_scores: Dict[str, torch.Tensor], + full_idx: int, + next_part_scores: Dict[str, torch.Tensor], + part_idx: int, + ) -> Dict[str, torch.Tensor]: + """Merge scores for new hypothesis. + + Args: + prev_scores (Dict[str, float]): + The previous hypothesis scores by `self.scorers` + next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` + full_idx (int): The next token id for `next_full_scores` + next_part_scores (Dict[str, torch.Tensor]): + scores of partial tokens by `self.part_scorers` + part_idx (int): The new token id for `next_part_scores` + + Returns: + Dict[str, torch.Tensor]: The new score dict. + Its keys are names of `self.full_scorers` and `self.part_scorers`. + Its values are scalar tensors by the scorers. + + """ + new_scores = dict() + for k, v in next_full_scores.items(): + new_scores[k] = prev_scores[k] + v[full_idx] + for k, v in next_part_scores.items(): + new_scores[k] = prev_scores[k] + v[part_idx] + return new_scores + + def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: + """Merge states for new hypothesis. + + Args: + states: states of `self.full_scorers` + part_states: states of `self.part_scorers` + part_idx (int): The new token id for `part_scores` + + Returns: + Dict[str, torch.Tensor]: The new score dict. + Its keys are names of `self.full_scorers` and `self.part_scorers`. + Its values are states of the scorers. + + """ + new_states = dict() + for k, v in states.items(): + new_states[k] = v + for k, d in self.part_scorers.items(): + new_states[k] = d.select_state(part_states[k], part_idx) + return new_states + + def search( + self, running_hyps: List[Hypothesis], + x: torch.Tensor, + x_mask: torch.Tensor = None, + pre_acoustic_embeds: torch.Tensor = None, + ) -> List[Hypothesis]: + """Search new tokens for running hypotheses and encoded speech x. + + Args: + running_hyps (List[Hypothesis]): Running hypotheses on beam + x (torch.Tensor): Encoded speech feature (T, D) + + Returns: + List[Hypotheses]: Best sorted hypotheses + + """ + best_hyps = [] + part_ids = torch.arange(self.n_vocab, device=x.device) # no pre-beam + for hyp in running_hyps: + # scoring + weighted_scores = torch.zeros(self.n_vocab, dtype=x.dtype, device=x.device) + scores, states = self.score_full(hyp, x, x_mask=x_mask, pre_acoustic_embeds=pre_acoustic_embeds) + for k in self.full_scorers: + weighted_scores += self.weights[k] * scores[k] + # partial scoring + if self.do_pre_beam: + pre_beam_scores = ( + weighted_scores + if self.pre_beam_score_key == "full" + else scores[self.pre_beam_score_key] + ) + part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] + part_scores, part_states = self.score_partial(hyp, part_ids, x) + for k in self.part_scorers: + weighted_scores[part_ids] += self.weights[k] * part_scores[k] + # add previous hyp score + weighted_scores += hyp.score + + # update hyps + for j, part_j in zip(*self.beam(weighted_scores, part_ids)): + # will be (2 x beam at most) + best_hyps.append( + Hypothesis( + score=weighted_scores[j], + yseq=self.append_token(hyp.yseq, j), + scores=self.merge_scores( + hyp.scores, scores, j, part_scores, part_j + ), + states=self.merge_states(states, part_states, part_j), + ) + ) + + # sort and prune 2 x beam -> beam + best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ + : min(len(best_hyps), self.beam_size) + ] + return best_hyps + + def forward( + self, x: torch.Tensor, + scama_mask: torch.Tensor = None, + pre_acoustic_embeds: torch.Tensor = None, + maxlenratio: float = 0.0, + minlenratio: float = 0.0, + maxlen: int = None, + minlen: int = 0, + ) -> List[Hypothesis]: + """Perform beam search. + + Args: + x (torch.Tensor): Encoded speech feature (T, D) + maxlenratio (float): Input length ratio to obtain max output length. + If maxlenratio=0.0 (default), it uses a end-detect function + to automatically find maximum hypothesis lengths + If maxlenratio<0.0, its absolute value is interpreted + as a constant max output length. + minlenratio (float): Input length ratio to obtain min output length. + + Returns: + list[Hypothesis]: N-best decoding results + + """ + if maxlen is None: + # set length bounds + if maxlenratio == 0: + maxlen = x.shape[0] + elif maxlenratio < 0: + maxlen = -1 * int(maxlenratio) + else: + maxlen = max(1, int(maxlenratio * x.size(0))) + minlen = int(minlenratio * x.size(0)) + + logging.info("decoder input length: " + str(x.shape[0])) + logging.info("max output length: " + str(maxlen)) + logging.info("min output length: " + str(minlen)) + + # main loop of prefix search + running_hyps = self.init_hyp(x) + ended_hyps = [] + for i in range(maxlen): + logging.debug("position " + str(i)) + mask_enc = None + if scama_mask is not None: + token_num_predictor = scama_mask.size(1) + token_id_slice = min(i, token_num_predictor-1) + mask_enc = scama_mask[:, token_id_slice:token_id_slice+1, :] + # if mask_enc.size(1) == 0: + # mask_enc = scama_mask[:, -2:-1, :] + # # mask_enc = torch.zeros_like(mask_enc) + pre_acoustic_embeds_cur = None + if pre_acoustic_embeds is not None: + b, t, d = pre_acoustic_embeds.size() + pad = torch.zeros((b, 1, d), dtype=pre_acoustic_embeds.dtype).to(device=pre_acoustic_embeds.device) + pre_acoustic_embeds = torch.cat((pre_acoustic_embeds, pad), dim=1) + token_id_slice = min(i, t) + pre_acoustic_embeds_cur = pre_acoustic_embeds[:, token_id_slice:token_id_slice+1, :] + + best = self.search(running_hyps, x, x_mask=mask_enc, pre_acoustic_embeds=pre_acoustic_embeds_cur) + # post process of one iteration + running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) + # end detection + if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): + logging.info(f"end detected at {i}") + break + if len(running_hyps) == 0: + logging.info("no hypothesis. Finish decoding.") + break + else: + logging.debug(f"remained hypotheses: {len(running_hyps)}") + + nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) + # check the number of hypotheses reaching to eos + if len(nbest_hyps) == 0: + logging.warning( + "there is no N-best results, perform recognition " + "again with smaller minlenratio." + ) + return ( + [] + if minlenratio < 0.1 + else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) + ) + + # report the best result + for x in nbest_hyps: + yseq = "".join([self.token_list[x] for x in x.yseq]) + logging.debug("nbest: y: {}, yseq: {}, score: {}".format(x.yseq, yseq, x.score)) + best = nbest_hyps[0] + for k, v in best.scores.items(): + logging.info( + f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" + ) + logging.info(f"total log probability: {best.score:.2f}") + logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") + logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") + if self.token_list is not None: + logging.info( + "best hypo: " + + "".join([self.token_list[x] for x in best.yseq[1:-1]]) + + "\n" + ) + return nbest_hyps + + def post_process( + self, + i: int, + maxlen: int, + maxlenratio: float, + running_hyps: List[Hypothesis], + ended_hyps: List[Hypothesis], + ) -> List[Hypothesis]: + """Perform post-processing of beam search iterations. + + Args: + i (int): The length of hypothesis tokens. + maxlen (int): The maximum length of tokens in beam search. + maxlenratio (int): The maximum length ratio in beam search. + running_hyps (List[Hypothesis]): The running hypotheses in beam search. + ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. + + Returns: + List[Hypothesis]: The new running hypotheses. + + """ + logging.debug(f"the number of running hypotheses: {len(running_hyps)}") + if self.token_list is not None: + logging.debug( + "best hypo: " + + "".join([self.token_list[x] for x in running_hyps[0].yseq[1:]]) + ) + # add eos in the final loop to avoid that there are no ended hyps + if i == maxlen - 1: + logging.info("adding in the last position in the loop") + running_hyps = [ + h._replace(yseq=self.append_token(h.yseq, self.eos)) + for h in running_hyps + ] + + # add ended hypotheses to a final list, and removed them from current hypotheses + # (this will be a problem, number of hyps < beam) + remained_hyps = [] + for hyp in running_hyps: + if hyp.yseq[-1] == self.eos: + # e.g., Word LM needs to add final score + for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): + s = d.final_score(hyp.states[k]) + hyp.scores[k] += s + hyp = hyp._replace(score=hyp.score + self.weights[k] * s) + ended_hyps.append(hyp) + else: + remained_hyps.append(hyp) + return remained_hyps + +class BeamSearchPara(torch.nn.Module): + """Beam search implementation.""" + + def __init__( + self, + scorers: Dict[str, ScorerInterface], + weights: Dict[str, float], + beam_size: int, + vocab_size: int, + sos: int, + eos: int, + token_list: List[str] = None, + pre_beam_ratio: float = 1.5, + pre_beam_score_key: str = None, + ): + """Initialize beam search. + + Args: + scorers (dict[str, ScorerInterface]): Dict of decoder modules + e.g., Decoder, CTCPrefixScorer, LM + The scorer will be ignored if it is `None` + weights (dict[str, float]): Dict of weights for each scorers + The scorer will be ignored if its weight is 0 + beam_size (int): The number of hypotheses kept during search + vocab_size (int): The number of vocabulary + sos (int): Start of sequence id + eos (int): End of sequence id + token_list (list[str]): List of tokens for debug log + pre_beam_score_key (str): key of scores to perform pre-beam search + pre_beam_ratio (float): beam size in the pre-beam search + will be `int(pre_beam_ratio * beam_size)` + + """ + super().__init__() + # set scorers + self.weights = weights + self.scorers = dict() + self.full_scorers = dict() + self.part_scorers = dict() + # this module dict is required for recursive cast + # `self.to(device, dtype)` in `recog.py` + self.nn_dict = torch.nn.ModuleDict() + for k, v in scorers.items(): + w = weights.get(k, 0) + if w == 0 or v is None: + continue + assert isinstance( + v, ScorerInterface + ), f"{k} ({type(v)}) does not implement ScorerInterface" + self.scorers[k] = v + if isinstance(v, PartialScorerInterface): + self.part_scorers[k] = v + else: + self.full_scorers[k] = v + if isinstance(v, torch.nn.Module): + self.nn_dict[k] = v + + # set configurations + self.sos = sos + self.eos = eos + self.token_list = token_list + self.pre_beam_size = int(pre_beam_ratio * beam_size) + self.beam_size = beam_size + self.n_vocab = vocab_size + if ( + pre_beam_score_key is not None + and pre_beam_score_key != "full" + and pre_beam_score_key not in self.full_scorers + ): + raise KeyError(f"{pre_beam_score_key} is not found in {self.full_scorers}") + self.pre_beam_score_key = pre_beam_score_key + self.do_pre_beam = ( + self.pre_beam_score_key is not None + and self.pre_beam_size < self.n_vocab + and len(self.part_scorers) > 0 + ) + + def init_hyp(self, x: torch.Tensor) -> List[Hypothesis]: + """Get an initial hypothesis data. + + Args: + x (torch.Tensor): The encoder output feature + + Returns: + Hypothesis: The initial hypothesis. + + """ + init_states = dict() + init_scores = dict() + for k, d in self.scorers.items(): + init_states[k] = d.init_state(x) + init_scores[k] = 0.0 + return [ + Hypothesis( + score=0.0, + scores=init_scores, + states=init_states, + yseq=torch.tensor([self.sos], device=x.device), + ) + ] + + @staticmethod + def append_token(xs: torch.Tensor, x: int) -> torch.Tensor: + """Append new token to prefix tokens. + + Args: + xs (torch.Tensor): The prefix token + x (int): The new token to append + + Returns: + torch.Tensor: New tensor contains: xs + [x] with xs.dtype and xs.device + + """ + x = torch.tensor([x], dtype=xs.dtype, device=xs.device) + return torch.cat((xs, x)) + + def score_full( + self, hyp: Hypothesis, x: torch.Tensor + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + """Score new hypothesis by `self.full_scorers`. + + Args: + hyp (Hypothesis): Hypothesis with prefix tokens to score + x (torch.Tensor): Corresponding input feature + + Returns: + Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of + score dict of `hyp` that has string keys of `self.full_scorers` + and tensor score values of shape: `(self.n_vocab,)`, + and state dict that has string keys + and state values of `self.full_scorers` + + """ + scores = dict() + states = dict() + for k, d in self.full_scorers.items(): + scores[k], states[k] = d.score(hyp.yseq, hyp.states[k], x) + return scores, states + + def score_partial( + self, hyp: Hypothesis, ids: torch.Tensor, x: torch.Tensor + ) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: + """Score new hypothesis by `self.part_scorers`. + + Args: + hyp (Hypothesis): Hypothesis with prefix tokens to score + ids (torch.Tensor): 1D tensor of new partial tokens to score + x (torch.Tensor): Corresponding input feature + + Returns: + Tuple[Dict[str, torch.Tensor], Dict[str, Any]]: Tuple of + score dict of `hyp` that has string keys of `self.part_scorers` + and tensor score values of shape: `(len(ids),)`, + and state dict that has string keys + and state values of `self.part_scorers` + + """ + scores = dict() + states = dict() + for k, d in self.part_scorers.items(): + scores[k], states[k] = d.score_partial(hyp.yseq, ids, hyp.states[k], x) + return scores, states + + def beam( + self, weighted_scores: torch.Tensor, ids: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute topk full token ids and partial token ids. + + Args: + weighted_scores (torch.Tensor): The weighted sum scores for each tokens. + Its shape is `(self.n_vocab,)`. + ids (torch.Tensor): The partial token ids to compute topk + + Returns: + Tuple[torch.Tensor, torch.Tensor]: + The topk full token ids and partial token ids. + Their shapes are `(self.beam_size,)` + + """ + # no pre beam performed + if weighted_scores.size(0) == ids.size(0): + top_ids = weighted_scores.topk(self.beam_size)[1] + return top_ids, top_ids + + # mask pruned in pre-beam not to select in topk + tmp = weighted_scores[ids] + weighted_scores[:] = -float("inf") + weighted_scores[ids] = tmp + top_ids = weighted_scores.topk(self.beam_size)[1] + local_ids = weighted_scores[ids].topk(self.beam_size)[1] + return top_ids, local_ids + + @staticmethod + def merge_scores( + prev_scores: Dict[str, float], + next_full_scores: Dict[str, torch.Tensor], + full_idx: int, + next_part_scores: Dict[str, torch.Tensor], + part_idx: int, + ) -> Dict[str, torch.Tensor]: + """Merge scores for new hypothesis. + + Args: + prev_scores (Dict[str, float]): + The previous hypothesis scores by `self.scorers` + next_full_scores (Dict[str, torch.Tensor]): scores by `self.full_scorers` + full_idx (int): The next token id for `next_full_scores` + next_part_scores (Dict[str, torch.Tensor]): + scores of partial tokens by `self.part_scorers` + part_idx (int): The new token id for `next_part_scores` + + Returns: + Dict[str, torch.Tensor]: The new score dict. + Its keys are names of `self.full_scorers` and `self.part_scorers`. + Its values are scalar tensors by the scorers. + + """ + new_scores = dict() + for k, v in next_full_scores.items(): + new_scores[k] = prev_scores[k] + v[full_idx] + for k, v in next_part_scores.items(): + new_scores[k] = prev_scores[k] + v[part_idx] + return new_scores + + def merge_states(self, states: Any, part_states: Any, part_idx: int) -> Any: + """Merge states for new hypothesis. + + Args: + states: states of `self.full_scorers` + part_states: states of `self.part_scorers` + part_idx (int): The new token id for `part_scores` + + Returns: + Dict[str, torch.Tensor]: The new score dict. + Its keys are names of `self.full_scorers` and `self.part_scorers`. + Its values are states of the scorers. + + """ + new_states = dict() + for k, v in states.items(): + new_states[k] = v + for k, d in self.part_scorers.items(): + new_states[k] = d.select_state(part_states[k], part_idx) + return new_states + + def search( + self, running_hyps: List[Hypothesis], x: torch.Tensor, am_score: torch.Tensor + ) -> List[Hypothesis]: + """Search new tokens for running hypotheses and encoded speech x. + + Args: + running_hyps (List[Hypothesis]): Running hypotheses on beam + x (torch.Tensor): Encoded speech feature (T, D) + + Returns: + List[Hypotheses]: Best sorted hypotheses + + """ + best_hyps = [] + part_ids = torch.arange(self.n_vocab, device=x.device) # no pre-beam + for hyp in running_hyps: + # scoring + weighted_scores = torch.zeros(self.n_vocab, dtype=x.dtype, device=x.device) + weighted_scores += am_score + scores, states = self.score_full(hyp, x) + for k in self.full_scorers: + weighted_scores += self.weights[k] * scores[k] + # partial scoring + if self.do_pre_beam: + pre_beam_scores = ( + weighted_scores + if self.pre_beam_score_key == "full" + else scores[self.pre_beam_score_key] + ) + part_ids = torch.topk(pre_beam_scores, self.pre_beam_size)[1] + part_scores, part_states = self.score_partial(hyp, part_ids, x) + for k in self.part_scorers: + weighted_scores[part_ids] += self.weights[k] * part_scores[k] + # add previous hyp score + weighted_scores += hyp.score + + # update hyps + for j, part_j in zip(*self.beam(weighted_scores, part_ids)): + # will be (2 x beam at most) + best_hyps.append( + Hypothesis( + score=weighted_scores[j], + yseq=self.append_token(hyp.yseq, j), + scores=self.merge_scores( + hyp.scores, scores, j, part_scores, part_j + ), + states=self.merge_states(states, part_states, part_j), + ) + ) + + # sort and prune 2 x beam -> beam + best_hyps = sorted(best_hyps, key=lambda x: x.score, reverse=True)[ + : min(len(best_hyps), self.beam_size) + ] + return best_hyps + + def forward( + self, x: torch.Tensor, am_scores: torch.Tensor, maxlenratio: float = 0.0, minlenratio: float = 0.0 + ) -> List[Hypothesis]: + """Perform beam search. + + Args: + x (torch.Tensor): Encoded speech feature (T, D) + maxlenratio (float): Input length ratio to obtain max output length. + If maxlenratio=0.0 (default), it uses a end-detect function + to automatically find maximum hypothesis lengths + If maxlenratio<0.0, its absolute value is interpreted + as a constant max output length. + minlenratio (float): Input length ratio to obtain min output length. + + Returns: + list[Hypothesis]: N-best decoding results + + """ + # set length bounds + maxlen = am_scores.shape[0] + logging.info("decoder input length: " + str(x.shape[0])) + logging.info("max output length: " + str(maxlen)) + + # main loop of prefix search + running_hyps = self.init_hyp(x) + ended_hyps = [] + for i in range(maxlen): + logging.debug("position " + str(i)) + best = self.search(running_hyps, x, am_scores[i]) + # post process of one iteration + running_hyps = self.post_process(i, maxlen, maxlenratio, best, ended_hyps) + # end detection + if maxlenratio == 0.0 and end_detect([h.asdict() for h in ended_hyps], i): + logging.info(f"end detected at {i}") + break + if len(running_hyps) == 0: + logging.info("no hypothesis. Finish decoding.") + break + else: + logging.debug(f"remained hypotheses: {len(running_hyps)}") + + nbest_hyps = sorted(ended_hyps, key=lambda x: x.score, reverse=True) + # check the number of hypotheses reaching to eos + if len(nbest_hyps) == 0: + logging.warning( + "there is no N-best results, perform recognition " + "again with smaller minlenratio." + ) + return ( + [] + if minlenratio < 0.1 + else self.forward(x, maxlenratio, max(0.0, minlenratio - 0.1)) + ) + + # report the best result + best = nbest_hyps[0] + for k, v in best.scores.items(): + logging.info( + f"{v:6.2f} * {self.weights[k]:3} = {v * self.weights[k]:6.2f} for {k}" + ) + logging.info(f"total log probability: {best.score:.2f}") + logging.info(f"normalized log probability: {best.score / len(best.yseq):.2f}") + logging.info(f"total number of ended hypotheses: {len(nbest_hyps)}") + if self.token_list is not None: + logging.info( + "best hypo: " + + "".join([self.token_list[x] for x in best.yseq[1:-1]]) + + "\n" + ) + return nbest_hyps + + def post_process( + self, + i: int, + maxlen: int, + maxlenratio: float, + running_hyps: List[Hypothesis], + ended_hyps: List[Hypothesis], + ) -> List[Hypothesis]: + """Perform post-processing of beam search iterations. + + Args: + i (int): The length of hypothesis tokens. + maxlen (int): The maximum length of tokens in beam search. + maxlenratio (int): The maximum length ratio in beam search. + running_hyps (List[Hypothesis]): The running hypotheses in beam search. + ended_hyps (List[Hypothesis]): The ended hypotheses in beam search. + + Returns: + List[Hypothesis]: The new running hypotheses. + + """ + logging.debug(f"the number of running hypotheses: {len(running_hyps)}") + if self.token_list is not None: + logging.debug( + "best hypo: " + + "".join([self.token_list[x] for x in running_hyps[0].yseq[1:]]) + ) + # add eos in the final loop to avoid that there are no ended hyps + if i == maxlen - 1: + logging.info("adding in the last position in the loop") + running_hyps = [ + h._replace(yseq=self.append_token(h.yseq, self.eos)) + for h in running_hyps + ] + + # add ended hypotheses to a final list, and removed them from current hypotheses + # (this will be a problem, number of hyps < beam) + remained_hyps = [] + for hyp in running_hyps: + if hyp.yseq[-1] == self.eos: + # e.g., Word LM needs to add final score + for k, d in chain(self.full_scorers.items(), self.part_scorers.items()): + s = d.final_score(hyp.states[k]) + hyp.scores[k] += s + hyp = hyp._replace(score=hyp.score + self.weights[k] * s) + ended_hyps.append(hyp) + else: + remained_hyps.append(hyp) + return remained_hyps + diff --git a/funasr/modules/dynamic_conv.py b/funasr/modules/dynamic_conv.py new file mode 100644 index 000000000..8a2a0c1ea --- /dev/null +++ b/funasr/modules/dynamic_conv.py @@ -0,0 +1,125 @@ +"""Dynamic Convolution module.""" + +import numpy +import torch +from torch import nn +import torch.nn.functional as F + + +MIN_VALUE = float(numpy.finfo(numpy.float32).min) + + +class DynamicConvolution(nn.Module): + """Dynamic Convolution layer. + + This implementation is based on + https://github.com/pytorch/fairseq/tree/master/fairseq + + Args: + wshare (int): the number of kernel of convolution + n_feat (int): the number of features + dropout_rate (float): dropout_rate + kernel_size (int): kernel size (length) + use_kernel_mask (bool): Use causal mask or not for convolution kernel + use_bias (bool): Use bias term or not. + + """ + + def __init__( + self, + wshare, + n_feat, + dropout_rate, + kernel_size, + use_kernel_mask=False, + use_bias=False, + ): + """Construct Dynamic Convolution layer.""" + super(DynamicConvolution, self).__init__() + + assert n_feat % wshare == 0 + self.wshare = wshare + self.use_kernel_mask = use_kernel_mask + self.dropout_rate = dropout_rate + self.kernel_size = kernel_size + self.attn = None + + # linear -> GLU -- -> lightconv -> linear + # \ / + # Linear + self.linear1 = nn.Linear(n_feat, n_feat * 2) + self.linear2 = nn.Linear(n_feat, n_feat) + self.linear_weight = nn.Linear(n_feat, self.wshare * 1 * kernel_size) + nn.init.xavier_uniform(self.linear_weight.weight) + self.act = nn.GLU() + + # dynamic conv related + self.use_bias = use_bias + if self.use_bias: + self.bias = nn.Parameter(torch.Tensor(n_feat)) + + def forward(self, query, key, value, mask): + """Forward of 'Dynamic Convolution'. + + This function takes query, key and value but uses only quert. + This is just for compatibility with self-attention layer (attention.py) + + Args: + query (torch.Tensor): (batch, time1, d_model) input tensor + key (torch.Tensor): (batch, time2, d_model) NOT USED + value (torch.Tensor): (batch, time2, d_model) NOT USED + mask (torch.Tensor): (batch, time1, time2) mask + + Return: + x (torch.Tensor): (batch, time1, d_model) output + + """ + # linear -> GLU -- -> lightconv -> linear + # \ / + # Linear + x = query + B, T, C = x.size() + H = self.wshare + k = self.kernel_size + + # first liner layer + x = self.linear1(x) + + # GLU activation + x = self.act(x) + + # get kernel of convolution + weight = self.linear_weight(x) # B x T x kH + weight = F.dropout(weight, self.dropout_rate, training=self.training) + weight = weight.view(B, T, H, k).transpose(1, 2).contiguous() # B x H x T x k + weight_new = torch.zeros(B * H * T * (T + k - 1), dtype=weight.dtype) + weight_new = weight_new.view(B, H, T, T + k - 1).fill_(float("-inf")) + weight_new = weight_new.to(x.device) # B x H x T x T+k-1 + weight_new.as_strided( + (B, H, T, k), ((T + k - 1) * T * H, (T + k - 1) * T, T + k, 1) + ).copy_(weight) + weight_new = weight_new.narrow(-1, int((k - 1) / 2), T) # B x H x T x T(k) + if self.use_kernel_mask: + kernel_mask = torch.tril(torch.ones(T, T, device=x.device)).unsqueeze(0) + weight_new = weight_new.masked_fill(kernel_mask == 0.0, float("-inf")) + weight_new = F.softmax(weight_new, dim=-1) + self.attn = weight_new + weight_new = weight_new.view(B * H, T, T) + + # convolution + x = x.transpose(1, 2).contiguous() # B x C x T + x = x.view(B * H, int(C / H), T).transpose(1, 2) + x = torch.bmm(weight_new, x) # BH x T x C/H + x = x.transpose(1, 2).contiguous().view(B, C, T) + + if self.use_bias: + x = x + self.bias.view(1, -1, 1) + x = x.transpose(1, 2) # B x T x C + + if mask is not None and not self.use_kernel_mask: + mask = mask.transpose(-1, -2) + x = x.masked_fill(mask == 0, 0.0) + + # second linear layer + x = self.linear2(x) + return x diff --git a/funasr/modules/dynamic_conv2d.py b/funasr/modules/dynamic_conv2d.py new file mode 100644 index 000000000..f8a4dd6e9 --- /dev/null +++ b/funasr/modules/dynamic_conv2d.py @@ -0,0 +1,138 @@ +"""Dynamic 2-Dimensional Convolution module.""" + +import numpy +import torch +from torch import nn +import torch.nn.functional as F + + +MIN_VALUE = float(numpy.finfo(numpy.float32).min) + + +class DynamicConvolution2D(nn.Module): + """Dynamic 2-Dimensional Convolution layer. + + This implementation is based on + https://github.com/pytorch/fairseq/tree/master/fairseq + + Args: + wshare (int): the number of kernel of convolution + n_feat (int): the number of features + dropout_rate (float): dropout_rate + kernel_size (int): kernel size (length) + use_kernel_mask (bool): Use causal mask or not for convolution kernel + use_bias (bool): Use bias term or not. + + """ + + def __init__( + self, + wshare, + n_feat, + dropout_rate, + kernel_size, + use_kernel_mask=False, + use_bias=False, + ): + """Construct Dynamic 2-Dimensional Convolution layer.""" + super(DynamicConvolution2D, self).__init__() + + assert n_feat % wshare == 0 + self.wshare = wshare + self.use_kernel_mask = use_kernel_mask + self.dropout_rate = dropout_rate + self.kernel_size = kernel_size + self.padding_size = int(kernel_size / 2) + self.attn_t = None + self.attn_f = None + + # linear -> GLU -- -> lightconv -> linear + # \ / + # Linear + self.linear1 = nn.Linear(n_feat, n_feat * 2) + self.linear2 = nn.Linear(n_feat * 2, n_feat) + self.linear_weight = nn.Linear(n_feat, self.wshare * 1 * kernel_size) + nn.init.xavier_uniform(self.linear_weight.weight) + self.linear_weight_f = nn.Linear(n_feat, kernel_size) + nn.init.xavier_uniform(self.linear_weight_f.weight) + self.act = nn.GLU() + + # dynamic conv related + self.use_bias = use_bias + if self.use_bias: + self.bias = nn.Parameter(torch.Tensor(n_feat)) + + def forward(self, query, key, value, mask): + """Forward of 'Dynamic 2-Dimensional Convolution'. + + This function takes query, key and value but uses only query. + This is just for compatibility with self-attention layer (attention.py) + + Args: + query (torch.Tensor): (batch, time1, d_model) input tensor + key (torch.Tensor): (batch, time2, d_model) NOT USED + value (torch.Tensor): (batch, time2, d_model) NOT USED + mask (torch.Tensor): (batch, time1, time2) mask + + Return: + x (torch.Tensor): (batch, time1, d_model) output + + """ + # linear -> GLU -- -> lightconv -> linear + # \ / + # Linear + x = query + B, T, C = x.size() + H = self.wshare + k = self.kernel_size + + # first liner layer + x = self.linear1(x) + + # GLU activation + x = self.act(x) + + # convolution of frequency axis + weight_f = self.linear_weight_f(x).view(B * T, 1, k) # B x T x k + self.attn_f = weight_f.view(B, T, k).unsqueeze(1) + xf = F.conv1d( + x.view(1, B * T, C), weight_f, padding=self.padding_size, groups=B * T + ) + xf = xf.view(B, T, C) + + # get kernel of convolution + weight = self.linear_weight(x) # B x T x kH + weight = F.dropout(weight, self.dropout_rate, training=self.training) + weight = weight.view(B, T, H, k).transpose(1, 2).contiguous() # B x H x T x k + weight_new = torch.zeros(B * H * T * (T + k - 1), dtype=weight.dtype) + weight_new = weight_new.view(B, H, T, T + k - 1).fill_(float("-inf")) + weight_new = weight_new.to(x.device) # B x H x T x T+k-1 + weight_new.as_strided( + (B, H, T, k), ((T + k - 1) * T * H, (T + k - 1) * T, T + k, 1) + ).copy_(weight) + weight_new = weight_new.narrow(-1, int((k - 1) / 2), T) # B x H x T x T(k) + if self.use_kernel_mask: + kernel_mask = torch.tril(torch.ones(T, T, device=x.device)).unsqueeze(0) + weight_new = weight_new.masked_fill(kernel_mask == 0.0, float("-inf")) + weight_new = F.softmax(weight_new, dim=-1) + self.attn_t = weight_new + weight_new = weight_new.view(B * H, T, T) + + # convolution + x = x.transpose(1, 2).contiguous() # B x C x T + x = x.view(B * H, int(C / H), T).transpose(1, 2) + x = torch.bmm(weight_new, x) + x = x.transpose(1, 2).contiguous().view(B, C, T) + + if self.use_bias: + x = x + self.bias.view(1, -1, 1) + x = x.transpose(1, 2) # B x T x C + x = torch.cat((x, xf), -1) # B x T x Cx2 + + if mask is not None and not self.use_kernel_mask: + mask = mask.transpose(-1, -2) + x = x.masked_fill(mask == 0, 0.0) + + # second linear layer + x = self.linear2(x) + return x diff --git a/funasr/modules/e2e_asr_common.py b/funasr/modules/e2e_asr_common.py new file mode 100644 index 000000000..92f90796a --- /dev/null +++ b/funasr/modules/e2e_asr_common.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +# Copyright 2017 Johns Hopkins University (Shinji Watanabe) +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Common functions for ASR.""" + +import json +import logging +import sys + +from itertools import groupby +import numpy as np +import six + + +def end_detect(ended_hyps, i, M=3, D_end=np.log(1 * np.exp(-10))): + """End detection. + + described in Eq. (50) of S. Watanabe et al + "Hybrid CTC/Attention Architecture for End-to-End Speech Recognition" + + :param ended_hyps: + :param i: + :param M: + :param D_end: + :return: + """ + if len(ended_hyps) == 0: + return False + count = 0 + best_hyp = sorted(ended_hyps, key=lambda x: x["score"], reverse=True)[0] + for m in six.moves.range(M): + # get ended_hyps with their length is i - m + hyp_length = i - m + hyps_same_length = [x for x in ended_hyps if len(x["yseq"]) == hyp_length] + if len(hyps_same_length) > 0: + best_hyp_same_length = sorted( + hyps_same_length, key=lambda x: x["score"], reverse=True + )[0] + if best_hyp_same_length["score"] - best_hyp["score"] < D_end: + count += 1 + + if count == M: + return True + else: + return False + + +# TODO(takaaki-hori): add different smoothing methods +def label_smoothing_dist(odim, lsm_type, transcript=None, blank=0): + """Obtain label distribution for loss smoothing. + + :param odim: + :param lsm_type: + :param blank: + :param transcript: + :return: + """ + if transcript is not None: + with open(transcript, "rb") as f: + trans_json = json.load(f)["utts"] + + if lsm_type == "unigram": + assert transcript is not None, ( + "transcript is required for %s label smoothing" % lsm_type + ) + labelcount = np.zeros(odim) + for k, v in trans_json.items(): + ids = np.array([int(n) for n in v["output"][0]["tokenid"].split()]) + # to avoid an error when there is no text in an uttrance + if len(ids) > 0: + labelcount[ids] += 1 + labelcount[odim - 1] = len(transcript) # count + labelcount[labelcount == 0] = 1 # flooring + labelcount[blank] = 0 # remove counts for blank + labeldist = labelcount.astype(np.float32) / np.sum(labelcount) + else: + logging.error("Error: unexpected label smoothing type: %s" % lsm_type) + sys.exit() + + return labeldist + + +def get_vgg2l_odim(idim, in_channel=3, out_channel=128): + """Return the output size of the VGG frontend. + + :param in_channel: input channel size + :param out_channel: output channel size + :return: output size + :rtype int + """ + idim = idim / in_channel + idim = np.ceil(np.array(idim, dtype=np.float32) / 2) # 1st max pooling + idim = np.ceil(np.array(idim, dtype=np.float32) / 2) # 2nd max pooling + return int(idim) * out_channel # numer of channels + + +class ErrorCalculator(object): + """Calculate CER and WER for E2E_ASR and CTC models during training. + + :param y_hats: numpy array with predicted text + :param y_pads: numpy array with true (target) text + :param char_list: + :param sym_space: + :param sym_blank: + :return: + """ + + def __init__( + self, char_list, sym_space, sym_blank, report_cer=False, report_wer=False + ): + """Construct an ErrorCalculator object.""" + super(ErrorCalculator, self).__init__() + + self.report_cer = report_cer + self.report_wer = report_wer + + self.char_list = char_list + self.space = sym_space + self.blank = sym_blank + self.idx_blank = self.char_list.index(self.blank) + if self.space in self.char_list: + self.idx_space = self.char_list.index(self.space) + else: + self.idx_space = None + + def __call__(self, ys_hat, ys_pad, is_ctc=False): + """Calculate sentence-level WER/CER score. + + :param torch.Tensor ys_hat: prediction (batch, seqlen) + :param torch.Tensor ys_pad: reference (batch, seqlen) + :param bool is_ctc: calculate CER score for CTC + :return: sentence-level WER score + :rtype float + :return: sentence-level CER score + :rtype float + """ + cer, wer = None, None + if is_ctc: + return self.calculate_cer_ctc(ys_hat, ys_pad) + elif not self.report_cer and not self.report_wer: + return cer, wer + + seqs_hat, seqs_true = self.convert_to_char(ys_hat, ys_pad) + if self.report_cer: + cer = self.calculate_cer(seqs_hat, seqs_true) + + if self.report_wer: + wer = self.calculate_wer(seqs_hat, seqs_true) + return cer, wer + + def calculate_cer_ctc(self, ys_hat, ys_pad): + """Calculate sentence-level CER score for CTC. + + :param torch.Tensor ys_hat: prediction (batch, seqlen) + :param torch.Tensor ys_pad: reference (batch, seqlen) + :return: average sentence-level CER score + :rtype float + """ + import editdistance + + cers, char_ref_lens = [], [] + for i, y in enumerate(ys_hat): + y_hat = [x[0] for x in groupby(y)] + y_true = ys_pad[i] + seq_hat, seq_true = [], [] + for idx in y_hat: + idx = int(idx) + if idx != -1 and idx != self.idx_blank and idx != self.idx_space: + seq_hat.append(self.char_list[int(idx)]) + + for idx in y_true: + idx = int(idx) + if idx != -1 and idx != self.idx_blank and idx != self.idx_space: + seq_true.append(self.char_list[int(idx)]) + + hyp_chars = "".join(seq_hat) + ref_chars = "".join(seq_true) + if len(ref_chars) > 0: + cers.append(editdistance.eval(hyp_chars, ref_chars)) + char_ref_lens.append(len(ref_chars)) + + cer_ctc = float(sum(cers)) / sum(char_ref_lens) if cers else None + return cer_ctc + + def convert_to_char(self, ys_hat, ys_pad): + """Convert index to character. + + :param torch.Tensor seqs_hat: prediction (batch, seqlen) + :param torch.Tensor seqs_true: reference (batch, seqlen) + :return: token list of prediction + :rtype list + :return: token list of reference + :rtype list + """ + seqs_hat, seqs_true = [], [] + for i, y_hat in enumerate(ys_hat): + y_true = ys_pad[i] + eos_true = np.where(y_true == -1)[0] + ymax = eos_true[0] if len(eos_true) > 0 else len(y_true) + # NOTE: padding index (-1) in y_true is used to pad y_hat + seq_hat = [self.char_list[int(idx)] for idx in y_hat[:ymax]] + seq_true = [self.char_list[int(idx)] for idx in y_true if int(idx) != -1] + seq_hat_text = "".join(seq_hat).replace(self.space, " ") + seq_hat_text = seq_hat_text.replace(self.blank, "") + seq_true_text = "".join(seq_true).replace(self.space, " ") + seqs_hat.append(seq_hat_text) + seqs_true.append(seq_true_text) + return seqs_hat, seqs_true + + def calculate_cer(self, seqs_hat, seqs_true): + """Calculate sentence-level CER score. + + :param list seqs_hat: prediction + :param list seqs_true: reference + :return: average sentence-level CER score + :rtype float + """ + import editdistance + + char_eds, char_ref_lens = [], [] + for i, seq_hat_text in enumerate(seqs_hat): + seq_true_text = seqs_true[i] + hyp_chars = seq_hat_text.replace(" ", "") + ref_chars = seq_true_text.replace(" ", "") + char_eds.append(editdistance.eval(hyp_chars, ref_chars)) + char_ref_lens.append(len(ref_chars)) + return float(sum(char_eds)) / sum(char_ref_lens) + + def calculate_wer(self, seqs_hat, seqs_true): + """Calculate sentence-level WER score. + + :param list seqs_hat: prediction + :param list seqs_true: reference + :return: average sentence-level WER score + :rtype float + """ + import editdistance + + word_eds, word_ref_lens = [], [] + for i, seq_hat_text in enumerate(seqs_hat): + seq_true_text = seqs_true[i] + hyp_words = seq_hat_text.split() + ref_words = seq_true_text.split() + word_eds.append(editdistance.eval(hyp_words, ref_words)) + word_ref_lens.append(len(ref_words)) + return float(sum(word_eds)) / sum(word_ref_lens) diff --git a/funasr/modules/embedding.py b/funasr/modules/embedding.py new file mode 100644 index 000000000..b61a61a88 --- /dev/null +++ b/funasr/modules/embedding.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Positional Encoding Module.""" + +import math +import torch + + +def _pre_hook( + state_dict, + prefix, + local_metadata, + strict, + missing_keys, + unexpected_keys, + error_msgs, +): + """Perform pre-hook in load_state_dict for backward compatibility. + + Note: + We saved self.pe until v.0.5.2 but we have omitted it later. + Therefore, we remove the item "pe" from `state_dict` for backward compatibility. + + """ + k = prefix + "pe" + if k in state_dict: + state_dict.pop(k) + + +class PositionalEncoding(torch.nn.Module): + """Positional encoding. + + Args: + d_model (int): Embedding dimension. + dropout_rate (float): Dropout rate. + max_len (int): Maximum input length. + reverse (bool): Whether to reverse the input position. Only for + the class LegacyRelPositionalEncoding. We remove it in the current + class RelPositionalEncoding. + """ + + def __init__(self, d_model, dropout_rate, max_len=5000, reverse=False): + """Construct an PositionalEncoding object.""" + super(PositionalEncoding, self).__init__() + self.d_model = d_model + self.reverse = reverse + self.xscale = math.sqrt(self.d_model) + self.dropout = torch.nn.Dropout(p=dropout_rate) + self.pe = None + self.extend_pe(torch.tensor(0.0).expand(1, max_len)) + self._register_load_state_dict_pre_hook(_pre_hook) + + def extend_pe(self, x): + """Reset the positional encodings.""" + if self.pe is not None: + if self.pe.size(1) >= x.size(1): + if self.pe.dtype != x.dtype or self.pe.device != x.device: + self.pe = self.pe.to(dtype=x.dtype, device=x.device) + return + pe = torch.zeros(x.size(1), self.d_model) + if self.reverse: + position = torch.arange( + x.size(1) - 1, -1, -1.0, dtype=torch.float32 + ).unsqueeze(1) + else: + position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, self.d_model, 2, dtype=torch.float32) + * -(math.log(10000.0) / self.d_model) + ) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.pe = pe.to(device=x.device, dtype=x.dtype) + + def forward(self, x: torch.Tensor): + """Add positional encoding. + + Args: + x (torch.Tensor): Input tensor (batch, time, `*`). + + Returns: + torch.Tensor: Encoded tensor (batch, time, `*`). + """ + self.extend_pe(x) + x = x * self.xscale + self.pe[:, : x.size(1)] + return self.dropout(x) + + +class ScaledPositionalEncoding(PositionalEncoding): + """Scaled positional encoding module. + + See Sec. 3.2 https://arxiv.org/abs/1809.08895 + + Args: + d_model (int): Embedding dimension. + dropout_rate (float): Dropout rate. + max_len (int): Maximum input length. + + """ + + def __init__(self, d_model, dropout_rate, max_len=5000): + """Initialize class.""" + super().__init__(d_model=d_model, dropout_rate=dropout_rate, max_len=max_len) + self.alpha = torch.nn.Parameter(torch.tensor(1.0)) + + def reset_parameters(self): + """Reset parameters.""" + self.alpha.data = torch.tensor(1.0) + + def forward(self, x): + """Add positional encoding. + + Args: + x (torch.Tensor): Input tensor (batch, time, `*`). + + Returns: + torch.Tensor: Encoded tensor (batch, time, `*`). + + """ + self.extend_pe(x) + x = x + self.alpha * self.pe[:, : x.size(1)] + return self.dropout(x) + + +class LearnableFourierPosEnc(torch.nn.Module): + """Learnable Fourier Features for Positional Encoding. + + See https://arxiv.org/pdf/2106.02795.pdf + + Args: + d_model (int): Embedding dimension. + dropout_rate (float): Dropout rate. + max_len (int): Maximum input length. + gamma (float): init parameter for the positional kernel variance + see https://arxiv.org/pdf/2106.02795.pdf. + apply_scaling (bool): Whether to scale the input before adding the pos encoding. + hidden_dim (int): if not None, we modulate the pos encodings with + an MLP whose hidden layer has hidden_dim neurons. + """ + + def __init__( + self, + d_model, + dropout_rate=0.0, + max_len=5000, + gamma=1.0, + apply_scaling=False, + hidden_dim=None, + ): + """Initialize class.""" + super(LearnableFourierPosEnc, self).__init__() + + self.d_model = d_model + + if apply_scaling: + self.xscale = math.sqrt(self.d_model) + else: + self.xscale = 1.0 + + self.dropout = torch.nn.Dropout(dropout_rate) + self.max_len = max_len + + self.gamma = gamma + if self.gamma is None: + self.gamma = self.d_model // 2 + + assert ( + d_model % 2 == 0 + ), "d_model should be divisible by two in order to use this layer." + self.w_r = torch.nn.Parameter(torch.empty(1, d_model // 2)) + self._reset() # init the weights + + self.hidden_dim = hidden_dim + if self.hidden_dim is not None: + self.mlp = torch.nn.Sequential( + torch.nn.Linear(d_model, hidden_dim), + torch.nn.GELU(), + torch.nn.Linear(hidden_dim, d_model), + ) + + def _reset(self): + self.w_r.data = torch.normal( + 0, (1 / math.sqrt(self.gamma)), (1, self.d_model // 2) + ) + + def extend_pe(self, x): + """Reset the positional encodings.""" + position_v = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1).to(x) + + cosine = torch.cos(torch.matmul(position_v, self.w_r)) + sine = torch.sin(torch.matmul(position_v, self.w_r)) + pos_enc = torch.cat((cosine, sine), -1) + pos_enc /= math.sqrt(self.d_model) + + if self.hidden_dim is None: + return pos_enc.unsqueeze(0) + else: + return self.mlp(pos_enc.unsqueeze(0)) + + def forward(self, x: torch.Tensor): + """Add positional encoding. + + Args: + x (torch.Tensor): Input tensor (batch, time, `*`). + + Returns: + torch.Tensor: Encoded tensor (batch, time, `*`). + """ + pe = self.extend_pe(x) + x = x * self.xscale + pe + return self.dropout(x) + + +class LegacyRelPositionalEncoding(PositionalEncoding): + """Relative positional encoding module (old version). + + Details can be found in https://github.com/espnet/espnet/pull/2816. + + See : Appendix B in https://arxiv.org/abs/1901.02860 + + Args: + d_model (int): Embedding dimension. + dropout_rate (float): Dropout rate. + max_len (int): Maximum input length. + + """ + + def __init__(self, d_model, dropout_rate, max_len=5000): + """Initialize class.""" + super().__init__( + d_model=d_model, + dropout_rate=dropout_rate, + max_len=max_len, + reverse=True, + ) + + def forward(self, x): + """Compute positional encoding. + + Args: + x (torch.Tensor): Input tensor (batch, time, `*`). + + Returns: + torch.Tensor: Encoded tensor (batch, time, `*`). + torch.Tensor: Positional embedding tensor (1, time, `*`). + + """ + self.extend_pe(x) + x = x * self.xscale + pos_emb = self.pe[:, : x.size(1)] + return self.dropout(x), self.dropout(pos_emb) + + +class RelPositionalEncoding(torch.nn.Module): + """Relative positional encoding module (new implementation). + + Details can be found in https://github.com/espnet/espnet/pull/2816. + + See : Appendix B in https://arxiv.org/abs/1901.02860 + + Args: + d_model (int): Embedding dimension. + dropout_rate (float): Dropout rate. + max_len (int): Maximum input length. + + """ + + def __init__(self, d_model, dropout_rate, max_len=5000): + """Construct an PositionalEncoding object.""" + super(RelPositionalEncoding, self).__init__() + self.d_model = d_model + self.xscale = math.sqrt(self.d_model) + self.dropout = torch.nn.Dropout(p=dropout_rate) + self.pe = None + self.extend_pe(torch.tensor(0.0).expand(1, max_len)) + + def extend_pe(self, x): + """Reset the positional encodings.""" + if self.pe is not None: + # self.pe contains both positive and negative parts + # the length of self.pe is 2 * input_len - 1 + if self.pe.size(1) >= x.size(1) * 2 - 1: + if self.pe.dtype != x.dtype or self.pe.device != x.device: + self.pe = self.pe.to(dtype=x.dtype, device=x.device) + return + # Suppose `i` means to the position of query vecotr and `j` means the + # position of key vector. We use position relative positions when keys + # are to the left (i>j) and negative relative positions otherwise (i= length: + if self.pe.dtype != dtype or self.pe.device != device: + self.pe = self.pe.to(dtype=dtype, device=device) + return + pe = torch.zeros(length, self.d_model) + position = torch.arange(0, length, dtype=torch.float32).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, self.d_model, 2, dtype=torch.float32) + * -(math.log(10000.0) / self.d_model) + ) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.pe = pe.to(device=device, dtype=dtype) + + def forward(self, x: torch.Tensor, start_idx: int = 0): + """Add positional encoding. + + Args: + x (torch.Tensor): Input tensor (batch, time, `*`). + + Returns: + torch.Tensor: Encoded tensor (batch, time, `*`). + + """ + self.extend_pe(x.size(1) + start_idx, x.device, x.dtype) + x = x * self.xscale + self.pe[:, start_idx : start_idx + x.size(1)] + return self.dropout(x) + +class SinusoidalPositionEncoder(torch.nn.Module): + ''' + + ''' + def __int__(self, d_model=80, dropout_rate=0.1): + pass + + def encode(self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32): + batch_size = positions.size(0) + positions = positions.type(dtype) + log_timescale_increment = torch.log(torch.tensor([10000], dtype=dtype)) / (depth / 2 - 1) + inv_timescales = torch.exp(torch.arange(depth / 2).type(dtype) * (-log_timescale_increment)) + inv_timescales = torch.reshape(inv_timescales, [batch_size, -1]) + scaled_time = torch.reshape(positions, [1, -1, 1]) * torch.reshape(inv_timescales, [1, 1, -1]) + encoding = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=2) + return encoding.type(dtype) + + def forward(self, x): + batch_size, timesteps, input_dim = x.size() + positions = torch.arange(1, timesteps+1)[None, :] + position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device) + + return x + position_encoding \ No newline at end of file diff --git a/funasr/modules/frontends/__init__.py b/funasr/modules/frontends/__init__.py new file mode 100644 index 000000000..b7f177368 --- /dev/null +++ b/funasr/modules/frontends/__init__.py @@ -0,0 +1 @@ +"""Initialize sub package.""" diff --git a/funasr/modules/frontends/beamformer.py b/funasr/modules/frontends/beamformer.py new file mode 100644 index 000000000..f3eccee4c --- /dev/null +++ b/funasr/modules/frontends/beamformer.py @@ -0,0 +1,84 @@ +import torch +from torch_complex import functional as FC +from torch_complex.tensor import ComplexTensor + + +def get_power_spectral_density_matrix( + xs: ComplexTensor, mask: torch.Tensor, normalization=True, eps: float = 1e-15 +) -> ComplexTensor: + """Return cross-channel power spectral density (PSD) matrix + + Args: + xs (ComplexTensor): (..., F, C, T) + mask (torch.Tensor): (..., F, C, T) + normalization (bool): + eps (float): + Returns + psd (ComplexTensor): (..., F, C, C) + + """ + # outer product: (..., C_1, T) x (..., C_2, T) -> (..., T, C, C_2) + psd_Y = FC.einsum("...ct,...et->...tce", [xs, xs.conj()]) + + # Averaging mask along C: (..., C, T) -> (..., T) + mask = mask.mean(dim=-2) + + # Normalized mask along T: (..., T) + if normalization: + # If assuming the tensor is padded with zero, the summation along + # the time axis is same regardless of the padding length. + mask = mask / (mask.sum(dim=-1, keepdim=True) + eps) + + # psd: (..., T, C, C) + psd = psd_Y * mask[..., None, None] + # (..., T, C, C) -> (..., C, C) + psd = psd.sum(dim=-3) + + return psd + + +def get_mvdr_vector( + psd_s: ComplexTensor, + psd_n: ComplexTensor, + reference_vector: torch.Tensor, + eps: float = 1e-15, +) -> ComplexTensor: + """Return the MVDR(Minimum Variance Distortionless Response) vector: + + h = (Npsd^-1 @ Spsd) / (Tr(Npsd^-1 @ Spsd)) @ u + + Reference: + On optimal frequency-domain multichannel linear filtering + for noise reduction; M. Souden et al., 2010; + https://ieeexplore.ieee.org/document/5089420 + + Args: + psd_s (ComplexTensor): (..., F, C, C) + psd_n (ComplexTensor): (..., F, C, C) + reference_vector (torch.Tensor): (..., C) + eps (float): + Returns: + beamform_vector (ComplexTensor)r: (..., F, C) + """ + # Add eps + C = psd_n.size(-1) + eye = torch.eye(C, dtype=psd_n.dtype, device=psd_n.device) + shape = [1 for _ in range(psd_n.dim() - 2)] + [C, C] + eye = eye.view(*shape) + psd_n += eps * eye + + # numerator: (..., C_1, C_2) x (..., C_2, C_3) -> (..., C_1, C_3) + numerator = FC.einsum("...ec,...cd->...ed", [psd_n.inverse(), psd_s]) + # ws: (..., C, C) / (...,) -> (..., C, C) + ws = numerator / (FC.trace(numerator)[..., None, None] + eps) + # h: (..., F, C_1, C_2) x (..., C_2) -> (..., F, C_1) + beamform_vector = FC.einsum("...fec,...c->...fe", [ws, reference_vector]) + return beamform_vector + + +def apply_beamforming_vector( + beamform_vector: ComplexTensor, mix: ComplexTensor +) -> ComplexTensor: + # (..., C) x (..., C, T) -> (..., T) + es = FC.einsum("...c,...ct->...t", [beamform_vector.conj(), mix]) + return es diff --git a/funasr/modules/frontends/dnn_beamformer.py b/funasr/modules/frontends/dnn_beamformer.py new file mode 100644 index 000000000..e75d771d3 --- /dev/null +++ b/funasr/modules/frontends/dnn_beamformer.py @@ -0,0 +1,172 @@ +"""DNN beamformer module.""" +from typing import Tuple + +import torch +from torch.nn import functional as F + +from funasr.modules.frontends.beamformer import apply_beamforming_vector +from funasr.modules.frontends.beamformer import get_mvdr_vector +from funasr.modules.frontends.beamformer import ( + get_power_spectral_density_matrix, # noqa: H301 +) +from funasr.modules.frontends.mask_estimator import MaskEstimator +from torch_complex.tensor import ComplexTensor + + +class DNN_Beamformer(torch.nn.Module): + """DNN mask based Beamformer + + Citation: + Multichannel End-to-end Speech Recognition; T. Ochiai et al., 2017; + https://arxiv.org/abs/1703.04783 + + """ + + def __init__( + self, + bidim, + btype="blstmp", + blayers=3, + bunits=300, + bprojs=320, + bnmask=2, + dropout_rate=0.0, + badim=320, + ref_channel: int = -1, + beamformer_type="mvdr", + ): + super().__init__() + self.mask = MaskEstimator( + btype, bidim, blayers, bunits, bprojs, dropout_rate, nmask=bnmask + ) + self.ref = AttentionReference(bidim, badim) + self.ref_channel = ref_channel + + self.nmask = bnmask + + if beamformer_type != "mvdr": + raise ValueError( + "Not supporting beamformer_type={}".format(beamformer_type) + ) + self.beamformer_type = beamformer_type + + def forward( + self, data: ComplexTensor, ilens: torch.LongTensor + ) -> Tuple[ComplexTensor, torch.LongTensor, ComplexTensor]: + """The forward function + + Notation: + B: Batch + C: Channel + T: Time or Sequence length + F: Freq + + Args: + data (ComplexTensor): (B, T, C, F) + ilens (torch.Tensor): (B,) + Returns: + enhanced (ComplexTensor): (B, T, F) + ilens (torch.Tensor): (B,) + + """ + + def apply_beamforming(data, ilens, psd_speech, psd_noise): + # u: (B, C) + if self.ref_channel < 0: + u, _ = self.ref(psd_speech, ilens) + else: + # (optional) Create onehot vector for fixed reference microphone + u = torch.zeros( + *(data.size()[:-3] + (data.size(-2),)), device=data.device + ) + u[..., self.ref_channel].fill_(1) + + ws = get_mvdr_vector(psd_speech, psd_noise, u) + enhanced = apply_beamforming_vector(ws, data) + + return enhanced, ws + + # data (B, T, C, F) -> (B, F, C, T) + data = data.permute(0, 3, 2, 1) + + # mask: (B, F, C, T) + masks, _ = self.mask(data, ilens) + assert self.nmask == len(masks) + + if self.nmask == 2: # (mask_speech, mask_noise) + mask_speech, mask_noise = masks + + psd_speech = get_power_spectral_density_matrix(data, mask_speech) + psd_noise = get_power_spectral_density_matrix(data, mask_noise) + + enhanced, ws = apply_beamforming(data, ilens, psd_speech, psd_noise) + + # (..., F, T) -> (..., T, F) + enhanced = enhanced.transpose(-1, -2) + mask_speech = mask_speech.transpose(-1, -3) + else: # multi-speaker case: (mask_speech1, ..., mask_noise) + mask_speech = list(masks[:-1]) + mask_noise = masks[-1] + + psd_speeches = [ + get_power_spectral_density_matrix(data, mask) for mask in mask_speech + ] + psd_noise = get_power_spectral_density_matrix(data, mask_noise) + + enhanced = [] + ws = [] + for i in range(self.nmask - 1): + psd_speech = psd_speeches.pop(i) + # treat all other speakers' psd_speech as noises + enh, w = apply_beamforming( + data, ilens, psd_speech, sum(psd_speeches) + psd_noise + ) + psd_speeches.insert(i, psd_speech) + + # (..., F, T) -> (..., T, F) + enh = enh.transpose(-1, -2) + mask_speech[i] = mask_speech[i].transpose(-1, -3) + + enhanced.append(enh) + ws.append(w) + + return enhanced, ilens, mask_speech + + +class AttentionReference(torch.nn.Module): + def __init__(self, bidim, att_dim): + super().__init__() + self.mlp_psd = torch.nn.Linear(bidim, att_dim) + self.gvec = torch.nn.Linear(att_dim, 1) + + def forward( + self, psd_in: ComplexTensor, ilens: torch.LongTensor, scaling: float = 2.0 + ) -> Tuple[torch.Tensor, torch.LongTensor]: + """The forward function + + Args: + psd_in (ComplexTensor): (B, F, C, C) + ilens (torch.Tensor): (B,) + scaling (float): + Returns: + u (torch.Tensor): (B, C) + ilens (torch.Tensor): (B,) + """ + B, _, C = psd_in.size()[:3] + assert psd_in.size(2) == psd_in.size(3), psd_in.size() + # psd_in: (B, F, C, C) + psd = psd_in.masked_fill( + torch.eye(C, dtype=torch.bool, device=psd_in.device), 0 + ) + # psd: (B, F, C, C) -> (B, C, F) + psd = (psd.sum(dim=-1) / (C - 1)).transpose(-1, -2) + + # Calculate amplitude + psd_feat = (psd.real**2 + psd.imag**2) ** 0.5 + + # (B, C, F) -> (B, C, F2) + mlp_psd = self.mlp_psd(psd_feat) + # (B, C, F2) -> (B, C, 1) -> (B, C) + e = self.gvec(torch.tanh(mlp_psd)).squeeze(-1) + u = F.softmax(scaling * e, dim=-1) + return u, ilens diff --git a/funasr/modules/frontends/dnn_wpe.py b/funasr/modules/frontends/dnn_wpe.py new file mode 100644 index 000000000..9596765c8 --- /dev/null +++ b/funasr/modules/frontends/dnn_wpe.py @@ -0,0 +1,93 @@ +from typing import Tuple + +from pytorch_wpe import wpe_one_iteration +import torch +from torch_complex.tensor import ComplexTensor + +from funasr.modules.frontends.mask_estimator import MaskEstimator +from funasr.modules.nets_utils import make_pad_mask + + +class DNN_WPE(torch.nn.Module): + def __init__( + self, + wtype: str = "blstmp", + widim: int = 257, + wlayers: int = 3, + wunits: int = 300, + wprojs: int = 320, + dropout_rate: float = 0.0, + taps: int = 5, + delay: int = 3, + use_dnn_mask: bool = True, + iterations: int = 1, + normalization: bool = False, + ): + super().__init__() + self.iterations = iterations + self.taps = taps + self.delay = delay + + self.normalization = normalization + self.use_dnn_mask = use_dnn_mask + + self.inverse_power = True + + if self.use_dnn_mask: + self.mask_est = MaskEstimator( + wtype, widim, wlayers, wunits, wprojs, dropout_rate, nmask=1 + ) + + def forward( + self, data: ComplexTensor, ilens: torch.LongTensor + ) -> Tuple[ComplexTensor, torch.LongTensor, ComplexTensor]: + """The forward function + + Notation: + B: Batch + C: Channel + T: Time or Sequence length + F: Freq or Some dimension of the feature vector + + Args: + data: (B, C, T, F) + ilens: (B,) + Returns: + data: (B, C, T, F) + ilens: (B,) + """ + # (B, T, C, F) -> (B, F, C, T) + enhanced = data = data.permute(0, 3, 2, 1) + mask = None + + for i in range(self.iterations): + # Calculate power: (..., C, T) + power = enhanced.real**2 + enhanced.imag**2 + if i == 0 and self.use_dnn_mask: + # mask: (B, F, C, T) + (mask,), _ = self.mask_est(enhanced, ilens) + if self.normalization: + # Normalize along T + mask = mask / mask.sum(dim=-1)[..., None] + # (..., C, T) * (..., C, T) -> (..., C, T) + power = power * mask + + # Averaging along the channel axis: (..., C, T) -> (..., T) + power = power.mean(dim=-2) + + # enhanced: (..., C, T) -> (..., C, T) + enhanced = wpe_one_iteration( + data.contiguous(), + power, + taps=self.taps, + delay=self.delay, + inverse_power=self.inverse_power, + ) + + enhanced.masked_fill_(make_pad_mask(ilens, enhanced.real), 0) + + # (B, F, C, T) -> (B, T, C, F) + enhanced = enhanced.permute(0, 3, 2, 1) + if mask is not None: + mask = mask.transpose(-1, -3) + return enhanced, ilens, mask diff --git a/funasr/modules/frontends/feature_transform.py b/funasr/modules/frontends/feature_transform.py new file mode 100644 index 000000000..353dca1a6 --- /dev/null +++ b/funasr/modules/frontends/feature_transform.py @@ -0,0 +1,263 @@ +from typing import List +from typing import Tuple +from typing import Union + +import librosa +import numpy as np +import torch +from torch_complex.tensor import ComplexTensor + +from funasr.modules.nets_utils import make_pad_mask + + +class FeatureTransform(torch.nn.Module): + def __init__( + self, + # Mel options, + fs: int = 16000, + n_fft: int = 512, + n_mels: int = 80, + fmin: float = 0.0, + fmax: float = None, + # Normalization + stats_file: str = None, + apply_uttmvn: bool = True, + uttmvn_norm_means: bool = True, + uttmvn_norm_vars: bool = False, + ): + super().__init__() + self.apply_uttmvn = apply_uttmvn + + self.logmel = LogMel(fs=fs, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax) + self.stats_file = stats_file + if stats_file is not None: + self.global_mvn = GlobalMVN(stats_file) + else: + self.global_mvn = None + + if self.apply_uttmvn is not None: + self.uttmvn = UtteranceMVN( + norm_means=uttmvn_norm_means, norm_vars=uttmvn_norm_vars + ) + else: + self.uttmvn = None + + def forward( + self, x: ComplexTensor, ilens: Union[torch.LongTensor, np.ndarray, List[int]] + ) -> Tuple[torch.Tensor, torch.LongTensor]: + # (B, T, F) or (B, T, C, F) + if x.dim() not in (3, 4): + raise ValueError(f"Input dim must be 3 or 4: {x.dim()}") + if not torch.is_tensor(ilens): + ilens = torch.from_numpy(np.asarray(ilens)).to(x.device) + + if x.dim() == 4: + # h: (B, T, C, F) -> h: (B, T, F) + if self.training: + # Select 1ch randomly + ch = np.random.randint(x.size(2)) + h = x[:, :, ch, :] + else: + # Use the first channel + h = x[:, :, 0, :] + else: + h = x + + # h: ComplexTensor(B, T, F) -> torch.Tensor(B, T, F) + h = h.real**2 + h.imag**2 + + h, _ = self.logmel(h, ilens) + if self.stats_file is not None: + h, _ = self.global_mvn(h, ilens) + if self.apply_uttmvn: + h, _ = self.uttmvn(h, ilens) + + return h, ilens + + +class LogMel(torch.nn.Module): + """Convert STFT to fbank feats + + The arguments is same as librosa.filters.mel + + Args: + fs: number > 0 [scalar] sampling rate of the incoming signal + n_fft: int > 0 [scalar] number of FFT components + n_mels: int > 0 [scalar] number of Mel bands to generate + fmin: float >= 0 [scalar] lowest frequency (in Hz) + fmax: float >= 0 [scalar] highest frequency (in Hz). + If `None`, use `fmax = fs / 2.0` + htk: use HTK formula instead of Slaney + norm: {None, 1, np.inf} [scalar] + if 1, divide the triangular mel weights by the width of the mel band + (area normalization). Otherwise, leave all the triangles aiming for + a peak value of 1.0 + + """ + + def __init__( + self, + fs: int = 16000, + n_fft: int = 512, + n_mels: int = 80, + fmin: float = 0.0, + fmax: float = None, + htk: bool = False, + norm=1, + ): + super().__init__() + + _mel_options = dict( + sr=fs, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax, htk=htk, norm=norm + ) + self.mel_options = _mel_options + + # Note(kamo): The mel matrix of librosa is different from kaldi. + melmat = librosa.filters.mel(**_mel_options) + # melmat: (D2, D1) -> (D1, D2) + self.register_buffer("melmat", torch.from_numpy(melmat.T).float()) + + def extra_repr(self): + return ", ".join(f"{k}={v}" for k, v in self.mel_options.items()) + + def forward( + self, feat: torch.Tensor, ilens: torch.LongTensor + ) -> Tuple[torch.Tensor, torch.LongTensor]: + # feat: (B, T, D1) x melmat: (D1, D2) -> mel_feat: (B, T, D2) + mel_feat = torch.matmul(feat, self.melmat) + + logmel_feat = (mel_feat + 1e-20).log() + # Zero padding + logmel_feat = logmel_feat.masked_fill(make_pad_mask(ilens, logmel_feat, 1), 0.0) + return logmel_feat, ilens + + +class GlobalMVN(torch.nn.Module): + """Apply global mean and variance normalization + + Args: + stats_file(str): npy file of 1-dim array or text file. + From the _first element to + the {(len(array) - 1) / 2}th element are treated as + the sum of features, + and the rest excluding the last elements are + treated as the sum of the square value of features, + and the last elements eqauls to the number of samples. + std_floor(float): + """ + + def __init__( + self, + stats_file: str, + norm_means: bool = True, + norm_vars: bool = True, + eps: float = 1.0e-20, + ): + super().__init__() + self.norm_means = norm_means + self.norm_vars = norm_vars + + self.stats_file = stats_file + stats = np.load(stats_file) + + stats = stats.astype(float) + assert (len(stats) - 1) % 2 == 0, stats.shape + + count = stats.flatten()[-1] + mean = stats[: (len(stats) - 1) // 2] / count + var = stats[(len(stats) - 1) // 2 : -1] / count - mean * mean + std = np.maximum(np.sqrt(var), eps) + + self.register_buffer("bias", torch.from_numpy(-mean.astype(np.float32))) + self.register_buffer("scale", torch.from_numpy(1 / std.astype(np.float32))) + + def extra_repr(self): + return ( + f"stats_file={self.stats_file}, " + f"norm_means={self.norm_means}, norm_vars={self.norm_vars}" + ) + + def forward( + self, x: torch.Tensor, ilens: torch.LongTensor + ) -> Tuple[torch.Tensor, torch.LongTensor]: + # feat: (B, T, D) + if self.norm_means: + x += self.bias.type_as(x) + x.masked_fill(make_pad_mask(ilens, x, 1), 0.0) + + if self.norm_vars: + x *= self.scale.type_as(x) + return x, ilens + + +class UtteranceMVN(torch.nn.Module): + def __init__( + self, norm_means: bool = True, norm_vars: bool = False, eps: float = 1.0e-20 + ): + super().__init__() + self.norm_means = norm_means + self.norm_vars = norm_vars + self.eps = eps + + def extra_repr(self): + return f"norm_means={self.norm_means}, norm_vars={self.norm_vars}" + + def forward( + self, x: torch.Tensor, ilens: torch.LongTensor + ) -> Tuple[torch.Tensor, torch.LongTensor]: + return utterance_mvn( + x, ilens, norm_means=self.norm_means, norm_vars=self.norm_vars, eps=self.eps + ) + + +def utterance_mvn( + x: torch.Tensor, + ilens: torch.LongTensor, + norm_means: bool = True, + norm_vars: bool = False, + eps: float = 1.0e-20, +) -> Tuple[torch.Tensor, torch.LongTensor]: + """Apply utterance mean and variance normalization + + Args: + x: (B, T, D), assumed zero padded + ilens: (B, T, D) + norm_means: + norm_vars: + eps: + + """ + ilens_ = ilens.type_as(x) + # mean: (B, D) + mean = x.sum(dim=1) / ilens_[:, None] + + if norm_means: + x -= mean[:, None, :] + x_ = x + else: + x_ = x - mean[:, None, :] + + # Zero padding + x_.masked_fill(make_pad_mask(ilens, x_, 1), 0.0) + if norm_vars: + var = x_.pow(2).sum(dim=1) / ilens_[:, None] + var = torch.clamp(var, min=eps) + x /= var.sqrt()[:, None, :] + x_ = x + return x_, ilens + + +def feature_transform_for(args, n_fft): + return FeatureTransform( + # Mel options, + fs=args.fbank_fs, + n_fft=n_fft, + n_mels=args.n_mels, + fmin=args.fbank_fmin, + fmax=args.fbank_fmax, + # Normalization + stats_file=args.stats_file, + apply_uttmvn=args.apply_uttmvn, + uttmvn_norm_means=args.uttmvn_norm_means, + uttmvn_norm_vars=args.uttmvn_norm_vars, + ) diff --git a/funasr/modules/frontends/frontend.py b/funasr/modules/frontends/frontend.py new file mode 100644 index 000000000..ab5ea3b79 --- /dev/null +++ b/funasr/modules/frontends/frontend.py @@ -0,0 +1,151 @@ +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + +import numpy +import torch +import torch.nn as nn +from torch_complex.tensor import ComplexTensor + +from funasr.modules.frontends.dnn_beamformer import DNN_Beamformer +from funasr.modules.frontends.dnn_wpe import DNN_WPE + + +class Frontend(nn.Module): + def __init__( + self, + idim: int, + # WPE options + use_wpe: bool = False, + wtype: str = "blstmp", + wlayers: int = 3, + wunits: int = 300, + wprojs: int = 320, + wdropout_rate: float = 0.0, + taps: int = 5, + delay: int = 3, + use_dnn_mask_for_wpe: bool = True, + # Beamformer options + use_beamformer: bool = False, + btype: str = "blstmp", + blayers: int = 3, + bunits: int = 300, + bprojs: int = 320, + bnmask: int = 2, + badim: int = 320, + ref_channel: int = -1, + bdropout_rate=0.0, + ): + super().__init__() + + self.use_beamformer = use_beamformer + self.use_wpe = use_wpe + self.use_dnn_mask_for_wpe = use_dnn_mask_for_wpe + # use frontend for all the data, + # e.g. in the case of multi-speaker speech separation + self.use_frontend_for_all = bnmask > 2 + + if self.use_wpe: + if self.use_dnn_mask_for_wpe: + # Use DNN for power estimation + # (Not observed significant gains) + iterations = 1 + else: + # Performing as conventional WPE, without DNN Estimator + iterations = 2 + + self.wpe = DNN_WPE( + wtype=wtype, + widim=idim, + wunits=wunits, + wprojs=wprojs, + wlayers=wlayers, + taps=taps, + delay=delay, + dropout_rate=wdropout_rate, + iterations=iterations, + use_dnn_mask=use_dnn_mask_for_wpe, + ) + else: + self.wpe = None + + if self.use_beamformer: + self.beamformer = DNN_Beamformer( + btype=btype, + bidim=idim, + bunits=bunits, + bprojs=bprojs, + blayers=blayers, + bnmask=bnmask, + dropout_rate=bdropout_rate, + badim=badim, + ref_channel=ref_channel, + ) + else: + self.beamformer = None + + def forward( + self, x: ComplexTensor, ilens: Union[torch.LongTensor, numpy.ndarray, List[int]] + ) -> Tuple[ComplexTensor, torch.LongTensor, Optional[ComplexTensor]]: + assert len(x) == len(ilens), (len(x), len(ilens)) + # (B, T, F) or (B, T, C, F) + if x.dim() not in (3, 4): + raise ValueError(f"Input dim must be 3 or 4: {x.dim()}") + if not torch.is_tensor(ilens): + ilens = torch.from_numpy(numpy.asarray(ilens)).to(x.device) + + mask = None + h = x + if h.dim() == 4: + if self.training: + choices = [(False, False)] if not self.use_frontend_for_all else [] + if self.use_wpe: + choices.append((True, False)) + + if self.use_beamformer: + choices.append((False, True)) + + use_wpe, use_beamformer = choices[numpy.random.randint(len(choices))] + + else: + use_wpe = self.use_wpe + use_beamformer = self.use_beamformer + + # 1. WPE + if use_wpe: + # h: (B, T, C, F) -> h: (B, T, C, F) + h, ilens, mask = self.wpe(h, ilens) + + # 2. Beamformer + if use_beamformer: + # h: (B, T, C, F) -> h: (B, T, F) + h, ilens, mask = self.beamformer(h, ilens) + + return h, ilens, mask + + +def frontend_for(args, idim): + return Frontend( + idim=idim, + # WPE options + use_wpe=args.use_wpe, + wtype=args.wtype, + wlayers=args.wlayers, + wunits=args.wunits, + wprojs=args.wprojs, + wdropout_rate=args.wdropout_rate, + taps=args.wpe_taps, + delay=args.wpe_delay, + use_dnn_mask_for_wpe=args.use_dnn_mask_for_wpe, + # Beamformer options + use_beamformer=args.use_beamformer, + btype=args.btype, + blayers=args.blayers, + bunits=args.bunits, + bprojs=args.bprojs, + bnmask=args.bnmask, + badim=args.badim, + ref_channel=args.ref_channel, + bdropout_rate=args.bdropout_rate, + ) diff --git a/funasr/modules/frontends/mask_estimator.py b/funasr/modules/frontends/mask_estimator.py new file mode 100644 index 000000000..cf4385769 --- /dev/null +++ b/funasr/modules/frontends/mask_estimator.py @@ -0,0 +1,77 @@ +from typing import Tuple + +import numpy as np +import torch +from torch.nn import functional as F +from torch_complex.tensor import ComplexTensor + +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.rnn.encoders import RNN +from funasr.modules.rnn.encoders import RNNP + + +class MaskEstimator(torch.nn.Module): + def __init__(self, type, idim, layers, units, projs, dropout, nmask=1): + super().__init__() + subsample = np.ones(layers + 1, dtype=np.int) + + typ = type.lstrip("vgg").rstrip("p") + if type[-1] == "p": + self.brnn = RNNP(idim, layers, units, projs, subsample, dropout, typ=typ) + else: + self.brnn = RNN(idim, layers, units, projs, dropout, typ=typ) + + self.type = type + self.nmask = nmask + self.linears = torch.nn.ModuleList( + [torch.nn.Linear(projs, idim) for _ in range(nmask)] + ) + + def forward( + self, xs: ComplexTensor, ilens: torch.LongTensor + ) -> Tuple[Tuple[torch.Tensor, ...], torch.LongTensor]: + """The forward function + + Args: + xs: (B, F, C, T) + ilens: (B,) + Returns: + hs (torch.Tensor): The hidden vector (B, F, C, T) + masks: A tuple of the masks. (B, F, C, T) + ilens: (B,) + """ + assert xs.size(0) == ilens.size(0), (xs.size(0), ilens.size(0)) + _, _, C, input_length = xs.size() + # (B, F, C, T) -> (B, C, T, F) + xs = xs.permute(0, 2, 3, 1) + + # Calculate amplitude: (B, C, T, F) -> (B, C, T, F) + xs = (xs.real**2 + xs.imag**2) ** 0.5 + # xs: (B, C, T, F) -> xs: (B * C, T, F) + xs = xs.contiguous().view(-1, xs.size(-2), xs.size(-1)) + # ilens: (B,) -> ilens_: (B * C) + ilens_ = ilens[:, None].expand(-1, C).contiguous().view(-1) + + # xs: (B * C, T, F) -> xs: (B * C, T, D) + xs, _, _ = self.brnn(xs, ilens_) + # xs: (B * C, T, D) -> xs: (B, C, T, D) + xs = xs.view(-1, C, xs.size(-2), xs.size(-1)) + + masks = [] + for linear in self.linears: + # xs: (B, C, T, D) -> mask:(B, C, T, F) + mask = linear(xs) + + mask = torch.sigmoid(mask) + # Zero padding + mask.masked_fill(make_pad_mask(ilens, mask, length_dim=2), 0) + + # (B, C, T, F) -> (B, F, C, T) + mask = mask.permute(0, 3, 1, 2) + + # Take cares of multi gpu cases: If input_length > max(ilens) + if mask.size(-1) < input_length: + mask = F.pad(mask, [0, input_length - mask.size(-1)], value=0) + masks.append(mask) + + return tuple(masks), ilens diff --git a/funasr/modules/layer_norm.py b/funasr/modules/layer_norm.py new file mode 100644 index 000000000..6e934e644 --- /dev/null +++ b/funasr/modules/layer_norm.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Layer normalization module.""" + +import torch + + +class LayerNorm(torch.nn.LayerNorm): + """Layer normalization module. + + Args: + nout (int): Output dim size. + dim (int): Dimension to be normalized. + + """ + + def __init__(self, nout, dim=-1): + """Construct an LayerNorm object.""" + super(LayerNorm, self).__init__(nout, eps=1e-12) + self.dim = dim + + def forward(self, x): + """Apply layer normalization. + + Args: + x (torch.Tensor): Input tensor. + + Returns: + torch.Tensor: Normalized tensor. + + """ + if self.dim == -1: + return super(LayerNorm, self).forward(x) + return ( + super(LayerNorm, self) + .forward(x.transpose(self.dim, -1)) + .transpose(self.dim, -1) + ) diff --git a/funasr/modules/lightconv.py b/funasr/modules/lightconv.py new file mode 100644 index 000000000..b24940259 --- /dev/null +++ b/funasr/modules/lightconv.py @@ -0,0 +1,112 @@ +"""Lightweight Convolution Module.""" + +import numpy +import torch +from torch import nn +import torch.nn.functional as F + + +MIN_VALUE = float(numpy.finfo(numpy.float32).min) + + +class LightweightConvolution(nn.Module): + """Lightweight Convolution layer. + + This implementation is based on + https://github.com/pytorch/fairseq/tree/master/fairseq + + Args: + wshare (int): the number of kernel of convolution + n_feat (int): the number of features + dropout_rate (float): dropout_rate + kernel_size (int): kernel size (length) + use_kernel_mask (bool): Use causal mask or not for convolution kernel + use_bias (bool): Use bias term or not. + + """ + + def __init__( + self, + wshare, + n_feat, + dropout_rate, + kernel_size, + use_kernel_mask=False, + use_bias=False, + ): + """Construct Lightweight Convolution layer.""" + super(LightweightConvolution, self).__init__() + + assert n_feat % wshare == 0 + self.wshare = wshare + self.use_kernel_mask = use_kernel_mask + self.dropout_rate = dropout_rate + self.kernel_size = kernel_size + self.padding_size = int(kernel_size / 2) + + # linear -> GLU -> lightconv -> linear + self.linear1 = nn.Linear(n_feat, n_feat * 2) + self.linear2 = nn.Linear(n_feat, n_feat) + self.act = nn.GLU() + + # lightconv related + self.weight = nn.Parameter( + torch.Tensor(self.wshare, 1, kernel_size).uniform_(0, 1) + ) + self.use_bias = use_bias + if self.use_bias: + self.bias = nn.Parameter(torch.Tensor(n_feat)) + + # mask of kernel + kernel_mask0 = torch.zeros(self.wshare, int(kernel_size / 2)) + kernel_mask1 = torch.ones(self.wshare, int(kernel_size / 2 + 1)) + self.kernel_mask = torch.cat((kernel_mask1, kernel_mask0), dim=-1).unsqueeze(1) + + def forward(self, query, key, value, mask): + """Forward of 'Lightweight Convolution'. + + This function takes query, key and value but uses only query. + This is just for compatibility with self-attention layer (attention.py) + + Args: + query (torch.Tensor): (batch, time1, d_model) input tensor + key (torch.Tensor): (batch, time2, d_model) NOT USED + value (torch.Tensor): (batch, time2, d_model) NOT USED + mask (torch.Tensor): (batch, time1, time2) mask + + Return: + x (torch.Tensor): (batch, time1, d_model) output + + """ + # linear -> GLU -> lightconv -> linear + x = query + B, T, C = x.size() + H = self.wshare + + # first liner layer + x = self.linear1(x) + + # GLU activation + x = self.act(x) + + # lightconv + x = x.transpose(1, 2).contiguous().view(-1, H, T) # B x C x T + weight = F.dropout(self.weight, self.dropout_rate, training=self.training) + if self.use_kernel_mask: + self.kernel_mask = self.kernel_mask.to(x.device) + weight = weight.masked_fill(self.kernel_mask == 0.0, float("-inf")) + weight = F.softmax(weight, dim=-1) + x = F.conv1d(x, weight, padding=self.padding_size, groups=self.wshare).view( + B, C, T + ) + if self.use_bias: + x = x + self.bias.view(1, -1, 1) + x = x.transpose(1, 2) # B x T x C + + if mask is not None and not self.use_kernel_mask: + mask = mask.transpose(-1, -2) + x = x.masked_fill(mask == 0, 0.0) + + # second linear layer + x = self.linear2(x) + return x diff --git a/funasr/modules/lightconv2d.py b/funasr/modules/lightconv2d.py new file mode 100644 index 000000000..294d23244 --- /dev/null +++ b/funasr/modules/lightconv2d.py @@ -0,0 +1,124 @@ +"""Lightweight 2-Dimensional Convolution module.""" + +import numpy +import torch +from torch import nn +import torch.nn.functional as F + + +MIN_VALUE = float(numpy.finfo(numpy.float32).min) + + +class LightweightConvolution2D(nn.Module): + """Lightweight 2-Dimensional Convolution layer. + + This implementation is based on + https://github.com/pytorch/fairseq/tree/master/fairseq + + Args: + wshare (int): the number of kernel of convolution + n_feat (int): the number of features + dropout_rate (float): dropout_rate + kernel_size (int): kernel size (length) + use_kernel_mask (bool): Use causal mask or not for convolution kernel + use_bias (bool): Use bias term or not. + + """ + + def __init__( + self, + wshare, + n_feat, + dropout_rate, + kernel_size, + use_kernel_mask=False, + use_bias=False, + ): + """Construct Lightweight 2-Dimensional Convolution layer.""" + super(LightweightConvolution2D, self).__init__() + + assert n_feat % wshare == 0 + self.wshare = wshare + self.use_kernel_mask = use_kernel_mask + self.dropout_rate = dropout_rate + self.kernel_size = kernel_size + self.padding_size = int(kernel_size / 2) + + # linear -> GLU -> lightconv -> linear + self.linear1 = nn.Linear(n_feat, n_feat * 2) + self.linear2 = nn.Linear(n_feat * 2, n_feat) + self.act = nn.GLU() + + # lightconv related + self.weight = nn.Parameter( + torch.Tensor(self.wshare, 1, kernel_size).uniform_(0, 1) + ) + self.weight_f = nn.Parameter(torch.Tensor(1, 1, kernel_size).uniform_(0, 1)) + self.use_bias = use_bias + if self.use_bias: + self.bias = nn.Parameter(torch.Tensor(n_feat)) + + # mask of kernel + kernel_mask0 = torch.zeros(self.wshare, int(kernel_size / 2)) + kernel_mask1 = torch.ones(self.wshare, int(kernel_size / 2 + 1)) + self.kernel_mask = torch.cat((kernel_mask1, kernel_mask0), dim=-1).unsqueeze(1) + + def forward(self, query, key, value, mask): + """Forward of 'Lightweight 2-Dimensional Convolution'. + + This function takes query, key and value but uses only query. + This is just for compatibility with self-attention layer (attention.py) + + Args: + query (torch.Tensor): (batch, time1, d_model) input tensor + key (torch.Tensor): (batch, time2, d_model) NOT USED + value (torch.Tensor): (batch, time2, d_model) NOT USED + mask (torch.Tensor): (batch, time1, time2) mask + + Return: + x (torch.Tensor): (batch, time1, d_model) output + + """ + # linear -> GLU -> lightconv -> linear + x = query + B, T, C = x.size() + H = self.wshare + + # first liner layer + x = self.linear1(x) + + # GLU activation + x = self.act(x) + + # convolution along frequency axis + weight_f = F.softmax(self.weight_f, dim=-1) + weight_f = F.dropout(weight_f, self.dropout_rate, training=self.training) + weight_new = torch.zeros( + B * T, 1, self.kernel_size, device=x.device, dtype=x.dtype + ).copy_(weight_f) + xf = F.conv1d( + x.view(1, B * T, C), weight_new, padding=self.padding_size, groups=B * T + ).view(B, T, C) + + # lightconv + x = x.transpose(1, 2).contiguous().view(-1, H, T) # B x C x T + weight = F.dropout(self.weight, self.dropout_rate, training=self.training) + if self.use_kernel_mask: + self.kernel_mask = self.kernel_mask.to(x.device) + weight = weight.masked_fill(self.kernel_mask == 0.0, float("-inf")) + weight = F.softmax(weight, dim=-1) + x = F.conv1d(x, weight, padding=self.padding_size, groups=self.wshare).view( + B, C, T + ) + if self.use_bias: + x = x + self.bias.view(1, -1, 1) + x = x.transpose(1, 2) # B x T x C + x = torch.cat((x, xf), -1) # B x T x Cx2 + + if mask is not None and not self.use_kernel_mask: + mask = mask.transpose(-1, -2) + x = x.masked_fill(mask == 0, 0.0) + + # second linear layer + x = self.linear2(x) + return x diff --git a/funasr/modules/mask.py b/funasr/modules/mask.py new file mode 100644 index 000000000..8f068e11c --- /dev/null +++ b/funasr/modules/mask.py @@ -0,0 +1,35 @@ +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Mask module.""" + +import torch + + +def subsequent_mask(size, device="cpu", dtype=torch.bool): + """Create mask for subsequent steps (size, size). + + :param int size: size of mask + :param str device: "cpu" or "cuda" or torch.Tensor.device + :param torch.dtype dtype: result dtype + :rtype: torch.Tensor + >>> subsequent_mask(3) + [[1, 0, 0], + [1, 1, 0], + [1, 1, 1]] + """ + ret = torch.ones(size, size, device=device, dtype=dtype) + return torch.tril(ret, out=ret) + + +def target_mask(ys_in_pad, ignore_id): + """Create mask for decoder self-attention. + + :param torch.Tensor ys_pad: batch of padded target sequences (B, Lmax) + :param int ignore_id: index of padding + :param torch.dtype dtype: result dtype + :rtype: torch.Tensor (B, Lmax, Lmax) + """ + ys_mask = ys_in_pad != ignore_id + m = subsequent_mask(ys_mask.size(-1), device=ys_mask.device).unsqueeze(0) + return ys_mask.unsqueeze(-2) & m diff --git a/funasr/modules/multi_layer_conv.py b/funasr/modules/multi_layer_conv.py new file mode 100644 index 000000000..5fb0717b0 --- /dev/null +++ b/funasr/modules/multi_layer_conv.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Tomoki Hayashi +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Layer modules for FFT block in FastSpeech (Feed-forward Transformer).""" + +import torch + + +class MultiLayeredConv1d(torch.nn.Module): + """Multi-layered conv1d for Transformer block. + + This is a module of multi-leyered conv1d designed + to replace positionwise feed-forward network + in Transforner block, which is introduced in + `FastSpeech: Fast, Robust and Controllable Text to Speech`_. + + .. _`FastSpeech: Fast, Robust and Controllable Text to Speech`: + https://arxiv.org/pdf/1905.09263.pdf + + """ + + def __init__(self, in_chans, hidden_chans, kernel_size, dropout_rate): + """Initialize MultiLayeredConv1d module. + + Args: + in_chans (int): Number of input channels. + hidden_chans (int): Number of hidden channels. + kernel_size (int): Kernel size of conv1d. + dropout_rate (float): Dropout rate. + + """ + super(MultiLayeredConv1d, self).__init__() + self.w_1 = torch.nn.Conv1d( + in_chans, + hidden_chans, + kernel_size, + stride=1, + padding=(kernel_size - 1) // 2, + ) + self.w_2 = torch.nn.Conv1d( + hidden_chans, + in_chans, + kernel_size, + stride=1, + padding=(kernel_size - 1) // 2, + ) + self.dropout = torch.nn.Dropout(dropout_rate) + + def forward(self, x): + """Calculate forward propagation. + + Args: + x (torch.Tensor): Batch of input tensors (B, T, in_chans). + + Returns: + torch.Tensor: Batch of output tensors (B, T, hidden_chans). + + """ + x = torch.relu(self.w_1(x.transpose(-1, 1))).transpose(-1, 1) + return self.w_2(self.dropout(x).transpose(-1, 1)).transpose(-1, 1) + + +class Conv1dLinear(torch.nn.Module): + """Conv1D + Linear for Transformer block. + + A variant of MultiLayeredConv1d, which replaces second conv-layer to linear. + + """ + + def __init__(self, in_chans, hidden_chans, kernel_size, dropout_rate): + """Initialize Conv1dLinear module. + + Args: + in_chans (int): Number of input channels. + hidden_chans (int): Number of hidden channels. + kernel_size (int): Kernel size of conv1d. + dropout_rate (float): Dropout rate. + + """ + super(Conv1dLinear, self).__init__() + self.w_1 = torch.nn.Conv1d( + in_chans, + hidden_chans, + kernel_size, + stride=1, + padding=(kernel_size - 1) // 2, + ) + self.w_2 = torch.nn.Linear(hidden_chans, in_chans) + self.dropout = torch.nn.Dropout(dropout_rate) + + def forward(self, x): + """Calculate forward propagation. + + Args: + x (torch.Tensor): Batch of input tensors (B, T, in_chans). + + Returns: + torch.Tensor: Batch of output tensors (B, T, hidden_chans). + + """ + x = torch.relu(self.w_1(x.transpose(-1, 1))).transpose(-1, 1) + return self.w_2(self.dropout(x)) diff --git a/funasr/modules/nets_utils.py b/funasr/modules/nets_utils.py new file mode 100644 index 000000000..6d77d69a6 --- /dev/null +++ b/funasr/modules/nets_utils.py @@ -0,0 +1,508 @@ +# -*- coding: utf-8 -*- + +"""Network related utility tools.""" + +import logging +from typing import Dict + +import numpy as np +import torch + + +def to_device(m, x): + """Send tensor into the device of the module. + + Args: + m (torch.nn.Module): Torch module. + x (Tensor): Torch tensor. + + Returns: + Tensor: Torch tensor located in the same place as torch module. + + """ + if isinstance(m, torch.nn.Module): + device = next(m.parameters()).device + elif isinstance(m, torch.Tensor): + device = m.device + else: + raise TypeError( + "Expected torch.nn.Module or torch.tensor, " f"bot got: {type(m)}" + ) + return x.to(device) + + +def pad_list(xs, pad_value): + """Perform padding for the list of tensors. + + Args: + xs (List): List of Tensors [(T_1, `*`), (T_2, `*`), ..., (T_B, `*`)]. + pad_value (float): Value for padding. + + Returns: + Tensor: Padded tensor (B, Tmax, `*`). + + Examples: + >>> x = [torch.ones(4), torch.ones(2), torch.ones(1)] + >>> x + [tensor([1., 1., 1., 1.]), tensor([1., 1.]), tensor([1.])] + >>> pad_list(x, 0) + tensor([[1., 1., 1., 1.], + [1., 1., 0., 0.], + [1., 0., 0., 0.]]) + + """ + n_batch = len(xs) + max_len = max(x.size(0) for x in xs) + pad = xs[0].new(n_batch, max_len, *xs[0].size()[1:]).fill_(pad_value) + + for i in range(n_batch): + pad[i, : xs[i].size(0)] = xs[i] + + return pad + + +def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None): + """Make mask tensor containing indices of padded part. + + Args: + lengths (LongTensor or List): Batch of lengths (B,). + xs (Tensor, optional): The reference tensor. + If set, masks will be the same shape as this tensor. + length_dim (int, optional): Dimension indicator of the above tensor. + See the example. + + Returns: + Tensor: Mask tensor containing indices of padded part. + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (including 1.2) + + Examples: + With only lengths. + + >>> lengths = [5, 3, 2] + >>> make_pad_mask(lengths) + masks = [[0, 0, 0, 0 ,0], + [0, 0, 0, 1, 1], + [0, 0, 1, 1, 1]] + + With the reference tensor. + + >>> xs = torch.zeros((3, 2, 4)) + >>> make_pad_mask(lengths, xs) + tensor([[[0, 0, 0, 0], + [0, 0, 0, 0]], + [[0, 0, 0, 1], + [0, 0, 0, 1]], + [[0, 0, 1, 1], + [0, 0, 1, 1]]], dtype=torch.uint8) + >>> xs = torch.zeros((3, 2, 6)) + >>> make_pad_mask(lengths, xs) + tensor([[[0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1]], + [[0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1]], + [[0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8) + + With the reference tensor and dimension indicator. + + >>> xs = torch.zeros((3, 6, 6)) + >>> make_pad_mask(lengths, xs, 1) + tensor([[[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1]], + [[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1]], + [[0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1]]], dtype=torch.uint8) + >>> make_pad_mask(lengths, xs, 2) + tensor([[[0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1], + [0, 0, 0, 0, 0, 1]], + [[0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1], + [0, 0, 0, 1, 1, 1]], + [[0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1], + [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8) + + """ + if length_dim == 0: + raise ValueError("length_dim cannot be 0: {}".format(length_dim)) + + if not isinstance(lengths, list): + lengths = lengths.tolist() + bs = int(len(lengths)) + if maxlen is None: + if xs is None: + maxlen = int(max(lengths)) + else: + maxlen = xs.size(length_dim) + else: + assert xs is None + assert maxlen >= int(max(lengths)) + + seq_range = torch.arange(0, maxlen, dtype=torch.int64) + seq_range_expand = seq_range.unsqueeze(0).expand(bs, maxlen) + seq_length_expand = seq_range_expand.new(lengths).unsqueeze(-1) + mask = seq_range_expand >= seq_length_expand + + if xs is not None: + assert xs.size(0) == bs, (xs.size(0), bs) + + if length_dim < 0: + length_dim = xs.dim() + length_dim + # ind = (:, None, ..., None, :, , None, ..., None) + ind = tuple( + slice(None) if i in (0, length_dim) else None for i in range(xs.dim()) + ) + mask = mask[ind].expand_as(xs).to(xs.device) + return mask + + +def make_non_pad_mask(lengths, xs=None, length_dim=-1): + """Make mask tensor containing indices of non-padded part. + + Args: + lengths (LongTensor or List): Batch of lengths (B,). + xs (Tensor, optional): The reference tensor. + If set, masks will be the same shape as this tensor. + length_dim (int, optional): Dimension indicator of the above tensor. + See the example. + + Returns: + ByteTensor: mask tensor containing indices of padded part. + dtype=torch.uint8 in PyTorch 1.2- + dtype=torch.bool in PyTorch 1.2+ (including 1.2) + + Examples: + With only lengths. + + >>> lengths = [5, 3, 2] + >>> make_non_pad_mask(lengths) + masks = [[1, 1, 1, 1 ,1], + [1, 1, 1, 0, 0], + [1, 1, 0, 0, 0]] + + With the reference tensor. + + >>> xs = torch.zeros((3, 2, 4)) + >>> make_non_pad_mask(lengths, xs) + tensor([[[1, 1, 1, 1], + [1, 1, 1, 1]], + [[1, 1, 1, 0], + [1, 1, 1, 0]], + [[1, 1, 0, 0], + [1, 1, 0, 0]]], dtype=torch.uint8) + >>> xs = torch.zeros((3, 2, 6)) + >>> make_non_pad_mask(lengths, xs) + tensor([[[1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0]], + [[1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0]], + [[1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0]]], dtype=torch.uint8) + + With the reference tensor and dimension indicator. + + >>> xs = torch.zeros((3, 6, 6)) + >>> make_non_pad_mask(lengths, xs, 1) + tensor([[[1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0]], + [[1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0]], + [[1, 1, 1, 1, 1, 1], + [1, 1, 1, 1, 1, 1], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0]]], dtype=torch.uint8) + >>> make_non_pad_mask(lengths, xs, 2) + tensor([[[1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0], + [1, 1, 1, 1, 1, 0]], + [[1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0], + [1, 1, 1, 0, 0, 0]], + [[1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0], + [1, 1, 0, 0, 0, 0]]], dtype=torch.uint8) + + """ + return ~make_pad_mask(lengths, xs, length_dim) + + +def mask_by_length(xs, lengths, fill=0): + """Mask tensor according to length. + + Args: + xs (Tensor): Batch of input tensor (B, `*`). + lengths (LongTensor or List): Batch of lengths (B,). + fill (int or float): Value to fill masked part. + + Returns: + Tensor: Batch of masked input tensor (B, `*`). + + Examples: + >>> x = torch.arange(5).repeat(3, 1) + 1 + >>> x + tensor([[1, 2, 3, 4, 5], + [1, 2, 3, 4, 5], + [1, 2, 3, 4, 5]]) + >>> lengths = [5, 3, 2] + >>> mask_by_length(x, lengths) + tensor([[1, 2, 3, 4, 5], + [1, 2, 3, 0, 0], + [1, 2, 0, 0, 0]]) + + """ + assert xs.size(0) == len(lengths) + ret = xs.data.new(*xs.size()).fill_(fill) + for i, l in enumerate(lengths): + ret[i, :l] = xs[i, :l] + return ret + + +def th_accuracy(pad_outputs, pad_targets, ignore_label): + """Calculate accuracy. + + Args: + pad_outputs (Tensor): Prediction tensors (B * Lmax, D). + pad_targets (LongTensor): Target label tensors (B, Lmax, D). + ignore_label (int): Ignore label id. + + Returns: + float: Accuracy value (0.0 - 1.0). + + """ + pad_pred = pad_outputs.view( + pad_targets.size(0), pad_targets.size(1), pad_outputs.size(1) + ).argmax(2) + mask = pad_targets != ignore_label + numerator = torch.sum( + pad_pred.masked_select(mask) == pad_targets.masked_select(mask) + ) + denominator = torch.sum(mask) + return float(numerator) / float(denominator) + + +def to_torch_tensor(x): + """Change to torch.Tensor or ComplexTensor from numpy.ndarray. + + Args: + x: Inputs. It should be one of numpy.ndarray, Tensor, ComplexTensor, and dict. + + Returns: + Tensor or ComplexTensor: Type converted inputs. + + Examples: + >>> xs = np.ones(3, dtype=np.float32) + >>> xs = to_torch_tensor(xs) + tensor([1., 1., 1.]) + >>> xs = torch.ones(3, 4, 5) + >>> assert to_torch_tensor(xs) is xs + >>> xs = {'real': xs, 'imag': xs} + >>> to_torch_tensor(xs) + ComplexTensor( + Real: + tensor([1., 1., 1.]) + Imag; + tensor([1., 1., 1.]) + ) + + """ + # If numpy, change to torch tensor + if isinstance(x, np.ndarray): + if x.dtype.kind == "c": + # Dynamically importing because torch_complex requires python3 + from torch_complex.tensor import ComplexTensor + + return ComplexTensor(x) + else: + return torch.from_numpy(x) + + # If {'real': ..., 'imag': ...}, convert to ComplexTensor + elif isinstance(x, dict): + # Dynamically importing because torch_complex requires python3 + from torch_complex.tensor import ComplexTensor + + if "real" not in x or "imag" not in x: + raise ValueError("has 'real' and 'imag' keys: {}".format(list(x))) + # Relative importing because of using python3 syntax + return ComplexTensor(x["real"], x["imag"]) + + # If torch.Tensor, as it is + elif isinstance(x, torch.Tensor): + return x + + else: + error = ( + "x must be numpy.ndarray, torch.Tensor or a dict like " + "{{'real': torch.Tensor, 'imag': torch.Tensor}}, " + "but got {}".format(type(x)) + ) + try: + from torch_complex.tensor import ComplexTensor + except Exception: + # If PY2 + raise ValueError(error) + else: + # If PY3 + if isinstance(x, ComplexTensor): + return x + else: + raise ValueError(error) + + +def get_subsample(train_args, mode, arch): + """Parse the subsampling factors from the args for the specified `mode` and `arch`. + + Args: + train_args: argument Namespace containing options. + mode: one of ('asr', 'mt', 'st') + arch: one of ('rnn', 'rnn-t', 'rnn_mix', 'rnn_mulenc', 'transformer') + + Returns: + np.ndarray / List[np.ndarray]: subsampling factors. + """ + if arch == "transformer": + return np.array([1]) + + elif mode == "mt" and arch == "rnn": + # +1 means input (+1) and layers outputs (train_args.elayer) + subsample = np.ones(train_args.elayers + 1, dtype=np.int) + logging.warning("Subsampling is not performed for machine translation.") + logging.info("subsample: " + " ".join([str(x) for x in subsample])) + return subsample + + elif ( + (mode == "asr" and arch in ("rnn", "rnn-t")) + or (mode == "mt" and arch == "rnn") + or (mode == "st" and arch == "rnn") + ): + subsample = np.ones(train_args.elayers + 1, dtype=np.int) + if train_args.etype.endswith("p") and not train_args.etype.startswith("vgg"): + ss = train_args.subsample.split("_") + for j in range(min(train_args.elayers + 1, len(ss))): + subsample[j] = int(ss[j]) + else: + logging.warning( + "Subsampling is not performed for vgg*. " + "It is performed in max pooling layers at CNN." + ) + logging.info("subsample: " + " ".join([str(x) for x in subsample])) + return subsample + + elif mode == "asr" and arch == "rnn_mix": + subsample = np.ones( + train_args.elayers_sd + train_args.elayers + 1, dtype=np.int + ) + if train_args.etype.endswith("p") and not train_args.etype.startswith("vgg"): + ss = train_args.subsample.split("_") + for j in range( + min(train_args.elayers_sd + train_args.elayers + 1, len(ss)) + ): + subsample[j] = int(ss[j]) + else: + logging.warning( + "Subsampling is not performed for vgg*. " + "It is performed in max pooling layers at CNN." + ) + logging.info("subsample: " + " ".join([str(x) for x in subsample])) + return subsample + + elif mode == "asr" and arch == "rnn_mulenc": + subsample_list = [] + for idx in range(train_args.num_encs): + subsample = np.ones(train_args.elayers[idx] + 1, dtype=np.int) + if train_args.etype[idx].endswith("p") and not train_args.etype[ + idx + ].startswith("vgg"): + ss = train_args.subsample[idx].split("_") + for j in range(min(train_args.elayers[idx] + 1, len(ss))): + subsample[j] = int(ss[j]) + else: + logging.warning( + "Encoder %d: Subsampling is not performed for vgg*. " + "It is performed in max pooling layers at CNN.", + idx + 1, + ) + logging.info("subsample: " + " ".join([str(x) for x in subsample])) + subsample_list.append(subsample) + return subsample_list + + else: + raise ValueError("Invalid options: mode={}, arch={}".format(mode, arch)) + + +def rename_state_dict( + old_prefix: str, new_prefix: str, state_dict: Dict[str, torch.Tensor] +): + """Replace keys of old prefix with new prefix in state dict.""" + # need this list not to break the dict iterator + old_keys = [k for k in state_dict if k.startswith(old_prefix)] + if len(old_keys) > 0: + logging.warning(f"Rename: {old_prefix} -> {new_prefix}") + for k in old_keys: + v = state_dict.pop(k) + new_k = k.replace(old_prefix, new_prefix) + state_dict[new_k] = v + + +class Swish(torch.nn.Module): + """Construct an Swish object.""" + + def forward(self, x): + """Return Swich activation function.""" + return x * torch.sigmoid(x) + + +def get_activation(act): + """Return activation function.""" + + activation_funcs = { + "hardtanh": torch.nn.Hardtanh, + "tanh": torch.nn.Tanh, + "relu": torch.nn.ReLU, + "selu": torch.nn.SELU, + "swish": Swish, + } + + return activation_funcs[act]() diff --git a/funasr/modules/positionwise_feed_forward.py b/funasr/modules/positionwise_feed_forward.py new file mode 100644 index 000000000..61b874fda --- /dev/null +++ b/funasr/modules/positionwise_feed_forward.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Positionwise feed forward layer definition.""" + +import torch + +from funasr.modules.layer_norm import LayerNorm + + +class PositionwiseFeedForward(torch.nn.Module): + """Positionwise feed forward layer. + + Args: + idim (int): Input dimenstion. + hidden_units (int): The number of hidden units. + dropout_rate (float): Dropout rate. + + """ + + def __init__(self, idim, hidden_units, dropout_rate, activation=torch.nn.ReLU()): + """Construct an PositionwiseFeedForward object.""" + super(PositionwiseFeedForward, self).__init__() + self.w_1 = torch.nn.Linear(idim, hidden_units) + self.w_2 = torch.nn.Linear(hidden_units, idim) + self.dropout = torch.nn.Dropout(dropout_rate) + self.activation = activation + + def forward(self, x): + """Forward function.""" + return self.w_2(self.dropout(self.activation(self.w_1(x)))) + + +class PositionwiseFeedForwardDecoderSANM(torch.nn.Module): + """Positionwise feed forward layer. + + Args: + idim (int): Input dimenstion. + hidden_units (int): The number of hidden units. + dropout_rate (float): Dropout rate. + + """ + + def __init__(self, idim, hidden_units, dropout_rate, adim=None, activation=torch.nn.ReLU()): + """Construct an PositionwiseFeedForward object.""" + super(PositionwiseFeedForwardDecoderSANM, self).__init__() + self.w_1 = torch.nn.Linear(idim, hidden_units) + self.w_2 = torch.nn.Linear(hidden_units, idim if adim is None else adim, bias=False) + self.dropout = torch.nn.Dropout(dropout_rate) + self.activation = activation + self.norm = LayerNorm(hidden_units) + + def forward(self, x): + """Forward function.""" + return self.w_2(self.norm(self.dropout(self.activation(self.w_1(x))))) diff --git a/funasr/modules/repeat.py b/funasr/modules/repeat.py new file mode 100644 index 000000000..a3d2676a8 --- /dev/null +++ b/funasr/modules/repeat.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Repeat the same layer definition.""" + +import torch + + +class MultiSequential(torch.nn.Sequential): + """Multi-input multi-output torch.nn.Sequential.""" + + def forward(self, *args): + """Repeat.""" + for m in self: + args = m(*args) + return args + + +def repeat(N, fn): + """Repeat module N times. + + Args: + N (int): Number of repeat time. + fn (Callable): Function to generate module. + + Returns: + MultiSequential: Repeated model instance. + + """ + return MultiSequential(*[fn(n) for n in range(N)]) diff --git a/funasr/modules/rnn/__init__.py b/funasr/modules/rnn/__init__.py new file mode 100644 index 000000000..b7f177368 --- /dev/null +++ b/funasr/modules/rnn/__init__.py @@ -0,0 +1 @@ +"""Initialize sub package.""" diff --git a/funasr/modules/rnn/argument.py b/funasr/modules/rnn/argument.py new file mode 100644 index 000000000..b4c89d25f --- /dev/null +++ b/funasr/modules/rnn/argument.py @@ -0,0 +1,156 @@ +# Copyright 2020 Hirofumi Inaguma +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Conformer common arguments.""" + + +def add_arguments_rnn_encoder_common(group): + """Define common arguments for RNN encoder.""" + group.add_argument( + "--etype", + default="blstmp", + type=str, + choices=[ + "lstm", + "blstm", + "lstmp", + "blstmp", + "vgglstmp", + "vggblstmp", + "vgglstm", + "vggblstm", + "gru", + "bgru", + "grup", + "bgrup", + "vgggrup", + "vggbgrup", + "vgggru", + "vggbgru", + ], + help="Type of encoder network architecture", + ) + group.add_argument( + "--elayers", + default=4, + type=int, + help="Number of encoder layers", + ) + group.add_argument( + "--eunits", + "-u", + default=300, + type=int, + help="Number of encoder hidden units", + ) + group.add_argument( + "--eprojs", default=320, type=int, help="Number of encoder projection units" + ) + group.add_argument( + "--subsample", + default="1", + type=str, + help="Subsample input frames x_y_z means " + "subsample every x frame at 1st layer, " + "every y frame at 2nd layer etc.", + ) + return group + + +def add_arguments_rnn_decoder_common(group): + """Define common arguments for RNN decoder.""" + group.add_argument( + "--dtype", + default="lstm", + type=str, + choices=["lstm", "gru"], + help="Type of decoder network architecture", + ) + group.add_argument( + "--dlayers", default=1, type=int, help="Number of decoder layers" + ) + group.add_argument( + "--dunits", default=320, type=int, help="Number of decoder hidden units" + ) + group.add_argument( + "--dropout-rate-decoder", + default=0.0, + type=float, + help="Dropout rate for the decoder", + ) + group.add_argument( + "--sampling-probability", + default=0.0, + type=float, + help="Ratio of predicted labels fed back to decoder", + ) + group.add_argument( + "--lsm-type", + const="", + default="", + type=str, + nargs="?", + choices=["", "unigram"], + help="Apply label smoothing with a specified distribution type", + ) + return group + + +def add_arguments_rnn_attention_common(group): + """Define common arguments for RNN attention.""" + group.add_argument( + "--atype", + default="dot", + type=str, + choices=[ + "noatt", + "dot", + "add", + "location", + "coverage", + "coverage_location", + "location2d", + "location_recurrent", + "multi_head_dot", + "multi_head_add", + "multi_head_loc", + "multi_head_multi_res_loc", + ], + help="Type of attention architecture", + ) + group.add_argument( + "--adim", + default=320, + type=int, + help="Number of attention transformation dimensions", + ) + group.add_argument( + "--awin", default=5, type=int, help="Window size for location2d attention" + ) + group.add_argument( + "--aheads", + default=4, + type=int, + help="Number of heads for multi head attention", + ) + group.add_argument( + "--aconv-chans", + default=-1, + type=int, + help="Number of attention convolution channels \ + (negative value indicates no location-aware attention)", + ) + group.add_argument( + "--aconv-filts", + default=100, + type=int, + help="Number of attention convolution filters \ + (negative value indicates no location-aware attention)", + ) + group.add_argument( + "--dropout-rate", + default=0.0, + type=float, + help="Dropout rate for the encoder", + ) + return group diff --git a/funasr/modules/rnn/attentions.py b/funasr/modules/rnn/attentions.py new file mode 100644 index 000000000..71d1786d6 --- /dev/null +++ b/funasr/modules/rnn/attentions.py @@ -0,0 +1,1808 @@ +"""Attention modules for RNN.""" + +import math +import six + +import torch +import torch.nn.functional as F + +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.nets_utils import to_device + + +def _apply_attention_constraint( + e, last_attended_idx, backward_window=1, forward_window=3 +): + """Apply monotonic attention constraint. + + This function apply the monotonic attention constraint + introduced in `Deep Voice 3: Scaling + Text-to-Speech with Convolutional Sequence Learning`_. + + Args: + e (Tensor): Attention energy before applying softmax (1, T). + last_attended_idx (int): The index of the inputs of the last attended [0, T]. + backward_window (int, optional): Backward window size in attention constraint. + forward_window (int, optional): Forward window size in attetion constraint. + + Returns: + Tensor: Monotonic constrained attention energy (1, T). + + .. _`Deep Voice 3: Scaling Text-to-Speech with Convolutional Sequence Learning`: + https://arxiv.org/abs/1710.07654 + + """ + if e.size(0) != 1: + raise NotImplementedError("Batch attention constraining is not yet supported.") + backward_idx = last_attended_idx - backward_window + forward_idx = last_attended_idx + forward_window + if backward_idx > 0: + e[:, :backward_idx] = -float("inf") + if forward_idx < e.size(1): + e[:, forward_idx:] = -float("inf") + return e + + +class NoAtt(torch.nn.Module): + """No attention""" + + def __init__(self): + super(NoAtt, self).__init__() + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.c = None + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.c = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev): + """NoAtt forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B, T_max, D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: dummy (does not use) + :param torch.Tensor att_prev: dummy (does not use) + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: previous attention weights + :rtype: torch.Tensor + """ + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + + # initialize attention weight with uniform dist. + if att_prev is None: + # if no bias, 0 0-pad goes 0 + mask = 1.0 - make_pad_mask(enc_hs_len).float() + att_prev = mask / mask.new(enc_hs_len).unsqueeze(-1) + att_prev = att_prev.to(self.enc_h) + self.c = torch.sum( + self.enc_h * att_prev.view(batch, self.h_length, 1), dim=1 + ) + + return self.c, att_prev + + +class AttDot(torch.nn.Module): + """Dot product attention + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param bool han_mode: flag to swith on mode of hierarchical attention + and not store pre_compute_enc_h + """ + + def __init__(self, eprojs, dunits, att_dim, han_mode=False): + super(AttDot, self).__init__() + self.mlp_enc = torch.nn.Linear(eprojs, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim) + + self.dunits = dunits + self.eprojs = eprojs + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev, scaling=2.0): + """AttDot forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: dummy (does not use) + :param torch.Tensor att_prev: dummy (does not use) + :param float scaling: scaling parameter before applying softmax + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: previous attention weight (B x T_max) + :rtype: torch.Tensor + """ + + batch = enc_hs_pad.size(0) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = torch.tanh(self.mlp_enc(self.enc_h)) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + e = torch.sum( + self.pre_compute_enc_h + * torch.tanh(self.mlp_dec(dec_z)).view(batch, 1, self.att_dim), + dim=2, + ) # utt x frame + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w = F.softmax(scaling * e, dim=1) + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) + return c, w + + +class AttAdd(torch.nn.Module): + """Additive attention + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param bool han_mode: flag to swith on mode of hierarchical attention + and not store pre_compute_enc_h + """ + + def __init__(self, eprojs, dunits, att_dim, han_mode=False): + super(AttAdd, self).__init__() + self.mlp_enc = torch.nn.Linear(eprojs, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) + self.gvec = torch.nn.Linear(att_dim, 1) + self.dunits = dunits + self.eprojs = eprojs + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev, scaling=2.0): + """AttAdd forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param torch.Tensor att_prev: dummy (does not use) + :param float scaling: scaling parameter before applying softmax + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: previous attention weights (B x T_max) + :rtype: torch.Tensor + """ + + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = self.mlp_enc(self.enc_h) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + # dec_z_tiled: utt x frame x att_dim + dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) + + # dot with gvec + # utt x frame x att_dim -> utt x frame + e = self.gvec(torch.tanh(self.pre_compute_enc_h + dec_z_tiled)).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w = F.softmax(scaling * e, dim=1) + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) + + return c, w + + +class AttLoc(torch.nn.Module): + """location-aware attention module. + + Reference: Attention-Based Models for Speech Recognition + (https://arxiv.org/pdf/1506.07503.pdf) + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param int aconv_chans: # channels of attention convolution + :param int aconv_filts: filter size of attention convolution + :param bool han_mode: flag to swith on mode of hierarchical attention + and not store pre_compute_enc_h + """ + + def __init__( + self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False + ): + super(AttLoc, self).__init__() + self.mlp_enc = torch.nn.Linear(eprojs, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) + self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) + self.loc_conv = torch.nn.Conv2d( + 1, + aconv_chans, + (1, 2 * aconv_filts + 1), + padding=(0, aconv_filts), + bias=False, + ) + self.gvec = torch.nn.Linear(att_dim, 1) + + self.dunits = dunits + self.eprojs = eprojs + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def forward( + self, + enc_hs_pad, + enc_hs_len, + dec_z, + att_prev, + scaling=2.0, + last_attended_idx=None, + backward_window=1, + forward_window=3, + ): + """Calculate AttLoc forward propagation. + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param torch.Tensor att_prev: previous attention weight (B x T_max) + :param float scaling: scaling parameter before applying softmax + :param torch.Tensor forward_window: + forward window size when constraining attention + :param int last_attended_idx: index of the inputs of the last attended + :param int backward_window: backward window size in attention constraint + :param int forward_window: forward window size in attetion constraint + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: previous attention weights (B x T_max) + :rtype: torch.Tensor + """ + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = self.mlp_enc(self.enc_h) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + # initialize attention weight with uniform dist. + if att_prev is None: + # if no bias, 0 0-pad goes 0 + att_prev = 1.0 - make_pad_mask(enc_hs_len).to( + device=dec_z.device, dtype=dec_z.dtype + ) + att_prev = att_prev / att_prev.new(enc_hs_len).unsqueeze(-1) + + # att_prev: utt x frame -> utt x 1 x 1 x frame + # -> utt x att_conv_chans x 1 x frame + att_conv = self.loc_conv(att_prev.view(batch, 1, 1, self.h_length)) + # att_conv: utt x att_conv_chans x 1 x frame -> utt x frame x att_conv_chans + att_conv = att_conv.squeeze(2).transpose(1, 2) + # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim + att_conv = self.mlp_att(att_conv) + + # dec_z_tiled: utt x frame x att_dim + dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) + + # dot with gvec + # utt x frame x att_dim -> utt x frame + e = self.gvec( + torch.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) + ).squeeze(2) + + # NOTE: consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + + # apply monotonic attention constraint (mainly for TTS) + if last_attended_idx is not None: + e = _apply_attention_constraint( + e, last_attended_idx, backward_window, forward_window + ) + + w = F.softmax(scaling * e, dim=1) + + # weighted sum over flames + # utt x hdim + c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) + + return c, w + + +class AttCov(torch.nn.Module): + """Coverage mechanism attention + + Reference: Get To The Point: Summarization with Pointer-Generator Network + (https://arxiv.org/abs/1704.04368) + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param bool han_mode: flag to swith on mode of hierarchical attention + and not store pre_compute_enc_h + """ + + def __init__(self, eprojs, dunits, att_dim, han_mode=False): + super(AttCov, self).__init__() + self.mlp_enc = torch.nn.Linear(eprojs, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) + self.wvec = torch.nn.Linear(1, att_dim) + self.gvec = torch.nn.Linear(att_dim, 1) + + self.dunits = dunits + self.eprojs = eprojs + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev_list, scaling=2.0): + """AttCov forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param list att_prev_list: list of previous attention weight + :param float scaling: scaling parameter before applying softmax + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: list of previous attention weights + :rtype: list + """ + + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = self.mlp_enc(self.enc_h) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + # initialize attention weight with uniform dist. + if att_prev_list is None: + # if no bias, 0 0-pad goes 0 + att_prev_list = to_device( + enc_hs_pad, (1.0 - make_pad_mask(enc_hs_len).float()) + ) + att_prev_list = [ + att_prev_list / att_prev_list.new(enc_hs_len).unsqueeze(-1) + ] + + # att_prev_list: L' * [B x T] => cov_vec B x T + cov_vec = sum(att_prev_list) + # cov_vec: B x T => B x T x 1 => B x T x att_dim + cov_vec = self.wvec(cov_vec.unsqueeze(-1)) + + # dec_z_tiled: utt x frame x att_dim + dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) + + # dot with gvec + # utt x frame x att_dim -> utt x frame + e = self.gvec( + torch.tanh(cov_vec + self.pre_compute_enc_h + dec_z_tiled) + ).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w = F.softmax(scaling * e, dim=1) + att_prev_list += [w] + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) + + return c, att_prev_list + + +class AttLoc2D(torch.nn.Module): + """2D location-aware attention + + This attention is an extended version of location aware attention. + It take not only one frame before attention weights, + but also earlier frames into account. + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param int aconv_chans: # channels of attention convolution + :param int aconv_filts: filter size of attention convolution + :param int att_win: attention window size (default=5) + :param bool han_mode: + flag to swith on mode of hierarchical attention and not store pre_compute_enc_h + """ + + def __init__( + self, eprojs, dunits, att_dim, att_win, aconv_chans, aconv_filts, han_mode=False + ): + super(AttLoc2D, self).__init__() + self.mlp_enc = torch.nn.Linear(eprojs, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) + self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) + self.loc_conv = torch.nn.Conv2d( + 1, + aconv_chans, + (att_win, 2 * aconv_filts + 1), + padding=(0, aconv_filts), + bias=False, + ) + self.gvec = torch.nn.Linear(att_dim, 1) + + self.dunits = dunits + self.eprojs = eprojs + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.aconv_chans = aconv_chans + self.att_win = att_win + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev, scaling=2.0): + """AttLoc2D forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param torch.Tensor att_prev: previous attention weight (B x att_win x T_max) + :param float scaling: scaling parameter before applying softmax + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: previous attention weights (B x att_win x T_max) + :rtype: torch.Tensor + """ + + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = self.mlp_enc(self.enc_h) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + # initialize attention weight with uniform dist. + if att_prev is None: + # B * [Li x att_win] + # if no bias, 0 0-pad goes 0 + att_prev = to_device(enc_hs_pad, (1.0 - make_pad_mask(enc_hs_len).float())) + att_prev = att_prev / att_prev.new(enc_hs_len).unsqueeze(-1) + att_prev = att_prev.unsqueeze(1).expand(-1, self.att_win, -1) + + # att_prev: B x att_win x Tmax -> B x 1 x att_win x Tmax -> B x C x 1 x Tmax + att_conv = self.loc_conv(att_prev.unsqueeze(1)) + # att_conv: B x C x 1 x Tmax -> B x Tmax x C + att_conv = att_conv.squeeze(2).transpose(1, 2) + # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim + att_conv = self.mlp_att(att_conv) + + # dec_z_tiled: utt x frame x att_dim + dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) + + # dot with gvec + # utt x frame x att_dim -> utt x frame + e = self.gvec( + torch.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) + ).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w = F.softmax(scaling * e, dim=1) + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) + + # update att_prev: B x att_win x Tmax -> B x att_win+1 x Tmax + # -> B x att_win x Tmax + att_prev = torch.cat([att_prev, w.unsqueeze(1)], dim=1) + att_prev = att_prev[:, 1:] + + return c, att_prev + + +class AttLocRec(torch.nn.Module): + """location-aware recurrent attention + + This attention is an extended version of location aware attention. + With the use of RNN, + it take the effect of the history of attention weights into account. + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param int aconv_chans: # channels of attention convolution + :param int aconv_filts: filter size of attention convolution + :param bool han_mode: + flag to swith on mode of hierarchical attention and not store pre_compute_enc_h + """ + + def __init__( + self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False + ): + super(AttLocRec, self).__init__() + self.mlp_enc = torch.nn.Linear(eprojs, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) + self.loc_conv = torch.nn.Conv2d( + 1, + aconv_chans, + (1, 2 * aconv_filts + 1), + padding=(0, aconv_filts), + bias=False, + ) + self.att_lstm = torch.nn.LSTMCell(aconv_chans, att_dim, bias=False) + self.gvec = torch.nn.Linear(att_dim, 1) + + self.dunits = dunits + self.eprojs = eprojs + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev_states, scaling=2.0): + """AttLocRec forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param tuple att_prev_states: previous attention weight and lstm states + ((B, T_max), ((B, att_dim), (B, att_dim))) + :param float scaling: scaling parameter before applying softmax + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: previous attention weights and lstm states (w, (hx, cx)) + ((B, T_max), ((B, att_dim), (B, att_dim))) + :rtype: tuple + """ + + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = self.mlp_enc(self.enc_h) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + if att_prev_states is None: + # initialize attention weight with uniform dist. + # if no bias, 0 0-pad goes 0 + att_prev = to_device(enc_hs_pad, (1.0 - make_pad_mask(enc_hs_len).float())) + att_prev = att_prev / att_prev.new(enc_hs_len).unsqueeze(-1) + + # initialize lstm states + att_h = enc_hs_pad.new_zeros(batch, self.att_dim) + att_c = enc_hs_pad.new_zeros(batch, self.att_dim) + att_states = (att_h, att_c) + else: + att_prev = att_prev_states[0] + att_states = att_prev_states[1] + + # B x 1 x 1 x T -> B x C x 1 x T + att_conv = self.loc_conv(att_prev.view(batch, 1, 1, self.h_length)) + # apply non-linear + att_conv = F.relu(att_conv) + # B x C x 1 x T -> B x C x 1 x 1 -> B x C + att_conv = F.max_pool2d(att_conv, (1, att_conv.size(3))).view(batch, -1) + + att_h, att_c = self.att_lstm(att_conv, att_states) + + # dec_z_tiled: utt x frame x att_dim + dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) + + # dot with gvec + # utt x frame x att_dim -> utt x frame + e = self.gvec( + torch.tanh(att_h.unsqueeze(1) + self.pre_compute_enc_h + dec_z_tiled) + ).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w = F.softmax(scaling * e, dim=1) + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) + + return c, (w, (att_h, att_c)) + + +class AttCovLoc(torch.nn.Module): + """Coverage mechanism location aware attention + + This attention is a combination of coverage and location-aware attentions. + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param int aconv_chans: # channels of attention convolution + :param int aconv_filts: filter size of attention convolution + :param bool han_mode: + flag to swith on mode of hierarchical attention and not store pre_compute_enc_h + """ + + def __init__( + self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False + ): + super(AttCovLoc, self).__init__() + self.mlp_enc = torch.nn.Linear(eprojs, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) + self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) + self.loc_conv = torch.nn.Conv2d( + 1, + aconv_chans, + (1, 2 * aconv_filts + 1), + padding=(0, aconv_filts), + bias=False, + ) + self.gvec = torch.nn.Linear(att_dim, 1) + + self.dunits = dunits + self.eprojs = eprojs + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.aconv_chans = aconv_chans + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev_list, scaling=2.0): + """AttCovLoc forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param list att_prev_list: list of previous attention weight + :param float scaling: scaling parameter before applying softmax + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: list of previous attention weights + :rtype: list + """ + + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = self.mlp_enc(self.enc_h) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + # initialize attention weight with uniform dist. + if att_prev_list is None: + # if no bias, 0 0-pad goes 0 + mask = 1.0 - make_pad_mask(enc_hs_len).float() + att_prev_list = [ + to_device(enc_hs_pad, mask / mask.new(enc_hs_len).unsqueeze(-1)) + ] + + # att_prev_list: L' * [B x T] => cov_vec B x T + cov_vec = sum(att_prev_list) + + # cov_vec: B x T -> B x 1 x 1 x T -> B x C x 1 x T + att_conv = self.loc_conv(cov_vec.view(batch, 1, 1, self.h_length)) + # att_conv: utt x att_conv_chans x 1 x frame -> utt x frame x att_conv_chans + att_conv = att_conv.squeeze(2).transpose(1, 2) + # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim + att_conv = self.mlp_att(att_conv) + + # dec_z_tiled: utt x frame x att_dim + dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) + + # dot with gvec + # utt x frame x att_dim -> utt x frame + e = self.gvec( + torch.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) + ).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w = F.softmax(scaling * e, dim=1) + att_prev_list += [w] + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) + + return c, att_prev_list + + +class AttMultiHeadDot(torch.nn.Module): + """Multi head dot product attention + + Reference: Attention is all you need + (https://arxiv.org/abs/1706.03762) + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int aheads: # heads of multi head attention + :param int att_dim_k: dimension k in multi head attention + :param int att_dim_v: dimension v in multi head attention + :param bool han_mode: flag to swith on mode of hierarchical attention + and not store pre_compute_k and pre_compute_v + """ + + def __init__(self, eprojs, dunits, aheads, att_dim_k, att_dim_v, han_mode=False): + super(AttMultiHeadDot, self).__init__() + self.mlp_q = torch.nn.ModuleList() + self.mlp_k = torch.nn.ModuleList() + self.mlp_v = torch.nn.ModuleList() + for _ in six.moves.range(aheads): + self.mlp_q += [torch.nn.Linear(dunits, att_dim_k)] + self.mlp_k += [torch.nn.Linear(eprojs, att_dim_k, bias=False)] + self.mlp_v += [torch.nn.Linear(eprojs, att_dim_v, bias=False)] + self.mlp_o = torch.nn.Linear(aheads * att_dim_v, eprojs, bias=False) + self.dunits = dunits + self.eprojs = eprojs + self.aheads = aheads + self.att_dim_k = att_dim_k + self.att_dim_v = att_dim_v + self.scaling = 1.0 / math.sqrt(att_dim_k) + self.h_length = None + self.enc_h = None + self.pre_compute_k = None + self.pre_compute_v = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_k = None + self.pre_compute_v = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev): + """AttMultiHeadDot forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param torch.Tensor att_prev: dummy (does not use) + :return: attention weighted encoder state (B x D_enc) + :rtype: torch.Tensor + :return: list of previous attention weight (B x T_max) * aheads + :rtype: list + """ + + batch = enc_hs_pad.size(0) + # pre-compute all k and v outside the decoder loop + if self.pre_compute_k is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_k = [ + torch.tanh(self.mlp_k[h](self.enc_h)) + for h in six.moves.range(self.aheads) + ] + + if self.pre_compute_v is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_v = [ + self.mlp_v[h](self.enc_h) for h in six.moves.range(self.aheads) + ] + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + c = [] + w = [] + for h in six.moves.range(self.aheads): + e = torch.sum( + self.pre_compute_k[h] + * torch.tanh(self.mlp_q[h](dec_z)).view(batch, 1, self.att_dim_k), + dim=2, + ) # utt x frame + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w += [F.softmax(self.scaling * e, dim=1)] + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c += [ + torch.sum( + self.pre_compute_v[h] * w[h].view(batch, self.h_length, 1), dim=1 + ) + ] + + # concat all of c + c = self.mlp_o(torch.cat(c, dim=1)) + + return c, w + + +class AttMultiHeadAdd(torch.nn.Module): + """Multi head additive attention + + Reference: Attention is all you need + (https://arxiv.org/abs/1706.03762) + + This attention is multi head attention using additive attention for each head. + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int aheads: # heads of multi head attention + :param int att_dim_k: dimension k in multi head attention + :param int att_dim_v: dimension v in multi head attention + :param bool han_mode: flag to swith on mode of hierarchical attention + and not store pre_compute_k and pre_compute_v + """ + + def __init__(self, eprojs, dunits, aheads, att_dim_k, att_dim_v, han_mode=False): + super(AttMultiHeadAdd, self).__init__() + self.mlp_q = torch.nn.ModuleList() + self.mlp_k = torch.nn.ModuleList() + self.mlp_v = torch.nn.ModuleList() + self.gvec = torch.nn.ModuleList() + for _ in six.moves.range(aheads): + self.mlp_q += [torch.nn.Linear(dunits, att_dim_k)] + self.mlp_k += [torch.nn.Linear(eprojs, att_dim_k, bias=False)] + self.mlp_v += [torch.nn.Linear(eprojs, att_dim_v, bias=False)] + self.gvec += [torch.nn.Linear(att_dim_k, 1)] + self.mlp_o = torch.nn.Linear(aheads * att_dim_v, eprojs, bias=False) + self.dunits = dunits + self.eprojs = eprojs + self.aheads = aheads + self.att_dim_k = att_dim_k + self.att_dim_v = att_dim_v + self.scaling = 1.0 / math.sqrt(att_dim_k) + self.h_length = None + self.enc_h = None + self.pre_compute_k = None + self.pre_compute_v = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_k = None + self.pre_compute_v = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev): + """AttMultiHeadAdd forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param torch.Tensor att_prev: dummy (does not use) + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: list of previous attention weight (B x T_max) * aheads + :rtype: list + """ + + batch = enc_hs_pad.size(0) + # pre-compute all k and v outside the decoder loop + if self.pre_compute_k is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_k = [ + self.mlp_k[h](self.enc_h) for h in six.moves.range(self.aheads) + ] + + if self.pre_compute_v is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_v = [ + self.mlp_v[h](self.enc_h) for h in six.moves.range(self.aheads) + ] + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + c = [] + w = [] + for h in six.moves.range(self.aheads): + e = self.gvec[h]( + torch.tanh( + self.pre_compute_k[h] + + self.mlp_q[h](dec_z).view(batch, 1, self.att_dim_k) + ) + ).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w += [F.softmax(self.scaling * e, dim=1)] + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c += [ + torch.sum( + self.pre_compute_v[h] * w[h].view(batch, self.h_length, 1), dim=1 + ) + ] + + # concat all of c + c = self.mlp_o(torch.cat(c, dim=1)) + + return c, w + + +class AttMultiHeadLoc(torch.nn.Module): + """Multi head location based attention + + Reference: Attention is all you need + (https://arxiv.org/abs/1706.03762) + + This attention is multi head attention using location-aware attention for each head. + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int aheads: # heads of multi head attention + :param int att_dim_k: dimension k in multi head attention + :param int att_dim_v: dimension v in multi head attention + :param int aconv_chans: # channels of attention convolution + :param int aconv_filts: filter size of attention convolution + :param bool han_mode: flag to swith on mode of hierarchical attention + and not store pre_compute_k and pre_compute_v + """ + + def __init__( + self, + eprojs, + dunits, + aheads, + att_dim_k, + att_dim_v, + aconv_chans, + aconv_filts, + han_mode=False, + ): + super(AttMultiHeadLoc, self).__init__() + self.mlp_q = torch.nn.ModuleList() + self.mlp_k = torch.nn.ModuleList() + self.mlp_v = torch.nn.ModuleList() + self.gvec = torch.nn.ModuleList() + self.loc_conv = torch.nn.ModuleList() + self.mlp_att = torch.nn.ModuleList() + for _ in six.moves.range(aheads): + self.mlp_q += [torch.nn.Linear(dunits, att_dim_k)] + self.mlp_k += [torch.nn.Linear(eprojs, att_dim_k, bias=False)] + self.mlp_v += [torch.nn.Linear(eprojs, att_dim_v, bias=False)] + self.gvec += [torch.nn.Linear(att_dim_k, 1)] + self.loc_conv += [ + torch.nn.Conv2d( + 1, + aconv_chans, + (1, 2 * aconv_filts + 1), + padding=(0, aconv_filts), + bias=False, + ) + ] + self.mlp_att += [torch.nn.Linear(aconv_chans, att_dim_k, bias=False)] + self.mlp_o = torch.nn.Linear(aheads * att_dim_v, eprojs, bias=False) + self.dunits = dunits + self.eprojs = eprojs + self.aheads = aheads + self.att_dim_k = att_dim_k + self.att_dim_v = att_dim_v + self.scaling = 1.0 / math.sqrt(att_dim_k) + self.h_length = None + self.enc_h = None + self.pre_compute_k = None + self.pre_compute_v = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_k = None + self.pre_compute_v = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev, scaling=2.0): + """AttMultiHeadLoc forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param torch.Tensor att_prev: + list of previous attention weight (B x T_max) * aheads + :param float scaling: scaling parameter before applying softmax + :return: attention weighted encoder state (B x D_enc) + :rtype: torch.Tensor + :return: list of previous attention weight (B x T_max) * aheads + :rtype: list + """ + + batch = enc_hs_pad.size(0) + # pre-compute all k and v outside the decoder loop + if self.pre_compute_k is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_k = [ + self.mlp_k[h](self.enc_h) for h in six.moves.range(self.aheads) + ] + + if self.pre_compute_v is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_v = [ + self.mlp_v[h](self.enc_h) for h in six.moves.range(self.aheads) + ] + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + if att_prev is None: + att_prev = [] + for _ in six.moves.range(self.aheads): + # if no bias, 0 0-pad goes 0 + mask = 1.0 - make_pad_mask(enc_hs_len).float() + att_prev += [ + to_device(enc_hs_pad, mask / mask.new(enc_hs_len).unsqueeze(-1)) + ] + + c = [] + w = [] + for h in six.moves.range(self.aheads): + att_conv = self.loc_conv[h](att_prev[h].view(batch, 1, 1, self.h_length)) + att_conv = att_conv.squeeze(2).transpose(1, 2) + att_conv = self.mlp_att[h](att_conv) + + e = self.gvec[h]( + torch.tanh( + self.pre_compute_k[h] + + att_conv + + self.mlp_q[h](dec_z).view(batch, 1, self.att_dim_k) + ) + ).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w += [F.softmax(scaling * e, dim=1)] + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c += [ + torch.sum( + self.pre_compute_v[h] * w[h].view(batch, self.h_length, 1), dim=1 + ) + ] + + # concat all of c + c = self.mlp_o(torch.cat(c, dim=1)) + + return c, w + + +class AttMultiHeadMultiResLoc(torch.nn.Module): + """Multi head multi resolution location based attention + + Reference: Attention is all you need + (https://arxiv.org/abs/1706.03762) + + This attention is multi head attention using location-aware attention for each head. + Furthermore, it uses different filter size for each head. + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int aheads: # heads of multi head attention + :param int att_dim_k: dimension k in multi head attention + :param int att_dim_v: dimension v in multi head attention + :param int aconv_chans: maximum # channels of attention convolution + each head use #ch = aconv_chans * (head + 1) / aheads + e.g. aheads=4, aconv_chans=100 => filter size = 25, 50, 75, 100 + :param int aconv_filts: filter size of attention convolution + :param bool han_mode: flag to swith on mode of hierarchical attention + and not store pre_compute_k and pre_compute_v + """ + + def __init__( + self, + eprojs, + dunits, + aheads, + att_dim_k, + att_dim_v, + aconv_chans, + aconv_filts, + han_mode=False, + ): + super(AttMultiHeadMultiResLoc, self).__init__() + self.mlp_q = torch.nn.ModuleList() + self.mlp_k = torch.nn.ModuleList() + self.mlp_v = torch.nn.ModuleList() + self.gvec = torch.nn.ModuleList() + self.loc_conv = torch.nn.ModuleList() + self.mlp_att = torch.nn.ModuleList() + for h in six.moves.range(aheads): + self.mlp_q += [torch.nn.Linear(dunits, att_dim_k)] + self.mlp_k += [torch.nn.Linear(eprojs, att_dim_k, bias=False)] + self.mlp_v += [torch.nn.Linear(eprojs, att_dim_v, bias=False)] + self.gvec += [torch.nn.Linear(att_dim_k, 1)] + afilts = aconv_filts * (h + 1) // aheads + self.loc_conv += [ + torch.nn.Conv2d( + 1, aconv_chans, (1, 2 * afilts + 1), padding=(0, afilts), bias=False + ) + ] + self.mlp_att += [torch.nn.Linear(aconv_chans, att_dim_k, bias=False)] + self.mlp_o = torch.nn.Linear(aheads * att_dim_v, eprojs, bias=False) + self.dunits = dunits + self.eprojs = eprojs + self.aheads = aheads + self.att_dim_k = att_dim_k + self.att_dim_v = att_dim_v + self.scaling = 1.0 / math.sqrt(att_dim_k) + self.h_length = None + self.enc_h = None + self.pre_compute_k = None + self.pre_compute_v = None + self.mask = None + self.han_mode = han_mode + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_k = None + self.pre_compute_v = None + self.mask = None + + def forward(self, enc_hs_pad, enc_hs_len, dec_z, att_prev): + """AttMultiHeadMultiResLoc forward + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param torch.Tensor att_prev: list of previous attention weight + (B x T_max) * aheads + :return: attention weighted encoder state (B x D_enc) + :rtype: torch.Tensor + :return: list of previous attention weight (B x T_max) * aheads + :rtype: list + """ + + batch = enc_hs_pad.size(0) + # pre-compute all k and v outside the decoder loop + if self.pre_compute_k is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_k = [ + self.mlp_k[h](self.enc_h) for h in six.moves.range(self.aheads) + ] + + if self.pre_compute_v is None or self.han_mode: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_v = [ + self.mlp_v[h](self.enc_h) for h in six.moves.range(self.aheads) + ] + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + if att_prev is None: + att_prev = [] + for _ in six.moves.range(self.aheads): + # if no bias, 0 0-pad goes 0 + mask = 1.0 - make_pad_mask(enc_hs_len).float() + att_prev += [ + to_device(enc_hs_pad, mask / mask.new(enc_hs_len).unsqueeze(-1)) + ] + + c = [] + w = [] + for h in six.moves.range(self.aheads): + att_conv = self.loc_conv[h](att_prev[h].view(batch, 1, 1, self.h_length)) + att_conv = att_conv.squeeze(2).transpose(1, 2) + att_conv = self.mlp_att[h](att_conv) + + e = self.gvec[h]( + torch.tanh( + self.pre_compute_k[h] + + att_conv + + self.mlp_q[h](dec_z).view(batch, 1, self.att_dim_k) + ) + ).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + w += [F.softmax(self.scaling * e, dim=1)] + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c += [ + torch.sum( + self.pre_compute_v[h] * w[h].view(batch, self.h_length, 1), dim=1 + ) + ] + + # concat all of c + c = self.mlp_o(torch.cat(c, dim=1)) + + return c, w + + +class AttForward(torch.nn.Module): + """Forward attention module. + + Reference: + Forward attention in sequence-to-sequence acoustic modeling for speech synthesis + (https://arxiv.org/pdf/1807.06736.pdf) + + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param int aconv_chans: # channels of attention convolution + :param int aconv_filts: filter size of attention convolution + """ + + def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts): + super(AttForward, self).__init__() + self.mlp_enc = torch.nn.Linear(eprojs, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) + self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) + self.loc_conv = torch.nn.Conv2d( + 1, + aconv_chans, + (1, 2 * aconv_filts + 1), + padding=(0, aconv_filts), + bias=False, + ) + self.gvec = torch.nn.Linear(att_dim, 1) + self.dunits = dunits + self.eprojs = eprojs + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def reset(self): + """reset states""" + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + + def forward( + self, + enc_hs_pad, + enc_hs_len, + dec_z, + att_prev, + scaling=1.0, + last_attended_idx=None, + backward_window=1, + forward_window=3, + ): + """Calculate AttForward forward propagation. + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B x T_max x D_enc) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B x D_dec) + :param torch.Tensor att_prev: attention weights of previous step + :param float scaling: scaling parameter before applying softmax + :param int last_attended_idx: index of the inputs of the last attended + :param int backward_window: backward window size in attention constraint + :param int forward_window: forward window size in attetion constraint + :return: attention weighted encoder state (B, D_enc) + :rtype: torch.Tensor + :return: previous attention weights (B x T_max) + :rtype: torch.Tensor + """ + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = self.mlp_enc(self.enc_h) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + if att_prev is None: + # initial attention will be [1, 0, 0, ...] + att_prev = enc_hs_pad.new_zeros(*enc_hs_pad.size()[:2]) + att_prev[:, 0] = 1.0 + + # att_prev: utt x frame -> utt x 1 x 1 x frame + # -> utt x att_conv_chans x 1 x frame + att_conv = self.loc_conv(att_prev.view(batch, 1, 1, self.h_length)) + # att_conv: utt x att_conv_chans x 1 x frame -> utt x frame x att_conv_chans + att_conv = att_conv.squeeze(2).transpose(1, 2) + # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim + att_conv = self.mlp_att(att_conv) + + # dec_z_tiled: utt x frame x att_dim + dec_z_tiled = self.mlp_dec(dec_z).unsqueeze(1) + + # dot with gvec + # utt x frame x att_dim -> utt x frame + e = self.gvec( + torch.tanh(self.pre_compute_enc_h + dec_z_tiled + att_conv) + ).squeeze(2) + + # NOTE: consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + + # apply monotonic attention constraint (mainly for TTS) + if last_attended_idx is not None: + e = _apply_attention_constraint( + e, last_attended_idx, backward_window, forward_window + ) + + w = F.softmax(scaling * e, dim=1) + + # forward attention + att_prev_shift = F.pad(att_prev, (1, 0))[:, :-1] + w = (att_prev + att_prev_shift) * w + # NOTE: clamp is needed to avoid nan gradient + w = F.normalize(torch.clamp(w, 1e-6), p=1, dim=1) + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c = torch.sum(self.enc_h * w.unsqueeze(-1), dim=1) + + return c, w + + +class AttForwardTA(torch.nn.Module): + """Forward attention with transition agent module. + + Reference: + Forward attention in sequence-to-sequence acoustic modeling for speech synthesis + (https://arxiv.org/pdf/1807.06736.pdf) + + :param int eunits: # units of encoder + :param int dunits: # units of decoder + :param int att_dim: attention dimension + :param int aconv_chans: # channels of attention convolution + :param int aconv_filts: filter size of attention convolution + :param int odim: output dimension + """ + + def __init__(self, eunits, dunits, att_dim, aconv_chans, aconv_filts, odim): + super(AttForwardTA, self).__init__() + self.mlp_enc = torch.nn.Linear(eunits, att_dim) + self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False) + self.mlp_ta = torch.nn.Linear(eunits + dunits + odim, 1) + self.mlp_att = torch.nn.Linear(aconv_chans, att_dim, bias=False) + self.loc_conv = torch.nn.Conv2d( + 1, + aconv_chans, + (1, 2 * aconv_filts + 1), + padding=(0, aconv_filts), + bias=False, + ) + self.gvec = torch.nn.Linear(att_dim, 1) + self.dunits = dunits + self.eunits = eunits + self.att_dim = att_dim + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + self.trans_agent_prob = 0.5 + + def reset(self): + self.h_length = None + self.enc_h = None + self.pre_compute_enc_h = None + self.mask = None + self.trans_agent_prob = 0.5 + + def forward( + self, + enc_hs_pad, + enc_hs_len, + dec_z, + att_prev, + out_prev, + scaling=1.0, + last_attended_idx=None, + backward_window=1, + forward_window=3, + ): + """Calculate AttForwardTA forward propagation. + + :param torch.Tensor enc_hs_pad: padded encoder hidden state (B, Tmax, eunits) + :param list enc_hs_len: padded encoder hidden state length (B) + :param torch.Tensor dec_z: decoder hidden state (B, dunits) + :param torch.Tensor att_prev: attention weights of previous step + :param torch.Tensor out_prev: decoder outputs of previous step (B, odim) + :param float scaling: scaling parameter before applying softmax + :param int last_attended_idx: index of the inputs of the last attended + :param int backward_window: backward window size in attention constraint + :param int forward_window: forward window size in attetion constraint + :return: attention weighted encoder state (B, dunits) + :rtype: torch.Tensor + :return: previous attention weights (B, Tmax) + :rtype: torch.Tensor + """ + batch = len(enc_hs_pad) + # pre-compute all h outside the decoder loop + if self.pre_compute_enc_h is None: + self.enc_h = enc_hs_pad # utt x frame x hdim + self.h_length = self.enc_h.size(1) + # utt x frame x att_dim + self.pre_compute_enc_h = self.mlp_enc(self.enc_h) + + if dec_z is None: + dec_z = enc_hs_pad.new_zeros(batch, self.dunits) + else: + dec_z = dec_z.view(batch, self.dunits) + + if att_prev is None: + # initial attention will be [1, 0, 0, ...] + att_prev = enc_hs_pad.new_zeros(*enc_hs_pad.size()[:2]) + att_prev[:, 0] = 1.0 + + # att_prev: utt x frame -> utt x 1 x 1 x frame + # -> utt x att_conv_chans x 1 x frame + att_conv = self.loc_conv(att_prev.view(batch, 1, 1, self.h_length)) + # att_conv: utt x att_conv_chans x 1 x frame -> utt x frame x att_conv_chans + att_conv = att_conv.squeeze(2).transpose(1, 2) + # att_conv: utt x frame x att_conv_chans -> utt x frame x att_dim + att_conv = self.mlp_att(att_conv) + + # dec_z_tiled: utt x frame x att_dim + dec_z_tiled = self.mlp_dec(dec_z).view(batch, 1, self.att_dim) + + # dot with gvec + # utt x frame x att_dim -> utt x frame + e = self.gvec( + torch.tanh(att_conv + self.pre_compute_enc_h + dec_z_tiled) + ).squeeze(2) + + # NOTE consider zero padding when compute w. + if self.mask is None: + self.mask = to_device(enc_hs_pad, make_pad_mask(enc_hs_len)) + e.masked_fill_(self.mask, -float("inf")) + + # apply monotonic attention constraint (mainly for TTS) + if last_attended_idx is not None: + e = _apply_attention_constraint( + e, last_attended_idx, backward_window, forward_window + ) + + w = F.softmax(scaling * e, dim=1) + + # forward attention + att_prev_shift = F.pad(att_prev, (1, 0))[:, :-1] + w = ( + self.trans_agent_prob * att_prev + + (1 - self.trans_agent_prob) * att_prev_shift + ) * w + # NOTE: clamp is needed to avoid nan gradient + w = F.normalize(torch.clamp(w, 1e-6), p=1, dim=1) + + # weighted sum over flames + # utt x hdim + # NOTE use bmm instead of sum(*) + c = torch.sum(self.enc_h * w.view(batch, self.h_length, 1), dim=1) + + # update transition agent prob + self.trans_agent_prob = torch.sigmoid( + self.mlp_ta(torch.cat([c, out_prev, dec_z], dim=1)) + ) + + return c, w + + +def att_for(args, num_att=1, han_mode=False): + """Instantiates an attention module given the program arguments + + :param Namespace args: The arguments + :param int num_att: number of attention modules + (in multi-speaker case, it can be 2 or more) + :param bool han_mode: switch on/off mode of hierarchical attention network (HAN) + :rtype torch.nn.Module + :return: The attention module + """ + att_list = torch.nn.ModuleList() + num_encs = getattr(args, "num_encs", 1) # use getattr to keep compatibility + aheads = getattr(args, "aheads", None) + awin = getattr(args, "awin", None) + aconv_chans = getattr(args, "aconv_chans", None) + aconv_filts = getattr(args, "aconv_filts", None) + + if num_encs == 1: + for i in range(num_att): + att = initial_att( + args.atype, + args.eprojs, + args.dunits, + aheads, + args.adim, + awin, + aconv_chans, + aconv_filts, + ) + att_list.append(att) + elif num_encs > 1: # no multi-speaker mode + if han_mode: + att = initial_att( + args.han_type, + args.eprojs, + args.dunits, + args.han_heads, + args.han_dim, + args.han_win, + args.han_conv_chans, + args.han_conv_filts, + han_mode=True, + ) + return att + else: + att_list = torch.nn.ModuleList() + for idx in range(num_encs): + att = initial_att( + args.atype[idx], + args.eprojs, + args.dunits, + aheads[idx], + args.adim[idx], + awin[idx], + aconv_chans[idx], + aconv_filts[idx], + ) + att_list.append(att) + else: + raise ValueError( + "Number of encoders needs to be more than one. {}".format(num_encs) + ) + return att_list + + +def initial_att( + atype, eprojs, dunits, aheads, adim, awin, aconv_chans, aconv_filts, han_mode=False +): + """Instantiates a single attention module + + :param str atype: attention type + :param int eprojs: # projection-units of encoder + :param int dunits: # units of decoder + :param int aheads: # heads of multi head attention + :param int adim: attention dimension + :param int awin: attention window size + :param int aconv_chans: # channels of attention convolution + :param int aconv_filts: filter size of attention convolution + :param bool han_mode: flag to swith on mode of hierarchical attention + :return: The attention module + """ + + if atype == "noatt": + att = NoAtt() + elif atype == "dot": + att = AttDot(eprojs, dunits, adim, han_mode) + elif atype == "add": + att = AttAdd(eprojs, dunits, adim, han_mode) + elif atype == "location": + att = AttLoc(eprojs, dunits, adim, aconv_chans, aconv_filts, han_mode) + elif atype == "location2d": + att = AttLoc2D(eprojs, dunits, adim, awin, aconv_chans, aconv_filts, han_mode) + elif atype == "location_recurrent": + att = AttLocRec(eprojs, dunits, adim, aconv_chans, aconv_filts, han_mode) + elif atype == "coverage": + att = AttCov(eprojs, dunits, adim, han_mode) + elif atype == "coverage_location": + att = AttCovLoc(eprojs, dunits, adim, aconv_chans, aconv_filts, han_mode) + elif atype == "multi_head_dot": + att = AttMultiHeadDot(eprojs, dunits, aheads, adim, adim, han_mode) + elif atype == "multi_head_add": + att = AttMultiHeadAdd(eprojs, dunits, aheads, adim, adim, han_mode) + elif atype == "multi_head_loc": + att = AttMultiHeadLoc( + eprojs, dunits, aheads, adim, adim, aconv_chans, aconv_filts, han_mode + ) + elif atype == "multi_head_multi_res_loc": + att = AttMultiHeadMultiResLoc( + eprojs, dunits, aheads, adim, adim, aconv_chans, aconv_filts, han_mode + ) + return att + + +def att_to_numpy(att_ws, att): + """Converts attention weights to a numpy array given the attention + + :param list att_ws: The attention weights + :param torch.nn.Module att: The attention + :rtype: np.ndarray + :return: The numpy array of the attention weights + """ + # convert to numpy array with the shape (B, Lmax, Tmax) + if isinstance(att, AttLoc2D): + # att_ws => list of previous concate attentions + att_ws = torch.stack([aw[:, -1] for aw in att_ws], dim=1).cpu().numpy() + elif isinstance(att, (AttCov, AttCovLoc)): + # att_ws => list of list of previous attentions + att_ws = ( + torch.stack([aw[idx] for idx, aw in enumerate(att_ws)], dim=1).cpu().numpy() + ) + elif isinstance(att, AttLocRec): + # att_ws => list of tuple of attention and hidden states + att_ws = torch.stack([aw[0] for aw in att_ws], dim=1).cpu().numpy() + elif isinstance( + att, + (AttMultiHeadDot, AttMultiHeadAdd, AttMultiHeadLoc, AttMultiHeadMultiResLoc), + ): + # att_ws => list of list of each head attention + n_heads = len(att_ws[0]) + att_ws_sorted_by_head = [] + for h in six.moves.range(n_heads): + att_ws_head = torch.stack([aw[h] for aw in att_ws], dim=1) + att_ws_sorted_by_head += [att_ws_head] + att_ws = torch.stack(att_ws_sorted_by_head, dim=1).cpu().numpy() + else: + # att_ws => list of attentions + att_ws = torch.stack(att_ws, dim=1).cpu().numpy() + return att_ws diff --git a/funasr/modules/rnn/decoders.py b/funasr/modules/rnn/decoders.py new file mode 100644 index 000000000..c5d886f30 --- /dev/null +++ b/funasr/modules/rnn/decoders.py @@ -0,0 +1,1211 @@ +"""RNN decoder module.""" +import logging +import math +import random +from argparse import Namespace + +import numpy as np +import six +import torch +import torch.nn.functional as F + +from funasr.modules.scorers.ctc_prefix_score import CTCPrefixScore +from funasr.modules.scorers.ctc_prefix_score import CTCPrefixScoreTH +from funasr.modules.scorers.scorer_interface import ScorerInterface +from funasr.modules.e2e_asr_common import end_detect +from funasr.modules.nets_utils import mask_by_length +from funasr.modules.nets_utils import pad_list +from funasr.modules.nets_utils import th_accuracy +from funasr.modules.nets_utils import to_device +from funasr.modules.rnn.attentions import att_to_numpy + +MAX_DECODER_OUTPUT = 5 +CTC_SCORING_RATIO = 1.5 + + +class Decoder(torch.nn.Module, ScorerInterface): + """Decoder module + + :param int eprojs: encoder projection units + :param int odim: dimension of outputs + :param str dtype: gru or lstm + :param int dlayers: decoder layers + :param int dunits: decoder units + :param int sos: start of sequence symbol id + :param int eos: end of sequence symbol id + :param torch.nn.Module att: attention module + :param int verbose: verbose level + :param list char_list: list of character strings + :param ndarray labeldist: distribution of label smoothing + :param float lsm_weight: label smoothing weight + :param float sampling_probability: scheduled sampling probability + :param float dropout: dropout rate + :param float context_residual: if True, use context vector for token generation + :param float replace_sos: use for multilingual (speech/text) translation + """ + + def __init__( + self, + eprojs, + odim, + dtype, + dlayers, + dunits, + sos, + eos, + att, + verbose=0, + char_list=None, + labeldist=None, + lsm_weight=0.0, + sampling_probability=0.0, + dropout=0.0, + context_residual=False, + replace_sos=False, + num_encs=1, + ): + + torch.nn.Module.__init__(self) + self.dtype = dtype + self.dunits = dunits + self.dlayers = dlayers + self.context_residual = context_residual + self.embed = torch.nn.Embedding(odim, dunits) + self.dropout_emb = torch.nn.Dropout(p=dropout) + + self.decoder = torch.nn.ModuleList() + self.dropout_dec = torch.nn.ModuleList() + self.decoder += [ + torch.nn.LSTMCell(dunits + eprojs, dunits) + if self.dtype == "lstm" + else torch.nn.GRUCell(dunits + eprojs, dunits) + ] + self.dropout_dec += [torch.nn.Dropout(p=dropout)] + for _ in six.moves.range(1, self.dlayers): + self.decoder += [ + torch.nn.LSTMCell(dunits, dunits) + if self.dtype == "lstm" + else torch.nn.GRUCell(dunits, dunits) + ] + self.dropout_dec += [torch.nn.Dropout(p=dropout)] + # NOTE: dropout is applied only for the vertical connections + # see https://arxiv.org/pdf/1409.2329.pdf + self.ignore_id = -1 + + if context_residual: + self.output = torch.nn.Linear(dunits + eprojs, odim) + else: + self.output = torch.nn.Linear(dunits, odim) + + self.loss = None + self.att = att + self.dunits = dunits + self.sos = sos + self.eos = eos + self.odim = odim + self.verbose = verbose + self.char_list = char_list + # for label smoothing + self.labeldist = labeldist + self.vlabeldist = None + self.lsm_weight = lsm_weight + self.sampling_probability = sampling_probability + self.dropout = dropout + self.num_encs = num_encs + + # for multilingual E2E-ST + self.replace_sos = replace_sos + + self.logzero = -10000000000.0 + + def zero_state(self, hs_pad): + return hs_pad.new_zeros(hs_pad.size(0), self.dunits) + + def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev): + if self.dtype == "lstm": + z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0])) + for i in six.moves.range(1, self.dlayers): + z_list[i], c_list[i] = self.decoder[i]( + self.dropout_dec[i - 1](z_list[i - 1]), (z_prev[i], c_prev[i]) + ) + else: + z_list[0] = self.decoder[0](ey, z_prev[0]) + for i in six.moves.range(1, self.dlayers): + z_list[i] = self.decoder[i]( + self.dropout_dec[i - 1](z_list[i - 1]), z_prev[i] + ) + return z_list, c_list + + def forward(self, hs_pad, hlens, ys_pad, strm_idx=0, lang_ids=None): + """Decoder forward + + :param torch.Tensor hs_pad: batch of padded hidden state sequences (B, Tmax, D) + [in multi-encoder case, + list of torch.Tensor, + [(B, Tmax_1, D), (B, Tmax_2, D), ..., ] ] + :param torch.Tensor hlens: batch of lengths of hidden state sequences (B) + [in multi-encoder case, list of torch.Tensor, + [(B), (B), ..., ] + :param torch.Tensor ys_pad: batch of padded character id sequence tensor + (B, Lmax) + :param int strm_idx: stream index indicates the index of decoding stream. + :param torch.Tensor lang_ids: batch of target language id tensor (B, 1) + :return: attention loss value + :rtype: torch.Tensor + :return: accuracy + :rtype: float + """ + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + hs_pad = [hs_pad] + hlens = [hlens] + + # TODO(kan-bayashi): need to make more smart way + ys = [y[y != self.ignore_id] for y in ys_pad] # parse padded ys + # attention index for the attention module + # in SPA (speaker parallel attention), + # att_idx is used to select attention module. In other cases, it is 0. + att_idx = min(strm_idx, len(self.att) - 1) + + # hlens should be list of list of integer + hlens = [list(map(int, hlens[idx])) for idx in range(self.num_encs)] + + self.loss = None + # prepare input and output word sequences with sos/eos IDs + eos = ys[0].new([self.eos]) + sos = ys[0].new([self.sos]) + if self.replace_sos: + ys_in = [torch.cat([idx, y], dim=0) for idx, y in zip(lang_ids, ys)] + else: + ys_in = [torch.cat([sos, y], dim=0) for y in ys] + ys_out = [torch.cat([y, eos], dim=0) for y in ys] + + # padding for ys with -1 + # pys: utt x olen + ys_in_pad = pad_list(ys_in, self.eos) + ys_out_pad = pad_list(ys_out, self.ignore_id) + + # get dim, length info + batch = ys_out_pad.size(0) + olength = ys_out_pad.size(1) + for idx in range(self.num_encs): + logging.info( + self.__class__.__name__ + + "Number of Encoder:{}; enc{}: input lengths: {}.".format( + self.num_encs, idx + 1, hlens[idx] + ) + ) + logging.info( + self.__class__.__name__ + + " output lengths: " + + str([y.size(0) for y in ys_out]) + ) + + # initialization + c_list = [self.zero_state(hs_pad[0])] + z_list = [self.zero_state(hs_pad[0])] + for _ in six.moves.range(1, self.dlayers): + c_list.append(self.zero_state(hs_pad[0])) + z_list.append(self.zero_state(hs_pad[0])) + z_all = [] + if self.num_encs == 1: + att_w = None + self.att[att_idx].reset() # reset pre-computation of h + else: + att_w_list = [None] * (self.num_encs + 1) # atts + han + att_c_list = [None] * (self.num_encs) # atts + for idx in range(self.num_encs + 1): + self.att[idx].reset() # reset pre-computation of h in atts and han + + # pre-computation of embedding + eys = self.dropout_emb(self.embed(ys_in_pad)) # utt x olen x zdim + + # loop for an output sequence + for i in six.moves.range(olength): + if self.num_encs == 1: + att_c, att_w = self.att[att_idx]( + hs_pad[0], hlens[0], self.dropout_dec[0](z_list[0]), att_w + ) + else: + for idx in range(self.num_encs): + att_c_list[idx], att_w_list[idx] = self.att[idx]( + hs_pad[idx], + hlens[idx], + self.dropout_dec[0](z_list[0]), + att_w_list[idx], + ) + hs_pad_han = torch.stack(att_c_list, dim=1) + hlens_han = [self.num_encs] * len(ys_in) + att_c, att_w_list[self.num_encs] = self.att[self.num_encs]( + hs_pad_han, + hlens_han, + self.dropout_dec[0](z_list[0]), + att_w_list[self.num_encs], + ) + if i > 0 and random.random() < self.sampling_probability: + logging.info(" scheduled sampling ") + z_out = self.output(z_all[-1]) + z_out = np.argmax(z_out.detach().cpu(), axis=1) + z_out = self.dropout_emb(self.embed(to_device(hs_pad[0], z_out))) + ey = torch.cat((z_out, att_c), dim=1) # utt x (zdim + hdim) + else: + ey = torch.cat((eys[:, i, :], att_c), dim=1) # utt x (zdim + hdim) + z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_list, c_list) + if self.context_residual: + z_all.append( + torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) + ) # utt x (zdim + hdim) + else: + z_all.append(self.dropout_dec[-1](z_list[-1])) # utt x (zdim) + + z_all = torch.stack(z_all, dim=1).view(batch * olength, -1) + # compute loss + y_all = self.output(z_all) + self.loss = F.cross_entropy( + y_all, + ys_out_pad.view(-1), + ignore_index=self.ignore_id, + reduction="mean", + ) + # compute perplexity + ppl = math.exp(self.loss.item()) + # -1: eos, which is removed in the loss computation + self.loss *= np.mean([len(x) for x in ys_in]) - 1 + acc = th_accuracy(y_all, ys_out_pad, ignore_label=self.ignore_id) + logging.info("att loss:" + "".join(str(self.loss.item()).split("\n"))) + + # show predicted character sequence for debug + if self.verbose > 0 and self.char_list is not None: + ys_hat = y_all.view(batch, olength, -1) + ys_true = ys_out_pad + for (i, y_hat), y_true in zip( + enumerate(ys_hat.detach().cpu().numpy()), ys_true.detach().cpu().numpy() + ): + if i == MAX_DECODER_OUTPUT: + break + idx_hat = np.argmax(y_hat[y_true != self.ignore_id], axis=1) + idx_true = y_true[y_true != self.ignore_id] + seq_hat = [self.char_list[int(idx)] for idx in idx_hat] + seq_true = [self.char_list[int(idx)] for idx in idx_true] + seq_hat = "".join(seq_hat) + seq_true = "".join(seq_true) + logging.info("groundtruth[%d]: " % i + seq_true) + logging.info("prediction [%d]: " % i + seq_hat) + + if self.labeldist is not None: + if self.vlabeldist is None: + self.vlabeldist = to_device(hs_pad[0], torch.from_numpy(self.labeldist)) + loss_reg = -torch.sum( + (F.log_softmax(y_all, dim=1) * self.vlabeldist).view(-1), dim=0 + ) / len(ys_in) + self.loss = (1.0 - self.lsm_weight) * self.loss + self.lsm_weight * loss_reg + + return self.loss, acc, ppl + + def recognize_beam(self, h, lpz, recog_args, char_list, rnnlm=None, strm_idx=0): + """beam search implementation + + :param torch.Tensor h: encoder hidden state (T, eprojs) + [in multi-encoder case, list of torch.Tensor, + [(T1, eprojs), (T2, eprojs), ...] ] + :param torch.Tensor lpz: ctc log softmax output (T, odim) + [in multi-encoder case, list of torch.Tensor, + [(T1, odim), (T2, odim), ...] ] + :param Namespace recog_args: argument Namespace containing options + :param char_list: list of character strings + :param torch.nn.Module rnnlm: language module + :param int strm_idx: + stream index for speaker parallel attention in multi-speaker case + :return: N-best decoding results + :rtype: list of dicts + """ + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + h = [h] + lpz = [lpz] + if self.num_encs > 1 and lpz is None: + lpz = [lpz] * self.num_encs + + for idx in range(self.num_encs): + logging.info( + "Number of Encoder:{}; enc{}: input lengths: {}.".format( + self.num_encs, idx + 1, h[0].size(0) + ) + ) + att_idx = min(strm_idx, len(self.att) - 1) + # initialization + c_list = [self.zero_state(h[0].unsqueeze(0))] + z_list = [self.zero_state(h[0].unsqueeze(0))] + for _ in six.moves.range(1, self.dlayers): + c_list.append(self.zero_state(h[0].unsqueeze(0))) + z_list.append(self.zero_state(h[0].unsqueeze(0))) + if self.num_encs == 1: + a = None + self.att[att_idx].reset() # reset pre-computation of h + else: + a = [None] * (self.num_encs + 1) # atts + han + att_w_list = [None] * (self.num_encs + 1) # atts + han + att_c_list = [None] * (self.num_encs) # atts + for idx in range(self.num_encs + 1): + self.att[idx].reset() # reset pre-computation of h in atts and han + + # search parms + beam = recog_args.beam_size + penalty = recog_args.penalty + ctc_weight = getattr(recog_args, "ctc_weight", False) # for NMT + + if lpz[0] is not None and self.num_encs > 1: + # weights-ctc, + # e.g. ctc_loss = w_1*ctc_1_loss + w_2 * ctc_2_loss + w_N * ctc_N_loss + weights_ctc_dec = recog_args.weights_ctc_dec / np.sum( + recog_args.weights_ctc_dec + ) # normalize + logging.info( + "ctc weights (decoding): " + " ".join([str(x) for x in weights_ctc_dec]) + ) + else: + weights_ctc_dec = [1.0] + + # preprate sos + if self.replace_sos and recog_args.tgt_lang: + y = char_list.index(recog_args.tgt_lang) + else: + y = self.sos + logging.info(" index: " + str(y)) + logging.info(" mark: " + char_list[y]) + vy = h[0].new_zeros(1).long() + + maxlen = np.amin([h[idx].size(0) for idx in range(self.num_encs)]) + if recog_args.maxlenratio != 0: + # maxlen >= 1 + maxlen = max(1, int(recog_args.maxlenratio * maxlen)) + minlen = int(recog_args.minlenratio * maxlen) + logging.info("max output length: " + str(maxlen)) + logging.info("min output length: " + str(minlen)) + + # initialize hypothesis + if rnnlm: + hyp = { + "score": 0.0, + "yseq": [y], + "c_prev": c_list, + "z_prev": z_list, + "a_prev": a, + "rnnlm_prev": None, + } + else: + hyp = { + "score": 0.0, + "yseq": [y], + "c_prev": c_list, + "z_prev": z_list, + "a_prev": a, + } + if lpz[0] is not None: + ctc_prefix_score = [ + CTCPrefixScore(lpz[idx].detach().numpy(), 0, self.eos, np) + for idx in range(self.num_encs) + ] + hyp["ctc_state_prev"] = [ + ctc_prefix_score[idx].initial_state() for idx in range(self.num_encs) + ] + hyp["ctc_score_prev"] = [0.0] * self.num_encs + if ctc_weight != 1.0: + # pre-pruning based on attention scores + ctc_beam = min(lpz[0].shape[-1], int(beam * CTC_SCORING_RATIO)) + else: + ctc_beam = lpz[0].shape[-1] + hyps = [hyp] + ended_hyps = [] + + for i in six.moves.range(maxlen): + logging.debug("position " + str(i)) + + hyps_best_kept = [] + for hyp in hyps: + vy[0] = hyp["yseq"][i] + ey = self.dropout_emb(self.embed(vy)) # utt list (1) x zdim + if self.num_encs == 1: + att_c, att_w = self.att[att_idx]( + h[0].unsqueeze(0), + [h[0].size(0)], + self.dropout_dec[0](hyp["z_prev"][0]), + hyp["a_prev"], + ) + else: + for idx in range(self.num_encs): + att_c_list[idx], att_w_list[idx] = self.att[idx]( + h[idx].unsqueeze(0), + [h[idx].size(0)], + self.dropout_dec[0](hyp["z_prev"][0]), + hyp["a_prev"][idx], + ) + h_han = torch.stack(att_c_list, dim=1) + att_c, att_w_list[self.num_encs] = self.att[self.num_encs]( + h_han, + [self.num_encs], + self.dropout_dec[0](hyp["z_prev"][0]), + hyp["a_prev"][self.num_encs], + ) + ey = torch.cat((ey, att_c), dim=1) # utt(1) x (zdim + hdim) + z_list, c_list = self.rnn_forward( + ey, z_list, c_list, hyp["z_prev"], hyp["c_prev"] + ) + + # get nbest local scores and their ids + if self.context_residual: + logits = self.output( + torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) + ) + else: + logits = self.output(self.dropout_dec[-1](z_list[-1])) + local_att_scores = F.log_softmax(logits, dim=1) + if rnnlm: + rnnlm_state, local_lm_scores = rnnlm.predict(hyp["rnnlm_prev"], vy) + local_scores = ( + local_att_scores + recog_args.lm_weight * local_lm_scores + ) + else: + local_scores = local_att_scores + + if lpz[0] is not None: + local_best_scores, local_best_ids = torch.topk( + local_att_scores, ctc_beam, dim=1 + ) + ctc_scores, ctc_states = ( + [None] * self.num_encs, + [None] * self.num_encs, + ) + for idx in range(self.num_encs): + ctc_scores[idx], ctc_states[idx] = ctc_prefix_score[idx]( + hyp["yseq"], local_best_ids[0], hyp["ctc_state_prev"][idx] + ) + local_scores = (1.0 - ctc_weight) * local_att_scores[ + :, local_best_ids[0] + ] + if self.num_encs == 1: + local_scores += ctc_weight * torch.from_numpy( + ctc_scores[0] - hyp["ctc_score_prev"][0] + ) + else: + for idx in range(self.num_encs): + local_scores += ( + ctc_weight + * weights_ctc_dec[idx] + * torch.from_numpy( + ctc_scores[idx] - hyp["ctc_score_prev"][idx] + ) + ) + if rnnlm: + local_scores += ( + recog_args.lm_weight * local_lm_scores[:, local_best_ids[0]] + ) + local_best_scores, joint_best_ids = torch.topk( + local_scores, beam, dim=1 + ) + local_best_ids = local_best_ids[:, joint_best_ids[0]] + else: + local_best_scores, local_best_ids = torch.topk( + local_scores, beam, dim=1 + ) + + for j in six.moves.range(beam): + new_hyp = {} + # [:] is needed! + new_hyp["z_prev"] = z_list[:] + new_hyp["c_prev"] = c_list[:] + if self.num_encs == 1: + new_hyp["a_prev"] = att_w[:] + else: + new_hyp["a_prev"] = [ + att_w_list[idx][:] for idx in range(self.num_encs + 1) + ] + new_hyp["score"] = hyp["score"] + local_best_scores[0, j] + new_hyp["yseq"] = [0] * (1 + len(hyp["yseq"])) + new_hyp["yseq"][: len(hyp["yseq"])] = hyp["yseq"] + new_hyp["yseq"][len(hyp["yseq"])] = int(local_best_ids[0, j]) + if rnnlm: + new_hyp["rnnlm_prev"] = rnnlm_state + if lpz[0] is not None: + new_hyp["ctc_state_prev"] = [ + ctc_states[idx][joint_best_ids[0, j]] + for idx in range(self.num_encs) + ] + new_hyp["ctc_score_prev"] = [ + ctc_scores[idx][joint_best_ids[0, j]] + for idx in range(self.num_encs) + ] + # will be (2 x beam) hyps at most + hyps_best_kept.append(new_hyp) + + hyps_best_kept = sorted( + hyps_best_kept, key=lambda x: x["score"], reverse=True + )[:beam] + + # sort and get nbest + hyps = hyps_best_kept + logging.debug("number of pruned hypotheses: " + str(len(hyps))) + logging.debug( + "best hypo: " + + "".join([char_list[int(x)] for x in hyps[0]["yseq"][1:]]) + ) + + # add eos in the final loop to avoid that there are no ended hyps + if i == maxlen - 1: + logging.info("adding in the last position in the loop") + for hyp in hyps: + hyp["yseq"].append(self.eos) + + # add ended hypotheses to a final list, + # and removed them from current hypotheses + # (this will be a problem, number of hyps < beam) + remained_hyps = [] + for hyp in hyps: + if hyp["yseq"][-1] == self.eos: + # only store the sequence that has more than minlen outputs + # also add penalty + if len(hyp["yseq"]) > minlen: + hyp["score"] += (i + 1) * penalty + if rnnlm: # Word LM needs to add final score + hyp["score"] += recog_args.lm_weight * rnnlm.final( + hyp["rnnlm_prev"] + ) + ended_hyps.append(hyp) + else: + remained_hyps.append(hyp) + + # end detection + if end_detect(ended_hyps, i) and recog_args.maxlenratio == 0.0: + logging.info("end detected at %d", i) + break + + hyps = remained_hyps + if len(hyps) > 0: + logging.debug("remaining hypotheses: " + str(len(hyps))) + else: + logging.info("no hypothesis. Finish decoding.") + break + + for hyp in hyps: + logging.debug( + "hypo: " + "".join([char_list[int(x)] for x in hyp["yseq"][1:]]) + ) + + logging.debug("number of ended hypotheses: " + str(len(ended_hyps))) + + nbest_hyps = sorted(ended_hyps, key=lambda x: x["score"], reverse=True)[ + : min(len(ended_hyps), recog_args.nbest) + ] + + # check number of hypotheses + if len(nbest_hyps) == 0: + logging.warning( + "there is no N-best results, " + "perform recognition again with smaller minlenratio." + ) + # should copy because Namespace will be overwritten globally + recog_args = Namespace(**vars(recog_args)) + recog_args.minlenratio = max(0.0, recog_args.minlenratio - 0.1) + if self.num_encs == 1: + return self.recognize_beam(h[0], lpz[0], recog_args, char_list, rnnlm) + else: + return self.recognize_beam(h, lpz, recog_args, char_list, rnnlm) + + logging.info("total log probability: " + str(nbest_hyps[0]["score"])) + logging.info( + "normalized log probability: " + + str(nbest_hyps[0]["score"] / len(nbest_hyps[0]["yseq"])) + ) + + # remove sos + return nbest_hyps + + def recognize_beam_batch( + self, + h, + hlens, + lpz, + recog_args, + char_list, + rnnlm=None, + normalize_score=True, + strm_idx=0, + lang_ids=None, + ): + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + h = [h] + hlens = [hlens] + lpz = [lpz] + if self.num_encs > 1 and lpz is None: + lpz = [lpz] * self.num_encs + + att_idx = min(strm_idx, len(self.att) - 1) + for idx in range(self.num_encs): + logging.info( + "Number of Encoder:{}; enc{}: input lengths: {}.".format( + self.num_encs, idx + 1, h[idx].size(1) + ) + ) + h[idx] = mask_by_length(h[idx], hlens[idx], 0.0) + + # search params + batch = len(hlens[0]) + beam = recog_args.beam_size + penalty = recog_args.penalty + ctc_weight = getattr(recog_args, "ctc_weight", 0) # for NMT + att_weight = 1.0 - ctc_weight + ctc_margin = getattr( + recog_args, "ctc_window_margin", 0 + ) # use getattr to keep compatibility + # weights-ctc, + # e.g. ctc_loss = w_1*ctc_1_loss + w_2 * ctc_2_loss + w_N * ctc_N_loss + if lpz[0] is not None and self.num_encs > 1: + weights_ctc_dec = recog_args.weights_ctc_dec / np.sum( + recog_args.weights_ctc_dec + ) # normalize + logging.info( + "ctc weights (decoding): " + " ".join([str(x) for x in weights_ctc_dec]) + ) + else: + weights_ctc_dec = [1.0] + + n_bb = batch * beam + pad_b = to_device(h[0], torch.arange(batch) * beam).view(-1, 1) + + max_hlen = np.amin([max(hlens[idx]) for idx in range(self.num_encs)]) + if recog_args.maxlenratio == 0: + maxlen = max_hlen + else: + maxlen = max(1, int(recog_args.maxlenratio * max_hlen)) + minlen = int(recog_args.minlenratio * max_hlen) + logging.info("max output length: " + str(maxlen)) + logging.info("min output length: " + str(minlen)) + + # initialization + c_prev = [ + to_device(h[0], torch.zeros(n_bb, self.dunits)) for _ in range(self.dlayers) + ] + z_prev = [ + to_device(h[0], torch.zeros(n_bb, self.dunits)) for _ in range(self.dlayers) + ] + c_list = [ + to_device(h[0], torch.zeros(n_bb, self.dunits)) for _ in range(self.dlayers) + ] + z_list = [ + to_device(h[0], torch.zeros(n_bb, self.dunits)) for _ in range(self.dlayers) + ] + vscores = to_device(h[0], torch.zeros(batch, beam)) + + rnnlm_state = None + if self.num_encs == 1: + a_prev = [None] + att_w_list, ctc_scorer, ctc_state = [None], [None], [None] + self.att[att_idx].reset() # reset pre-computation of h + else: + a_prev = [None] * (self.num_encs + 1) # atts + han + att_w_list = [None] * (self.num_encs + 1) # atts + han + att_c_list = [None] * (self.num_encs) # atts + ctc_scorer, ctc_state = [None] * (self.num_encs), [None] * (self.num_encs) + for idx in range(self.num_encs + 1): + self.att[idx].reset() # reset pre-computation of h in atts and han + + if self.replace_sos and recog_args.tgt_lang: + logging.info(" index: " + str(char_list.index(recog_args.tgt_lang))) + logging.info(" mark: " + recog_args.tgt_lang) + yseq = [ + [char_list.index(recog_args.tgt_lang)] for _ in six.moves.range(n_bb) + ] + elif lang_ids is not None: + # NOTE: used for evaluation during training + yseq = [ + [lang_ids[b // recog_args.beam_size]] for b in six.moves.range(n_bb) + ] + else: + logging.info(" index: " + str(self.sos)) + logging.info(" mark: " + char_list[self.sos]) + yseq = [[self.sos] for _ in six.moves.range(n_bb)] + + accum_odim_ids = [self.sos for _ in six.moves.range(n_bb)] + stop_search = [False for _ in six.moves.range(batch)] + nbest_hyps = [[] for _ in six.moves.range(batch)] + ended_hyps = [[] for _ in range(batch)] + + exp_hlens = [ + hlens[idx].repeat(beam).view(beam, batch).transpose(0, 1).contiguous() + for idx in range(self.num_encs) + ] + exp_hlens = [exp_hlens[idx].view(-1).tolist() for idx in range(self.num_encs)] + exp_h = [ + h[idx].unsqueeze(1).repeat(1, beam, 1, 1).contiguous() + for idx in range(self.num_encs) + ] + exp_h = [ + exp_h[idx].view(n_bb, h[idx].size()[1], h[idx].size()[2]) + for idx in range(self.num_encs) + ] + + if lpz[0] is not None: + scoring_num = min( + int(beam * CTC_SCORING_RATIO) + if att_weight > 0.0 and not lpz[0].is_cuda + else 0, + lpz[0].size(-1), + ) + ctc_scorer = [ + CTCPrefixScoreTH( + lpz[idx], + hlens[idx], + 0, + self.eos, + margin=ctc_margin, + ) + for idx in range(self.num_encs) + ] + + for i in six.moves.range(maxlen): + logging.debug("position " + str(i)) + + vy = to_device(h[0], torch.LongTensor(self._get_last_yseq(yseq))) + ey = self.dropout_emb(self.embed(vy)) + if self.num_encs == 1: + att_c, att_w = self.att[att_idx]( + exp_h[0], exp_hlens[0], self.dropout_dec[0](z_prev[0]), a_prev[0] + ) + att_w_list = [att_w] + else: + for idx in range(self.num_encs): + att_c_list[idx], att_w_list[idx] = self.att[idx]( + exp_h[idx], + exp_hlens[idx], + self.dropout_dec[0](z_prev[0]), + a_prev[idx], + ) + exp_h_han = torch.stack(att_c_list, dim=1) + att_c, att_w_list[self.num_encs] = self.att[self.num_encs]( + exp_h_han, + [self.num_encs] * n_bb, + self.dropout_dec[0](z_prev[0]), + a_prev[self.num_encs], + ) + ey = torch.cat((ey, att_c), dim=1) + + # attention decoder + z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_prev, c_prev) + if self.context_residual: + logits = self.output( + torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) + ) + else: + logits = self.output(self.dropout_dec[-1](z_list[-1])) + local_scores = att_weight * F.log_softmax(logits, dim=1) + + # rnnlm + if rnnlm: + rnnlm_state, local_lm_scores = rnnlm.buff_predict(rnnlm_state, vy, n_bb) + local_scores = local_scores + recog_args.lm_weight * local_lm_scores + + # ctc + if ctc_scorer[0]: + local_scores[:, 0] = self.logzero # avoid choosing blank + part_ids = ( + torch.topk(local_scores, scoring_num, dim=-1)[1] + if scoring_num > 0 + else None + ) + for idx in range(self.num_encs): + att_w = att_w_list[idx] + att_w_ = att_w if isinstance(att_w, torch.Tensor) else att_w[0] + local_ctc_scores, ctc_state[idx] = ctc_scorer[idx]( + yseq, ctc_state[idx], part_ids, att_w_ + ) + local_scores = ( + local_scores + + ctc_weight * weights_ctc_dec[idx] * local_ctc_scores + ) + + local_scores = local_scores.view(batch, beam, self.odim) + if i == 0: + local_scores[:, 1:, :] = self.logzero + + # accumulate scores + eos_vscores = local_scores[:, :, self.eos] + vscores + vscores = vscores.view(batch, beam, 1).repeat(1, 1, self.odim) + vscores[:, :, self.eos] = self.logzero + vscores = (vscores + local_scores).view(batch, -1) + + # global pruning + accum_best_scores, accum_best_ids = torch.topk(vscores, beam, 1) + accum_odim_ids = ( + torch.fmod(accum_best_ids, self.odim).view(-1).data.cpu().tolist() + ) + accum_padded_beam_ids = ( + (accum_best_ids // self.odim + pad_b).view(-1).data.cpu().tolist() + ) + + y_prev = yseq[:][:] + yseq = self._index_select_list(yseq, accum_padded_beam_ids) + yseq = self._append_ids(yseq, accum_odim_ids) + vscores = accum_best_scores + vidx = to_device(h[0], torch.LongTensor(accum_padded_beam_ids)) + + a_prev = [] + num_atts = self.num_encs if self.num_encs == 1 else self.num_encs + 1 + for idx in range(num_atts): + if isinstance(att_w_list[idx], torch.Tensor): + _a_prev = torch.index_select( + att_w_list[idx].view(n_bb, *att_w_list[idx].shape[1:]), 0, vidx + ) + elif isinstance(att_w_list[idx], list): + # handle the case of multi-head attention + _a_prev = [ + torch.index_select(att_w_one.view(n_bb, -1), 0, vidx) + for att_w_one in att_w_list[idx] + ] + else: + # handle the case of location_recurrent when return is a tuple + _a_prev_ = torch.index_select( + att_w_list[idx][0].view(n_bb, -1), 0, vidx + ) + _h_prev_ = torch.index_select( + att_w_list[idx][1][0].view(n_bb, -1), 0, vidx + ) + _c_prev_ = torch.index_select( + att_w_list[idx][1][1].view(n_bb, -1), 0, vidx + ) + _a_prev = (_a_prev_, (_h_prev_, _c_prev_)) + a_prev.append(_a_prev) + z_prev = [ + torch.index_select(z_list[li].view(n_bb, -1), 0, vidx) + for li in range(self.dlayers) + ] + c_prev = [ + torch.index_select(c_list[li].view(n_bb, -1), 0, vidx) + for li in range(self.dlayers) + ] + + # pick ended hyps + if i >= minlen: + k = 0 + penalty_i = (i + 1) * penalty + thr = accum_best_scores[:, -1] + for samp_i in six.moves.range(batch): + if stop_search[samp_i]: + k = k + beam + continue + for beam_j in six.moves.range(beam): + _vscore = None + if eos_vscores[samp_i, beam_j] > thr[samp_i]: + yk = y_prev[k][:] + if len(yk) <= min( + hlens[idx][samp_i] for idx in range(self.num_encs) + ): + _vscore = eos_vscores[samp_i][beam_j] + penalty_i + elif i == maxlen - 1: + yk = yseq[k][:] + _vscore = vscores[samp_i][beam_j] + penalty_i + if _vscore: + yk.append(self.eos) + if rnnlm: + _vscore += recog_args.lm_weight * rnnlm.final( + rnnlm_state, index=k + ) + _score = _vscore.data.cpu().numpy() + ended_hyps[samp_i].append( + {"yseq": yk, "vscore": _vscore, "score": _score} + ) + k = k + 1 + + # end detection + stop_search = [ + stop_search[samp_i] or end_detect(ended_hyps[samp_i], i) + for samp_i in six.moves.range(batch) + ] + stop_search_summary = list(set(stop_search)) + if len(stop_search_summary) == 1 and stop_search_summary[0]: + break + + if rnnlm: + rnnlm_state = self._index_select_lm_state(rnnlm_state, 0, vidx) + if ctc_scorer[0]: + for idx in range(self.num_encs): + ctc_state[idx] = ctc_scorer[idx].index_select_state( + ctc_state[idx], accum_best_ids + ) + + torch.cuda.empty_cache() + + dummy_hyps = [ + {"yseq": [self.sos, self.eos], "score": np.array([-float("inf")])} + ] + ended_hyps = [ + ended_hyps[samp_i] if len(ended_hyps[samp_i]) != 0 else dummy_hyps + for samp_i in six.moves.range(batch) + ] + if normalize_score: + for samp_i in six.moves.range(batch): + for x in ended_hyps[samp_i]: + x["score"] /= len(x["yseq"]) + + nbest_hyps = [ + sorted(ended_hyps[samp_i], key=lambda x: x["score"], reverse=True)[ + : min(len(ended_hyps[samp_i]), recog_args.nbest) + ] + for samp_i in six.moves.range(batch) + ] + + return nbest_hyps + + def calculate_all_attentions(self, hs_pad, hlen, ys_pad, strm_idx=0, lang_ids=None): + """Calculate all of attentions + + :param torch.Tensor hs_pad: batch of padded hidden state sequences + (B, Tmax, D) + in multi-encoder case, list of torch.Tensor, + [(B, Tmax_1, D), (B, Tmax_2, D), ..., ] ] + :param torch.Tensor hlen: batch of lengths of hidden state sequences (B) + [in multi-encoder case, list of torch.Tensor, + [(B), (B), ..., ] + :param torch.Tensor ys_pad: + batch of padded character id sequence tensor (B, Lmax) + :param int strm_idx: + stream index for parallel speaker attention in multi-speaker case + :param torch.Tensor lang_ids: batch of target language id tensor (B, 1) + :return: attention weights with the following shape, + 1) multi-head case => attention weights (B, H, Lmax, Tmax), + 2) multi-encoder case => + [(B, Lmax, Tmax1), (B, Lmax, Tmax2), ..., (B, Lmax, NumEncs)] + 3) other case => attention weights (B, Lmax, Tmax). + :rtype: float ndarray + """ + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + hs_pad = [hs_pad] + hlen = [hlen] + + # TODO(kan-bayashi): need to make more smart way + ys = [y[y != self.ignore_id] for y in ys_pad] # parse padded ys + att_idx = min(strm_idx, len(self.att) - 1) + + # hlen should be list of list of integer + hlen = [list(map(int, hlen[idx])) for idx in range(self.num_encs)] + + self.loss = None + # prepare input and output word sequences with sos/eos IDs + eos = ys[0].new([self.eos]) + sos = ys[0].new([self.sos]) + if self.replace_sos: + ys_in = [torch.cat([idx, y], dim=0) for idx, y in zip(lang_ids, ys)] + else: + ys_in = [torch.cat([sos, y], dim=0) for y in ys] + ys_out = [torch.cat([y, eos], dim=0) for y in ys] + + # padding for ys with -1 + # pys: utt x olen + ys_in_pad = pad_list(ys_in, self.eos) + ys_out_pad = pad_list(ys_out, self.ignore_id) + + # get length info + olength = ys_out_pad.size(1) + + # initialization + c_list = [self.zero_state(hs_pad[0])] + z_list = [self.zero_state(hs_pad[0])] + for _ in six.moves.range(1, self.dlayers): + c_list.append(self.zero_state(hs_pad[0])) + z_list.append(self.zero_state(hs_pad[0])) + att_ws = [] + if self.num_encs == 1: + att_w = None + self.att[att_idx].reset() # reset pre-computation of h + else: + att_w_list = [None] * (self.num_encs + 1) # atts + han + att_c_list = [None] * (self.num_encs) # atts + for idx in range(self.num_encs + 1): + self.att[idx].reset() # reset pre-computation of h in atts and han + + # pre-computation of embedding + eys = self.dropout_emb(self.embed(ys_in_pad)) # utt x olen x zdim + + # loop for an output sequence + for i in six.moves.range(olength): + if self.num_encs == 1: + att_c, att_w = self.att[att_idx]( + hs_pad[0], hlen[0], self.dropout_dec[0](z_list[0]), att_w + ) + att_ws.append(att_w) + else: + for idx in range(self.num_encs): + att_c_list[idx], att_w_list[idx] = self.att[idx]( + hs_pad[idx], + hlen[idx], + self.dropout_dec[0](z_list[0]), + att_w_list[idx], + ) + hs_pad_han = torch.stack(att_c_list, dim=1) + hlen_han = [self.num_encs] * len(ys_in) + att_c, att_w_list[self.num_encs] = self.att[self.num_encs]( + hs_pad_han, + hlen_han, + self.dropout_dec[0](z_list[0]), + att_w_list[self.num_encs], + ) + att_ws.append(att_w_list.copy()) + ey = torch.cat((eys[:, i, :], att_c), dim=1) # utt x (zdim + hdim) + z_list, c_list = self.rnn_forward(ey, z_list, c_list, z_list, c_list) + + if self.num_encs == 1: + # convert to numpy array with the shape (B, Lmax, Tmax) + att_ws = att_to_numpy(att_ws, self.att[att_idx]) + else: + _att_ws = [] + for idx, ws in enumerate(zip(*att_ws)): + ws = att_to_numpy(ws, self.att[idx]) + _att_ws.append(ws) + att_ws = _att_ws + return att_ws + + @staticmethod + def _get_last_yseq(exp_yseq): + last = [] + for y_seq in exp_yseq: + last.append(y_seq[-1]) + return last + + @staticmethod + def _append_ids(yseq, ids): + if isinstance(ids, list): + for i, j in enumerate(ids): + yseq[i].append(j) + else: + for i in range(len(yseq)): + yseq[i].append(ids) + return yseq + + @staticmethod + def _index_select_list(yseq, lst): + new_yseq = [] + for i in lst: + new_yseq.append(yseq[i][:]) + return new_yseq + + @staticmethod + def _index_select_lm_state(rnnlm_state, dim, vidx): + if isinstance(rnnlm_state, dict): + new_state = {} + for k, v in rnnlm_state.items(): + new_state[k] = [torch.index_select(vi, dim, vidx) for vi in v] + elif isinstance(rnnlm_state, list): + new_state = [] + for i in vidx: + new_state.append(rnnlm_state[int(i)][:]) + return new_state + + # scorer interface methods + def init_state(self, x): + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + x = [x] + + c_list = [self.zero_state(x[0].unsqueeze(0))] + z_list = [self.zero_state(x[0].unsqueeze(0))] + for _ in six.moves.range(1, self.dlayers): + c_list.append(self.zero_state(x[0].unsqueeze(0))) + z_list.append(self.zero_state(x[0].unsqueeze(0))) + # TODO(karita): support strm_index for `asr_mix` + strm_index = 0 + att_idx = min(strm_index, len(self.att) - 1) + if self.num_encs == 1: + a = None + self.att[att_idx].reset() # reset pre-computation of h + else: + a = [None] * (self.num_encs + 1) # atts + han + for idx in range(self.num_encs + 1): + self.att[idx].reset() # reset pre-computation of h in atts and han + return dict( + c_prev=c_list[:], + z_prev=z_list[:], + a_prev=a, + workspace=(att_idx, z_list, c_list), + ) + + def score(self, yseq, state, x): + # to support mutiple encoder asr mode, in single encoder mode, + # convert torch.Tensor to List of torch.Tensor + if self.num_encs == 1: + x = [x] + + att_idx, z_list, c_list = state["workspace"] + vy = yseq[-1].unsqueeze(0) + ey = self.dropout_emb(self.embed(vy)) # utt list (1) x zdim + if self.num_encs == 1: + att_c, att_w = self.att[att_idx]( + x[0].unsqueeze(0), + [x[0].size(0)], + self.dropout_dec[0](state["z_prev"][0]), + state["a_prev"], + ) + else: + att_w = [None] * (self.num_encs + 1) # atts + han + att_c_list = [None] * (self.num_encs) # atts + for idx in range(self.num_encs): + att_c_list[idx], att_w[idx] = self.att[idx]( + x[idx].unsqueeze(0), + [x[idx].size(0)], + self.dropout_dec[0](state["z_prev"][0]), + state["a_prev"][idx], + ) + h_han = torch.stack(att_c_list, dim=1) + att_c, att_w[self.num_encs] = self.att[self.num_encs]( + h_han, + [self.num_encs], + self.dropout_dec[0](state["z_prev"][0]), + state["a_prev"][self.num_encs], + ) + ey = torch.cat((ey, att_c), dim=1) # utt(1) x (zdim + hdim) + z_list, c_list = self.rnn_forward( + ey, z_list, c_list, state["z_prev"], state["c_prev"] + ) + if self.context_residual: + logits = self.output( + torch.cat((self.dropout_dec[-1](z_list[-1]), att_c), dim=-1) + ) + else: + logits = self.output(self.dropout_dec[-1](z_list[-1])) + logp = F.log_softmax(logits, dim=1).squeeze(0) + return ( + logp, + dict( + c_prev=c_list[:], + z_prev=z_list[:], + a_prev=att_w, + workspace=(att_idx, z_list, c_list), + ), + ) + + +def decoder_for(args, odim, sos, eos, att, labeldist): + return Decoder( + args.eprojs, + odim, + args.dtype, + args.dlayers, + args.dunits, + sos, + eos, + att, + args.verbose, + args.char_list, + labeldist, + args.lsm_weight, + args.sampling_probability, + args.dropout_rate_decoder, + getattr(args, "context_residual", False), # use getattr to keep compatibility + getattr(args, "replace_sos", False), # use getattr to keep compatibility + getattr(args, "num_encs", 1), + ) # use getattr to keep compatibility diff --git a/funasr/modules/rnn/encoders.py b/funasr/modules/rnn/encoders.py new file mode 100644 index 000000000..a7320c282 --- /dev/null +++ b/funasr/modules/rnn/encoders.py @@ -0,0 +1,372 @@ +import logging + +import numpy as np +import six +import torch +import torch.nn.functional as F +from torch.nn.utils.rnn import pack_padded_sequence +from torch.nn.utils.rnn import pad_packed_sequence + +from funasr.modules.e2e_asr_common import get_vgg2l_odim +from funasr.modules.nets_utils import make_pad_mask +from funasr.modules.nets_utils import to_device + + +class RNNP(torch.nn.Module): + """RNN with projection layer module + + :param int idim: dimension of inputs + :param int elayers: number of encoder layers + :param int cdim: number of rnn units (resulted in cdim * 2 if bidirectional) + :param int hdim: number of projection units + :param np.ndarray subsample: list of subsampling numbers + :param float dropout: dropout rate + :param str typ: The RNN type + """ + + def __init__(self, idim, elayers, cdim, hdim, subsample, dropout, typ="blstm"): + super(RNNP, self).__init__() + bidir = typ[0] == "b" + for i in six.moves.range(elayers): + if i == 0: + inputdim = idim + else: + inputdim = hdim + + RNN = torch.nn.LSTM if "lstm" in typ else torch.nn.GRU + rnn = RNN( + inputdim, cdim, num_layers=1, bidirectional=bidir, batch_first=True + ) + + setattr(self, "%s%d" % ("birnn" if bidir else "rnn", i), rnn) + + # bottleneck layer to merge + if bidir: + setattr(self, "bt%d" % i, torch.nn.Linear(2 * cdim, hdim)) + else: + setattr(self, "bt%d" % i, torch.nn.Linear(cdim, hdim)) + + self.elayers = elayers + self.cdim = cdim + self.subsample = subsample + self.typ = typ + self.bidir = bidir + self.dropout = dropout + + def forward(self, xs_pad, ilens, prev_state=None): + """RNNP forward + + :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, idim) + :param torch.Tensor ilens: batch of lengths of input sequences (B) + :param torch.Tensor prev_state: batch of previous RNN states + :return: batch of hidden state sequences (B, Tmax, hdim) + :rtype: torch.Tensor + """ + logging.debug(self.__class__.__name__ + " input lengths: " + str(ilens)) + elayer_states = [] + for layer in six.moves.range(self.elayers): + if not isinstance(ilens, torch.Tensor): + ilens = torch.tensor(ilens) + xs_pack = pack_padded_sequence(xs_pad, ilens.cpu(), batch_first=True) + rnn = getattr(self, ("birnn" if self.bidir else "rnn") + str(layer)) + rnn.flatten_parameters() + if prev_state is not None and rnn.bidirectional: + prev_state = reset_backward_rnn_state(prev_state) + ys, states = rnn( + xs_pack, hx=None if prev_state is None else prev_state[layer] + ) + elayer_states.append(states) + # ys: utt list of frame x cdim x 2 (2: means bidirectional) + ys_pad, ilens = pad_packed_sequence(ys, batch_first=True) + sub = self.subsample[layer + 1] + if sub > 1: + ys_pad = ys_pad[:, ::sub] + ilens = torch.tensor([int(i + 1) // sub for i in ilens]) + # (sum _utt frame_utt) x dim + projection_layer = getattr(self, "bt%d" % layer) + projected = projection_layer(ys_pad.contiguous().view(-1, ys_pad.size(2))) + xs_pad = projected.view(ys_pad.size(0), ys_pad.size(1), -1) + if layer < self.elayers - 1: + xs_pad = torch.tanh(F.dropout(xs_pad, p=self.dropout)) + + return xs_pad, ilens, elayer_states # x: utt list of frame x dim + + +class RNN(torch.nn.Module): + """RNN module + + :param int idim: dimension of inputs + :param int elayers: number of encoder layers + :param int cdim: number of rnn units (resulted in cdim * 2 if bidirectional) + :param int hdim: number of final projection units + :param float dropout: dropout rate + :param str typ: The RNN type + """ + + def __init__(self, idim, elayers, cdim, hdim, dropout, typ="blstm"): + super(RNN, self).__init__() + bidir = typ[0] == "b" + self.nbrnn = ( + torch.nn.LSTM( + idim, + cdim, + elayers, + batch_first=True, + dropout=dropout, + bidirectional=bidir, + ) + if "lstm" in typ + else torch.nn.GRU( + idim, + cdim, + elayers, + batch_first=True, + dropout=dropout, + bidirectional=bidir, + ) + ) + if bidir: + self.l_last = torch.nn.Linear(cdim * 2, hdim) + else: + self.l_last = torch.nn.Linear(cdim, hdim) + self.typ = typ + + def forward(self, xs_pad, ilens, prev_state=None): + """RNN forward + + :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, D) + :param torch.Tensor ilens: batch of lengths of input sequences (B) + :param torch.Tensor prev_state: batch of previous RNN states + :return: batch of hidden state sequences (B, Tmax, eprojs) + :rtype: torch.Tensor + """ + logging.debug(self.__class__.__name__ + " input lengths: " + str(ilens)) + if not isinstance(ilens, torch.Tensor): + ilens = torch.tensor(ilens) + xs_pack = pack_padded_sequence(xs_pad, ilens.cpu(), batch_first=True) + self.nbrnn.flatten_parameters() + if prev_state is not None and self.nbrnn.bidirectional: + # We assume that when previous state is passed, + # it means that we're streaming the input + # and therefore cannot propagate backward BRNN state + # (otherwise it goes in the wrong direction) + prev_state = reset_backward_rnn_state(prev_state) + ys, states = self.nbrnn(xs_pack, hx=prev_state) + # ys: utt list of frame x cdim x 2 (2: means bidirectional) + ys_pad, ilens = pad_packed_sequence(ys, batch_first=True) + # (sum _utt frame_utt) x dim + projected = torch.tanh( + self.l_last(ys_pad.contiguous().view(-1, ys_pad.size(2))) + ) + xs_pad = projected.view(ys_pad.size(0), ys_pad.size(1), -1) + return xs_pad, ilens, states # x: utt list of frame x dim + + +def reset_backward_rnn_state(states): + """Sets backward BRNN states to zeroes + + Useful in processing of sliding windows over the inputs + """ + if isinstance(states, (list, tuple)): + for state in states: + state[1::2] = 0.0 + else: + states[1::2] = 0.0 + return states + + +class VGG2L(torch.nn.Module): + """VGG-like module + + :param int in_channel: number of input channels + """ + + def __init__(self, in_channel=1): + super(VGG2L, self).__init__() + # CNN layer (VGG motivated) + self.conv1_1 = torch.nn.Conv2d(in_channel, 64, 3, stride=1, padding=1) + self.conv1_2 = torch.nn.Conv2d(64, 64, 3, stride=1, padding=1) + self.conv2_1 = torch.nn.Conv2d(64, 128, 3, stride=1, padding=1) + self.conv2_2 = torch.nn.Conv2d(128, 128, 3, stride=1, padding=1) + + self.in_channel = in_channel + + def forward(self, xs_pad, ilens, **kwargs): + """VGG2L forward + + :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, D) + :param torch.Tensor ilens: batch of lengths of input sequences (B) + :return: batch of padded hidden state sequences (B, Tmax // 4, 128 * D // 4) + :rtype: torch.Tensor + """ + logging.debug(self.__class__.__name__ + " input lengths: " + str(ilens)) + + # x: utt x frame x dim + # xs_pad = F.pad_sequence(xs_pad) + + # x: utt x 1 (input channel num) x frame x dim + xs_pad = xs_pad.view( + xs_pad.size(0), + xs_pad.size(1), + self.in_channel, + xs_pad.size(2) // self.in_channel, + ).transpose(1, 2) + + # NOTE: max_pool1d ? + xs_pad = F.relu(self.conv1_1(xs_pad)) + xs_pad = F.relu(self.conv1_2(xs_pad)) + xs_pad = F.max_pool2d(xs_pad, 2, stride=2, ceil_mode=True) + + xs_pad = F.relu(self.conv2_1(xs_pad)) + xs_pad = F.relu(self.conv2_2(xs_pad)) + xs_pad = F.max_pool2d(xs_pad, 2, stride=2, ceil_mode=True) + if torch.is_tensor(ilens): + ilens = ilens.cpu().numpy() + else: + ilens = np.array(ilens, dtype=np.float32) + ilens = np.array(np.ceil(ilens / 2), dtype=np.int64) + ilens = np.array( + np.ceil(np.array(ilens, dtype=np.float32) / 2), dtype=np.int64 + ).tolist() + + # x: utt_list of frame (remove zeropaded frames) x (input channel num x dim) + xs_pad = xs_pad.transpose(1, 2) + xs_pad = xs_pad.contiguous().view( + xs_pad.size(0), xs_pad.size(1), xs_pad.size(2) * xs_pad.size(3) + ) + return xs_pad, ilens, None # no state in this layer + + +class Encoder(torch.nn.Module): + """Encoder module + + :param str etype: type of encoder network + :param int idim: number of dimensions of encoder network + :param int elayers: number of layers of encoder network + :param int eunits: number of lstm units of encoder network + :param int eprojs: number of projection units of encoder network + :param np.ndarray subsample: list of subsampling numbers + :param float dropout: dropout rate + :param int in_channel: number of input channels + """ + + def __init__( + self, etype, idim, elayers, eunits, eprojs, subsample, dropout, in_channel=1 + ): + super(Encoder, self).__init__() + typ = etype.lstrip("vgg").rstrip("p") + if typ not in ["lstm", "gru", "blstm", "bgru"]: + logging.error("Error: need to specify an appropriate encoder architecture") + + if etype.startswith("vgg"): + if etype[-1] == "p": + self.enc = torch.nn.ModuleList( + [ + VGG2L(in_channel), + RNNP( + get_vgg2l_odim(idim, in_channel=in_channel), + elayers, + eunits, + eprojs, + subsample, + dropout, + typ=typ, + ), + ] + ) + logging.info("Use CNN-VGG + " + typ.upper() + "P for encoder") + else: + self.enc = torch.nn.ModuleList( + [ + VGG2L(in_channel), + RNN( + get_vgg2l_odim(idim, in_channel=in_channel), + elayers, + eunits, + eprojs, + dropout, + typ=typ, + ), + ] + ) + logging.info("Use CNN-VGG + " + typ.upper() + " for encoder") + self.conv_subsampling_factor = 4 + else: + if etype[-1] == "p": + self.enc = torch.nn.ModuleList( + [RNNP(idim, elayers, eunits, eprojs, subsample, dropout, typ=typ)] + ) + logging.info(typ.upper() + " with every-layer projection for encoder") + else: + self.enc = torch.nn.ModuleList( + [RNN(idim, elayers, eunits, eprojs, dropout, typ=typ)] + ) + logging.info(typ.upper() + " without projection for encoder") + self.conv_subsampling_factor = 1 + + def forward(self, xs_pad, ilens, prev_states=None): + """Encoder forward + + :param torch.Tensor xs_pad: batch of padded input sequences (B, Tmax, D) + :param torch.Tensor ilens: batch of lengths of input sequences (B) + :param torch.Tensor prev_state: batch of previous encoder hidden states (?, ...) + :return: batch of hidden state sequences (B, Tmax, eprojs) + :rtype: torch.Tensor + """ + if prev_states is None: + prev_states = [None] * len(self.enc) + assert len(prev_states) == len(self.enc) + + current_states = [] + for module, prev_state in zip(self.enc, prev_states): + xs_pad, ilens, states = module(xs_pad, ilens, prev_state=prev_state) + current_states.append(states) + + # make mask to remove bias value in padded part + mask = to_device(xs_pad, make_pad_mask(ilens).unsqueeze(-1)) + + return xs_pad.masked_fill(mask, 0.0), ilens, current_states + + +def encoder_for(args, idim, subsample): + """Instantiates an encoder module given the program arguments + + :param Namespace args: The arguments + :param int or List of integer idim: dimension of input, e.g. 83, or + List of dimensions of inputs, e.g. [83,83] + :param List or List of List subsample: subsample factors, e.g. [1,2,2,1,1], or + List of subsample factors of each encoder. + e.g. [[1,2,2,1,1], [1,2,2,1,1]] + :rtype torch.nn.Module + :return: The encoder module + """ + num_encs = getattr(args, "num_encs", 1) # use getattr to keep compatibility + if num_encs == 1: + # compatible with single encoder asr mode + return Encoder( + args.etype, + idim, + args.elayers, + args.eunits, + args.eprojs, + subsample, + args.dropout_rate, + ) + elif num_encs >= 1: + enc_list = torch.nn.ModuleList() + for idx in range(num_encs): + enc = Encoder( + args.etype[idx], + idim[idx], + args.elayers[idx], + args.eunits[idx], + args.eprojs, + subsample[idx], + args.dropout_rate[idx], + ) + enc_list.append(enc) + return enc_list + else: + raise ValueError( + "Number of encoders needs to be more than one. {}".format(num_encs) + ) diff --git a/funasr/modules/scorers/__init__.py b/funasr/modules/scorers/__init__.py new file mode 100644 index 000000000..b7f177368 --- /dev/null +++ b/funasr/modules/scorers/__init__.py @@ -0,0 +1 @@ +"""Initialize sub package.""" diff --git a/funasr/modules/scorers/ctc.py b/funasr/modules/scorers/ctc.py new file mode 100644 index 000000000..61deace59 --- /dev/null +++ b/funasr/modules/scorers/ctc.py @@ -0,0 +1,158 @@ +"""ScorerInterface implementation for CTC.""" + +import numpy as np +import torch + +from funasr.modules.scorers.ctc_prefix_score import CTCPrefixScore +from funasr.modules.scorers.ctc_prefix_score import CTCPrefixScoreTH +from funasr.modules.scorers.scorer_interface import BatchPartialScorerInterface + + +class CTCPrefixScorer(BatchPartialScorerInterface): + """Decoder interface wrapper for CTCPrefixScore.""" + + def __init__(self, ctc: torch.nn.Module, eos: int): + """Initialize class. + + Args: + ctc (torch.nn.Module): The CTC implementation. + For example, :class:`espnet.nets.pytorch_backend.ctc.CTC` + eos (int): The end-of-sequence id. + + """ + self.ctc = ctc + self.eos = eos + self.impl = None + + def init_state(self, x: torch.Tensor): + """Get an initial state for decoding. + + Args: + x (torch.Tensor): The encoded feature tensor + + Returns: initial state + + """ + logp = self.ctc.log_softmax(x.unsqueeze(0)).detach().squeeze(0).cpu().numpy() + # TODO(karita): use CTCPrefixScoreTH + self.impl = CTCPrefixScore(logp, 0, self.eos, np) + return 0, self.impl.initial_state() + + def select_state(self, state, i, new_id=None): + """Select state with relative ids in the main beam search. + + Args: + state: Decoder state for prefix tokens + i (int): Index to select a state in the main beam search + new_id (int): New label id to select a state if necessary + + Returns: + state: pruned state + + """ + if type(state) == tuple: + if len(state) == 2: # for CTCPrefixScore + sc, st = state + return sc[i], st[i] + else: # for CTCPrefixScoreTH (need new_id > 0) + r, log_psi, f_min, f_max, scoring_idmap = state + s = log_psi[i, new_id].expand(log_psi.size(1)) + if scoring_idmap is not None: + return r[:, :, i, scoring_idmap[i, new_id]], s, f_min, f_max + else: + return r[:, :, i, new_id], s, f_min, f_max + return None if state is None else state[i] + + def score_partial(self, y, ids, state, x): + """Score new token. + + Args: + y (torch.Tensor): 1D prefix token + next_tokens (torch.Tensor): torch.int64 next token to score + state: decoder state for prefix tokens + x (torch.Tensor): 2D encoder feature that generates ys + + Returns: + tuple[torch.Tensor, Any]: + Tuple of a score tensor for y that has a shape `(len(next_tokens),)` + and next state for ys + + """ + prev_score, state = state + presub_score, new_st = self.impl(y.cpu(), ids.cpu(), state) + tscore = torch.as_tensor( + presub_score - prev_score, device=x.device, dtype=x.dtype + ) + return tscore, (presub_score, new_st) + + def batch_init_state(self, x: torch.Tensor): + """Get an initial state for decoding. + + Args: + x (torch.Tensor): The encoded feature tensor + + Returns: initial state + + """ + logp = self.ctc.log_softmax(x.unsqueeze(0)) # assuming batch_size = 1 + xlen = torch.tensor([logp.size(1)]) + self.impl = CTCPrefixScoreTH(logp, xlen, 0, self.eos) + return None + + def batch_score_partial(self, y, ids, state, x): + """Score new token. + + Args: + y (torch.Tensor): 1D prefix token + ids (torch.Tensor): torch.int64 next token to score + state: decoder state for prefix tokens + x (torch.Tensor): 2D encoder feature that generates ys + + Returns: + tuple[torch.Tensor, Any]: + Tuple of a score tensor for y that has a shape `(len(next_tokens),)` + and next state for ys + + """ + batch_state = ( + ( + torch.stack([s[0] for s in state], dim=2), + torch.stack([s[1] for s in state]), + state[0][2], + state[0][3], + ) + if state[0] is not None + else None + ) + return self.impl(y, batch_state, ids) + + def extend_prob(self, x: torch.Tensor): + """Extend probs for decoding. + + This extension is for streaming decoding + as in Eq (14) in https://arxiv.org/abs/2006.14941 + + Args: + x (torch.Tensor): The encoded feature tensor + + """ + logp = self.ctc.log_softmax(x.unsqueeze(0)) + self.impl.extend_prob(logp) + + def extend_state(self, state): + """Extend state for decoding. + + This extension is for streaming decoding + as in Eq (14) in https://arxiv.org/abs/2006.14941 + + Args: + state: The states of hyps + + Returns: exteded state + + """ + new_state = [] + for s in state: + new_state.append(self.impl.extend_state(s)) + + return new_state diff --git a/funasr/modules/scorers/ctc_prefix_score.py b/funasr/modules/scorers/ctc_prefix_score.py new file mode 100644 index 000000000..0c67ecd09 --- /dev/null +++ b/funasr/modules/scorers/ctc_prefix_score.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python3 + +# Copyright 2018 Mitsubishi Electric Research Labs (Takaaki Hori) +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import torch + +import numpy as np +import six + + +class CTCPrefixScoreTH(object): + """Batch processing of CTCPrefixScore + + which is based on Algorithm 2 in WATANABE et al. + "HYBRID CTC/ATTENTION ARCHITECTURE FOR END-TO-END SPEECH RECOGNITION," + but extended to efficiently compute the label probablities for multiple + hypotheses simultaneously + See also Seki et al. "Vectorized Beam Search for CTC-Attention-Based + Speech Recognition," In INTERSPEECH (pp. 3825-3829), 2019. + """ + + def __init__(self, x, xlens, blank, eos, margin=0): + """Construct CTC prefix scorer + + :param torch.Tensor x: input label posterior sequences (B, T, O) + :param torch.Tensor xlens: input lengths (B,) + :param int blank: blank label id + :param int eos: end-of-sequence id + :param int margin: margin parameter for windowing (0 means no windowing) + """ + # In the comment lines, + # we assume T: input_length, B: batch size, W: beam width, O: output dim. + self.logzero = -10000000000.0 + self.blank = blank + self.eos = eos + self.batch = x.size(0) + self.input_length = x.size(1) + self.odim = x.size(2) + self.dtype = x.dtype + self.device = ( + torch.device("cuda:%d" % x.get_device()) + if x.is_cuda + else torch.device("cpu") + ) + # Pad the rest of posteriors in the batch + # TODO(takaaki-hori): need a better way without for-loops + for i, l in enumerate(xlens): + if l < self.input_length: + x[i, l:, :] = self.logzero + x[i, l:, blank] = 0 + # Reshape input x + xn = x.transpose(0, 1) # (B, T, O) -> (T, B, O) + xb = xn[:, :, self.blank].unsqueeze(2).expand(-1, -1, self.odim) + self.x = torch.stack([xn, xb]) # (2, T, B, O) + self.end_frames = torch.as_tensor(xlens) - 1 + + # Setup CTC windowing + self.margin = margin + if margin > 0: + self.frame_ids = torch.arange( + self.input_length, dtype=self.dtype, device=self.device + ) + # Base indices for index conversion + self.idx_bh = None + self.idx_b = torch.arange(self.batch, device=self.device) + self.idx_bo = (self.idx_b * self.odim).unsqueeze(1) + + def __call__(self, y, state, scoring_ids=None, att_w=None): + """Compute CTC prefix scores for next labels + + :param list y: prefix label sequences + :param tuple state: previous CTC state + :param torch.Tensor pre_scores: scores for pre-selection of hypotheses (BW, O) + :param torch.Tensor att_w: attention weights to decide CTC window + :return new_state, ctc_local_scores (BW, O) + """ + output_length = len(y[0]) - 1 # ignore sos + last_ids = [yi[-1] for yi in y] # last output label ids + n_bh = len(last_ids) # batch * hyps + n_hyps = n_bh // self.batch # assuming each utterance has the same # of hyps + self.scoring_num = scoring_ids.size(-1) if scoring_ids is not None else 0 + # prepare state info + if state is None: + r_prev = torch.full( + (self.input_length, 2, self.batch, n_hyps), + self.logzero, + dtype=self.dtype, + device=self.device, + ) + r_prev[:, 1] = torch.cumsum(self.x[0, :, :, self.blank], 0).unsqueeze(2) + r_prev = r_prev.view(-1, 2, n_bh) + s_prev = 0.0 + f_min_prev = 0 + f_max_prev = 1 + else: + r_prev, s_prev, f_min_prev, f_max_prev = state + + # select input dimensions for scoring + if self.scoring_num > 0: + scoring_idmap = torch.full( + (n_bh, self.odim), -1, dtype=torch.long, device=self.device + ) + snum = self.scoring_num + if self.idx_bh is None or n_bh > len(self.idx_bh): + self.idx_bh = torch.arange(n_bh, device=self.device).view(-1, 1) + scoring_idmap[self.idx_bh[:n_bh], scoring_ids] = torch.arange( + snum, device=self.device + ) + scoring_idx = ( + scoring_ids + self.idx_bo.repeat(1, n_hyps).view(-1, 1) + ).view(-1) + x_ = torch.index_select( + self.x.view(2, -1, self.batch * self.odim), 2, scoring_idx + ).view(2, -1, n_bh, snum) + else: + scoring_ids = None + scoring_idmap = None + snum = self.odim + x_ = self.x.unsqueeze(3).repeat(1, 1, 1, n_hyps, 1).view(2, -1, n_bh, snum) + + # new CTC forward probs are prepared as a (T x 2 x BW x S) tensor + # that corresponds to r_t^n(h) and r_t^b(h) in a batch. + r = torch.full( + (self.input_length, 2, n_bh, snum), + self.logzero, + dtype=self.dtype, + device=self.device, + ) + if output_length == 0: + r[0, 0] = x_[0, 0] + + r_sum = torch.logsumexp(r_prev, 1) + log_phi = r_sum.unsqueeze(2).repeat(1, 1, snum) + if scoring_ids is not None: + for idx in range(n_bh): + pos = scoring_idmap[idx, last_ids[idx]] + if pos >= 0: + log_phi[:, idx, pos] = r_prev[:, 1, idx] + else: + for idx in range(n_bh): + log_phi[:, idx, last_ids[idx]] = r_prev[:, 1, idx] + + # decide start and end frames based on attention weights + if att_w is not None and self.margin > 0: + f_arg = torch.matmul(att_w, self.frame_ids) + f_min = max(int(f_arg.min().cpu()), f_min_prev) + f_max = max(int(f_arg.max().cpu()), f_max_prev) + start = min(f_max_prev, max(f_min - self.margin, output_length, 1)) + end = min(f_max + self.margin, self.input_length) + else: + f_min = f_max = 0 + start = max(output_length, 1) + end = self.input_length + + # compute forward probabilities log(r_t^n(h)) and log(r_t^b(h)) + for t in range(start, end): + rp = r[t - 1] + rr = torch.stack([rp[0], log_phi[t - 1], rp[0], rp[1]]).view( + 2, 2, n_bh, snum + ) + r[t] = torch.logsumexp(rr, 1) + x_[:, t] + + # compute log prefix probabilities log(psi) + log_phi_x = torch.cat((log_phi[0].unsqueeze(0), log_phi[:-1]), dim=0) + x_[0] + if scoring_ids is not None: + log_psi = torch.full( + (n_bh, self.odim), self.logzero, dtype=self.dtype, device=self.device + ) + log_psi_ = torch.logsumexp( + torch.cat((log_phi_x[start:end], r[start - 1, 0].unsqueeze(0)), dim=0), + dim=0, + ) + for si in range(n_bh): + log_psi[si, scoring_ids[si]] = log_psi_[si] + else: + log_psi = torch.logsumexp( + torch.cat((log_phi_x[start:end], r[start - 1, 0].unsqueeze(0)), dim=0), + dim=0, + ) + + for si in range(n_bh): + log_psi[si, self.eos] = r_sum[self.end_frames[si // n_hyps], si] + + # exclude blank probs + log_psi[:, self.blank] = self.logzero + + return (log_psi - s_prev), (r, log_psi, f_min, f_max, scoring_idmap) + + def index_select_state(self, state, best_ids): + """Select CTC states according to best ids + + :param state : CTC state + :param best_ids : index numbers selected by beam pruning (B, W) + :return selected_state + """ + r, s, f_min, f_max, scoring_idmap = state + # convert ids to BHO space + n_bh = len(s) + n_hyps = n_bh // self.batch + vidx = (best_ids + (self.idx_b * (n_hyps * self.odim)).view(-1, 1)).view(-1) + # select hypothesis scores + s_new = torch.index_select(s.view(-1), 0, vidx) + s_new = s_new.view(-1, 1).repeat(1, self.odim).view(n_bh, self.odim) + # convert ids to BHS space (S: scoring_num) + if scoring_idmap is not None: + snum = self.scoring_num + hyp_idx = (best_ids // self.odim + (self.idx_b * n_hyps).view(-1, 1)).view( + -1 + ) + label_ids = torch.fmod(best_ids, self.odim).view(-1) + score_idx = scoring_idmap[hyp_idx, label_ids] + score_idx[score_idx == -1] = 0 + vidx = score_idx + hyp_idx * snum + else: + snum = self.odim + # select forward probabilities + r_new = torch.index_select(r.view(-1, 2, n_bh * snum), 2, vidx).view( + -1, 2, n_bh + ) + return r_new, s_new, f_min, f_max + + def extend_prob(self, x): + """Extend CTC prob. + + :param torch.Tensor x: input label posterior sequences (B, T, O) + """ + + if self.x.shape[1] < x.shape[1]: # self.x (2,T,B,O); x (B,T,O) + # Pad the rest of posteriors in the batch + # TODO(takaaki-hori): need a better way without for-loops + xlens = [x.size(1)] + for i, l in enumerate(xlens): + if l < self.input_length: + x[i, l:, :] = self.logzero + x[i, l:, self.blank] = 0 + tmp_x = self.x + xn = x.transpose(0, 1) # (B, T, O) -> (T, B, O) + xb = xn[:, :, self.blank].unsqueeze(2).expand(-1, -1, self.odim) + self.x = torch.stack([xn, xb]) # (2, T, B, O) + self.x[:, : tmp_x.shape[1], :, :] = tmp_x + self.input_length = x.size(1) + self.end_frames = torch.as_tensor(xlens) - 1 + + def extend_state(self, state): + """Compute CTC prefix state. + + + :param state : CTC state + :return ctc_state + """ + + if state is None: + # nothing to do + return state + else: + r_prev, s_prev, f_min_prev, f_max_prev = state + + r_prev_new = torch.full( + (self.input_length, 2), + self.logzero, + dtype=self.dtype, + device=self.device, + ) + start = max(r_prev.shape[0], 1) + r_prev_new[0:start] = r_prev + for t in six.moves.range(start, self.input_length): + r_prev_new[t, 1] = r_prev_new[t - 1, 1] + self.x[0, t, :, self.blank] + + return (r_prev_new, s_prev, f_min_prev, f_max_prev) + + +class CTCPrefixScore(object): + """Compute CTC label sequence scores + + which is based on Algorithm 2 in WATANABE et al. + "HYBRID CTC/ATTENTION ARCHITECTURE FOR END-TO-END SPEECH RECOGNITION," + but extended to efficiently compute the probablities of multiple labels + simultaneously + """ + + def __init__(self, x, blank, eos, xp): + self.xp = xp + self.logzero = -10000000000.0 + self.blank = blank + self.eos = eos + self.input_length = len(x) + self.x = x + + def initial_state(self): + """Obtain an initial CTC state + + :return: CTC state + """ + # initial CTC state is made of a frame x 2 tensor that corresponds to + # r_t^n() and r_t^b(), where 0 and 1 of axis=1 represent + # superscripts n and b (non-blank and blank), respectively. + r = self.xp.full((self.input_length, 2), self.logzero, dtype=np.float32) + r[0, 1] = self.x[0, self.blank] + for i in six.moves.range(1, self.input_length): + r[i, 1] = r[i - 1, 1] + self.x[i, self.blank] + return r + + def __call__(self, y, cs, r_prev): + """Compute CTC prefix scores for next labels + + :param y : prefix label sequence + :param cs : array of next labels + :param r_prev: previous CTC state + :return ctc_scores, ctc_states + """ + # initialize CTC states + output_length = len(y) - 1 # ignore sos + # new CTC states are prepared as a frame x (n or b) x n_labels tensor + # that corresponds to r_t^n(h) and r_t^b(h). + r = self.xp.ndarray((self.input_length, 2, len(cs)), dtype=np.float32) + xs = self.x[:, cs] + if output_length == 0: + r[0, 0] = xs[0] + r[0, 1] = self.logzero + else: + r[output_length - 1] = self.logzero + + # prepare forward probabilities for the last label + r_sum = self.xp.logaddexp( + r_prev[:, 0], r_prev[:, 1] + ) # log(r_t^n(g) + r_t^b(g)) + last = y[-1] + if output_length > 0 and last in cs: + log_phi = self.xp.ndarray((self.input_length, len(cs)), dtype=np.float32) + for i in six.moves.range(len(cs)): + log_phi[:, i] = r_sum if cs[i] != last else r_prev[:, 1] + else: + log_phi = r_sum + + # compute forward probabilities log(r_t^n(h)), log(r_t^b(h)), + # and log prefix probabilities log(psi) + start = max(output_length, 1) + log_psi = r[start - 1, 0] + for t in six.moves.range(start, self.input_length): + r[t, 0] = self.xp.logaddexp(r[t - 1, 0], log_phi[t - 1]) + xs[t] + r[t, 1] = ( + self.xp.logaddexp(r[t - 1, 0], r[t - 1, 1]) + self.x[t, self.blank] + ) + log_psi = self.xp.logaddexp(log_psi, log_phi[t - 1] + xs[t]) + + # get P(...eos|X) that ends with the prefix itself + eos_pos = self.xp.where(cs == self.eos)[0] + if len(eos_pos) > 0: + log_psi[eos_pos] = r_sum[-1] # log(r_T^n(g) + r_T^b(g)) + + # exclude blank probs + blank_pos = self.xp.where(cs == self.blank)[0] + if len(blank_pos) > 0: + log_psi[blank_pos] = self.logzero + + # return the log prefix probability and CTC states, where the label axis + # of the CTC states is moved to the first axis to slice it easily + return log_psi, self.xp.rollaxis(r, 2) diff --git a/funasr/modules/scorers/length_bonus.py b/funasr/modules/scorers/length_bonus.py new file mode 100644 index 000000000..7f576e04f --- /dev/null +++ b/funasr/modules/scorers/length_bonus.py @@ -0,0 +1,61 @@ +"""Length bonus module.""" +from typing import Any +from typing import List +from typing import Tuple + +import torch + +from funasr.modules.scorers.scorer_interface import BatchScorerInterface + + +class LengthBonus(BatchScorerInterface): + """Length bonus in beam search.""" + + def __init__(self, n_vocab: int): + """Initialize class. + + Args: + n_vocab (int): The number of tokens in vocabulary for beam search + + """ + self.n = n_vocab + + def score(self, y, state, x): + """Score new token. + + Args: + y (torch.Tensor): 1D torch.int64 prefix tokens. + state: Scorer state for prefix tokens + x (torch.Tensor): 2D encoder feature that generates ys. + + Returns: + tuple[torch.Tensor, Any]: Tuple of + torch.float32 scores for next token (n_vocab) + and None + + """ + return torch.tensor([1.0], device=x.device, dtype=x.dtype).expand(self.n), None + + def batch_score( + self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor + ) -> Tuple[torch.Tensor, List[Any]]: + """Score new token batch. + + Args: + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). + + Returns: + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, n_vocab)` + and next state list for ys. + + """ + return ( + torch.tensor([1.0], device=xs.device, dtype=xs.dtype).expand( + ys.shape[0], self.n + ), + None, + ) diff --git a/funasr/modules/scorers/scorer_interface.py b/funasr/modules/scorers/scorer_interface.py new file mode 100644 index 000000000..946ec6be3 --- /dev/null +++ b/funasr/modules/scorers/scorer_interface.py @@ -0,0 +1,188 @@ +"""Scorer interface module.""" + +from typing import Any +from typing import List +from typing import Tuple + +import torch +import warnings + + +class ScorerInterface: + """Scorer interface for beam search. + + The scorer performs scoring of the all tokens in vocabulary. + + Examples: + * Search heuristics + * :class:`espnet.nets.scorers.length_bonus.LengthBonus` + * Decoder networks of the sequence-to-sequence models + * :class:`espnet.nets.pytorch_backend.nets.transformer.decoder.Decoder` + * :class:`espnet.nets.pytorch_backend.nets.rnn.decoders.Decoder` + * Neural language models + * :class:`espnet.nets.pytorch_backend.lm.transformer.TransformerLM` + * :class:`espnet.nets.pytorch_backend.lm.default.DefaultRNNLM` + * :class:`espnet.nets.pytorch_backend.lm.seq_rnn.SequentialRNNLM` + + """ + + def init_state(self, x: torch.Tensor) -> Any: + """Get an initial state for decoding (optional). + + Args: + x (torch.Tensor): The encoded feature tensor + + Returns: initial state + + """ + return None + + def select_state(self, state: Any, i: int, new_id: int = None) -> Any: + """Select state with relative ids in the main beam search. + + Args: + state: Decoder state for prefix tokens + i (int): Index to select a state in the main beam search + new_id (int): New label index to select a state if necessary + + Returns: + state: pruned state + + """ + return None if state is None else state[i] + + def score( + self, y: torch.Tensor, state: Any, x: torch.Tensor + ) -> Tuple[torch.Tensor, Any]: + """Score new token (required). + + Args: + y (torch.Tensor): 1D torch.int64 prefix tokens. + state: Scorer state for prefix tokens + x (torch.Tensor): The encoder feature that generates ys. + + Returns: + tuple[torch.Tensor, Any]: Tuple of + scores for next token that has a shape of `(n_vocab)` + and next state for ys + + """ + raise NotImplementedError + + def final_score(self, state: Any) -> float: + """Score eos (optional). + + Args: + state: Scorer state for prefix tokens + + Returns: + float: final score + + """ + return 0.0 + + +class BatchScorerInterface(ScorerInterface): + """Batch scorer interface.""" + + def batch_init_state(self, x: torch.Tensor) -> Any: + """Get an initial state for decoding (optional). + + Args: + x (torch.Tensor): The encoded feature tensor + + Returns: initial state + + """ + return self.init_state(x) + + def batch_score( + self, ys: torch.Tensor, states: List[Any], xs: torch.Tensor + ) -> Tuple[torch.Tensor, List[Any]]: + """Score new token batch (required). + + Args: + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). + + Returns: + tuple[torch.Tensor, List[Any]]: Tuple of + batchfied scores for next token with shape of `(n_batch, n_vocab)` + and next state list for ys. + + """ + warnings.warn( + "{} batch score is implemented through for loop not parallelized".format( + self.__class__.__name__ + ) + ) + scores = list() + outstates = list() + for i, (y, state, x) in enumerate(zip(ys, states, xs)): + score, outstate = self.score(y, state, x) + outstates.append(outstate) + scores.append(score) + scores = torch.cat(scores, 0).view(ys.shape[0], -1) + return scores, outstates + + +class PartialScorerInterface(ScorerInterface): + """Partial scorer interface for beam search. + + The partial scorer performs scoring when non-partial scorer finished scoring, + and receives pre-pruned next tokens to score because it is too heavy to score + all the tokens. + + Examples: + * Prefix search for connectionist-temporal-classification models + * :class:`espnet.nets.scorers.ctc.CTCPrefixScorer` + + """ + + def score_partial( + self, y: torch.Tensor, next_tokens: torch.Tensor, state: Any, x: torch.Tensor + ) -> Tuple[torch.Tensor, Any]: + """Score new token (required). + + Args: + y (torch.Tensor): 1D prefix token + next_tokens (torch.Tensor): torch.int64 next token to score + state: decoder state for prefix tokens + x (torch.Tensor): The encoder feature that generates ys + + Returns: + tuple[torch.Tensor, Any]: + Tuple of a score tensor for y that has a shape `(len(next_tokens),)` + and next state for ys + + """ + raise NotImplementedError + + +class BatchPartialScorerInterface(BatchScorerInterface, PartialScorerInterface): + """Batch partial scorer interface for beam search.""" + + def batch_score_partial( + self, + ys: torch.Tensor, + next_tokens: torch.Tensor, + states: List[Any], + xs: torch.Tensor, + ) -> Tuple[torch.Tensor, Any]: + """Score new token (required). + + Args: + ys (torch.Tensor): torch.int64 prefix tokens (n_batch, ylen). + next_tokens (torch.Tensor): torch.int64 tokens to score (n_batch, n_token). + states (List[Any]): Scorer states for prefix tokens. + xs (torch.Tensor): + The encoder feature that generates ys (n_batch, xlen, n_feat). + + Returns: + tuple[torch.Tensor, Any]: + Tuple of a score tensor for ys that has a shape `(n_batch, n_vocab)` + and next states for ys + """ + raise NotImplementedError diff --git a/funasr/modules/streaming_utils/chunk_utilis.py b/funasr/modules/streaming_utils/chunk_utilis.py new file mode 100644 index 000000000..ea37c68cc --- /dev/null +++ b/funasr/modules/streaming_utils/chunk_utilis.py @@ -0,0 +1,390 @@ + +import torch +import numpy as np +import math +from funasr.modules.nets_utils import make_pad_mask +import logging +import torch.nn.functional as F +from funasr.modules.streaming_utils.utils import sequence_mask + + + +class overlap_chunk(): + """ + author: Speech Lab, Alibaba Group, China + San-m: Memory equipped self-attention for end-to-end speech recognition + https://arxiv.org/abs/2006.01713 + + """ + def __init__(self, + chunk_size: tuple = (16,), + stride: tuple = (10,), + pad_left: tuple = (0,), + encoder_att_look_back_factor: tuple = (1,), + shfit_fsmn: int = 0, + decoder_att_look_back_factor: tuple = (1,), + ): + + pad_left = self.check_chunk_size_args(chunk_size, pad_left) + encoder_att_look_back_factor = self.check_chunk_size_args(chunk_size, encoder_att_look_back_factor) + decoder_att_look_back_factor = self.check_chunk_size_args(chunk_size, decoder_att_look_back_factor) + self.chunk_size, self.stride, self.pad_left, self.encoder_att_look_back_factor, self.decoder_att_look_back_factor \ + = chunk_size, stride, pad_left, encoder_att_look_back_factor, decoder_att_look_back_factor + self.shfit_fsmn = shfit_fsmn + self.x_add_mask = None + self.x_rm_mask = None + self.x_len = None + self.mask_shfit_chunk = None + self.mask_chunk_predictor = None + self.mask_att_chunk_encoder = None + self.mask_shift_att_chunk_decoder = None + self.chunk_outs = None + self.chunk_size_cur, self.stride_cur, self.pad_left_cur, self.encoder_att_look_back_factor_cur, self.chunk_size_pad_shift_cur \ + = None, None, None, None, None + + def check_chunk_size_args(self, chunk_size, x): + if len(x) < len(chunk_size): + x = [x[0] for i in chunk_size] + return x + + def get_chunk_size(self, + ind: int = 0 + ): + # with torch.no_grad: + chunk_size, stride, pad_left, encoder_att_look_back_factor, decoder_att_look_back_factor = \ + self.chunk_size[ind], self.stride[ind], self.pad_left[ind], self.encoder_att_look_back_factor[ind], self.decoder_att_look_back_factor[ind] + self.chunk_size_cur, self.stride_cur, self.pad_left_cur, self.encoder_att_look_back_factor_cur, self.chunk_size_pad_shift_cur, self.decoder_att_look_back_factor_cur \ + = chunk_size, stride, pad_left, encoder_att_look_back_factor, chunk_size + self.shfit_fsmn, decoder_att_look_back_factor + return self.chunk_size_cur, self.stride_cur, self.pad_left_cur, self.encoder_att_look_back_factor_cur, self.chunk_size_pad_shift_cur + + def random_choice(self, training=True, decoding_ind=None): + chunk_num = len(self.chunk_size) + ind = 0 + if training and chunk_num > 1: + ind = torch.randint(0, chunk_num-1, ()).cpu().item() + if not training and decoding_ind is not None: + ind = int(decoding_ind) + + return ind + + + + + def gen_chunk_mask(self, x_len, ind=0, num_units=1, num_units_predictor=1): + + with torch.no_grad(): + x_len = x_len.cpu().numpy() + x_len_max = x_len.max() + + chunk_size, stride, pad_left, encoder_att_look_back_factor, chunk_size_pad_shift = self.get_chunk_size(ind) + shfit_fsmn = self.shfit_fsmn + pad_right = chunk_size - stride - pad_left + + chunk_num_batch = np.ceil(x_len/stride).astype(np.int32) + x_len_chunk = (chunk_num_batch-1) * chunk_size_pad_shift + shfit_fsmn + pad_left + 0 + x_len - (chunk_num_batch-1) * stride + x_len_chunk = x_len_chunk.astype(x_len.dtype) + x_len_chunk_max = x_len_chunk.max() + + chunk_num = int(math.ceil(x_len_max/stride)) + dtype = np.int32 + max_len_for_x_mask_tmp = max(chunk_size, x_len_max + pad_left) + x_add_mask = np.zeros([0, max_len_for_x_mask_tmp], dtype=dtype) + x_rm_mask = np.zeros([max_len_for_x_mask_tmp, 0], dtype=dtype) + mask_shfit_chunk = np.zeros([0, num_units], dtype=dtype) + mask_chunk_predictor = np.zeros([0, num_units_predictor], dtype=dtype) + mask_shift_att_chunk_decoder = np.zeros([0, 1], dtype=dtype) + mask_att_chunk_encoder = np.zeros([0, chunk_num*chunk_size_pad_shift], dtype=dtype) + for chunk_ids in range(chunk_num): + # x_mask add + fsmn_padding = np.zeros((shfit_fsmn, max_len_for_x_mask_tmp), dtype=dtype) + x_mask_cur = np.diag(np.ones(chunk_size, dtype=np.float32)) + x_mask_pad_left = np.zeros((chunk_size, chunk_ids * stride), dtype=dtype) + x_mask_pad_right = np.zeros((chunk_size, max_len_for_x_mask_tmp), dtype=dtype) + x_cur_pad = np.concatenate([x_mask_pad_left, x_mask_cur, x_mask_pad_right], axis=1) + x_cur_pad = x_cur_pad[:chunk_size, :max_len_for_x_mask_tmp] + x_add_mask_fsmn = np.concatenate([fsmn_padding, x_cur_pad], axis=0) + x_add_mask = np.concatenate([x_add_mask, x_add_mask_fsmn], axis=0) + + # x_mask rm + fsmn_padding = np.zeros((max_len_for_x_mask_tmp, shfit_fsmn),dtype=dtype) + padding_mask_left = np.zeros((max_len_for_x_mask_tmp, pad_left),dtype=dtype) + padding_mask_right = np.zeros((max_len_for_x_mask_tmp, pad_right), dtype=dtype) + x_mask_cur = np.diag(np.ones(stride, dtype=dtype)) + x_mask_cur_pad_top = np.zeros((chunk_ids*stride, stride), dtype=dtype) + x_mask_cur_pad_bottom = np.zeros((max_len_for_x_mask_tmp, stride), dtype=dtype) + x_rm_mask_cur = np.concatenate([x_mask_cur_pad_top, x_mask_cur, x_mask_cur_pad_bottom], axis=0) + x_rm_mask_cur = x_rm_mask_cur[:max_len_for_x_mask_tmp, :stride] + x_rm_mask_cur_fsmn = np.concatenate([fsmn_padding, padding_mask_left, x_rm_mask_cur, padding_mask_right], axis=1) + x_rm_mask = np.concatenate([x_rm_mask, x_rm_mask_cur_fsmn], axis=1) + + # fsmn_padding_mask + pad_shfit_mask = np.zeros([shfit_fsmn, num_units], dtype=dtype) + ones_1 = np.ones([chunk_size, num_units], dtype=dtype) + mask_shfit_chunk_cur = np.concatenate([pad_shfit_mask, ones_1], axis=0) + mask_shfit_chunk = np.concatenate([mask_shfit_chunk, mask_shfit_chunk_cur], axis=0) + + # predictor mask + zeros_1 = np.zeros([shfit_fsmn + pad_left, num_units_predictor], dtype=dtype) + ones_2 = np.ones([stride, num_units_predictor], dtype=dtype) + zeros_3 = np.zeros([chunk_size - stride - pad_left, num_units_predictor], dtype=dtype) + ones_zeros = np.concatenate([ones_2, zeros_3], axis=0) + mask_chunk_predictor_cur = np.concatenate([zeros_1, ones_zeros], axis=0) + mask_chunk_predictor = np.concatenate([mask_chunk_predictor, mask_chunk_predictor_cur], axis=0) + + # encoder att mask + zeros_1_top = np.zeros([shfit_fsmn, chunk_num*chunk_size_pad_shift], dtype=dtype) + + zeros_2_num = max(chunk_ids - encoder_att_look_back_factor, 0) + zeros_2 = np.zeros([chunk_size, zeros_2_num*chunk_size_pad_shift], dtype=dtype) + + encoder_att_look_back_num = max(chunk_ids - zeros_2_num, 0) + zeros_2_left = np.zeros([chunk_size, shfit_fsmn], dtype=dtype) + ones_2_mid = np.ones([stride, stride], dtype=dtype) + zeros_2_bottom = np.zeros([chunk_size-stride, stride], dtype=dtype) + zeros_2_right = np.zeros([chunk_size, chunk_size-stride], dtype=dtype) + ones_2 = np.concatenate([ones_2_mid, zeros_2_bottom], axis=0) + ones_2 = np.concatenate([zeros_2_left, ones_2, zeros_2_right], axis=1) + ones_2 = np.tile(ones_2, [1, encoder_att_look_back_num]) + + zeros_3_left = np.zeros([chunk_size, shfit_fsmn], dtype=dtype) + ones_3_right = np.ones([chunk_size, chunk_size], dtype=dtype) + ones_3 = np.concatenate([zeros_3_left, ones_3_right], axis=1) + + zeros_remain_num = max(chunk_num - 1 - chunk_ids, 0) + zeros_remain = np.zeros([chunk_size, zeros_remain_num*chunk_size_pad_shift], dtype=dtype) + + ones2_bottom = np.concatenate([zeros_2, ones_2, ones_3, zeros_remain], axis=1) + mask_att_chunk_encoder_cur = np.concatenate([zeros_1_top, ones2_bottom], axis=0) + mask_att_chunk_encoder = np.concatenate([mask_att_chunk_encoder, mask_att_chunk_encoder_cur], axis=0) + + + # decoder fsmn_shift_att_mask + zeros_1 = np.zeros([shfit_fsmn, 1]) + ones_1 = np.ones([chunk_size, 1]) + mask_shift_att_chunk_decoder_cur = np.concatenate([zeros_1, ones_1], axis=0) + mask_shift_att_chunk_decoder = np.concatenate( + [mask_shift_att_chunk_decoder, mask_shift_att_chunk_decoder_cur], axis=0) + + self.x_add_mask = x_add_mask[:x_len_chunk_max, :x_len_max+pad_left] + self.x_len_chunk = x_len_chunk + self.x_rm_mask = x_rm_mask[:x_len_max, :x_len_chunk_max] + self.x_len = x_len + self.mask_shfit_chunk = mask_shfit_chunk[:x_len_chunk_max, :] + self.mask_chunk_predictor = mask_chunk_predictor[:x_len_chunk_max, :] + self.mask_att_chunk_encoder = mask_att_chunk_encoder[:x_len_chunk_max, :x_len_chunk_max] + self.mask_shift_att_chunk_decoder = mask_shift_att_chunk_decoder[:x_len_chunk_max, :] + self.chunk_outs = (self.x_add_mask, + self.x_len_chunk, + self.x_rm_mask, + self.x_len, + self.mask_shfit_chunk, + self.mask_chunk_predictor, + self.mask_att_chunk_encoder, + self.mask_shift_att_chunk_decoder) + + return self.chunk_outs + + + def split_chunk(self, x, x_len, chunk_outs): + """ + :param x: (b, t, d) + :param x_length: (b) + :param ind: int + :return: + """ + x = x[:, :x_len.max(), :] + b, t, d = x.size() + x_len_mask = (~make_pad_mask(x_len, maxlen=t)).to( + x.device) + x *= x_len_mask[:, :, None] + + x_add_mask = self.get_x_add_mask(chunk_outs, x.device, dtype=x.dtype) + x_len_chunk = self.get_x_len_chunk(chunk_outs, x_len.device, dtype=x_len.dtype) + pad = (0, 0, self.pad_left_cur, 0) + x = F.pad(x, pad, "constant", 0.0) + b, t, d = x.size() + x = torch.transpose(x, 1, 0) + x = torch.reshape(x, [t, -1]) + x_chunk = torch.mm(x_add_mask, x) + x_chunk = torch.reshape(x_chunk, [-1, b, d]).transpose(1, 0) + + return x_chunk, x_len_chunk + + def remove_chunk(self, x_chunk, x_len_chunk, chunk_outs): + x_chunk = x_chunk[:, :x_len_chunk.max(), :] + b, t, d = x_chunk.size() + x_len_chunk_mask = (~make_pad_mask(x_len_chunk, maxlen=t)).to( + x_chunk.device) + x_chunk *= x_len_chunk_mask[:, :, None] + + x_rm_mask = self.get_x_rm_mask(chunk_outs, x_chunk.device, dtype=x_chunk.dtype) + x_len = self.get_x_len(chunk_outs, x_len_chunk.device, dtype=x_len_chunk.dtype) + x_chunk = torch.transpose(x_chunk, 1, 0) + x_chunk = torch.reshape(x_chunk, [t, -1]) + x = torch.mm(x_rm_mask, x_chunk) + x = torch.reshape(x, [-1, b, d]).transpose(1, 0) + + return x, x_len + + def get_x_add_mask(self, chunk_outs=None, device='cpu', idx=0, dtype=torch.float32): + with torch.no_grad(): + x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] + x = torch.from_numpy(x).type(dtype).to(device) + return x + + def get_x_len_chunk(self, chunk_outs=None, device='cpu', idx=1, dtype=torch.float32): + with torch.no_grad(): + x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] + x = torch.from_numpy(x).type(dtype).to(device) + return x + + + def get_x_rm_mask(self, chunk_outs=None, device='cpu', idx=2, dtype=torch.float32): + with torch.no_grad(): + x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] + x = torch.from_numpy(x).type(dtype).to(device) + return x + + def get_x_len(self, chunk_outs=None, device='cpu', idx=3, dtype=torch.float32): + with torch.no_grad(): + x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] + x = torch.from_numpy(x).type(dtype).to(device) + return x + + + def get_mask_shfit_chunk(self, chunk_outs=None, device='cpu', batch_size=1, num_units=1, idx=4, dtype=torch.float32): + with torch.no_grad(): + x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] + x = np.tile(x[None, :, :, ], [batch_size, 1, num_units]) + x = torch.from_numpy(x).type(dtype).to(device) + return x + + def get_mask_chunk_predictor(self, chunk_outs=None, device='cpu', batch_size=1, num_units=1, idx=5, dtype=torch.float32): + with torch.no_grad(): + x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] + x = np.tile(x[None, :, :, ], [batch_size, 1, num_units]) + x = torch.from_numpy(x).type(dtype).to(device) + return x + + def get_mask_att_chunk_encoder(self, chunk_outs=None, device='cpu', batch_size=1, idx=6, dtype=torch.float32): + with torch.no_grad(): + x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] + x = np.tile(x[None, :, :, ], [batch_size, 1, 1]) + x = torch.from_numpy(x).type(dtype).to(device) + return x + + def get_mask_shift_att_chunk_decoder(self, chunk_outs=None, device='cpu', batch_size=1, idx=7, dtype=torch.float32): + with torch.no_grad(): + x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx] + x = np.tile(x[None, None, :, 0], [batch_size, 1, 1]) + x = torch.from_numpy(x).type(dtype).to(device) + return x + + + +def build_scama_mask_for_cross_attention_decoder( + predictor_alignments: torch.Tensor, + encoder_sequence_length: torch.Tensor, + chunk_size: int = 5, + encoder_chunk_size: int = 5, + attention_chunk_center_bias: int = 0, + attention_chunk_size: int = 1, + attention_chunk_type: str = 'chunk', + step=None, + predictor_mask_chunk_hopping: torch.Tensor = None, + decoder_att_look_back_factor: int = 1, + mask_shift_att_chunk_decoder: torch.Tensor = None, + target_length: torch.Tensor = None, + is_training=True, + dtype: torch.dtype = torch.float32): + with torch.no_grad(): + device = predictor_alignments.device + batch_size, chunk_num = predictor_alignments.size() + maximum_encoder_length = encoder_sequence_length.max().item() + int_type = predictor_alignments.dtype + if not is_training: + target_length = predictor_alignments.sum(dim=-1).type(encoder_sequence_length.dtype) + maximum_target_length = target_length.max() + predictor_alignments_cumsum = torch.cumsum(predictor_alignments, dim=1) + predictor_alignments_cumsum = predictor_alignments_cumsum[:, None, :].repeat(1, maximum_target_length, 1) + + + index = torch.ones([batch_size, maximum_target_length], dtype=int_type).to(device) + index = torch.cumsum(index, dim=1) + index = index[:, :, None].repeat(1, 1, chunk_num) + + index_div = torch.floor(torch.divide(predictor_alignments_cumsum, index)).type(int_type) + index_div_bool_zeros = index_div == 0 + index_div_bool_zeros_count = torch.sum(index_div_bool_zeros.type(int_type), dim=-1) + 1 + + index_div_bool_zeros_count = torch.clip(index_div_bool_zeros_count, min=1, max=chunk_num) + + index_div_bool_zeros_count *= chunk_size + index_div_bool_zeros_count += attention_chunk_center_bias + index_div_bool_zeros_count = torch.clip(index_div_bool_zeros_count-1, min=0, max=maximum_encoder_length) + index_div_bool_zeros_count_ori = index_div_bool_zeros_count + + index_div_bool_zeros_count = (torch.floor(index_div_bool_zeros_count / encoder_chunk_size)+1)*encoder_chunk_size + max_len_chunk = math.ceil(maximum_encoder_length / encoder_chunk_size) * encoder_chunk_size + + mask_flip, mask_flip2 = None, None + if attention_chunk_size is not None: + index_div_bool_zeros_count_beg = index_div_bool_zeros_count - attention_chunk_size + index_div_bool_zeros_count_beg = torch.clip(index_div_bool_zeros_count_beg, 0, max_len_chunk) + index_div_bool_zeros_count_beg_mask = sequence_mask(index_div_bool_zeros_count_beg, maxlen=max_len_chunk, dtype=int_type, device=device) + mask_flip = 1 - index_div_bool_zeros_count_beg_mask + attention_chunk_size2 = attention_chunk_size * (decoder_att_look_back_factor+1) + index_div_bool_zeros_count_beg = index_div_bool_zeros_count - attention_chunk_size2 + + index_div_bool_zeros_count_beg = torch.clip(index_div_bool_zeros_count_beg, 0, max_len_chunk) + index_div_bool_zeros_count_beg_mask = sequence_mask(index_div_bool_zeros_count_beg, maxlen=max_len_chunk, dtype=int_type, device=device) + mask_flip2 = 1 - index_div_bool_zeros_count_beg_mask + + mask = sequence_mask(index_div_bool_zeros_count, maxlen=max_len_chunk, dtype=dtype, device=device) + + if predictor_mask_chunk_hopping is not None: + b, k, t = mask.size() + predictor_mask_chunk_hopping = predictor_mask_chunk_hopping[:, None, :, 0].repeat(1, k, 1) + + mask_mask_flip = mask + if mask_flip is not None: + mask_mask_flip = mask_flip * mask + + def _fn(): + mask_sliced = mask[:b, :k, encoder_chunk_size:t] + zero_pad_right = torch.zeros([b, k, encoder_chunk_size], dtype=mask_sliced.dtype).to(device) + mask_sliced = torch.cat([mask_sliced, zero_pad_right], dim=2) + _, _, tt = predictor_mask_chunk_hopping.size() + pad_right_p = max_len_chunk - tt + predictor_mask_chunk_hopping_pad = torch.nn.functional.pad(predictor_mask_chunk_hopping, [0, pad_right_p], "constant", 0) + masked = mask_sliced * predictor_mask_chunk_hopping_pad + + mask_true = mask_mask_flip + masked + return mask_true + + mask = _fn() if t > chunk_size else mask_mask_flip + + + + if mask_flip2 is not None: + mask *= mask_flip2 + + mask_target = sequence_mask(target_length, maxlen=maximum_target_length, dtype=mask.dtype, device=device) + mask = mask[:, :maximum_target_length, :] * mask_target[:, :, None] + + + + mask_len = sequence_mask(encoder_sequence_length, maxlen=maximum_encoder_length, dtype=mask.dtype, device=device) + mask = mask[:, :, :maximum_encoder_length] * mask_len[:, None, :] + + + + + if attention_chunk_type == 'full': + mask = torch.ones_like(mask).to(device) + if mask_shift_att_chunk_decoder is not None: + mask = mask * mask_shift_att_chunk_decoder + mask = mask[:, :maximum_target_length, :maximum_encoder_length].type(dtype).to(device) + + return mask + diff --git a/funasr/modules/streaming_utils/utils.py b/funasr/modules/streaming_utils/utils.py new file mode 100644 index 000000000..dd76de923 --- /dev/null +++ b/funasr/modules/streaming_utils/utils.py @@ -0,0 +1,47 @@ +import torch +from torch.nn import functional as F + +import numpy as np + +def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None): + if maxlen is None: + maxlen = lengths.max() + row_vector = torch.arange(0, maxlen, 1).to(lengths.device) + matrix = torch.unsqueeze(lengths, dim=-1) + mask = row_vector < matrix + mask = mask.detach() + + return mask.type(dtype).to(device) if device is not None else mask.type(dtype) + +def apply_cmvn(inputs, mvn): + device = inputs.device + dtype = inputs.dtype + frame, dim = inputs.shape + meams = np.tile(mvn[0:1, :dim], (frame, 1)) + vars = np.tile(mvn[1:2, :dim], (frame, 1)) + inputs -= torch.from_numpy(meams).type(dtype).to(device) + inputs *= torch.from_numpy(vars).type(dtype).to(device) + + return inputs.type(torch.float32) + + + + +def drop_and_add(inputs: torch.Tensor, + outputs: torch.Tensor, + training: bool, + dropout_rate: float = 0.1, + stoch_layer_coeff: float = 1.0): + + + + outputs = F.dropout(outputs, p=dropout_rate, training=training, inplace=True) + outputs *= stoch_layer_coeff + + input_dim = inputs.size(-1) + output_dim = outputs.size(-1) + + if input_dim == output_dim: + outputs += inputs + return outputs + diff --git a/funasr/modules/subsampling.py b/funasr/modules/subsampling.py new file mode 100644 index 000000000..f9a1c16e6 --- /dev/null +++ b/funasr/modules/subsampling.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Copyright 2019 Shigeki Karita +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Subsampling layer definition.""" + +import torch +import torch.nn.functional as F +from funasr.modules.embedding import PositionalEncoding + + +class TooShortUttError(Exception): + """Raised when the utt is too short for subsampling. + + Args: + message (str): Message for error catch + actual_size (int): the short size that cannot pass the subsampling + limit (int): the limit size for subsampling + + """ + + def __init__(self, message, actual_size, limit): + """Construct a TooShortUttError for error handler.""" + super().__init__(message) + self.actual_size = actual_size + self.limit = limit + + +def check_short_utt(ins, size): + """Check if the utterance is too short for subsampling.""" + if isinstance(ins, Conv2dSubsampling2) and size < 3: + return True, 3 + if isinstance(ins, Conv2dSubsampling) and size < 7: + return True, 7 + if isinstance(ins, Conv2dSubsampling6) and size < 11: + return True, 11 + if isinstance(ins, Conv2dSubsampling8) and size < 15: + return True, 15 + return False, -1 + + +class Conv2dSubsampling(torch.nn.Module): + """Convolutional 2D subsampling (to 1/4 length). + + Args: + idim (int): Input dimension. + odim (int): Output dimension. + dropout_rate (float): Dropout rate. + pos_enc (torch.nn.Module): Custom position encoding layer. + + """ + + def __init__(self, idim, odim, dropout_rate, pos_enc=None): + """Construct an Conv2dSubsampling object.""" + super(Conv2dSubsampling, self).__init__() + self.conv = torch.nn.Sequential( + torch.nn.Conv2d(1, odim, 3, 2), + torch.nn.ReLU(), + torch.nn.Conv2d(odim, odim, 3, 2), + torch.nn.ReLU(), + ) + self.out = torch.nn.Sequential( + torch.nn.Linear(odim * (((idim - 1) // 2 - 1) // 2), odim), + pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), + ) + + def forward(self, x, x_mask): + """Subsample x. + + Args: + x (torch.Tensor): Input tensor (#batch, time, idim). + x_mask (torch.Tensor): Input mask (#batch, 1, time). + + Returns: + torch.Tensor: Subsampled tensor (#batch, time', odim), + where time' = time // 4. + torch.Tensor: Subsampled mask (#batch, 1, time'), + where time' = time // 4. + + """ + x = x.unsqueeze(1) # (b, c, t, f) + x = self.conv(x) + b, c, t, f = x.size() + x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) + if x_mask is None: + return x, None + return x, x_mask[:, :, :-2:2][:, :, :-2:2] + + def __getitem__(self, key): + """Get item. + + When reset_parameters() is called, if use_scaled_pos_enc is used, + return the positioning encoding. + + """ + if key != -1: + raise NotImplementedError("Support only `-1` (for `reset_parameters`).") + return self.out[key] + + +class Conv2dSubsampling2(torch.nn.Module): + """Convolutional 2D subsampling (to 1/2 length). + + Args: + idim (int): Input dimension. + odim (int): Output dimension. + dropout_rate (float): Dropout rate. + pos_enc (torch.nn.Module): Custom position encoding layer. + + """ + + def __init__(self, idim, odim, dropout_rate, pos_enc=None): + """Construct an Conv2dSubsampling2 object.""" + super(Conv2dSubsampling2, self).__init__() + self.conv = torch.nn.Sequential( + torch.nn.Conv2d(1, odim, 3, 2), + torch.nn.ReLU(), + torch.nn.Conv2d(odim, odim, 3, 1), + torch.nn.ReLU(), + ) + self.out = torch.nn.Sequential( + torch.nn.Linear(odim * (((idim - 1) // 2 - 2)), odim), + pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), + ) + + def forward(self, x, x_mask): + """Subsample x. + + Args: + x (torch.Tensor): Input tensor (#batch, time, idim). + x_mask (torch.Tensor): Input mask (#batch, 1, time). + + Returns: + torch.Tensor: Subsampled tensor (#batch, time', odim), + where time' = time // 2. + torch.Tensor: Subsampled mask (#batch, 1, time'), + where time' = time // 2. + + """ + x = x.unsqueeze(1) # (b, c, t, f) + x = self.conv(x) + b, c, t, f = x.size() + x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) + if x_mask is None: + return x, None + return x, x_mask[:, :, :-2:2][:, :, :-2:1] + + def __getitem__(self, key): + """Get item. + + When reset_parameters() is called, if use_scaled_pos_enc is used, + return the positioning encoding. + + """ + if key != -1: + raise NotImplementedError("Support only `-1` (for `reset_parameters`).") + return self.out[key] + + +class Conv2dSubsampling6(torch.nn.Module): + """Convolutional 2D subsampling (to 1/6 length). + + Args: + idim (int): Input dimension. + odim (int): Output dimension. + dropout_rate (float): Dropout rate. + pos_enc (torch.nn.Module): Custom position encoding layer. + + """ + + def __init__(self, idim, odim, dropout_rate, pos_enc=None): + """Construct an Conv2dSubsampling6 object.""" + super(Conv2dSubsampling6, self).__init__() + self.conv = torch.nn.Sequential( + torch.nn.Conv2d(1, odim, 3, 2), + torch.nn.ReLU(), + torch.nn.Conv2d(odim, odim, 5, 3), + torch.nn.ReLU(), + ) + self.out = torch.nn.Sequential( + torch.nn.Linear(odim * (((idim - 1) // 2 - 2) // 3), odim), + pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), + ) + + def forward(self, x, x_mask): + """Subsample x. + + Args: + x (torch.Tensor): Input tensor (#batch, time, idim). + x_mask (torch.Tensor): Input mask (#batch, 1, time). + + Returns: + torch.Tensor: Subsampled tensor (#batch, time', odim), + where time' = time // 6. + torch.Tensor: Subsampled mask (#batch, 1, time'), + where time' = time // 6. + + """ + x = x.unsqueeze(1) # (b, c, t, f) + x = self.conv(x) + b, c, t, f = x.size() + x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) + if x_mask is None: + return x, None + return x, x_mask[:, :, :-2:2][:, :, :-4:3] + + +class Conv2dSubsampling8(torch.nn.Module): + """Convolutional 2D subsampling (to 1/8 length). + + Args: + idim (int): Input dimension. + odim (int): Output dimension. + dropout_rate (float): Dropout rate. + pos_enc (torch.nn.Module): Custom position encoding layer. + + """ + + def __init__(self, idim, odim, dropout_rate, pos_enc=None): + """Construct an Conv2dSubsampling8 object.""" + super(Conv2dSubsampling8, self).__init__() + self.conv = torch.nn.Sequential( + torch.nn.Conv2d(1, odim, 3, 2), + torch.nn.ReLU(), + torch.nn.Conv2d(odim, odim, 3, 2), + torch.nn.ReLU(), + torch.nn.Conv2d(odim, odim, 3, 2), + torch.nn.ReLU(), + ) + self.out = torch.nn.Sequential( + torch.nn.Linear(odim * ((((idim - 1) // 2 - 1) // 2 - 1) // 2), odim), + pos_enc if pos_enc is not None else PositionalEncoding(odim, dropout_rate), + ) + + def forward(self, x, x_mask): + """Subsample x. + + Args: + x (torch.Tensor): Input tensor (#batch, time, idim). + x_mask (torch.Tensor): Input mask (#batch, 1, time). + + Returns: + torch.Tensor: Subsampled tensor (#batch, time', odim), + where time' = time // 8. + torch.Tensor: Subsampled mask (#batch, 1, time'), + where time' = time // 8. + + """ + x = x.unsqueeze(1) # (b, c, t, f) + x = self.conv(x) + b, c, t, f = x.size() + x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) + if x_mask is None: + return x, None + return x, x_mask[:, :, :-2:2][:, :, :-2:2][:, :, :-2:2] + +class Conv1dSubsampling(torch.nn.Module): + """Convolutional 1D subsampling (to 1/2 length). + + Args: + idim (int): Input dimension. + odim (int): Output dimension. + dropout_rate (float): Dropout rate. + pos_enc (torch.nn.Module): Custom position encoding layer. + + """ + + def __init__(self, idim, odim, kernel_size, stride, pad): + super(Conv1dSubsampling, self).__init__() + self.conv = torch.nn.Conv1d(idim, odim, kernel_size, stride) + self.pad_fn = torch.nn.ConstantPad1d(pad, 0.0) + self.stride = stride + self.odim = odim + + def output_size(self) -> int: + return self.odim + + def forward(self, x, x_len): + """Subsample x. + + """ + x = x.transpose(1, 2) # (b, d ,t) + x = self.pad_fn(x) + x = F.relu(self.conv(x)) + x = x.transpose(1, 2) # (b, t ,d) + + if x_len is None: + + return x, None + x_len = (x_len - 1) // self.stride + 1 + return x, x_len + + def __getitem__(self, key): + """Get item. + + When reset_parameters() is called, if use_scaled_pos_enc is used, + return the positioning encoding. + + """ + if key != -1: + raise NotImplementedError("Support only `-1` (for `reset_parameters`).") + return self.out[key] diff --git a/funasr/modules/subsampling_without_posenc.py b/funasr/modules/subsampling_without_posenc.py new file mode 100644 index 000000000..239d3f1ad --- /dev/null +++ b/funasr/modules/subsampling_without_posenc.py @@ -0,0 +1,61 @@ +# Copyright 2020 Emiru Tsunoo +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Subsampling layer definition.""" + +import math +import torch + + +class Conv2dSubsamplingWOPosEnc(torch.nn.Module): + """Convolutional 2D subsampling. + + Args: + idim (int): Input dimension. + odim (int): Output dimension. + dropout_rate (float): Dropout rate. + kernels (list): kernel sizes + strides (list): stride sizes + + """ + + def __init__(self, idim, odim, dropout_rate, kernels, strides): + """Construct an Conv2dSubsamplingWOPosEnc object.""" + assert len(kernels) == len(strides) + super().__init__() + conv = [] + olen = idim + for i, (k, s) in enumerate(zip(kernels, strides)): + conv += [ + torch.nn.Conv2d(1 if i == 0 else odim, odim, k, s), + torch.nn.ReLU(), + ] + olen = math.floor((olen - k) / s + 1) + self.conv = torch.nn.Sequential(*conv) + self.out = torch.nn.Linear(odim * olen, odim) + self.strides = strides + self.kernels = kernels + + def forward(self, x, x_mask): + """Subsample x. + + Args: + x (torch.Tensor): Input tensor (#batch, time, idim). + x_mask (torch.Tensor): Input mask (#batch, 1, time). + + Returns: + torch.Tensor: Subsampled tensor (#batch, time', odim), + where time' = time // 4. + torch.Tensor: Subsampled mask (#batch, 1, time'), + where time' = time // 4. + + """ + x = x.unsqueeze(1) # (b, c, t, f) + x = self.conv(x) + b, c, t, f = x.size() + x = self.out(x.transpose(1, 2).contiguous().view(b, t, c * f)) + if x_mask is None: + return x, None + for k, s in zip(self.kernels, self.strides): + x_mask = x_mask[:, :, : -k + 1 : s] + return x, x_mask diff --git a/funasr/optimizers/__init__.py b/funasr/optimizers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/optimizers/sgd.py b/funasr/optimizers/sgd.py new file mode 100644 index 000000000..3f0d3d1c9 --- /dev/null +++ b/funasr/optimizers/sgd.py @@ -0,0 +1,32 @@ +import torch +from typeguard import check_argument_types + + +class SGD(torch.optim.SGD): + """Thin inheritance of torch.optim.SGD to bind the required arguments, 'lr' + + Note that + the arguments of the optimizer invoked by AbsTask.main() + must have default value except for 'param'. + + I can't understand why only SGD.lr doesn't have the default value. + """ + + def __init__( + self, + params, + lr: float = 0.1, + momentum: float = 0.0, + dampening: float = 0.0, + weight_decay: float = 0.0, + nesterov: bool = False, + ): + assert check_argument_types() + super().__init__( + params, + lr=lr, + momentum=momentum, + dampening=dampening, + weight_decay=weight_decay, + nesterov=nesterov, + ) diff --git a/funasr/samplers/__init__.py b/funasr/samplers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/samplers/abs_sampler.py b/funasr/samplers/abs_sampler.py new file mode 100644 index 000000000..2f7aa539b --- /dev/null +++ b/funasr/samplers/abs_sampler.py @@ -0,0 +1,19 @@ +from abc import ABC +from abc import abstractmethod +from typing import Iterator +from typing import Tuple + +from torch.utils.data import Sampler + + +class AbsSampler(Sampler, ABC): + @abstractmethod + def __len__(self) -> int: + raise NotImplementedError + + @abstractmethod + def __iter__(self) -> Iterator[Tuple[str, ...]]: + raise NotImplementedError + + def generate(self, seed): + return list(self) diff --git a/funasr/samplers/build_batch_sampler.py b/funasr/samplers/build_batch_sampler.py new file mode 100644 index 000000000..edda6ba02 --- /dev/null +++ b/funasr/samplers/build_batch_sampler.py @@ -0,0 +1,167 @@ +from typing import List +from typing import Sequence +from typing import Tuple +from typing import Union + +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.samplers.abs_sampler import AbsSampler +from funasr.samplers.folded_batch_sampler import FoldedBatchSampler +from funasr.samplers.length_batch_sampler import LengthBatchSampler +from funasr.samplers.num_elements_batch_sampler import NumElementsBatchSampler +from funasr.samplers.sorted_batch_sampler import SortedBatchSampler +from funasr.samplers.unsorted_batch_sampler import UnsortedBatchSampler + + +BATCH_TYPES = dict( + unsorted="UnsortedBatchSampler has nothing in particular feature and " + "just creates mini-batches which has constant batch_size. " + "This sampler doesn't require any length " + "information for each feature. " + "'key_file' is just a text file which describes each sample name." + "\n\n" + " utterance_id_a\n" + " utterance_id_b\n" + " utterance_id_c\n" + "\n" + "The fist column is referred, so 'shape file' can be used, too.\n\n" + " utterance_id_a 100,80\n" + " utterance_id_b 400,80\n" + " utterance_id_c 512,80\n", + sorted="SortedBatchSampler sorts samples by the length of the first input " + " in order to make each sample in a mini-batch has close length. " + "This sampler requires a text file which describes the length for each sample " + "\n\n" + " utterance_id_a 1000\n" + " utterance_id_b 1453\n" + " utterance_id_c 1241\n" + "\n" + "The first element of feature dimensions is referred, " + "so 'shape_file' can be also used.\n\n" + " utterance_id_a 1000,80\n" + " utterance_id_b 1453,80\n" + " utterance_id_c 1241,80\n", + folded="FoldedBatchSampler supports variable batch_size. " + "The batch_size is decided by\n" + " batch_size = base_batch_size // (L // fold_length)\n" + "L is referred to the largest length of samples in the mini-batch. " + "This samples requires length information as same as SortedBatchSampler\n", + length="LengthBatchSampler supports variable batch_size. " + "This sampler makes mini-batches which have same number of 'bins' as possible " + "counting by the total lengths of each feature in the mini-batch. " + "This sampler requires a text file which describes the length for each sample. " + "\n\n" + " utterance_id_a 1000\n" + " utterance_id_b 1453\n" + " utterance_id_c 1241\n" + "\n" + "The first element of feature dimensions is referred, " + "so 'shape_file' can be also used.\n\n" + " utterance_id_a 1000,80\n" + " utterance_id_b 1453,80\n" + " utterance_id_c 1241,80\n", + numel="NumElementsBatchSampler supports variable batch_size. " + "Just like LengthBatchSampler, this sampler makes mini-batches" + " which have same number of 'bins' as possible " + "counting by the total number of elements of each feature " + "instead of the length. " + "Thus this sampler requires the full information of the dimension of the features. " + "\n\n" + " utterance_id_a 1000,80\n" + " utterance_id_b 1453,80\n" + " utterance_id_c 1241,80\n", +) + + +def build_batch_sampler( + type: str, + batch_size: int, + batch_bins: int, + shape_files: Union[Tuple[str, ...], List[str]], + sort_in_batch: str = "descending", + sort_batch: str = "ascending", + drop_last: bool = False, + min_batch_size: int = 1, + fold_lengths: Sequence[int] = (), + padding: bool = True, + utt2category_file: str = None, +) -> AbsSampler: + """Helper function to instantiate BatchSampler. + + Args: + type: mini-batch type. "unsorted", "sorted", "folded", "numel", or, "length" + batch_size: The mini-batch size. Used for "unsorted", "sorted", "folded" mode + batch_bins: Used for "numel" model + shape_files: Text files describing the length and dimension + of each features. e.g. uttA 1330,80 + sort_in_batch: + sort_batch: + drop_last: + min_batch_size: Used for "numel" or "folded" mode + fold_lengths: Used for "folded" mode + padding: Whether sequences are input as a padded tensor or not. + used for "numel" mode + """ + assert check_argument_types() + if len(shape_files) == 0: + raise ValueError("No shape file are given") + + if type == "unsorted": + retval = UnsortedBatchSampler( + batch_size=batch_size, key_file=shape_files[0], drop_last=drop_last + ) + + elif type == "sorted": + retval = SortedBatchSampler( + batch_size=batch_size, + shape_file=shape_files[0], + sort_in_batch=sort_in_batch, + sort_batch=sort_batch, + drop_last=drop_last, + ) + + elif type == "folded": + if len(fold_lengths) != len(shape_files): + raise ValueError( + f"The number of fold_lengths must be equal to " + f"the number of shape_files: " + f"{len(fold_lengths)} != {len(shape_files)}" + ) + retval = FoldedBatchSampler( + batch_size=batch_size, + shape_files=shape_files, + fold_lengths=fold_lengths, + sort_in_batch=sort_in_batch, + sort_batch=sort_batch, + drop_last=drop_last, + min_batch_size=min_batch_size, + utt2category_file=utt2category_file, + ) + + elif type == "numel": + retval = NumElementsBatchSampler( + batch_bins=batch_bins, + shape_files=shape_files, + sort_in_batch=sort_in_batch, + sort_batch=sort_batch, + drop_last=drop_last, + padding=padding, + min_batch_size=min_batch_size, + ) + + elif type == "length": + retval = LengthBatchSampler( + batch_bins=batch_bins, + shape_files=shape_files, + sort_in_batch=sort_in_batch, + sort_batch=sort_batch, + drop_last=drop_last, + padding=padding, + min_batch_size=min_batch_size, + ) + + else: + raise ValueError(f"Not supported: {type}") + assert check_return_type(retval) + return retval diff --git a/funasr/samplers/folded_batch_sampler.py b/funasr/samplers/folded_batch_sampler.py new file mode 100644 index 000000000..48e960454 --- /dev/null +++ b/funasr/samplers/folded_batch_sampler.py @@ -0,0 +1,156 @@ +from typing import Iterator +from typing import List +from typing import Sequence +from typing import Tuple +from typing import Union + +from typeguard import check_argument_types + +from funasr.fileio.read_text import load_num_sequence_text +from funasr.fileio.read_text import read_2column_text +from funasr.samplers.abs_sampler import AbsSampler + + +class FoldedBatchSampler(AbsSampler): + def __init__( + self, + batch_size: int, + shape_files: Union[Tuple[str, ...], List[str]], + fold_lengths: Sequence[int], + min_batch_size: int = 1, + sort_in_batch: str = "descending", + sort_batch: str = "ascending", + drop_last: bool = False, + utt2category_file: str = None, + ): + assert check_argument_types() + assert batch_size > 0 + if sort_batch != "ascending" and sort_batch != "descending": + raise ValueError( + f"sort_batch must be ascending or descending: {sort_batch}" + ) + if sort_in_batch != "descending" and sort_in_batch != "ascending": + raise ValueError( + f"sort_in_batch must be ascending or descending: {sort_in_batch}" + ) + + self.batch_size = batch_size + self.shape_files = shape_files + self.sort_in_batch = sort_in_batch + self.sort_batch = sort_batch + self.drop_last = drop_last + + # utt2shape: (Length, ...) + # uttA 100,... + # uttB 201,... + utt2shapes = [ + load_num_sequence_text(s, loader_type="csv_int") for s in shape_files + ] + + first_utt2shape = utt2shapes[0] + for s, d in zip(shape_files, utt2shapes): + if set(d) != set(first_utt2shape): + raise RuntimeError( + f"keys are mismatched between {s} != {shape_files[0]}" + ) + + # Sort samples in ascending order + # (shape order should be like (Length, Dim)) + keys = sorted(first_utt2shape, key=lambda k: first_utt2shape[k][0]) + if len(keys) == 0: + raise RuntimeError(f"0 lines found: {shape_files[0]}") + + category2utt = {} + if utt2category_file is not None: + utt2category = read_2column_text(utt2category_file) + if set(utt2category) != set(first_utt2shape): + raise RuntimeError( + "keys are mismatched between " + f"{utt2category_file} != {shape_files[0]}" + ) + for k in keys: + category2utt.setdefault(utt2category[k], []).append(k) + else: + category2utt["default_category"] = keys + + self.batch_list = [] + for d, v in category2utt.items(): + category_keys = v + # Decide batch-sizes + start = 0 + batch_sizes = [] + while True: + k = category_keys[start] + factor = max(int(d[k][0] / m) for d, m in zip(utt2shapes, fold_lengths)) + bs = max(min_batch_size, int(batch_size / (1 + factor))) + if self.drop_last and start + bs > len(category_keys): + # This if-block avoids 0-batches + if len(self.batch_list) > 0: + break + + bs = min(len(category_keys) - start, bs) + batch_sizes.append(bs) + start += bs + if start >= len(category_keys): + break + + if len(batch_sizes) == 0: + # Maybe we can't reach here + raise RuntimeError("0 batches") + + # If the last batch-size is smaller than minimum batch_size, + # the samples are redistributed to the other mini-batches + if len(batch_sizes) > 1 and batch_sizes[-1] < min_batch_size: + for i in range(batch_sizes.pop(-1)): + batch_sizes[-(i % len(batch_sizes)) - 2] += 1 + + if not self.drop_last: + # Bug check + assert sum(batch_sizes) == len( + category_keys + ), f"{sum(batch_sizes)} != {len(category_keys)}" + + # Set mini-batch + cur_batch_list = [] + start = 0 + for bs in batch_sizes: + assert len(category_keys) >= start + bs, "Bug" + minibatch_keys = category_keys[start : start + bs] + start += bs + if sort_in_batch == "descending": + minibatch_keys.reverse() + elif sort_in_batch == "ascending": + # Key are already sorted in ascending + pass + else: + raise ValueError( + "sort_in_batch must be ascending or " + f"descending: {sort_in_batch}" + ) + cur_batch_list.append(tuple(minibatch_keys)) + + if sort_batch == "ascending": + pass + elif sort_batch == "descending": + cur_batch_list.reverse() + else: + raise ValueError( + f"sort_batch must be ascending or descending: {sort_batch}" + ) + self.batch_list.extend(cur_batch_list) + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f"N-batch={len(self)}, " + f"batch_size={self.batch_size}, " + f"shape_files={self.shape_files}, " + f"sort_in_batch={self.sort_in_batch}, " + f"sort_batch={self.sort_batch})" + ) + + def __len__(self): + return len(self.batch_list) + + def __iter__(self) -> Iterator[Tuple[str, ...]]: + return iter(self.batch_list) diff --git a/funasr/samplers/length_batch_sampler.py b/funasr/samplers/length_batch_sampler.py new file mode 100644 index 000000000..cdf0e5809 --- /dev/null +++ b/funasr/samplers/length_batch_sampler.py @@ -0,0 +1,143 @@ +from typing import Iterator +from typing import List +from typing import Tuple +from typing import Union + +from typeguard import check_argument_types + +from funasr.fileio.read_text import load_num_sequence_text +from funasr.samplers.abs_sampler import AbsSampler + + +class LengthBatchSampler(AbsSampler): + def __init__( + self, + batch_bins: int, + shape_files: Union[Tuple[str, ...], List[str]], + min_batch_size: int = 1, + sort_in_batch: str = "descending", + sort_batch: str = "ascending", + drop_last: bool = False, + padding: bool = True, + ): + assert check_argument_types() + assert batch_bins > 0 + if sort_batch != "ascending" and sort_batch != "descending": + raise ValueError( + f"sort_batch must be ascending or descending: {sort_batch}" + ) + if sort_in_batch != "descending" and sort_in_batch != "ascending": + raise ValueError( + f"sort_in_batch must be ascending or descending: {sort_in_batch}" + ) + + self.batch_bins = batch_bins + self.shape_files = shape_files + self.sort_in_batch = sort_in_batch + self.sort_batch = sort_batch + self.drop_last = drop_last + + # utt2shape: (Length, ...) + # uttA 100,... + # uttB 201,... + utt2shapes = [ + load_num_sequence_text(s, loader_type="csv_int") for s in shape_files + ] + + first_utt2shape = utt2shapes[0] + for s, d in zip(shape_files, utt2shapes): + if set(d) != set(first_utt2shape): + raise RuntimeError( + f"keys are mismatched between {s} != {shape_files[0]}" + ) + + # Sort samples in ascending order + # (shape order should be like (Length, Dim)) + keys = sorted(first_utt2shape, key=lambda k: first_utt2shape[k][0]) + if len(keys) == 0: + raise RuntimeError(f"0 lines found: {shape_files[0]}") + + # Decide batch-sizes + batch_sizes = [] + current_batch_keys = [] + for key in keys: + current_batch_keys.append(key) + # shape: (Length, dim1, dim2, ...) + if padding: + # bins = bs x max_length + bins = sum(len(current_batch_keys) * sh[key][0] for sh in utt2shapes) + else: + # bins = sum of lengths + bins = sum(d[k][0] for k in current_batch_keys for d in utt2shapes) + + if bins > batch_bins and len(current_batch_keys) >= min_batch_size: + batch_sizes.append(len(current_batch_keys)) + current_batch_keys = [] + else: + if len(current_batch_keys) != 0 and ( + not self.drop_last or len(batch_sizes) == 0 + ): + batch_sizes.append(len(current_batch_keys)) + + if len(batch_sizes) == 0: + # Maybe we can't reach here + raise RuntimeError("0 batches") + + # If the last batch-size is smaller than minimum batch_size, + # the samples are redistributed to the other mini-batches + if len(batch_sizes) > 1 and batch_sizes[-1] < min_batch_size: + for i in range(batch_sizes.pop(-1)): + batch_sizes[-(i % len(batch_sizes)) - 1] += 1 + + if not self.drop_last: + # Bug check + assert sum(batch_sizes) == len(keys), f"{sum(batch_sizes)} != {len(keys)}" + + # Set mini-batch + self.batch_list = [] + iter_bs = iter(batch_sizes) + bs = next(iter_bs) + minibatch_keys = [] + for key in keys: + minibatch_keys.append(key) + if len(minibatch_keys) == bs: + if sort_in_batch == "descending": + minibatch_keys.reverse() + elif sort_in_batch == "ascending": + # Key are already sorted in ascending + pass + else: + raise ValueError( + "sort_in_batch must be ascending" + f" or descending: {sort_in_batch}" + ) + self.batch_list.append(tuple(minibatch_keys)) + minibatch_keys = [] + try: + bs = next(iter_bs) + except StopIteration: + break + + if sort_batch == "ascending": + pass + elif sort_batch == "descending": + self.batch_list.reverse() + else: + raise ValueError( + f"sort_batch must be ascending or descending: {sort_batch}" + ) + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f"N-batch={len(self)}, " + f"batch_bins={self.batch_bins}, " + f"sort_in_batch={self.sort_in_batch}, " + f"sort_batch={self.sort_batch})" + ) + + def __len__(self): + return len(self.batch_list) + + def __iter__(self) -> Iterator[Tuple[str, ...]]: + return iter(self.batch_list) diff --git a/funasr/samplers/num_elements_batch_sampler.py b/funasr/samplers/num_elements_batch_sampler.py new file mode 100644 index 000000000..0ffad9289 --- /dev/null +++ b/funasr/samplers/num_elements_batch_sampler.py @@ -0,0 +1,160 @@ +from typing import Iterator +from typing import List +from typing import Tuple +from typing import Union + +import numpy as np +from typeguard import check_argument_types + +from funasr.fileio.read_text import load_num_sequence_text +from funasr.samplers.abs_sampler import AbsSampler + + +class NumElementsBatchSampler(AbsSampler): + def __init__( + self, + batch_bins: int, + shape_files: Union[Tuple[str, ...], List[str]], + min_batch_size: int = 1, + sort_in_batch: str = "descending", + sort_batch: str = "ascending", + drop_last: bool = False, + padding: bool = True, + ): + assert check_argument_types() + assert batch_bins > 0 + if sort_batch != "ascending" and sort_batch != "descending": + raise ValueError( + f"sort_batch must be ascending or descending: {sort_batch}" + ) + if sort_in_batch != "descending" and sort_in_batch != "ascending": + raise ValueError( + f"sort_in_batch must be ascending or descending: {sort_in_batch}" + ) + + self.batch_bins = batch_bins + self.shape_files = shape_files + self.sort_in_batch = sort_in_batch + self.sort_batch = sort_batch + self.drop_last = drop_last + + # utt2shape: (Length, ...) + # uttA 100,... + # uttB 201,... + utt2shapes = [ + load_num_sequence_text(s, loader_type="csv_int") for s in shape_files + ] + + first_utt2shape = utt2shapes[0] + for s, d in zip(shape_files, utt2shapes): + if set(d) != set(first_utt2shape): + raise RuntimeError( + f"keys are mismatched between {s} != {shape_files[0]}" + ) + + # Sort samples in ascending order + # (shape order should be like (Length, Dim)) + keys = sorted(first_utt2shape, key=lambda k: first_utt2shape[k][0]) + if len(keys) == 0: + raise RuntimeError(f"0 lines found: {shape_files[0]}") + if padding: + # If padding case, the feat-dim must be same over whole corpus, + # therefore the first sample is referred + feat_dims = [np.prod(d[keys[0]][1:]) for d in utt2shapes] + else: + feat_dims = None + + # Decide batch-sizes + batch_sizes = [] + current_batch_keys = [] + for key in keys: + current_batch_keys.append(key) + # shape: (Length, dim1, dim2, ...) + if padding: + for d, s in zip(utt2shapes, shape_files): + if tuple(d[key][1:]) != tuple(d[keys[0]][1:]): + raise RuntimeError( + "If padding=True, the " + f"feature dimension must be unified: {s}", + ) + bins = sum( + len(current_batch_keys) * sh[key][0] * d + for sh, d in zip(utt2shapes, feat_dims) + ) + else: + bins = sum( + np.prod(d[k]) for k in current_batch_keys for d in utt2shapes + ) + + if bins > batch_bins and len(current_batch_keys) >= min_batch_size: + batch_sizes.append(len(current_batch_keys)) + current_batch_keys = [] + else: + if len(current_batch_keys) != 0 and ( + not self.drop_last or len(batch_sizes) == 0 + ): + batch_sizes.append(len(current_batch_keys)) + + if len(batch_sizes) == 0: + # Maybe we can't reach here + raise RuntimeError("0 batches") + + # If the last batch-size is smaller than minimum batch_size, + # the samples are redistributed to the other mini-batches + if len(batch_sizes) > 1 and batch_sizes[-1] < min_batch_size: + for i in range(batch_sizes.pop(-1)): + batch_sizes[-(i % len(batch_sizes)) - 1] += 1 + + if not self.drop_last: + # Bug check + assert sum(batch_sizes) == len(keys), f"{sum(batch_sizes)} != {len(keys)}" + + # Set mini-batch + self.batch_list = [] + iter_bs = iter(batch_sizes) + bs = next(iter_bs) + minibatch_keys = [] + for key in keys: + minibatch_keys.append(key) + if len(minibatch_keys) == bs: + if sort_in_batch == "descending": + minibatch_keys.reverse() + elif sort_in_batch == "ascending": + # Key are already sorted in ascending + pass + else: + raise ValueError( + "sort_in_batch must be ascending" + f" or descending: {sort_in_batch}" + ) + + self.batch_list.append(tuple(minibatch_keys)) + minibatch_keys = [] + try: + bs = next(iter_bs) + except StopIteration: + break + + if sort_batch == "ascending": + pass + elif sort_batch == "descending": + self.batch_list.reverse() + else: + raise ValueError( + f"sort_batch must be ascending or descending: {sort_batch}" + ) + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f"N-batch={len(self)}, " + f"batch_bins={self.batch_bins}, " + f"sort_in_batch={self.sort_in_batch}, " + f"sort_batch={self.sort_batch})" + ) + + def __len__(self): + return len(self.batch_list) + + def __iter__(self) -> Iterator[Tuple[str, ...]]: + return iter(self.batch_list) diff --git a/funasr/samplers/sorted_batch_sampler.py b/funasr/samplers/sorted_batch_sampler.py new file mode 100644 index 000000000..d6c3b4111 --- /dev/null +++ b/funasr/samplers/sorted_batch_sampler.py @@ -0,0 +1,95 @@ +import logging +from typing import Iterator +from typing import Tuple + +from typeguard import check_argument_types + +from funasr.fileio.read_text import load_num_sequence_text +from funasr.samplers.abs_sampler import AbsSampler + + +class SortedBatchSampler(AbsSampler): + """BatchSampler with sorted samples by length. + + Args: + batch_size: + shape_file: + sort_in_batch: 'descending', 'ascending' or None. + sort_batch: + """ + + def __init__( + self, + batch_size: int, + shape_file: str, + sort_in_batch: str = "descending", + sort_batch: str = "ascending", + drop_last: bool = False, + ): + assert check_argument_types() + assert batch_size > 0 + self.batch_size = batch_size + self.shape_file = shape_file + self.sort_in_batch = sort_in_batch + self.sort_batch = sort_batch + self.drop_last = drop_last + + # utt2shape: (Length, ...) + # uttA 100,... + # uttB 201,... + utt2shape = load_num_sequence_text(shape_file, loader_type="csv_int") + if sort_in_batch == "descending": + # Sort samples in descending order (required by RNN) + keys = sorted(utt2shape, key=lambda k: -utt2shape[k][0]) + elif sort_in_batch == "ascending": + # Sort samples in ascending order + keys = sorted(utt2shape, key=lambda k: utt2shape[k][0]) + else: + raise ValueError( + f"sort_in_batch must be either one of " + f"ascending, descending, or None: {sort_in_batch}" + ) + if len(keys) == 0: + raise RuntimeError(f"0 lines found: {shape_file}") + + # Apply max(, 1) to avoid 0-batches + N = max(len(keys) // batch_size, 1) + if not self.drop_last: + # Split keys evenly as possible as. Note that If N != 1, + # the these batches always have size of batch_size at minimum. + self.batch_list = [ + keys[i * len(keys) // N : (i + 1) * len(keys) // N] for i in range(N) + ] + else: + self.batch_list = [ + tuple(keys[i * batch_size : (i + 1) * batch_size]) for i in range(N) + ] + + if len(self.batch_list) == 0: + logging.warning(f"{shape_file} is empty") + + if sort_in_batch != sort_batch: + if sort_batch not in ("ascending", "descending"): + raise ValueError( + f"sort_batch must be ascending or descending: {sort_batch}" + ) + self.batch_list.reverse() + + if len(self.batch_list) == 0: + raise RuntimeError("0 batches") + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f"N-batch={len(self)}, " + f"batch_size={self.batch_size}, " + f"shape_file={self.shape_file}, " + f"sort_in_batch={self.sort_in_batch}, " + f"sort_batch={self.sort_batch})" + ) + + def __len__(self): + return len(self.batch_list) + + def __iter__(self) -> Iterator[Tuple[str, ...]]: + return iter(self.batch_list) diff --git a/funasr/samplers/unsorted_batch_sampler.py b/funasr/samplers/unsorted_batch_sampler.py new file mode 100644 index 000000000..349e526c7 --- /dev/null +++ b/funasr/samplers/unsorted_batch_sampler.py @@ -0,0 +1,91 @@ +import logging +from typing import Iterator +from typing import Tuple + +from typeguard import check_argument_types + +from funasr.fileio.read_text import read_2column_text +from funasr.samplers.abs_sampler import AbsSampler + + +class UnsortedBatchSampler(AbsSampler): + """BatchSampler with constant batch-size. + + Any sorting is not done in this class, + so no length information is required, + This class is convenient for decoding mode, + or not seq2seq learning e.g. classification. + + Args: + batch_size: + key_file: + """ + + def __init__( + self, + batch_size: int, + key_file: str, + drop_last: bool = False, + utt2category_file: str = None, + ): + assert check_argument_types() + assert batch_size > 0 + self.batch_size = batch_size + self.key_file = key_file + self.drop_last = drop_last + + # utt2shape: + # uttA + # uttB + utt2any = read_2column_text(key_file) + if len(utt2any) == 0: + logging.warning(f"{key_file} is empty") + # In this case the, the first column in only used + keys = list(utt2any) + if len(keys) == 0: + raise RuntimeError(f"0 lines found: {key_file}") + + category2utt = {} + if utt2category_file is not None: + utt2category = read_2column_text(utt2category_file) + if set(utt2category) != set(keys): + raise RuntimeError( + f"keys are mismatched between {utt2category_file} != {key_file}" + ) + for k, v in utt2category.items(): + category2utt.setdefault(v, []).append(k) + else: + category2utt["default_category"] = keys + + self.batch_list = [] + for d, v in category2utt.items(): + category_keys = v + # Apply max(, 1) to avoid 0-batches + N = max(len(category_keys) // batch_size, 1) + if not self.drop_last: + # Split keys evenly as possible as. Note that If N != 1, + # the these batches always have size of batch_size at minimum. + cur_batch_list = [ + category_keys[i * len(keys) // N : (i + 1) * len(keys) // N] + for i in range(N) + ] + else: + cur_batch_list = [ + tuple(category_keys[i * batch_size : (i + 1) * batch_size]) + for i in range(N) + ] + self.batch_list.extend(cur_batch_list) + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f"N-batch={len(self)}, " + f"batch_size={self.batch_size}, " + f"key_file={self.key_file}, " + ) + + def __len__(self): + return len(self.batch_list) + + def __iter__(self) -> Iterator[Tuple[str, ...]]: + return iter(self.batch_list) diff --git a/funasr/schedulers/__init__.py b/funasr/schedulers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/schedulers/abs_scheduler.py b/funasr/schedulers/abs_scheduler.py new file mode 100644 index 000000000..7395f259c --- /dev/null +++ b/funasr/schedulers/abs_scheduler.py @@ -0,0 +1,84 @@ +from abc import ABC +from abc import abstractmethod + +import torch.optim.lr_scheduler as L + + +class AbsScheduler(ABC): + @abstractmethod + def step(self, epoch: int = None): + pass + + @abstractmethod + def state_dict(self): + pass + + @abstractmethod + def load_state_dict(self, state): + pass + + +# If you need to define custom scheduler, please inherit these classes +class AbsBatchStepScheduler(AbsScheduler): + @abstractmethod + def step(self, epoch: int = None): + pass + + @abstractmethod + def state_dict(self): + pass + + @abstractmethod + def load_state_dict(self, state): + pass + + +class AbsEpochStepScheduler(AbsScheduler): + @abstractmethod + def step(self, epoch: int = None): + pass + + @abstractmethod + def state_dict(self): + pass + + @abstractmethod + def load_state_dict(self, state): + pass + + +class AbsValEpochStepScheduler(AbsEpochStepScheduler): + @abstractmethod + def step(self, val, epoch: int = None): + pass + + @abstractmethod + def state_dict(self): + pass + + @abstractmethod + def load_state_dict(self, state): + pass + + +# Create alias type to check the type +# Note(kamo): Currently PyTorch doesn't provide the base class +# to judge these classes. +AbsValEpochStepScheduler.register(L.ReduceLROnPlateau) +for s in [ + L.ReduceLROnPlateau, + L.LambdaLR, + L.StepLR, + L.MultiStepLR, + L.MultiStepLR, + L.ExponentialLR, + L.CosineAnnealingLR, +]: + AbsEpochStepScheduler.register(s) + +AbsBatchStepScheduler.register(L.CyclicLR) +for s in [ + L.OneCycleLR, + L.CosineAnnealingWarmRestarts, +]: + AbsBatchStepScheduler.register(s) diff --git a/funasr/schedulers/noam_lr.py b/funasr/schedulers/noam_lr.py new file mode 100644 index 000000000..80df019ac --- /dev/null +++ b/funasr/schedulers/noam_lr.py @@ -0,0 +1,65 @@ +"""Noam learning rate scheduler module.""" +from typing import Union +import warnings + +import torch +from torch.optim.lr_scheduler import _LRScheduler +from typeguard import check_argument_types + +from funasr.schedulers.abs_scheduler import AbsBatchStepScheduler + + +class NoamLR(_LRScheduler, AbsBatchStepScheduler): + """The LR scheduler proposed by Noam + + Ref: + "Attention Is All You Need", https://arxiv.org/pdf/1706.03762.pdf + + FIXME(kamo): PyTorch doesn't provide _LRScheduler as public class, + thus the behaviour isn't guaranteed at forward PyTorch version. + + NOTE(kamo): The "model_size" in original implementation is derived from + the model, but in this implementation, this parameter is a constant value. + You need to change it if the model is changed. + + """ + + def __init__( + self, + optimizer: torch.optim.Optimizer, + model_size: Union[int, float] = 320, + warmup_steps: Union[int, float] = 25000, + last_epoch: int = -1, + ): + assert check_argument_types() + self.model_size = model_size + self.warmup_steps = warmup_steps + + lr = list(optimizer.param_groups)[0]["lr"] + new_lr = self.lr_for_WarmupLR(lr) + warnings.warn( + f"NoamLR is deprecated. " + f"Use WarmupLR(warmup_steps={warmup_steps}) with Optimizer(lr={new_lr})", + ) + + # __init__() must be invoked before setting field + # because step() is also invoked in __init__() + super().__init__(optimizer, last_epoch) + + def lr_for_WarmupLR(self, lr: float) -> float: + return lr / self.model_size**0.5 / self.warmup_steps**0.5 + + def __repr__(self): + return ( + f"{self.__class__.__name__}(model_size={self.model_size}, " + f"warmup_steps={self.warmup_steps})" + ) + + def get_lr(self): + step_num = self.last_epoch + 1 + return [ + lr + * self.model_size**-0.5 + * min(step_num**-0.5, step_num * self.warmup_steps**-1.5) + for lr in self.base_lrs + ] diff --git a/funasr/schedulers/warmup_lr.py b/funasr/schedulers/warmup_lr.py new file mode 100644 index 000000000..dbf3aca53 --- /dev/null +++ b/funasr/schedulers/warmup_lr.py @@ -0,0 +1,50 @@ +"""Warm up learning rate scheduler module.""" +from typing import Union + +import torch +from torch.optim.lr_scheduler import _LRScheduler +from typeguard import check_argument_types + +from funasr.schedulers.abs_scheduler import AbsBatchStepScheduler + + +class WarmupLR(_LRScheduler, AbsBatchStepScheduler): + """The WarmupLR scheduler + + This scheduler is almost same as NoamLR Scheduler except for following difference: + + NoamLR: + lr = optimizer.lr * model_size ** -0.5 + * min(step ** -0.5, step * warmup_step ** -1.5) + WarmupLR: + lr = optimizer.lr * warmup_step ** 0.5 + * min(step ** -0.5, step * warmup_step ** -1.5) + + Note that the maximum lr equals to optimizer.lr in this scheduler. + + """ + + def __init__( + self, + optimizer: torch.optim.Optimizer, + warmup_steps: Union[int, float] = 25000, + last_epoch: int = -1, + ): + assert check_argument_types() + self.warmup_steps = warmup_steps + + # __init__() must be invoked before setting field + # because step() is also invoked in __init__() + super().__init__(optimizer, last_epoch) + + def __repr__(self): + return f"{self.__class__.__name__}(warmup_steps={self.warmup_steps})" + + def get_lr(self): + step_num = self.last_epoch + 1 + return [ + lr + * self.warmup_steps**0.5 + * min(step_num**-0.5, step_num * self.warmup_steps**-1.5) + for lr in self.base_lrs + ] diff --git a/funasr/tasks/__init__.py b/funasr/tasks/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/tasks/abs_task.py b/funasr/tasks/abs_task.py new file mode 100644 index 000000000..5ea78c349 --- /dev/null +++ b/funasr/tasks/abs_task.py @@ -0,0 +1,1795 @@ +# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Abstract task module.""" +import argparse +import functools +import logging +import os +import sys +from abc import ABC +from abc import abstractmethod +from dataclasses import dataclass +from distutils.version import LooseVersion +from io import BytesIO +from pathlib import Path +from typing import Any +from typing import Callable +from typing import Dict +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import humanfriendly +import numpy as np +import torch +import torch.multiprocessing +import torch.nn +import torch.optim +import yaml +from torch.utils.data import DataLoader +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr import __version__ +from funasr.datasets.dataset import AbsDataset +from funasr.datasets.dataset import DATA_TYPES +from funasr.datasets.dataset import ESPnetDataset +from funasr.datasets.iterable_dataset import IterableESPnetDataset +from funasr.iterators.abs_iter_factory import AbsIterFactory +from funasr.iterators.chunk_iter_factory import ChunkIterFactory +from funasr.iterators.multiple_iter_factory import MultipleIterFactory +from funasr.iterators.sequence_iter_factory import SequenceIterFactory +from funasr.optimizers.sgd import SGD +from funasr.samplers.build_batch_sampler import BATCH_TYPES +from funasr.samplers.build_batch_sampler import build_batch_sampler +from funasr.samplers.unsorted_batch_sampler import UnsortedBatchSampler +from funasr.schedulers.noam_lr import NoamLR +from funasr.schedulers.warmup_lr import WarmupLR +from funasr.torch_utils.load_pretrained_model import load_pretrained_model +from funasr.torch_utils.model_summary import model_summary +from funasr.torch_utils.pytorch_version import pytorch_cudnn_version +from funasr.torch_utils.set_all_random_seed import set_all_random_seed +from funasr.train.abs_espnet_model import AbsESPnetModel +from funasr.train.class_choices import ClassChoices +from funasr.train.distributed_utils import DistributedOption +from funasr.train.trainer import Trainer +from funasr.utils import config_argparse +from funasr.utils.build_dataclass import build_dataclass +from funasr.utils.cli_utils import get_commandline_args +from funasr.utils.get_default_kwargs import get_default_kwargs +from funasr.utils.nested_dict_action import NestedDictAction +from funasr.utils.types import humanfriendly_parse_size_or_none +from funasr.utils.types import int_or_none +from funasr.utils.types import str2bool +from funasr.utils.types import str2triple_str +from funasr.utils.types import str_or_int +from funasr.utils.types import str_or_none +from funasr.utils.yaml_no_alias_safe_dump import yaml_no_alias_safe_dump + +try: + import wandb +except Exception: + wandb = None + +if LooseVersion(torch.__version__) >= LooseVersion("1.5.0"): + pass +else: + pass + +optim_classes = dict( + adam=torch.optim.Adam, + adamw=torch.optim.AdamW, + sgd=SGD, + adadelta=torch.optim.Adadelta, + adagrad=torch.optim.Adagrad, + adamax=torch.optim.Adamax, + asgd=torch.optim.ASGD, + lbfgs=torch.optim.LBFGS, + rmsprop=torch.optim.RMSprop, + rprop=torch.optim.Rprop, +) +if LooseVersion(torch.__version__) >= LooseVersion("1.10.0"): + # From 1.10.0, RAdam is officially supported + optim_classes.update( + radam=torch.optim.RAdam, + ) +try: + import torch_optimizer + + optim_classes.update( + accagd=torch_optimizer.AccSGD, + adabound=torch_optimizer.AdaBound, + adamod=torch_optimizer.AdaMod, + diffgrad=torch_optimizer.DiffGrad, + lamb=torch_optimizer.Lamb, + novograd=torch_optimizer.NovoGrad, + pid=torch_optimizer.PID, + # torch_optimizer<=0.0.1a10 doesn't support + # qhadam=torch_optimizer.QHAdam, + qhm=torch_optimizer.QHM, + sgdw=torch_optimizer.SGDW, + yogi=torch_optimizer.Yogi, + ) + if LooseVersion(torch_optimizer.__version__) < LooseVersion("0.2.0"): + # From 0.2.0, RAdam is dropped + optim_classes.update( + radam=torch_optimizer.RAdam, + ) + del torch_optimizer +except ImportError: + pass +try: + import apex + + optim_classes.update( + fusedadam=apex.optimizers.FusedAdam, + fusedlamb=apex.optimizers.FusedLAMB, + fusednovograd=apex.optimizers.FusedNovoGrad, + fusedsgd=apex.optimizers.FusedSGD, + ) + del apex +except ImportError: + pass +try: + import fairscale +except ImportError: + fairscale = None + +scheduler_classes = dict( + ReduceLROnPlateau=torch.optim.lr_scheduler.ReduceLROnPlateau, + lambdalr=torch.optim.lr_scheduler.LambdaLR, + steplr=torch.optim.lr_scheduler.StepLR, + multisteplr=torch.optim.lr_scheduler.MultiStepLR, + exponentiallr=torch.optim.lr_scheduler.ExponentialLR, + CosineAnnealingLR=torch.optim.lr_scheduler.CosineAnnealingLR, + noamlr=NoamLR, + warmuplr=WarmupLR, + cycliclr=torch.optim.lr_scheduler.CyclicLR, + onecyclelr=torch.optim.lr_scheduler.OneCycleLR, + CosineAnnealingWarmRestarts=torch.optim.lr_scheduler.CosineAnnealingWarmRestarts, +) +# To lower keys +optim_classes = {k.lower(): v for k, v in optim_classes.items()} +scheduler_classes = {k.lower(): v for k, v in scheduler_classes.items()} + + +@dataclass +class IteratorOptions: + preprocess_fn: callable + collate_fn: callable + data_path_and_name_and_type: list + shape_files: list + batch_size: int + batch_bins: int + batch_type: str + max_cache_size: float + max_cache_fd: int + distributed: bool + num_batches: Optional[int] + num_iters_per_epoch: Optional[int] + train: bool + + +class AbsTask(ABC): + # Use @staticmethod, or @classmethod, + # instead of instance method to avoid God classes + + # If you need more than one optimizers, change this value in inheritance + num_optimizers: int = 1 + trainer = Trainer + class_choices_list: List[ClassChoices] = [] + + def __init__(self): + raise RuntimeError("This class can't be instantiated.") + + @classmethod + @abstractmethod + def add_task_arguments(cls, parser: argparse.ArgumentParser): + pass + + @classmethod + @abstractmethod + def build_collate_fn( + cls, args: argparse.Namespace, train: bool + ) -> Callable[[Sequence[Dict[str, np.ndarray]]], Dict[str, torch.Tensor]]: + """Return "collate_fn", which is a callable object and given to DataLoader. + + >>> from torch.utils.data import DataLoader + >>> loader = DataLoader(collate_fn=cls.build_collate_fn(args, train=True), ...) + + In many cases, you can use our common collate_fn. + """ + raise NotImplementedError + + @classmethod + @abstractmethod + def build_preprocess_fn( + cls, args: argparse.Namespace, train: bool + ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]: + raise NotImplementedError + + @classmethod + @abstractmethod + def required_data_names( + cls, train: bool = True, inference: bool = False + ) -> Tuple[str, ...]: + """Define the required names by Task + + This function is used by + >>> cls.check_task_requirements() + If your model is defined as following, + + >>> from funasr.train.abs_espnet_model import AbsESPnetModel + >>> class Model(AbsESPnetModel): + ... def forward(self, input, output, opt=None): pass + + then "required_data_names" should be as + + >>> required_data_names = ('input', 'output') + """ + raise NotImplementedError + + @classmethod + @abstractmethod + def optional_data_names( + cls, train: bool = True, inference: bool = False + ) -> Tuple[str, ...]: + """Define the optional names by Task + + This function is used by + >>> cls.check_task_requirements() + If your model is defined as follows, + + >>> from funasr.train.abs_espnet_model import AbsESPnetModel + >>> class Model(AbsESPnetModel): + ... def forward(self, input, output, opt=None): pass + + then "optional_data_names" should be as + + >>> optional_data_names = ('opt',) + """ + raise NotImplementedError + + @classmethod + @abstractmethod + def build_model(cls, args: argparse.Namespace) -> AbsESPnetModel: + raise NotImplementedError + + @classmethod + def get_parser(cls) -> config_argparse.ArgumentParser: + assert check_argument_types() + + class ArgumentDefaultsRawTextHelpFormatter( + argparse.RawTextHelpFormatter, + argparse.ArgumentDefaultsHelpFormatter, + ): + pass + + parser = config_argparse.ArgumentParser( + description="base parser", + formatter_class=ArgumentDefaultsRawTextHelpFormatter, + ) + + # NOTE(kamo): Use '_' instead of '-' to avoid confusion. + # I think '-' looks really confusing if it's written in yaml. + + # NOTE(kamo): add_arguments(..., required=True) can't be used + # to provide --print_config mode. Instead of it, do as + parser.set_defaults(required=["output_dir"]) + + group = parser.add_argument_group("Common configuration") + + group.add_argument( + "--print_config", + action="store_true", + help="Print the config file and exit", + ) + group.add_argument( + "--log_level", + type=lambda x: x.upper(), + default="INFO", + choices=("ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"), + help="The verbose level of logging", + ) + group.add_argument( + "--dry_run", + type=str2bool, + default=False, + help="Perform process without training", + ) + group.add_argument( + "--iterator_type", + type=str, + choices=["sequence", "chunk", "task", "none"], + default="sequence", + help="Specify iterator type", + ) + + group.add_argument("--output_dir", type=str_or_none, default=None) + group.add_argument( + "--ngpu", + type=int, + default=0, + help="The number of gpus. 0 indicates CPU mode", + ) + group.add_argument("--seed", type=int, default=0, help="Random seed") + group.add_argument( + "--num_workers", + type=int, + default=1, + help="The number of workers used for DataLoader", + ) + group.add_argument( + "--num_att_plot", + type=int, + default=3, + help="The number images to plot the outputs from attention. " + "This option makes sense only when attention-based model. " + "We can also disable the attention plot by setting it 0", + ) + + group = parser.add_argument_group("distributed training related") + group.add_argument( + "--dist_backend", + default="nccl", + type=str, + help="distributed backend", + ) + group.add_argument( + "--dist_init_method", + type=str, + default="env://", + help='if init_method="env://", env values of "MASTER_PORT", "MASTER_ADDR", ' + '"WORLD_SIZE", and "RANK" are referred.', + ) + group.add_argument( + "--dist_world_size", + default=None, + type=int_or_none, + help="number of nodes for distributed training", + ) + group.add_argument( + "--dist_rank", + type=int_or_none, + default=None, + help="node rank for distributed training", + ) + group.add_argument( + # Not starting with "dist_" for compatibility to launch.py + "--local_rank", + type=int_or_none, + default=None, + help="local rank for distributed training. This option is used if " + "--multiprocessing_distributed=false", + ) + group.add_argument( + "--dist_master_addr", + default=None, + type=str_or_none, + help="The master address for distributed training. " + "This value is used when dist_init_method == 'env://'", + ) + group.add_argument( + "--dist_master_port", + default=None, + type=int_or_none, + help="The master port for distributed training" + "This value is used when dist_init_method == 'env://'", + ) + group.add_argument( + "--dist_launcher", + default=None, + type=str_or_none, + choices=["slurm", "mpi", None], + help="The launcher type for distributed training", + ) + group.add_argument( + "--multiprocessing_distributed", + default=False, + type=str2bool, + help="Use multi-processing distributed training to launch " + "N processes per node, which has N GPUs. This is the " + "fastest way to use PyTorch for either single node or " + "multi node data parallel training", + ) + group.add_argument( + "--unused_parameters", + type=str2bool, + default=False, + help="Whether to use the find_unused_parameters in " + "torch.nn.parallel.DistributedDataParallel ", + ) + group.add_argument( + "--sharded_ddp", + default=False, + type=str2bool, + help="Enable sharded training provided by fairscale", + ) + + group = parser.add_argument_group("cudnn mode related") + group.add_argument( + "--cudnn_enabled", + type=str2bool, + default=torch.backends.cudnn.enabled, + help="Enable CUDNN", + ) + group.add_argument( + "--cudnn_benchmark", + type=str2bool, + default=torch.backends.cudnn.benchmark, + help="Enable cudnn-benchmark mode", + ) + group.add_argument( + "--cudnn_deterministic", + type=str2bool, + default=True, + help="Enable cudnn-deterministic mode", + ) + + group = parser.add_argument_group("collect stats mode related") + group.add_argument( + "--collect_stats", + type=str2bool, + default=False, + help='Perform on "collect stats" mode', + ) + group.add_argument( + "--write_collected_feats", + type=str2bool, + default=False, + help='Write the output features from the model when "collect stats" mode', + ) + + group = parser.add_argument_group("Trainer related") + group.add_argument( + "--max_epoch", + type=int, + default=40, + help="The maximum number epoch to train", + ) + group.add_argument( + "--max_update", + type=int, + default=sys.maxsize, + help="The maximum number update step to train", + ) + group.add_argument( + "--patience", + type=int_or_none, + default=None, + help="Number of epochs to wait without improvement " + "before stopping the training", + ) + group.add_argument( + "--val_scheduler_criterion", + type=str, + nargs=2, + default=("valid", "loss"), + help="The criterion used for the value given to the lr scheduler. " + 'Give a pair referring the phase, "train" or "valid",' + 'and the criterion name. The mode specifying "min" or "max" can ' + "be changed by --scheduler_conf", + ) + group.add_argument( + "--early_stopping_criterion", + type=str, + nargs=3, + default=("valid", "loss", "min"), + help="The criterion used for judging of early stopping. " + 'Give a pair referring the phase, "train" or "valid",' + 'the criterion name and the mode, "min" or "max", e.g. "acc,max".', + ) + group.add_argument( + "--best_model_criterion", + type=str2triple_str, + nargs="+", + default=[ + ("train", "loss", "min"), + ("valid", "loss", "min"), + ("train", "acc", "max"), + ("valid", "acc", "max"), + ], + help="The criterion used for judging of the best model. " + 'Give a pair referring the phase, "train" or "valid",' + 'the criterion name, and the mode, "min" or "max", e.g. "acc,max".', + ) + group.add_argument( + "--keep_nbest_models", + type=int, + nargs="+", + default=[10], + help="Remove previous snapshots excluding the n-best scored epochs", + ) + group.add_argument( + "--nbest_averaging_interval", + type=int, + default=0, + help="The epoch interval to apply model averaging and save nbest models", + ) + group.add_argument( + "--grad_clip", + type=float, + default=5.0, + help="Gradient norm threshold to clip", + ) + group.add_argument( + "--grad_clip_type", + type=float, + default=2.0, + help="The type of the used p-norm for gradient clip. Can be inf", + ) + group.add_argument( + "--grad_noise", + type=str2bool, + default=False, + help="The flag to switch to use noise injection to " + "gradients during training", + ) + group.add_argument( + "--accum_grad", + type=int, + default=1, + help="The number of gradient accumulation", + ) + group.add_argument( + "--no_forward_run", + type=str2bool, + default=False, + help="Just only iterating data loading without " + "model forwarding and training", + ) + group.add_argument( + "--resume", + type=str2bool, + default=False, + help="Enable resuming if checkpoint is existing", + ) + group.add_argument( + "--train_dtype", + default="float32", + choices=["float16", "float32", "float64"], + help="Data type for training.", + ) + group.add_argument( + "--use_amp", + type=str2bool, + default=False, + help="Enable Automatic Mixed Precision. This feature requires pytorch>=1.6", + ) + group.add_argument( + "--log_interval", + type=int_or_none, + default=None, + help="Show the logs every the number iterations in each epochs at the " + "training phase. If None is given, it is decided according the number " + "of training samples automatically .", + ) + group.add_argument( + "--use_tensorboard", + type=str2bool, + default=True, + help="Enable tensorboard logging", + ) + group.add_argument( + "--use_wandb", + type=str2bool, + default=False, + help="Enable wandb logging", + ) + group.add_argument( + "--wandb_project", + type=str, + default=None, + help="Specify wandb project", + ) + group.add_argument( + "--wandb_id", + type=str, + default=None, + help="Specify wandb id", + ) + group.add_argument( + "--wandb_entity", + type=str, + default=None, + help="Specify wandb entity", + ) + group.add_argument( + "--wandb_name", + type=str, + default=None, + help="Specify wandb run name", + ) + group.add_argument( + "--wandb_model_log_interval", + type=int, + default=-1, + help="Set the model log period", + ) + group.add_argument( + "--detect_anomaly", + type=str2bool, + default=False, + help="Set torch.autograd.set_detect_anomaly", + ) + + group = parser.add_argument_group("Pretraining model related") + group.add_argument("--pretrain_path", help="This option is obsoleted") + group.add_argument( + "--init_param", + type=str, + default=[], + nargs="*", + help="Specify the file path used for initialization of parameters. " + "The format is ':::', " + "where file_path is the model file path, " + "src_key specifies the key of model states to be used in the model file, " + "dst_key specifies the attribute of the model to be initialized, " + "and exclude_keys excludes keys of model states for the initialization." + "e.g.\n" + " # Load all parameters" + " --init_param some/where/model.pth\n" + " # Load only decoder parameters" + " --init_param some/where/model.pth:decoder:decoder\n" + " # Load only decoder parameters excluding decoder.embed" + " --init_param some/where/model.pth:decoder:decoder:decoder.embed\n" + " --init_param some/where/model.pth:decoder:decoder:decoder.embed\n", + ) + group.add_argument( + "--ignore_init_mismatch", + type=str2bool, + default=False, + help="Ignore size mismatch when loading pre-trained model", + ) + group.add_argument( + "--freeze_param", + type=str, + default=[], + nargs="*", + help="Freeze parameters", + ) + + group = parser.add_argument_group("BatchSampler related") + group.add_argument( + "--num_iters_per_epoch", + type=int_or_none, + default=None, + help="Restrict the number of iterations for training per epoch", + ) + group.add_argument( + "--batch_size", + type=int, + default=20, + help="The mini-batch size used for training. Used if batch_type='unsorted'," + " 'sorted', or 'folded'.", + ) + group.add_argument( + "--valid_batch_size", + type=int_or_none, + default=None, + help="If not given, the value of --batch_size is used", + ) + group.add_argument( + "--batch_bins", + type=int, + default=1000000, + help="The number of batch bins. Used if batch_type='length' or 'numel'", + ) + group.add_argument( + "--valid_batch_bins", + type=int_or_none, + default=None, + help="If not given, the value of --batch_bins is used", + ) + + group.add_argument("--train_shape_file", type=str, action="append", default=[]) + group.add_argument("--valid_shape_file", type=str, action="append", default=[]) + + group = parser.add_argument_group("Sequence iterator related") + _batch_type_help = "" + for key, value in BATCH_TYPES.items(): + _batch_type_help += f'"{key}":\n{value}\n' + group.add_argument( + "--batch_type", + type=str, + default="folded", + choices=list(BATCH_TYPES), + help=_batch_type_help, + ) + group.add_argument( + "--valid_batch_type", + type=str_or_none, + default=None, + choices=list(BATCH_TYPES) + [None], + help="If not given, the value of --batch_type is used", + ) + group.add_argument("--fold_length", type=int, action="append", default=[]) + group.add_argument( + "--sort_in_batch", + type=str, + default="descending", + choices=["descending", "ascending"], + help="Sort the samples in each mini-batches by the sample " + 'lengths. To enable this, "shape_file" must have the length information.', + ) + group.add_argument( + "--sort_batch", + type=str, + default="descending", + choices=["descending", "ascending"], + help="Sort mini-batches by the sample lengths", + ) + group.add_argument( + "--multiple_iterator", + type=str2bool, + default=False, + help="Use multiple iterator mode", + ) + + group = parser.add_argument_group("Chunk iterator related") + group.add_argument( + "--chunk_length", + type=str_or_int, + default=500, + help="Specify chunk length. e.g. '300', '300,400,500', or '300-400'." + "If multiple numbers separated by command are given, " + "one of them is selected randomly for each samples. " + "If two numbers are given with '-', it indicates the range of the choices. " + "Note that if the sequence length is shorter than the all chunk_lengths, " + "the sample is discarded. ", + ) + group.add_argument( + "--chunk_shift_ratio", + type=float, + default=0.5, + help="Specify the shift width of chunks. If it's less than 1, " + "allows the overlapping and if bigger than 1, there are some gaps " + "between each chunk.", + ) + group.add_argument( + "--num_cache_chunks", + type=int, + default=1024, + help="Shuffle in the specified number of chunks and generate mini-batches " + "More larger this value, more randomness can be obtained.", + ) + + group = parser.add_argument_group("Dataset related") + _data_path_and_name_and_type_help = ( + "Give three words splitted by comma. It's used for the training data. " + "e.g. '--train_data_path_and_name_and_type some/path/a.scp,foo,sound'. " + "The first value, some/path/a.scp, indicates the file path, " + "and the second, foo, is the key name used for the mini-batch data, " + "and the last, sound, decides the file type. " + "This option is repeatable, so you can input any number of features " + "for your task. Supported file types are as follows:\n\n" + ) + for key, dic in DATA_TYPES.items(): + _data_path_and_name_and_type_help += f'"{key}":\n{dic["help"]}\n\n' + + # for large dataset + group.add_argument( + "--dataset_type", + type=str, + default="small", + help="whether to use dataloader for large dataset", + ) + parser.add_argument( + "--dataset_conf", + action=NestedDictAction, + default=dict(), + help=f"The keyword arguments for dataset", + ) + group.add_argument( + "--train_data_file", + type=str, + default=None, + help="train_list for large dataset", + ) + group.add_argument( + "--valid_data_file", + type=str, + default=None, + help="valid_list for large dataset", + ) + + group.add_argument( + "--train_data_path_and_name_and_type", + type=str2triple_str, + action="append", + default=[], + help=_data_path_and_name_and_type_help, + ) + group.add_argument( + "--valid_data_path_and_name_and_type", + type=str2triple_str, + action="append", + default=[], + ) + group.add_argument( + "--allow_variable_data_keys", + type=str2bool, + default=False, + help="Allow the arbitrary keys for mini-batch with ignoring " + "the task requirements", + ) + group.add_argument( + "--max_cache_size", + type=humanfriendly.parse_size, + default=0.0, + help="The maximum cache size for data loader. e.g. 10MB, 20GB.", + ) + group.add_argument( + "--max_cache_fd", + type=int, + default=32, + help="The maximum number of file descriptors to be kept " + "as opened for ark files. " + "This feature is only valid when data type is 'kaldi_ark'.", + ) + group.add_argument( + "--valid_max_cache_size", + type=humanfriendly_parse_size_or_none, + default=None, + help="The maximum cache size for validation data loader. e.g. 10MB, 20GB. " + "If None, the 5 percent size of --max_cache_size", + ) + + group = parser.add_argument_group("Optimizer related") + for i in range(1, cls.num_optimizers + 1): + suf = "" if i == 1 else str(i) + group.add_argument( + f"--optim{suf}", + type=lambda x: x.lower(), + default="adadelta", + choices=list(optim_classes), + help="The optimizer type", + ) + group.add_argument( + f"--optim{suf}_conf", + action=NestedDictAction, + default=dict(), + help="The keyword arguments for optimizer", + ) + group.add_argument( + f"--scheduler{suf}", + type=lambda x: str_or_none(x.lower()), + default=None, + choices=list(scheduler_classes) + [None], + help="The lr scheduler type", + ) + group.add_argument( + f"--scheduler{suf}_conf", + action=NestedDictAction, + default=dict(), + help="The keyword arguments for lr scheduler", + ) + + # for training on PAI + group = parser.add_argument_group("PAI training related") + group.add_argument( + "--use_pai", + type=str2bool, + default=False, + help="flag to indicate whether training on PAI", + ) + group.add_argument( + "--num_worker_count", + type=int, + default=1, + help="The number of machines on PAI.", + ) + group.add_argument( + "--access_key_id", + type=str, + default=None, + help="The username for oss.", + ) + group.add_argument( + "--access_key_secret", + type=str, + default=None, + help="The password for oss.", + ) + group.add_argument( + "--endpoint", + type=str, + default=None, + help="The endpoint for oss.", + ) + group.add_argument( + "--bucket_name", + type=str, + default=None, + help="The bucket name for oss.", + ) + group.add_argument( + "--oss_bucket", + default=None, + help="oss bucket.", + ) + + cls.trainer.add_arguments(parser) + cls.add_task_arguments(parser) + + assert check_return_type(parser) + return parser + + @classmethod + def build_optimizers( + cls, + args: argparse.Namespace, + model: torch.nn.Module, + ) -> List[torch.optim.Optimizer]: + if cls.num_optimizers != 1: + raise RuntimeError( + "build_optimizers() must be overridden if num_optimizers != 1" + ) + + optim_class = optim_classes.get(args.optim) + if optim_class is None: + raise ValueError(f"must be one of {list(optim_classes)}: {args.optim}") + if args.sharded_ddp: + if fairscale is None: + raise RuntimeError("Requiring fairscale. Do 'pip install fairscale'") + optim = fairscale.optim.oss.OSS( + params=model.parameters(), optim=optim_class, **args.optim_conf + ) + else: + optim = optim_class(model.parameters(), **args.optim_conf) + + optimizers = [optim] + return optimizers + + @classmethod + def exclude_opts(cls) -> Tuple[str, ...]: + """The options not to be shown by --print_config""" + return "required", "print_config", "config", "ngpu" + + @classmethod + def get_default_config(cls) -> Dict[str, Any]: + """Return the configuration as dict. + + This method is used by print_config() + """ + + def get_class_type(name: str, classes: dict): + _cls = classes.get(name) + if _cls is None: + raise ValueError(f"must be one of {list(classes)}: {name}") + return _cls + + # This method is used only for --print_config + assert check_argument_types() + parser = cls.get_parser() + args, _ = parser.parse_known_args() + config = vars(args) + # Excludes the options not to be shown + for k in AbsTask.exclude_opts(): + config.pop(k) + + for i in range(1, cls.num_optimizers + 1): + suf = "" if i == 1 else str(i) + name = config[f"optim{suf}"] + optim_class = get_class_type(name, optim_classes) + conf = get_default_kwargs(optim_class) + # Overwrite the default by the arguments, + conf.update(config[f"optim{suf}_conf"]) + # and set it again + config[f"optim{suf}_conf"] = conf + + name = config[f"scheduler{suf}"] + if name is not None: + scheduler_class = get_class_type(name, scheduler_classes) + conf = get_default_kwargs(scheduler_class) + # Overwrite the default by the arguments, + conf.update(config[f"scheduler{suf}_conf"]) + # and set it again + config[f"scheduler{suf}_conf"] = conf + + for class_choices in cls.class_choices_list: + if getattr(args, class_choices.name) is not None: + class_obj = class_choices.get_class(getattr(args, class_choices.name)) + conf = get_default_kwargs(class_obj) + name = class_choices.name + # Overwrite the default by the arguments, + conf.update(config[f"{name}_conf"]) + # and set it again + config[f"{name}_conf"] = conf + return config + + @classmethod + def check_required_command_args(cls, args: argparse.Namespace): + assert check_argument_types() + for k in vars(args): + if "-" in k: + raise RuntimeError(f'Use "_" instead of "-": parser.get_parser("{k}")') + + required = ", ".join( + f"--{a}" for a in args.required if getattr(args, a) is None + ) + + if len(required) != 0: + parser = cls.get_parser() + parser.print_help(file=sys.stderr) + p = Path(sys.argv[0]).name + print(file=sys.stderr) + print( + f"{p}: error: the following arguments are required: " f"{required}", + file=sys.stderr, + ) + sys.exit(2) + + @classmethod + def check_task_requirements( + cls, + dataset: Union[AbsDataset, IterableESPnetDataset], + allow_variable_data_keys: bool, + train: bool, + inference: bool = False, + ) -> None: + """Check if the dataset satisfy the requirement of current Task""" + assert check_argument_types() + mes = ( + f"If you intend to use an additional input, modify " + f'"{cls.__name__}.required_data_names()" or ' + f'"{cls.__name__}.optional_data_names()". ' + f"Otherwise you need to set --allow_variable_data_keys true " + ) + + for k in cls.required_data_names(train, inference): + if not dataset.has_name(k): + raise RuntimeError( + f'"{cls.required_data_names(train, inference)}" are required for' + f' {cls.__name__}. but "{dataset.names()}" are input.\n{mes}' + ) + if not allow_variable_data_keys: + task_keys = cls.required_data_names( + train, inference + ) + cls.optional_data_names(train, inference) + for k in dataset.names(): + if k not in task_keys: + raise RuntimeError( + f"The data-name must be one of {task_keys} " + f'for {cls.__name__}: "{k}" is not allowed.\n{mes}' + ) + + @classmethod + def print_config(cls, file=sys.stdout) -> None: + assert check_argument_types() + # Shows the config: e.g. python train.py asr --print_config + config = cls.get_default_config() + file.write(yaml_no_alias_safe_dump(config, indent=4, sort_keys=False)) + + @classmethod + def main(cls, args: argparse.Namespace = None, cmd: Sequence[str] = None): + assert check_argument_types() + print(get_commandline_args(), file=sys.stderr) + if args is None: + parser = cls.get_parser() + args = parser.parse_args(cmd) + args.version = __version__ + if args.pretrain_path is not None: + raise RuntimeError("--pretrain_path is deprecated. Use --init_param") + if args.print_config: + cls.print_config() + sys.exit(0) + cls.check_required_command_args(args) + + if not args.distributed or not args.multiprocessing_distributed: + cls.main_worker(args) + else: + assert args.ngpu > 1 + cls.main_worker(args) + + @classmethod + def main_worker(cls, args: argparse.Namespace): + assert check_argument_types() + + # 0. Init distributed process + distributed_option = build_dataclass(DistributedOption, args) + # Setting distributed_option.dist_rank, etc. + if args.use_pai: + distributed_option.init_options_pai() + else: + distributed_option.init_options() + + # NOTE(kamo): Don't use logging before invoking logging.basicConfig() + if not distributed_option.distributed or distributed_option.dist_rank == 0: + if not distributed_option.distributed: + _rank = "" + else: + _rank = ( + f":{distributed_option.dist_rank}/" + f"{distributed_option.dist_world_size}" + ) + + # NOTE(kamo): + # logging.basicConfig() is invoked in main_worker() instead of main() + # because it can be invoked only once in a process. + # FIXME(kamo): Should we use logging.getLogger()? + logging.basicConfig( + level=args.log_level, + format=f"[{os.uname()[1].split('.')[0]}]" + f" %(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", + ) + else: + # Suppress logging if RANK != 0 + logging.basicConfig( + level="ERROR", + format=f"[{os.uname()[1].split('.')[0]}]" + f" %(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", + ) + # Invoking torch.distributed.init_process_group + if args.use_pai: + distributed_option.init_torch_distributed_pai(args) + else: + distributed_option.init_torch_distributed(args) + + # 1. Set random-seed + set_all_random_seed(args.seed) + torch.backends.cudnn.enabled = args.cudnn_enabled + torch.backends.cudnn.benchmark = args.cudnn_benchmark + torch.backends.cudnn.deterministic = args.cudnn_deterministic + if args.detect_anomaly: + logging.info("Invoking torch.autograd.set_detect_anomaly(True)") + torch.autograd.set_detect_anomaly(args.detect_anomaly) + + # 2. Build model + model = cls.build_model(args=args) + if not isinstance(model, AbsESPnetModel): + raise RuntimeError( + f"model must inherit {AbsESPnetModel.__name__}, but got {type(model)}" + ) + model = model.to( + dtype=getattr(torch, args.train_dtype), + device="cuda" if args.ngpu > 0 else "cpu", + ) + for t in args.freeze_param: + for k, p in model.named_parameters(): + if k.startswith(t + ".") or k == t: + logging.info(f"Setting {k}.requires_grad = False") + p.requires_grad = False + + # 3. Build optimizer + optimizers = cls.build_optimizers(args, model=model) + + # 4. Build schedulers + schedulers = [] + for i, optim in enumerate(optimizers, 1): + suf = "" if i == 1 else str(i) + name = getattr(args, f"scheduler{suf}") + conf = getattr(args, f"scheduler{suf}_conf") + if name is not None: + cls_ = scheduler_classes.get(name) + if cls_ is None: + raise ValueError( + f"must be one of {list(scheduler_classes)}: {name}" + ) + scheduler = cls_(optim, **conf) + else: + scheduler = None + + schedulers.append(scheduler) + + logging.info(pytorch_cudnn_version()) + logging.info(model_summary(model)) + for i, (o, s) in enumerate(zip(optimizers, schedulers), 1): + suf = "" if i == 1 else str(i) + logging.info(f"Optimizer{suf}:\n{o}") + logging.info(f"Scheduler{suf}: {s}") + + # 5. Dump "args" to config.yaml + # NOTE(kamo): "args" should be saved after object-buildings are done + # because they are allowed to modify "args". + output_dir = Path(args.output_dir) + if not distributed_option.distributed or distributed_option.dist_rank == 0: + output_dir.mkdir(parents=True, exist_ok=True) + with (output_dir / "config.yaml").open("w", encoding="utf-8") as f: + logging.info( + f'Saving the configuration in {output_dir / "config.yaml"}' + ) + if args.use_pai: + buffer = BytesIO() + torch.save({"config": vars(args)}, buffer) + args.oss_bucket.put_object(os.path.join(args.output_dir, "config.dict"), buffer.getvalue()) + else: + yaml_no_alias_safe_dump(vars(args), f, indent=4, sort_keys=False) + + if args.dry_run: + pass + else: + logging.info("Training args: {}".format(args)) + # 6. Loads pre-trained model + for p in args.init_param: + logging.info(f"Loading pretrained params from {p}") + load_pretrained_model( + model=model, + init_param=p, + ignore_init_mismatch=args.ignore_init_mismatch, + # NOTE(kamo): "cuda" for torch.load always indicates cuda:0 + # in PyTorch<=1.4 + map_location=f"cuda:{torch.cuda.current_device()}" + if args.ngpu > 0 + else "cpu", + oss_bucket=args.oss_bucket, + ) + + # 7. Build iterator factories + if args.dataset_type == "large": + from funasr.datasets.large_datasets.build_dataloader import ArkDataLoader + train_iter_factory = ArkDataLoader(args.train_data_file, args.token_list, + args.config, mode="train") + valid_iter_factory = ArkDataLoader(args.valid_data_file, args.token_list, + args.config, mode="eval") + elif args.dataset_type == "small": + train_iter_factory = cls.build_iter_factory( + args=args, + distributed_option=distributed_option, + mode="train", + ) + valid_iter_factory = cls.build_iter_factory( + args=args, + distributed_option=distributed_option, + mode="valid", + ) + else: + raise ValueError(f"Not supported dataset_type={args.dataset_type}") + + if args.scheduler == "tri_stage": + for scheduler in schedulers: + scheduler.init_tri_stage_scheudler(max_update=args.max_update) + + # 8. Start training + if args.use_wandb: + if wandb is None: + raise RuntimeError("Please install wandb") + + try: + wandb.login() + except wandb.errors.UsageError: + logging.info("wandb not configured! run `wandb login` to enable") + args.use_wandb = False + + if args.use_wandb: + if ( + not distributed_option.distributed + or distributed_option.dist_rank == 0 + ): + if args.wandb_project is None: + project = "FunASR_" + cls.__name__ + else: + project = args.wandb_project + + if args.wandb_name is None: + name = str(Path(".").resolve()).replace("/", "_") + else: + name = args.wandb_name + + wandb.init( + entity=args.wandb_entity, + project=project, + name=name, + dir=output_dir, + id=args.wandb_id, + resume="allow", + ) + wandb.config.update(args) + else: + # wandb also supports grouping for distributed training, + # but we only logs aggregated data, + # so it's enough to perform on rank0 node. + args.use_wandb = False + + # Don't give args to trainer.run() directly!!! + # Instead of it, define "Options" object and build here. + trainer_options = cls.trainer.build_options(args) + cls.trainer.run( + model=model, + optimizers=optimizers, + schedulers=schedulers, + train_iter_factory=train_iter_factory, + valid_iter_factory=valid_iter_factory, + trainer_options=trainer_options, + distributed_option=distributed_option, + ) + + if args.use_wandb and wandb.run: + wandb.finish() + + @classmethod + def build_iter_options( + cls, + args: argparse.Namespace, + distributed_option: DistributedOption, + mode: str, + ): + if mode == "train": + preprocess_fn = cls.build_preprocess_fn(args, train=True) + collate_fn = cls.build_collate_fn(args, train=True) + data_path_and_name_and_type = args.train_data_path_and_name_and_type + shape_files = args.train_shape_file + batch_size = args.batch_size + batch_bins = args.batch_bins + batch_type = args.batch_type + max_cache_size = args.max_cache_size + max_cache_fd = args.max_cache_fd + distributed = distributed_option.distributed + num_batches = None + num_iters_per_epoch = args.num_iters_per_epoch + train = True + + elif mode == "valid": + preprocess_fn = cls.build_preprocess_fn(args, train=False) + collate_fn = cls.build_collate_fn(args, train=False) + data_path_and_name_and_type = args.valid_data_path_and_name_and_type + shape_files = args.valid_shape_file + + if args.valid_batch_type is None: + batch_type = args.batch_type + else: + batch_type = args.valid_batch_type + if args.valid_batch_size is None: + batch_size = args.batch_size + else: + batch_size = args.valid_batch_size + if args.valid_batch_bins is None: + batch_bins = args.batch_bins + else: + batch_bins = args.valid_batch_bins + if args.valid_max_cache_size is None: + # Cache 5% of maximum size for validation loader + max_cache_size = 0.05 * args.max_cache_size + else: + max_cache_size = args.valid_max_cache_size + max_cache_fd = args.max_cache_fd + distributed = distributed_option.distributed + num_batches = None + num_iters_per_epoch = None + train = False + else: + raise NotImplementedError(f"mode={mode}") + + return IteratorOptions( + preprocess_fn=preprocess_fn, + collate_fn=collate_fn, + data_path_and_name_and_type=data_path_and_name_and_type, + shape_files=shape_files, + batch_type=batch_type, + batch_size=batch_size, + batch_bins=batch_bins, + num_batches=num_batches, + max_cache_size=max_cache_size, + max_cache_fd=max_cache_fd, + distributed=distributed, + num_iters_per_epoch=num_iters_per_epoch, + train=train, + ) + + @classmethod + def build_iter_factory( + cls, + args: argparse.Namespace, + distributed_option: DistributedOption, + mode: str, + kwargs: dict = None, + ) -> AbsIterFactory: + """Build a factory object of mini-batch iterator. + + This object is invoked at every epochs to build the iterator for each epoch + as following: + + >>> iter_factory = cls.build_iter_factory(...) + >>> for epoch in range(1, max_epoch): + ... for keys, batch in iter_fatory.build_iter(epoch): + ... model(**batch) + + The mini-batches for each epochs are fully controlled by this class. + Note that the random seed used for shuffling is decided as "seed + epoch" and + the generated mini-batches can be reproduces when resuming. + + Note that the definition of "epoch" doesn't always indicate + to run out of the whole training corpus. + "--num_iters_per_epoch" option restricts the number of iterations for each epoch + and the rest of samples for the originally epoch are left for the next epoch. + e.g. If The number of mini-batches equals to 4, the following two are same: + + - 1 epoch without "--num_iters_per_epoch" + - 4 epoch with "--num_iters_per_epoch" == 4 + + """ + assert check_argument_types() + iter_options = cls.build_iter_options(args, distributed_option, mode) + + # Overwrite iter_options if any kwargs is given + if kwargs is not None: + for k, v in kwargs.items(): + setattr(iter_options, k, v) + + if args.iterator_type == "sequence": + return cls.build_sequence_iter_factory( + args=args, + iter_options=iter_options, + mode=mode, + ) + elif args.iterator_type == "chunk": + return cls.build_chunk_iter_factory( + args=args, + iter_options=iter_options, + mode=mode, + ) + elif args.iterator_type == "task": + return cls.build_task_iter_factory( + args=args, + iter_options=iter_options, + mode=mode, + ) + else: + raise RuntimeError(f"Not supported: iterator_type={args.iterator_type}") + + @classmethod + def build_sequence_iter_factory( + cls, args: argparse.Namespace, iter_options: IteratorOptions, mode: str + ) -> AbsIterFactory: + assert check_argument_types() + + dataset = ESPnetDataset( + iter_options.data_path_and_name_and_type, + float_dtype=args.train_dtype, + preprocess=iter_options.preprocess_fn, + max_cache_size=iter_options.max_cache_size, + max_cache_fd=iter_options.max_cache_fd, + ) + cls.check_task_requirements( + dataset, args.allow_variable_data_keys, train=iter_options.train + ) + + if Path( + Path(iter_options.data_path_and_name_and_type[0][0]).parent, "utt2category" + ).exists(): + utt2category_file = str( + Path( + Path(iter_options.data_path_and_name_and_type[0][0]).parent, + "utt2category", + ) + ) + else: + utt2category_file = None + batch_sampler = build_batch_sampler( + type=iter_options.batch_type, + shape_files=iter_options.shape_files, + fold_lengths=args.fold_length, + batch_size=iter_options.batch_size, + batch_bins=iter_options.batch_bins, + sort_in_batch=args.sort_in_batch, + sort_batch=args.sort_batch, + drop_last=False, + min_batch_size=torch.distributed.get_world_size() + if iter_options.distributed + else 1, + utt2category_file=utt2category_file, + ) + + batches = list(batch_sampler) + if iter_options.num_batches is not None: + batches = batches[: iter_options.num_batches] + + bs_list = [len(batch) for batch in batches] + + logging.info(f"[{mode}] dataset:\n{dataset}") + logging.info(f"[{mode}] Batch sampler: {batch_sampler}") + logging.info( + f"[{mode}] mini-batch sizes summary: N-batch={len(bs_list)}, " + f"mean={np.mean(bs_list):.1f}, min={np.min(bs_list)}, max={np.max(bs_list)}" + ) + + if args.scheduler == "tri_stage" and mode == "train": + args.max_update = len(bs_list) * args.max_epoch + logging.info("Max update: {}".format(args.max_update)) + + if iter_options.distributed: + world_size = torch.distributed.get_world_size() + rank = torch.distributed.get_rank() + for batch in batches: + if len(batch) < world_size: + raise RuntimeError( + f"The batch-size must be equal or more than world_size: " + f"{len(batch)} < {world_size}" + ) + batches = [batch[rank::world_size] for batch in batches] + + return SequenceIterFactory( + dataset=dataset, + batches=batches, + seed=args.seed, + num_iters_per_epoch=iter_options.num_iters_per_epoch, + shuffle=iter_options.train, + num_workers=args.num_workers, + collate_fn=iter_options.collate_fn, + pin_memory=args.ngpu > 0, + ) + + @classmethod + def build_chunk_iter_factory( + cls, + args: argparse.Namespace, + iter_options: IteratorOptions, + mode: str, + ) -> AbsIterFactory: + assert check_argument_types() + + dataset = ESPnetDataset( + iter_options.data_path_and_name_and_type, + float_dtype=args.train_dtype, + preprocess=iter_options.preprocess_fn, + max_cache_size=iter_options.max_cache_size, + max_cache_fd=iter_options.max_cache_fd, + ) + cls.check_task_requirements( + dataset, args.allow_variable_data_keys, train=iter_options.train + ) + + if len(iter_options.shape_files) == 0: + key_file = iter_options.data_path_and_name_and_type[0][0] + else: + key_file = iter_options.shape_files[0] + + batch_sampler = UnsortedBatchSampler(batch_size=1, key_file=key_file) + batches = list(batch_sampler) + if iter_options.num_batches is not None: + batches = batches[: iter_options.num_batches] + logging.info(f"[{mode}] dataset:\n{dataset}") + + if iter_options.distributed: + world_size = torch.distributed.get_world_size() + rank = torch.distributed.get_rank() + if len(batches) < world_size: + raise RuntimeError("Number of samples is smaller than world_size") + if iter_options.batch_size < world_size: + raise RuntimeError("batch_size must be equal or more than world_size") + + if rank < iter_options.batch_size % world_size: + batch_size = iter_options.batch_size // world_size + 1 + else: + batch_size = iter_options.batch_size // world_size + num_cache_chunks = args.num_cache_chunks // world_size + # NOTE(kamo): Split whole corpus by sample numbers without considering + # each of the lengths, therefore the number of iteration counts are not + # always equal to each other and the iterations are limitted + # by the fewest iterations. + # i.e. the samples over the counts are discarded. + batches = batches[rank::world_size] + else: + batch_size = iter_options.batch_size + num_cache_chunks = args.num_cache_chunks + + return ChunkIterFactory( + dataset=dataset, + batches=batches, + seed=args.seed, + batch_size=batch_size, + # For chunk iterator, + # --num_iters_per_epoch doesn't indicate the number of iterations, + # but indicates the number of samples. + num_samples_per_epoch=iter_options.num_iters_per_epoch, + shuffle=iter_options.train, + num_workers=args.num_workers, + collate_fn=iter_options.collate_fn, + pin_memory=args.ngpu > 0, + chunk_length=args.chunk_length, + chunk_shift_ratio=args.chunk_shift_ratio, + num_cache_chunks=num_cache_chunks, + ) + + # NOTE(kamo): Not abstract class + @classmethod + def build_task_iter_factory( + cls, + args: argparse.Namespace, + iter_options: IteratorOptions, + mode: str, + ) -> AbsIterFactory: + """Build task specific iterator factory + + Example: + + >>> class YourTask(AbsTask): + ... @classmethod + ... def add_task_arguments(cls, parser: argparse.ArgumentParser): + ... parser.set_defaults(iterator_type="task") + ... + ... @classmethod + ... def build_task_iter_factory( + ... cls, + ... args: argparse.Namespace, + ... iter_options: IteratorOptions, + ... mode: str, + ... ): + ... return FooIterFactory(...) + ... + ... @classmethod + ... def build_iter_options( + .... args: argparse.Namespace, + ... distributed_option: DistributedOption, + ... mode: str + ... ): + ... # if you need to customize options object + """ + raise NotImplementedError + + @classmethod + def build_multiple_iter_factory( + cls, args: argparse.Namespace, distributed_option: DistributedOption, mode: str + ): + assert check_argument_types() + iter_options = cls.build_iter_options(args, distributed_option, mode) + assert len(iter_options.data_path_and_name_and_type) > 0, len( + iter_options.data_path_and_name_and_type + ) + + # 1. Sanity check + num_splits = None + for path in [ + path for path, _, _ in iter_options.data_path_and_name_and_type + ] + list(iter_options.shape_files): + if not Path(path).is_dir(): + raise RuntimeError(f"{path} is not a directory") + p = Path(path) / "num_splits" + if not p.exists(): + raise FileNotFoundError(f"{p} is not found") + with p.open() as f: + _num_splits = int(f.read()) + if num_splits is not None and num_splits != _num_splits: + raise RuntimeError( + f"Number of splits are mismathed: " + f"{iter_options.data_path_and_name_and_type[0][0]} and {path}" + ) + num_splits = _num_splits + + for i in range(num_splits): + p = Path(path) / f"split.{i}" + if not p.exists(): + raise FileNotFoundError(f"{p} is not found") + + # 2. Create functions to build an iter factory for each splits + data_path_and_name_and_type_list = [ + [ + (str(Path(p) / f"split.{i}"), n, t) + for p, n, t in iter_options.data_path_and_name_and_type + ] + for i in range(num_splits) + ] + shape_files_list = [ + [str(Path(s) / f"split.{i}") for s in iter_options.shape_files] + for i in range(num_splits) + ] + num_iters_per_epoch_list = [ + (iter_options.num_iters_per_epoch + i) // num_splits + if iter_options.num_iters_per_epoch is not None + else None + for i in range(num_splits) + ] + max_cache_size = iter_options.max_cache_size / num_splits + + # Note that iter-factories are built for each epoch at runtime lazily. + build_funcs = [ + functools.partial( + cls.build_iter_factory, + args, + distributed_option, + mode, + kwargs=dict( + data_path_and_name_and_type=_data_path_and_name_and_type, + shape_files=_shape_files, + num_iters_per_epoch=_num_iters_per_epoch, + max_cache_size=max_cache_size, + ), + ) + for ( + _data_path_and_name_and_type, + _shape_files, + _num_iters_per_epoch, + ) in zip( + data_path_and_name_and_type_list, + shape_files_list, + num_iters_per_epoch_list, + ) + ] + + # 3. Build MultipleIterFactory + return MultipleIterFactory( + build_funcs=build_funcs, shuffle=iter_options.train, seed=args.seed + ) + + @classmethod + def build_streaming_iterator( + cls, + data_path_and_name_and_type, + preprocess_fn, + collate_fn, + key_file: str = None, + batch_size: int = 1, + dtype: str = np.float32, + num_workers: int = 1, + allow_variable_data_keys: bool = False, + ngpu: int = 0, + inference: bool = False, + ) -> DataLoader: + """Build DataLoader using iterable dataset""" + assert check_argument_types() + # For backward compatibility for pytorch DataLoader + if collate_fn is not None: + kwargs = dict(collate_fn=collate_fn) + else: + kwargs = {} + + dataset = IterableESPnetDataset( + data_path_and_name_and_type, + float_dtype=dtype, + preprocess=preprocess_fn, + key_file=key_file, + ) + if dataset.apply_utt2category: + kwargs.update(batch_size=1) + else: + kwargs.update(batch_size=batch_size) + + cls.check_task_requirements( + dataset, allow_variable_data_keys, train=False, inference=inference + ) + + return DataLoader( + dataset=dataset, + pin_memory=ngpu > 0, + num_workers=num_workers, + **kwargs, + ) + + # ~~~~~~~~~ The methods below are mainly used for inference ~~~~~~~~~ + @classmethod + def build_model_from_file( + cls, + config_file: Union[Path, str] = None, + model_file: Union[Path, str] = None, + device: str = "cpu", + ) -> Tuple[AbsESPnetModel, argparse.Namespace]: + """Build model from the files. + + This method is used for inference or fine-tuning. + + Args: + config_file: The yaml file saved when training. + model_file: The model file saved when training. + device: Device type, "cpu", "cuda", or "cuda:N". + + """ + assert check_argument_types() + if config_file is None: + assert model_file is not None, ( + "The argument 'model_file' must be provided " + "if the argument 'config_file' is not specified." + ) + config_file = Path(model_file).parent / "config.yaml" + else: + config_file = Path(config_file) + + with config_file.open("r", encoding="utf-8") as f: + args = yaml.safe_load(f) + args = argparse.Namespace(**args) + model = cls.build_model(args) + if not isinstance(model, AbsESPnetModel): + raise RuntimeError( + f"model must inherit {AbsESPnetModel.__name__}, but got {type(model)}" + ) + model.to(device) + if model_file is not None: + if device == "cuda": + # NOTE(kamo): "cuda" for torch.load always indicates cuda:0 + # in PyTorch<=1.4 + device = f"cuda:{torch.cuda.current_device()}" + model.load_state_dict(torch.load(model_file, map_location=device)) + + return model, args diff --git a/funasr/tasks/asr.py b/funasr/tasks/asr.py new file mode 100644 index 000000000..9367ed32e --- /dev/null +++ b/funasr/tasks/asr.py @@ -0,0 +1,879 @@ +import argparse +import logging +from typing import Callable +from typing import Collection +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple + +import numpy as np +import torch +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.datasets.collate_fn import CommonCollateFn +from funasr.datasets.preprocessor import CommonPreprocessor +from funasr.models.ctc import CTC +from funasr.models.decoder.abs_decoder import AbsDecoder +from funasr.models.decoder.rnn_decoder import RNNDecoder +from funasr.models.decoder.transformer_decoder import ( + DynamicConvolution2DTransformerDecoder, # noqa: H301 +) +from funasr.models.decoder.transformer_decoder import DynamicConvolutionTransformerDecoder +from funasr.models.decoder.transformer_decoder import ( + LightweightConvolution2DTransformerDecoder, # noqa: H301 +) +from funasr.models.decoder.transformer_decoder import ( + LightweightConvolutionTransformerDecoder, # noqa: H301 +) +from funasr.models.decoder.transformer_decoder import TransformerDecoder +from funasr.models.encoder.abs_encoder import AbsEncoder +from funasr.models.encoder.conformer_encoder import ConformerEncoder +from funasr.models.encoder.rnn_encoder import RNNEncoder +from funasr.models.encoder.transformer_encoder import TransformerEncoder +from funasr.models.frontend.abs_frontend import AbsFrontend +from funasr.models.frontend.default import DefaultFrontend +from funasr.models.frontend.fused import FusedFrontends +from funasr.models.frontend.s3prl import S3prlFrontend +from funasr.models.frontend.windowing import SlidingWindow +from funasr.models.postencoder.abs_postencoder import AbsPostEncoder +from funasr.models.postencoder.hugging_face_transformers_postencoder import ( + HuggingFaceTransformersPostEncoder, # noqa: H301 +) +from funasr.models.preencoder.abs_preencoder import AbsPreEncoder +from funasr.models.preencoder.linear import LinearProjection +from funasr.models.preencoder.sinc import LightweightSincConvs +from funasr.models.specaug.abs_specaug import AbsSpecAug +from funasr.models.specaug.specaug import SpecAug +from funasr.layers.abs_normalize import AbsNormalize +from funasr.layers.global_mvn import GlobalMVN +from funasr.layers.utterance_mvn import UtteranceMVN +from funasr.tasks.abs_task import AbsTask +from funasr.text.phoneme_tokenizer import g2p_choices +from funasr.torch_utils.initialize import initialize +from funasr.train.abs_espnet_model import AbsESPnetModel +from funasr.train.class_choices import ClassChoices +from funasr.train.trainer import Trainer +from funasr.utils.get_default_kwargs import get_default_kwargs +from funasr.utils.nested_dict_action import NestedDictAction +from funasr.utils.types import float_or_none +from funasr.utils.types import int_or_none +from funasr.utils.types import str2bool +from funasr.utils.types import str_or_none + +from funasr.models.specaug.specaug import SpecAugLFR +from funasr.models.predictor.cif import CifPredictor, CifPredictorV2 +from funasr.modules.subsampling import Conv1dSubsampling +from funasr.models.e2e_asr import ESPnetASRModel +from funasr.models.e2e_uni_asr import UniASR +from funasr.models.encoder.sanm_encoder import SANMEncoder, SANMEncoderChunkOpt +from funasr.models.decoder.sanm_decoder import ParaformerSANMDecoder, FsmnDecoderSCAMAOpt +from funasr.models.e2e_asr_paraformer import Paraformer, ParaformerBert +from funasr.models.decoder.transformer_decoder import ParaformerDecoderSAN + +frontend_choices = ClassChoices( + name="frontend", + classes=dict( + default=DefaultFrontend, + sliding_window=SlidingWindow, + s3prl=S3prlFrontend, + fused=FusedFrontends, + ), + type_check=AbsFrontend, + default="default", +) +specaug_choices = ClassChoices( + name="specaug", + classes=dict( + specaug=SpecAug, + specaug_lfr=SpecAugLFR, + ), + type_check=AbsSpecAug, + default=None, + optional=True, +) +normalize_choices = ClassChoices( + "normalize", + classes=dict( + global_mvn=GlobalMVN, + utterance_mvn=UtteranceMVN, + ), + type_check=AbsNormalize, + default=None, + optional=True, +) +model_choices = ClassChoices( + "model", + classes=dict( + asr=ESPnetASRModel, + uniasr=UniASR, + paraformer=Paraformer, + paraformer_bert=ParaformerBert, + ), + type_check=AbsESPnetModel, + default="asr", +) +preencoder_choices = ClassChoices( + name="preencoder", + classes=dict( + sinc=LightweightSincConvs, + linear=LinearProjection, + ), + type_check=AbsPreEncoder, + default=None, + optional=True, +) +encoder_choices = ClassChoices( + "encoder", + classes=dict( + conformer=ConformerEncoder, + transformer=TransformerEncoder, + rnn=RNNEncoder, + sanm=SANMEncoder, + sanm_chunk_opt=SANMEncoderChunkOpt, + ), + type_check=AbsEncoder, + default="rnn", +) +encoder_choices2 = ClassChoices( + "encoder2", + classes=dict( + conformer=ConformerEncoder, + transformer=TransformerEncoder, + rnn=RNNEncoder, + sanm=SANMEncoder, + sanm_chunk_opt=SANMEncoderChunkOpt, + ), + type_check=AbsEncoder, + default="rnn", +) +postencoder_choices = ClassChoices( + name="postencoder", + classes=dict( + hugging_face_transformers=HuggingFaceTransformersPostEncoder, + ), + type_check=AbsPostEncoder, + default=None, + optional=True, +) +decoder_choices = ClassChoices( + "decoder", + classes=dict( + transformer=TransformerDecoder, + lightweight_conv=LightweightConvolutionTransformerDecoder, + lightweight_conv2d=LightweightConvolution2DTransformerDecoder, + dynamic_conv=DynamicConvolutionTransformerDecoder, + dynamic_conv2d=DynamicConvolution2DTransformerDecoder, + rnn=RNNDecoder, + fsmn_scama_opt=FsmnDecoderSCAMAOpt, + paraformer_decoder_sanm=ParaformerSANMDecoder, + paraformer_decoder_san=ParaformerDecoderSAN, + ), + type_check=AbsDecoder, + default="rnn", +) +decoder_choices2 = ClassChoices( + "decoder2", + classes=dict( + transformer=TransformerDecoder, + lightweight_conv=LightweightConvolutionTransformerDecoder, + lightweight_conv2d=LightweightConvolution2DTransformerDecoder, + dynamic_conv=DynamicConvolutionTransformerDecoder, + dynamic_conv2d=DynamicConvolution2DTransformerDecoder, + rnn=RNNDecoder, + fsmn_scama_opt=FsmnDecoderSCAMAOpt, + paraformer_decoder_sanm=ParaformerSANMDecoder, + ), + type_check=AbsDecoder, + default="rnn", +) +predictor_choices = ClassChoices( + name="predictor", + classes=dict( + cif_predictor=CifPredictor, + ctc_predictor=None, + cif_predictor_v2=CifPredictorV2, + ), + type_check=None, + default="cif_predictor", + optional=True, +) +predictor_choices2 = ClassChoices( + name="predictor2", + classes=dict( + cif_predictor=CifPredictor, + ctc_predictor=None, + cif_predictor_v2=CifPredictorV2, + ), + type_check=None, + default="cif_predictor", + optional=True, +) +stride_conv_choices = ClassChoices( + name="stride_conv", + classes=dict( + stride_conv1d=Conv1dSubsampling + ), + type_check=None, + default="stride_conv1d", + optional=True, +) + + +class ASRTask(AbsTask): + # If you need more than one optimizers, change this value + num_optimizers: int = 1 + + # Add variable objects configurations + class_choices_list = [ + # --frontend and --frontend_conf + frontend_choices, + # --specaug and --specaug_conf + specaug_choices, + # --normalize and --normalize_conf + normalize_choices, + # --model and --model_conf + model_choices, + # --preencoder and --preencoder_conf + preencoder_choices, + # --encoder and --encoder_conf + encoder_choices, + # --postencoder and --postencoder_conf + postencoder_choices, + # --decoder and --decoder_conf + decoder_choices, + ] + + # If you need to modify train() or eval() procedures, change Trainer class here + trainer = Trainer + + @classmethod + def add_task_arguments(cls, parser: argparse.ArgumentParser): + group = parser.add_argument_group(description="Task related") + + # NOTE(kamo): add_arguments(..., required=True) can't be used + # to provide --print_config mode. Instead of it, do as + required = parser.get_default("required") + required += ["token_list"] + + group.add_argument( + "--token_list", + type=str_or_none, + default=None, + help="A text mapping int-id to token", + ) + group.add_argument( + "--split_with_space", + type=str2bool, + default=True, + help="whether to split text using ", + ) + group.add_argument( + "--init", + type=lambda x: str_or_none(x.lower()), + default=None, + help="The initialization method", + choices=[ + "chainer", + "xavier_uniform", + "xavier_normal", + "kaiming_uniform", + "kaiming_normal", + None, + ], + ) + + group.add_argument( + "--input_size", + type=int_or_none, + default=None, + help="The number of input dimension of the feature", + ) + + group.add_argument( + "--ctc_conf", + action=NestedDictAction, + default=get_default_kwargs(CTC), + help="The keyword arguments for CTC class.", + ) + group.add_argument( + "--joint_net_conf", + action=NestedDictAction, + default=None, + help="The keyword arguments for joint network class.", + ) + + group = parser.add_argument_group(description="Preprocess related") + group.add_argument( + "--use_preprocessor", + type=str2bool, + default=True, + help="Apply preprocessing to data or not", + ) + group.add_argument( + "--token_type", + type=str, + default="bpe", + choices=["bpe", "char", "word", "phn"], + help="The text will be tokenized " "in the specified level token", + ) + group.add_argument( + "--bpemodel", + type=str_or_none, + default=None, + help="The model file of sentencepiece", + ) + parser.add_argument( + "--non_linguistic_symbols", + type=str_or_none, + default=None, + help="non_linguistic_symbols file path", + ) + parser.add_argument( + "--cleaner", + type=str_or_none, + choices=[None, "tacotron", "jaconv", "vietnamese"], + default=None, + help="Apply text cleaning", + ) + parser.add_argument( + "--g2p", + type=str_or_none, + choices=g2p_choices, + default=None, + help="Specify g2p method if --token_type=phn", + ) + parser.add_argument( + "--speech_volume_normalize", + type=float_or_none, + default=None, + help="Scale the maximum amplitude to the given value.", + ) + parser.add_argument( + "--rir_scp", + type=str_or_none, + default=None, + help="The file path of rir scp file.", + ) + parser.add_argument( + "--rir_apply_prob", + type=float, + default=1.0, + help="THe probability for applying RIR convolution.", + ) + parser.add_argument( + "--noise_scp", + type=str_or_none, + default=None, + help="The file path of noise scp file.", + ) + parser.add_argument( + "--noise_apply_prob", + type=float, + default=1.0, + help="The probability applying Noise adding.", + ) + parser.add_argument( + "--noise_db_range", + type=str, + default="13_15", + help="The range of noise decibel level.", + ) + + for class_choices in cls.class_choices_list: + # Append -- and --_conf. + # e.g. --encoder and --encoder_conf + class_choices.add_arguments(group) + + @classmethod + def build_collate_fn( + cls, args: argparse.Namespace, train: bool + ) -> Callable[ + [Collection[Tuple[str, Dict[str, np.ndarray]]]], + Tuple[List[str], Dict[str, torch.Tensor]], + ]: + assert check_argument_types() + # NOTE(kamo): int value = 0 is reserved by CTC-blank symbol + return CommonCollateFn(float_pad_value=0.0, int_pad_value=-1) + + @classmethod + def build_preprocess_fn( + cls, args: argparse.Namespace, train: bool + ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]: + assert check_argument_types() + if args.use_preprocessor: + retval = CommonPreprocessor( + train=train, + token_type=args.token_type, + token_list=args.token_list, + bpemodel=args.bpemodel, + non_linguistic_symbols=args.non_linguistic_symbols, + text_cleaner=args.cleaner, + g2p_type=args.g2p, + split_with_space=args.split_with_space if hasattr(args, "split_with_space") else False, + # NOTE(kamo): Check attribute existence for backward compatibility + rir_scp=args.rir_scp if hasattr(args, "rir_scp") else None, + rir_apply_prob=args.rir_apply_prob + if hasattr(args, "rir_apply_prob") + else 1.0, + noise_scp=args.noise_scp if hasattr(args, "noise_scp") else None, + noise_apply_prob=args.noise_apply_prob + if hasattr(args, "noise_apply_prob") + else 1.0, + noise_db_range=args.noise_db_range + if hasattr(args, "noise_db_range") + else "13_15", + speech_volume_normalize=args.speech_volume_normalize + if hasattr(args, "rir_scp") + else None, + ) + else: + retval = None + assert check_return_type(retval) + return retval + + @classmethod + def required_data_names( + cls, train: bool = True, inference: bool = False + ) -> Tuple[str, ...]: + if not inference: + retval = ("speech", "text") + else: + # Recognition mode + retval = ("speech",) + return retval + + @classmethod + def optional_data_names( + cls, train: bool = True, inference: bool = False + ) -> Tuple[str, ...]: + retval = () + assert check_return_type(retval) + return retval + + @classmethod + def build_model(cls, args: argparse.Namespace): + assert check_argument_types() + if isinstance(args.token_list, str): + with open(args.token_list, encoding="utf-8") as f: + token_list = [line.rstrip() for line in f] + + # Overwriting token_list to keep it as "portable". + args.token_list = list(token_list) + elif isinstance(args.token_list, (tuple, list)): + token_list = list(args.token_list) + else: + raise RuntimeError("token_list must be str or list") + vocab_size = len(token_list) + logging.info(f"Vocabulary size: {vocab_size}") + + # 1. frontend + if args.input_size is None: + # Extract features in the model + frontend_class = frontend_choices.get_class(args.frontend) + frontend = frontend_class(**args.frontend_conf) + input_size = frontend.output_size() + else: + # Give features from data-loader + args.frontend = None + args.frontend_conf = {} + frontend = None + input_size = args.input_size + + # 2. Data augmentation for spectrogram + if args.specaug is not None: + specaug_class = specaug_choices.get_class(args.specaug) + specaug = specaug_class(**args.specaug_conf) + else: + specaug = None + + # 3. Normalization layer + if args.normalize is not None: + normalize_class = normalize_choices.get_class(args.normalize) + normalize = normalize_class(**args.normalize_conf) + else: + normalize = None + + # 4. Pre-encoder input block + # NOTE(kan-bayashi): Use getattr to keep the compatibility + if getattr(args, "preencoder", None) is not None: + preencoder_class = preencoder_choices.get_class(args.preencoder) + preencoder = preencoder_class(**args.preencoder_conf) + input_size = preencoder.output_size() + else: + preencoder = None + + # 5. Encoder + encoder_class = encoder_choices.get_class(args.encoder) + encoder = encoder_class(input_size=input_size, **args.encoder_conf) + + # 6. Post-encoder block + # NOTE(kan-bayashi): Use getattr to keep the compatibility + encoder_output_size = encoder.output_size() + if getattr(args, "postencoder", None) is not None: + postencoder_class = postencoder_choices.get_class(args.postencoder) + postencoder = postencoder_class( + input_size=encoder_output_size, **args.postencoder_conf + ) + encoder_output_size = postencoder.output_size() + else: + postencoder = None + + # 7. Decoder + decoder_class = decoder_choices.get_class(args.decoder) + decoder = decoder_class( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + **args.decoder_conf, + ) + + # 8. CTC + ctc = CTC( + odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf + ) + + # 9. Build model + try: + model_class = model_choices.get_class(args.model) + except AttributeError: + model_class = model_choices.get_class("asr") + model = model_class( + vocab_size=vocab_size, + frontend=frontend, + specaug=specaug, + normalize=normalize, + preencoder=preencoder, + encoder=encoder, + postencoder=postencoder, + decoder=decoder, + ctc=ctc, + token_list=token_list, + **args.model_conf, + ) + + # 10. Initialize + if args.init is not None: + initialize(model, args.init) + + assert check_return_type(model) + return model + + +class ASRTaskUniASR(ASRTask): + # If you need more than one optimizers, change this value + num_optimizers: int = 1 + + # Add variable objects configurations + class_choices_list = [ + # --frontend and --frontend_conf + frontend_choices, + # --specaug and --specaug_conf + specaug_choices, + # --normalize and --normalize_conf + normalize_choices, + # --model and --model_conf + model_choices, + # --preencoder and --preencoder_conf + preencoder_choices, + # --encoder and --encoder_conf + encoder_choices, + # --postencoder and --postencoder_conf + postencoder_choices, + # --decoder and --decoder_conf + decoder_choices, + # --predictor and --predictor_conf + predictor_choices, + # --encoder2 and --encoder2_conf + encoder_choices2, + # --decoder2 and --decoder2_conf + decoder_choices2, + # --predictor2 and --predictor2_conf + predictor_choices2, + # --stride_conv and --stride_conv_conf + stride_conv_choices, + ] + + # If you need to modify train() or eval() procedures, change Trainer class here + trainer = Trainer + + @classmethod + def build_model(cls, args: argparse.Namespace): + assert check_argument_types() + if isinstance(args.token_list, str): + with open(args.token_list, encoding="utf-8") as f: + token_list = [line.rstrip() for line in f] + + # Overwriting token_list to keep it as "portable". + args.token_list = list(token_list) + elif isinstance(args.token_list, (tuple, list)): + token_list = list(args.token_list) + else: + raise RuntimeError("token_list must be str or list") + vocab_size = len(token_list) + logging.info(f"Vocabulary size: {vocab_size}") + + # 1. frontend + if args.input_size is None: + # Extract features in the model + frontend_class = frontend_choices.get_class(args.frontend) + frontend = frontend_class(**args.frontend_conf) + input_size = frontend.output_size() + else: + # Give features from data-loader + args.frontend = None + args.frontend_conf = {} + frontend = None + input_size = args.input_size + + # 2. Data augmentation for spectrogram + if args.specaug is not None: + specaug_class = specaug_choices.get_class(args.specaug) + specaug = specaug_class(**args.specaug_conf) + else: + specaug = None + + # 3. Normalization layer + if args.normalize is not None: + normalize_class = normalize_choices.get_class(args.normalize) + normalize = normalize_class(**args.normalize_conf) + else: + normalize = None + + # 4. Pre-encoder input block + # NOTE(kan-bayashi): Use getattr to keep the compatibility + if getattr(args, "preencoder", None) is not None: + preencoder_class = preencoder_choices.get_class(args.preencoder) + preencoder = preencoder_class(**args.preencoder_conf) + input_size = preencoder.output_size() + else: + preencoder = None + + # 5. Encoder + encoder_class = encoder_choices.get_class(args.encoder) + encoder = encoder_class(input_size=input_size, **args.encoder_conf) + encoder_output_size = encoder.output_size() + + stride_conv_class = stride_conv_choices.get_class(args.stride_conv) + stride_conv = stride_conv_class(**args.stride_conv_conf, idim=input_size + encoder_output_size, + odim=input_size + encoder_output_size) + stride_conv_output_size = stride_conv.output_size() + + # 6. Encoder2 + encoder_class2 = encoder_choices2.get_class(args.encoder2) + encoder2 = encoder_class2(input_size=stride_conv_output_size, **args.encoder2_conf) + + # 7. Post-encoder block + # NOTE(kan-bayashi): Use getattr to keep the compatibility + encoder_output_size2 = encoder2.output_size() + if getattr(args, "postencoder", None) is not None: + postencoder_class = postencoder_choices.get_class(args.postencoder) + postencoder = postencoder_class( + input_size=encoder_output_size, **args.postencoder_conf + ) + encoder_output_size = postencoder.output_size() + else: + postencoder = None + + # 8. Decoder & Decoder2 + decoder_class = decoder_choices.get_class(args.decoder) + decoder_class2 = decoder_choices2.get_class(args.decoder2) + decoder = decoder_class( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + **args.decoder_conf, + ) + decoder2 = decoder_class2( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size2, + **args.decoder2_conf, + ) + + # 9. CTC + ctc = CTC( + odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf + ) + ctc2 = CTC( + odim=vocab_size, encoder_output_size=encoder_output_size2, **args.ctc_conf + ) + + # 10. Predictor + predictor_class = predictor_choices.get_class(args.predictor) + predictor = predictor_class(**args.predictor_conf) + + predictor_class = predictor_choices2.get_class(args.predictor2) + predictor2 = predictor_class(**args.predictor2_conf) + + # 11. Build model + try: + model_class = model_choices.get_class(args.model) + except AttributeError: + model_class = model_choices.get_class("asr") + model = model_class( + vocab_size=vocab_size, + frontend=frontend, + specaug=specaug, + normalize=normalize, + preencoder=preencoder, + encoder=encoder, + postencoder=postencoder, + decoder=decoder, + ctc=ctc, + token_list=token_list, + predictor=predictor, + ctc2=ctc2, + encoder2=encoder2, + decoder2=decoder2, + predictor2=predictor2, + stride_conv=stride_conv, + **args.model_conf, + ) + + # 12. Initialize + if args.init is not None: + initialize(model, args.init) + + assert check_return_type(model) + return model + + +class ASRTaskParaformer(ASRTask): + # If you need more than one optimizers, change this value + num_optimizers: int = 1 + + # Add variable objects configurations + class_choices_list = [ + # --frontend and --frontend_conf + frontend_choices, + # --specaug and --specaug_conf + specaug_choices, + # --normalize and --normalize_conf + normalize_choices, + # --model and --model_conf + model_choices, + # --preencoder and --preencoder_conf + preencoder_choices, + # --encoder and --encoder_conf + encoder_choices, + # --postencoder and --postencoder_conf + postencoder_choices, + # --decoder and --decoder_conf + decoder_choices, + # --predictor and --predictor_conf + predictor_choices, + ] + + # If you need to modify train() or eval() procedures, change Trainer class here + trainer = Trainer + + @classmethod + def build_model(cls, args: argparse.Namespace): + assert check_argument_types() + if isinstance(args.token_list, str): + with open(args.token_list, encoding="utf-8") as f: + token_list = [line.rstrip() for line in f] + + # Overwriting token_list to keep it as "portable". + args.token_list = list(token_list) + elif isinstance(args.token_list, (tuple, list)): + token_list = list(args.token_list) + else: + raise RuntimeError("token_list must be str or list") + vocab_size = len(token_list) + logging.info(f"Vocabulary size: {vocab_size }") + + # 1. frontend + if args.input_size is None: + # Extract features in the model + frontend_class = frontend_choices.get_class(args.frontend) + frontend = frontend_class(**args.frontend_conf) + input_size = frontend.output_size() + else: + # Give features from data-loader + args.frontend = None + args.frontend_conf = {} + frontend = None + input_size = args.input_size + + # 2. Data augmentation for spectrogram + if args.specaug is not None: + specaug_class = specaug_choices.get_class(args.specaug) + specaug = specaug_class(**args.specaug_conf) + else: + specaug = None + + # 3. Normalization layer + if args.normalize is not None: + normalize_class = normalize_choices.get_class(args.normalize) + normalize = normalize_class(**args.normalize_conf) + else: + normalize = None + + # 4. Pre-encoder input block + # NOTE(kan-bayashi): Use getattr to keep the compatibility + if getattr(args, "preencoder", None) is not None: + preencoder_class = preencoder_choices.get_class(args.preencoder) + preencoder = preencoder_class(**args.preencoder_conf) + input_size = preencoder.output_size() + else: + preencoder = None + + # 5. Encoder + encoder_class = encoder_choices.get_class(args.encoder) + encoder = encoder_class(input_size=input_size, **args.encoder_conf) + + # 6. Post-encoder block + # NOTE(kan-bayashi): Use getattr to keep the compatibility + encoder_output_size = encoder.output_size() + if getattr(args, "postencoder", None) is not None: + postencoder_class = postencoder_choices.get_class(args.postencoder) + postencoder = postencoder_class( + input_size=encoder_output_size, **args.postencoder_conf + ) + encoder_output_size = postencoder.output_size() + else: + postencoder = None + + # 7. Decoder + decoder_class = decoder_choices.get_class(args.decoder) + decoder = decoder_class( + vocab_size=vocab_size, + encoder_output_size=encoder_output_size, + **args.decoder_conf, + ) + + # 8. CTC + ctc = CTC( + odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf + ) + + # 9. Predictor + predictor_class = predictor_choices.get_class(args.predictor) + predictor = predictor_class(**args.predictor_conf) + + # 10. Build model + try: + model_class = model_choices.get_class(args.model) + except AttributeError: + model_class = model_choices.get_class("asr") + model = model_class( + vocab_size=vocab_size, + frontend=frontend, + specaug=specaug, + normalize=normalize, + preencoder=preencoder, + encoder=encoder, + postencoder=postencoder, + decoder=decoder, + ctc=ctc, + token_list=token_list, + predictor=predictor, + **args.model_conf, + ) + + # 11. Initialize + if args.init is not None: + initialize(model, args.init) + + assert check_return_type(model) + return model diff --git a/funasr/tasks/lm.py b/funasr/tasks/lm.py new file mode 100644 index 000000000..46b9fe089 --- /dev/null +++ b/funasr/tasks/lm.py @@ -0,0 +1,211 @@ +import argparse +import logging +from typing import Callable +from typing import Collection +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple + +import numpy as np +import torch +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.datasets.collate_fn import CommonCollateFn +from funasr.datasets.preprocessor import CommonPreprocessor +from funasr.lm.abs_model import AbsLM +from funasr.lm.espnet_model import ESPnetLanguageModel +from funasr.lm.seq_rnn_lm import SequentialRNNLM +from funasr.lm.transformer_lm import TransformerLM +from funasr.tasks.abs_task import AbsTask +from funasr.text.phoneme_tokenizer import g2p_choices +from funasr.torch_utils.initialize import initialize +from funasr.train.class_choices import ClassChoices +from funasr.train.trainer import Trainer +from funasr.utils.get_default_kwargs import get_default_kwargs +from funasr.utils.nested_dict_action import NestedDictAction +from funasr.utils.types import str2bool +from funasr.utils.types import str_or_none + +lm_choices = ClassChoices( + "lm", + classes=dict( + seq_rnn=SequentialRNNLM, + transformer=TransformerLM, + ), + type_check=AbsLM, + default="seq_rnn", +) + + +class LMTask(AbsTask): + # If you need more than one optimizers, change this value + num_optimizers: int = 1 + + # Add variable objects configurations + class_choices_list = [lm_choices] + + # If you need to modify train() or eval() procedures, change Trainer class here + trainer = Trainer + + @classmethod + def add_task_arguments(cls, parser: argparse.ArgumentParser): + # NOTE(kamo): Use '_' instead of '-' to avoid confusion + assert check_argument_types() + group = parser.add_argument_group(description="Task related") + + # NOTE(kamo): add_arguments(..., required=True) can't be used + # to provide --print_config mode. Instead of it, do as + required = parser.get_default("required") + required += ["token_list"] + + group.add_argument( + "--token_list", + type=str_or_none, + default=None, + help="A text mapping int-id to token", + ) + group.add_argument( + "--init", + type=lambda x: str_or_none(x.lower()), + default=None, + help="The initialization method", + choices=[ + "chainer", + "xavier_uniform", + "xavier_normal", + "kaiming_uniform", + "kaiming_normal", + None, + ], + ) + group.add_argument( + "--model_conf", + action=NestedDictAction, + default=get_default_kwargs(ESPnetLanguageModel), + help="The keyword arguments for model class.", + ) + + group = parser.add_argument_group(description="Preprocess related") + group.add_argument( + "--use_preprocessor", + type=str2bool, + default=True, + help="Apply preprocessing to data or not", + ) + group.add_argument( + "--token_type", + type=str, + default="bpe", + choices=["bpe", "char", "word"], + help="", + ) + group.add_argument( + "--bpemodel", + type=str_or_none, + default=None, + help="The model file fo sentencepiece", + ) + parser.add_argument( + "--non_linguistic_symbols", + type=str_or_none, + help="non_linguistic_symbols file path", + ) + parser.add_argument( + "--cleaner", + type=str_or_none, + choices=[None, "tacotron", "jaconv", "vietnamese"], + default=None, + help="Apply text cleaning", + ) + parser.add_argument( + "--g2p", + type=str_or_none, + choices=g2p_choices, + default=None, + help="Specify g2p method if --token_type=phn", + ) + + for class_choices in cls.class_choices_list: + class_choices.add_arguments(group) + + assert check_return_type(parser) + return parser + + @classmethod + def build_collate_fn( + cls, args: argparse.Namespace, train: bool + ) -> Callable[ + [Collection[Tuple[str, Dict[str, np.ndarray]]]], + Tuple[List[str], Dict[str, torch.Tensor]], + ]: + assert check_argument_types() + return CommonCollateFn(int_pad_value=0) + + @classmethod + def build_preprocess_fn( + cls, args: argparse.Namespace, train: bool + ) -> Optional[Callable[[str, Dict[str, np.array]], Dict[str, np.ndarray]]]: + assert check_argument_types() + if args.use_preprocessor: + retval = CommonPreprocessor( + train=train, + token_type=args.token_type, + token_list=args.token_list, + bpemodel=args.bpemodel, + text_cleaner=args.cleaner, + g2p_type=args.g2p, + non_linguistic_symbols=args.non_linguistic_symbols, + ) + else: + retval = None + assert check_return_type(retval) + return retval + + @classmethod + def required_data_names( + cls, train: bool = True, inference: bool = False + ) -> Tuple[str, ...]: + retval = ("text",) + return retval + + @classmethod + def optional_data_names( + cls, train: bool = True, inference: bool = False + ) -> Tuple[str, ...]: + retval = () + return retval + + @classmethod + def build_model(cls, args: argparse.Namespace) -> ESPnetLanguageModel: + assert check_argument_types() + if isinstance(args.token_list, str): + with open(args.token_list, encoding="utf-8") as f: + token_list = [line.rstrip() for line in f] + + # "args" is saved as it is in a yaml file by BaseTask.main(). + # Overwriting token_list to keep it as "portable". + args.token_list = token_list.copy() + elif isinstance(args.token_list, (tuple, list)): + token_list = args.token_list.copy() + else: + raise RuntimeError("token_list must be str or dict") + + vocab_size = len(token_list) + logging.info(f"Vocabulary size: {vocab_size}") + + # 1. Build LM model + lm_class = lm_choices.get_class(args.lm) + lm = lm_class(vocab_size=vocab_size, **args.lm_conf) + + # 2. Build ESPnetModel + # Assume the last-id is sos_and_eos + model = ESPnetLanguageModel(lm=lm, vocab_size=vocab_size, **args.model_conf) + + # 3. Initialize + if args.init is not None: + initialize(model, args.init) + + assert check_return_type(model) + return model diff --git a/funasr/text/__init__.py b/funasr/text/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/text/abs_tokenizer.py b/funasr/text/abs_tokenizer.py new file mode 100644 index 000000000..fc2ccb3c3 --- /dev/null +++ b/funasr/text/abs_tokenizer.py @@ -0,0 +1,14 @@ +from abc import ABC +from abc import abstractmethod +from typing import Iterable +from typing import List + + +class AbsTokenizer(ABC): + @abstractmethod + def text2tokens(self, line: str) -> List[str]: + raise NotImplementedError + + @abstractmethod + def tokens2text(self, tokens: Iterable[str]) -> str: + raise NotImplementedError diff --git a/funasr/text/build_tokenizer.py b/funasr/text/build_tokenizer.py new file mode 100644 index 000000000..8e29d3ed5 --- /dev/null +++ b/funasr/text/build_tokenizer.py @@ -0,0 +1,63 @@ +from pathlib import Path +from typing import Iterable +from typing import Union + +from typeguard import check_argument_types + +from funasr.text.abs_tokenizer import AbsTokenizer +from funasr.text.char_tokenizer import CharTokenizer +from funasr.text.phoneme_tokenizer import PhonemeTokenizer +from funasr.text.sentencepiece_tokenizer import SentencepiecesTokenizer +from funasr.text.word_tokenizer import WordTokenizer + + +def build_tokenizer( + token_type: str, + bpemodel: Union[Path, str, Iterable[str]] = None, + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + remove_non_linguistic_symbols: bool = False, + space_symbol: str = "", + delimiter: str = None, + g2p_type: str = None, +) -> AbsTokenizer: + """A helper function to instantiate Tokenizer""" + assert check_argument_types() + if token_type == "bpe": + if bpemodel is None: + raise ValueError('bpemodel is required if token_type = "bpe"') + + if remove_non_linguistic_symbols: + raise RuntimeError( + "remove_non_linguistic_symbols is not implemented for token_type=bpe" + ) + return SentencepiecesTokenizer(bpemodel) + + elif token_type == "word": + if remove_non_linguistic_symbols and non_linguistic_symbols is not None: + return WordTokenizer( + delimiter=delimiter, + non_linguistic_symbols=non_linguistic_symbols, + remove_non_linguistic_symbols=True, + ) + else: + return WordTokenizer(delimiter=delimiter) + + elif token_type == "char": + return CharTokenizer( + non_linguistic_symbols=non_linguistic_symbols, + space_symbol=space_symbol, + remove_non_linguistic_symbols=remove_non_linguistic_symbols, + ) + + elif token_type == "phn": + return PhonemeTokenizer( + g2p_type=g2p_type, + non_linguistic_symbols=non_linguistic_symbols, + space_symbol=space_symbol, + remove_non_linguistic_symbols=remove_non_linguistic_symbols, + ) + + else: + raise ValueError( + f"token_mode must be one of bpe, word, char or phn: " f"{token_type}" + ) diff --git a/funasr/text/char_tokenizer.py b/funasr/text/char_tokenizer.py new file mode 100644 index 000000000..00ae42732 --- /dev/null +++ b/funasr/text/char_tokenizer.py @@ -0,0 +1,62 @@ +from pathlib import Path +from typing import Iterable +from typing import List +from typing import Union +import warnings + +from typeguard import check_argument_types + +from funasr.text.abs_tokenizer import AbsTokenizer + + +class CharTokenizer(AbsTokenizer): + def __init__( + self, + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + space_symbol: str = "", + remove_non_linguistic_symbols: bool = False, + ): + assert check_argument_types() + self.space_symbol = space_symbol + if non_linguistic_symbols is None: + self.non_linguistic_symbols = set() + elif isinstance(non_linguistic_symbols, (Path, str)): + non_linguistic_symbols = Path(non_linguistic_symbols) + try: + with non_linguistic_symbols.open("r", encoding="utf-8") as f: + self.non_linguistic_symbols = set(line.rstrip() for line in f) + except FileNotFoundError: + warnings.warn(f"{non_linguistic_symbols} doesn't exist.") + self.non_linguistic_symbols = set() + else: + self.non_linguistic_symbols = set(non_linguistic_symbols) + self.remove_non_linguistic_symbols = remove_non_linguistic_symbols + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f'space_symbol="{self.space_symbol}"' + f'non_linguistic_symbols="{self.non_linguistic_symbols}"' + f")" + ) + + def text2tokens(self, line: Union[str, list]) -> List[str]: + tokens = [] + while len(line) != 0: + for w in self.non_linguistic_symbols: + if line.startswith(w): + if not self.remove_non_linguistic_symbols: + tokens.append(line[: len(w)]) + line = line[len(w) :] + break + else: + t = line[0] + if t == " ": + t = "" + tokens.append(t) + line = line[1:] + return tokens + + def tokens2text(self, tokens: Iterable[str]) -> str: + tokens = [t if t != self.space_symbol else " " for t in tokens] + return "".join(tokens) diff --git a/funasr/text/cleaner.py b/funasr/text/cleaner.py new file mode 100644 index 000000000..be26940c4 --- /dev/null +++ b/funasr/text/cleaner.py @@ -0,0 +1,48 @@ +from typing import Collection + +from jaconv import jaconv +import tacotron_cleaner.cleaners +from typeguard import check_argument_types + +try: + from vietnamese_cleaner import vietnamese_cleaners +except ImportError: + vietnamese_cleaners = None + + +class TextCleaner: + """Text cleaner. + + Examples: + >>> cleaner = TextCleaner("tacotron") + >>> cleaner("(Hello-World); & jr. & dr.") + 'HELLO WORLD, AND JUNIOR AND DOCTOR' + + """ + + def __init__(self, cleaner_types: Collection[str] = None): + assert check_argument_types() + + if cleaner_types is None: + self.cleaner_types = [] + elif isinstance(cleaner_types, str): + self.cleaner_types = [cleaner_types] + else: + self.cleaner_types = list(cleaner_types) + + def __call__(self, text: str) -> str: + for t in self.cleaner_types: + if t == "tacotron": + text = tacotron_cleaner.cleaners.custom_english_cleaners(text) + elif t == "jaconv": + text = jaconv.normalize(text) + elif t == "vietnamese": + if vietnamese_cleaners is None: + raise RuntimeError("Please install underthesea") + text = vietnamese_cleaners.vietnamese_cleaner(text) + elif t == "korean_cleaner": + text = KoreanCleaner.normalize_text(text) + else: + raise RuntimeError(f"Not supported: type={t}") + + return text diff --git a/funasr/text/korean_cleaner.py b/funasr/text/korean_cleaner.py new file mode 100644 index 000000000..ee556d42a --- /dev/null +++ b/funasr/text/korean_cleaner.py @@ -0,0 +1,77 @@ +# Referenced from https://github.com/hccho2/Tacotron-Wavenet-Vocoder-Korean + +import re + + +class KoreanCleaner: + @classmethod + def _normalize_numbers(cls, text): + number_to_kor = { + "0": "영", + "1": "일", + "2": "이", + "3": "삼", + "4": "사", + "5": "오", + "6": "육", + "7": "칠", + "8": "팔", + "9": "구", + } + new_text = "".join( + number_to_kor[char] if char in number_to_kor.keys() else char + for char in text + ) + return new_text + + @classmethod + def _normalize_english_text(cls, text): + upper_alphabet_to_kor = { + "A": "에이", + "B": "비", + "C": "씨", + "D": "디", + "E": "이", + "F": "에프", + "G": "지", + "H": "에이치", + "I": "아이", + "J": "제이", + "K": "케이", + "L": "엘", + "M": "엠", + "N": "엔", + "O": "오", + "P": "피", + "Q": "큐", + "R": "알", + "S": "에스", + "T": "티", + "U": "유", + "V": "브이", + "W": "더블유", + "X": "엑스", + "Y": "와이", + "Z": "지", + } + new_text = re.sub("[a-z]+", lambda x: str.upper(x.group()), text) + new_text = "".join( + upper_alphabet_to_kor[char] + if char in upper_alphabet_to_kor.keys() + else char + for char in new_text + ) + + return new_text + + @classmethod + def normalize_text(cls, text): + # stage 0 : text strip + text = text.strip() + + # stage 1 : normalize numbers + text = cls._normalize_numbers(text) + + # stage 2 : normalize english text + text = cls._normalize_english_text(text) + return text diff --git a/funasr/text/phoneme_tokenizer.py b/funasr/text/phoneme_tokenizer.py new file mode 100644 index 000000000..d424b40a8 --- /dev/null +++ b/funasr/text/phoneme_tokenizer.py @@ -0,0 +1,528 @@ +import logging +from pathlib import Path +import re +from typing import Iterable +from typing import List +from typing import Optional +from typing import Union +import warnings + +# import g2p_en +import jamo +from typeguard import check_argument_types + +from funasr.text.abs_tokenizer import AbsTokenizer + + +g2p_choices = [ + None, + "g2p_en", + "g2p_en_no_space", + "pyopenjtalk", + "pyopenjtalk_kana", + "pyopenjtalk_accent", + "pyopenjtalk_accent_with_pause", + "pyopenjtalk_prosody", + "pypinyin_g2p", + "pypinyin_g2p_phone", + "espeak_ng_arabic", + "espeak_ng_german", + "espeak_ng_french", + "espeak_ng_spanish", + "espeak_ng_russian", + "espeak_ng_greek", + "espeak_ng_finnish", + "espeak_ng_hungarian", + "espeak_ng_dutch", + "espeak_ng_english_us_vits", + "espeak_ng_hindi", + "g2pk", + "g2pk_no_space", + "korean_jaso", + "korean_jaso_no_space", +] + + +def split_by_space(text) -> List[str]: + if " " in text: + text = text.replace(" ", " ") + return [c.replace("", " ") for c in text.split(" ")] + else: + return text.split(" ") + + +def pyopenjtalk_g2p(text) -> List[str]: + import pyopenjtalk + + # phones is a str object separated by space + phones = pyopenjtalk.g2p(text, kana=False) + phones = phones.split(" ") + return phones + + +def pyopenjtalk_g2p_accent(text) -> List[str]: + import pyopenjtalk + import re + + phones = [] + for labels in pyopenjtalk.run_frontend(text)[1]: + p = re.findall(r"\-(.*?)\+.*?\/A:([0-9\-]+).*?\/F:.*?_([0-9]+)", labels) + if len(p) == 1: + phones += [p[0][0], p[0][2], p[0][1]] + return phones + + +def pyopenjtalk_g2p_accent_with_pause(text) -> List[str]: + import pyopenjtalk + import re + + phones = [] + for labels in pyopenjtalk.run_frontend(text)[1]: + if labels.split("-")[1].split("+")[0] == "pau": + phones += ["pau"] + continue + p = re.findall(r"\-(.*?)\+.*?\/A:([0-9\-]+).*?\/F:.*?_([0-9]+)", labels) + if len(p) == 1: + phones += [p[0][0], p[0][2], p[0][1]] + return phones + + +def pyopenjtalk_g2p_kana(text) -> List[str]: + import pyopenjtalk + + kanas = pyopenjtalk.g2p(text, kana=True) + return list(kanas) + + +def pyopenjtalk_g2p_prosody(text: str, drop_unvoiced_vowels: bool = True) -> List[str]: + """Extract phoneme + prosoody symbol sequence from input full-context labels. + + The algorithm is based on `Prosodic features control by symbols as input of + sequence-to-sequence acoustic modeling for neural TTS`_ with some r9y9's tweaks. + + Args: + text (str): Input text. + drop_unvoiced_vowels (bool): whether to drop unvoiced vowels. + + Returns: + List[str]: List of phoneme + prosody symbols. + + Examples: + >>> from funasr.text.phoneme_tokenizer import pyopenjtalk_g2p_prosody + >>> pyopenjtalk_g2p_prosody("こんにちは。") + ['^', 'k', 'o', '[', 'N', 'n', 'i', 'ch', 'i', 'w', 'a', '$'] + + .. _`Prosodic features control by symbols as input of sequence-to-sequence acoustic + modeling for neural TTS`: https://doi.org/10.1587/transinf.2020EDP7104 + + """ + import pyopenjtalk + + labels = pyopenjtalk.run_frontend(text)[1] + N = len(labels) + + phones = [] + for n in range(N): + lab_curr = labels[n] + + # current phoneme + p3 = re.search(r"\-(.*?)\+", lab_curr).group(1) + + # deal unvoiced vowels as normal vowels + if drop_unvoiced_vowels and p3 in "AEIOU": + p3 = p3.lower() + + # deal with sil at the beginning and the end of text + if p3 == "sil": + assert n == 0 or n == N - 1 + if n == 0: + phones.append("^") + elif n == N - 1: + # check question form or not + e3 = _numeric_feature_by_regex(r"!(\d+)_", lab_curr) + if e3 == 0: + phones.append("$") + elif e3 == 1: + phones.append("?") + continue + elif p3 == "pau": + phones.append("_") + continue + else: + phones.append(p3) + + # accent type and position info (forward or backward) + a1 = _numeric_feature_by_regex(r"/A:([0-9\-]+)\+", lab_curr) + a2 = _numeric_feature_by_regex(r"\+(\d+)\+", lab_curr) + a3 = _numeric_feature_by_regex(r"\+(\d+)/", lab_curr) + + # number of mora in accent phrase + f1 = _numeric_feature_by_regex(r"/F:(\d+)_", lab_curr) + + a2_next = _numeric_feature_by_regex(r"\+(\d+)\+", labels[n + 1]) + # accent phrase border + if a3 == 1 and a2_next == 1 and p3 in "aeiouAEIOUNcl": + phones.append("#") + # pitch falling + elif a1 == 0 and a2_next == a2 + 1 and a2 != f1: + phones.append("]") + # pitch rising + elif a2 == 1 and a2_next == 2: + phones.append("[") + + return phones + + +def _numeric_feature_by_regex(regex, s): + match = re.search(regex, s) + if match is None: + return -50 + return int(match.group(1)) + + +def pypinyin_g2p(text) -> List[str]: + from pypinyin import pinyin + from pypinyin import Style + + phones = [phone[0] for phone in pinyin(text, style=Style.TONE3)] + return phones + + +def pypinyin_g2p_phone(text) -> List[str]: + from pypinyin import pinyin + from pypinyin import Style + from pypinyin.style._utils import get_finals + from pypinyin.style._utils import get_initials + + phones = [ + p + for phone in pinyin(text, style=Style.TONE3) + for p in [ + get_initials(phone[0], strict=True), + get_finals(phone[0], strict=True), + ] + if len(p) != 0 + ] + return phones + + +class G2p_en: + """On behalf of g2p_en.G2p. + + g2p_en.G2p isn't pickalable and it can't be copied to the other processes + via multiprocessing module. + As a workaround, g2p_en.G2p is instantiated upon calling this class. + + """ + + def __init__(self, no_space: bool = False): + self.no_space = no_space + self.g2p = None + + def __call__(self, text) -> List[str]: + if self.g2p is None: + self.g2p = g2p_en.G2p() + + phones = self.g2p(text) + if self.no_space: + # remove space which represents word serapater + phones = list(filter(lambda s: s != " ", phones)) + return phones + + +class G2pk: + """On behalf of g2pk.G2p. + + g2pk.G2p isn't pickalable and it can't be copied to the other processes + via multiprocessing module. + As a workaround, g2pk.G2p is instantiated upon calling this class. + + """ + + def __init__( + self, descritive=False, group_vowels=False, to_syl=False, no_space=False + ): + self.descritive = descritive + self.group_vowels = group_vowels + self.to_syl = to_syl + self.no_space = no_space + self.g2p = None + + def __call__(self, text) -> List[str]: + if self.g2p is None: + import g2pk + + self.g2p = g2pk.G2p() + + phones = list( + self.g2p( + text, + descriptive=self.descritive, + group_vowels=self.group_vowels, + to_syl=self.to_syl, + ) + ) + if self.no_space: + # remove space which represents word serapater + phones = list(filter(lambda s: s != " ", phones)) + return phones + + +class Jaso: + PUNC = "!'(),-.:;?" + SPACE = " " + + JAMO_LEADS = "".join([chr(_) for _ in range(0x1100, 0x1113)]) + JAMO_VOWELS = "".join([chr(_) for _ in range(0x1161, 0x1176)]) + JAMO_TAILS = "".join([chr(_) for _ in range(0x11A8, 0x11C3)]) + + VALID_CHARS = JAMO_LEADS + JAMO_VOWELS + JAMO_TAILS + PUNC + SPACE + + def __init__(self, space_symbol=" ", no_space=False): + self.space_symbol = space_symbol + self.no_space = no_space + + def _text_to_jaso(self, line: str) -> List[str]: + jasos = list(jamo.hangul_to_jamo(line)) + return jasos + + def _remove_non_korean_characters(self, tokens): + new_tokens = [token for token in tokens if token in self.VALID_CHARS] + return new_tokens + + def __call__(self, text) -> List[str]: + graphemes = [x for x in self._text_to_jaso(text)] + graphemes = self._remove_non_korean_characters(graphemes) + + if self.no_space: + graphemes = list(filter(lambda s: s != " ", graphemes)) + else: + graphemes = [x if x != " " else self.space_symbol for x in graphemes] + return graphemes + + +class Phonemizer: + """Phonemizer module for various languages. + + This is wrapper module of https://github.com/bootphon/phonemizer. + You can define various g2p modules by specifying options for phonemizer. + + See available options: + https://github.com/bootphon/phonemizer/blob/master/phonemizer/phonemize.py#L32 + + """ + + def __init__( + self, + backend, + word_separator: Optional[str] = None, + syllable_separator: Optional[str] = None, + phone_separator: Optional[str] = " ", + strip=False, + split_by_single_token: bool = False, + **phonemizer_kwargs, + ): + # delayed import + from phonemizer.backend import BACKENDS + from phonemizer.separator import Separator + + self.separator = Separator( + word=word_separator, + syllable=syllable_separator, + phone=phone_separator, + ) + + # define logger to suppress the warning in phonemizer + logger = logging.getLogger("phonemizer") + logger.setLevel(logging.ERROR) + self.phonemizer = BACKENDS[backend]( + **phonemizer_kwargs, + logger=logger, + ) + self.strip = strip + self.split_by_single_token = split_by_single_token + + def __call__(self, text) -> List[str]: + tokens = self.phonemizer.phonemize( + [text], + separator=self.separator, + strip=self.strip, + njobs=1, + )[0] + if not self.split_by_single_token: + return tokens.split() + else: + # "a: ab" -> ["a", ":", "", "a", "b"] + # TODO(kan-bayashi): space replacement should be dealt in PhonemeTokenizer + return [c.replace(" ", "") for c in tokens] + + +class PhonemeTokenizer(AbsTokenizer): + def __init__( + self, + g2p_type: Union[None, str], + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + space_symbol: str = "", + remove_non_linguistic_symbols: bool = False, + ): + assert check_argument_types() + if g2p_type is None: + self.g2p = split_by_space + elif g2p_type == "g2p_en": + self.g2p = G2p_en(no_space=False) + elif g2p_type == "g2p_en_no_space": + self.g2p = G2p_en(no_space=True) + elif g2p_type == "pyopenjtalk": + self.g2p = pyopenjtalk_g2p + elif g2p_type == "pyopenjtalk_kana": + self.g2p = pyopenjtalk_g2p_kana + elif g2p_type == "pyopenjtalk_accent": + self.g2p = pyopenjtalk_g2p_accent + elif g2p_type == "pyopenjtalk_accent_with_pause": + self.g2p = pyopenjtalk_g2p_accent_with_pause + elif g2p_type == "pyopenjtalk_prosody": + self.g2p = pyopenjtalk_g2p_prosody + elif g2p_type == "pypinyin_g2p": + self.g2p = pypinyin_g2p + elif g2p_type == "pypinyin_g2p_phone": + self.g2p = pypinyin_g2p_phone + elif g2p_type == "espeak_ng_arabic": + self.g2p = Phonemizer( + language="ar", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_german": + self.g2p = Phonemizer( + language="de", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_french": + self.g2p = Phonemizer( + language="fr-fr", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_spanish": + self.g2p = Phonemizer( + language="es", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_russian": + self.g2p = Phonemizer( + language="ru", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_greek": + self.g2p = Phonemizer( + language="el", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_finnish": + self.g2p = Phonemizer( + language="fi", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_hungarian": + self.g2p = Phonemizer( + language="hu", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_dutch": + self.g2p = Phonemizer( + language="nl", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "espeak_ng_hindi": + self.g2p = Phonemizer( + language="hi", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + ) + elif g2p_type == "g2pk": + self.g2p = G2pk(no_space=False) + elif g2p_type == "g2pk_no_space": + self.g2p = G2pk(no_space=True) + elif g2p_type == "espeak_ng_english_us_vits": + # VITS official implementation-like processing + # Reference: https://github.com/jaywalnut310/vits + self.g2p = Phonemizer( + language="en-us", + backend="espeak", + with_stress=True, + preserve_punctuation=True, + strip=True, + word_separator=" ", + phone_separator="", + split_by_single_token=True, + ) + elif g2p_type == "korean_jaso": + self.g2p = Jaso(space_symbol=space_symbol, no_space=False) + elif g2p_type == "korean_jaso_no_space": + self.g2p = Jaso(no_space=True) + else: + raise NotImplementedError(f"Not supported: g2p_type={g2p_type}") + + self.g2p_type = g2p_type + self.space_symbol = space_symbol + if non_linguistic_symbols is None: + self.non_linguistic_symbols = set() + elif isinstance(non_linguistic_symbols, (Path, str)): + non_linguistic_symbols = Path(non_linguistic_symbols) + try: + with non_linguistic_symbols.open("r", encoding="utf-8") as f: + self.non_linguistic_symbols = set(line.rstrip() for line in f) + except FileNotFoundError: + warnings.warn(f"{non_linguistic_symbols} doesn't exist.") + self.non_linguistic_symbols = set() + else: + self.non_linguistic_symbols = set(non_linguistic_symbols) + self.remove_non_linguistic_symbols = remove_non_linguistic_symbols + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f'g2p_type="{self.g2p_type}", ' + f'space_symbol="{self.space_symbol}", ' + f'non_linguistic_symbols="{self.non_linguistic_symbols}"' + ")" + ) + + def text2tokens(self, line: str) -> List[str]: + tokens = [] + while len(line) != 0: + for w in self.non_linguistic_symbols: + if line.startswith(w): + if not self.remove_non_linguistic_symbols: + tokens.append(line[: len(w)]) + line = line[len(w) :] + break + else: + t = line[0] + tokens.append(t) + line = line[1:] + + line = "".join(tokens) + tokens = self.g2p(line) + return tokens + + def tokens2text(self, tokens: Iterable[str]) -> str: + # phoneme type is not invertible + return "".join(tokens) diff --git a/funasr/text/sentencepiece_tokenizer.py b/funasr/text/sentencepiece_tokenizer.py new file mode 100644 index 000000000..e4cc15272 --- /dev/null +++ b/funasr/text/sentencepiece_tokenizer.py @@ -0,0 +1,38 @@ +from pathlib import Path +from typing import Iterable +from typing import List +from typing import Union + +import sentencepiece as spm +from typeguard import check_argument_types + +from funasr.text.abs_tokenizer import AbsTokenizer + + +class SentencepiecesTokenizer(AbsTokenizer): + def __init__(self, model: Union[Path, str]): + assert check_argument_types() + self.model = str(model) + # NOTE(kamo): + # Don't build SentencePieceProcessor in __init__() + # because it's not picklable and it may cause following error, + # "TypeError: can't pickle SwigPyObject objects", + # when giving it as argument of "multiprocessing.Process()". + self.sp = None + + def __repr__(self): + return f'{self.__class__.__name__}(model="{self.model}")' + + def _build_sentence_piece_processor(self): + # Build SentencePieceProcessor lazily. + if self.sp is None: + self.sp = spm.SentencePieceProcessor() + self.sp.load(self.model) + + def text2tokens(self, line: str) -> List[str]: + self._build_sentence_piece_processor() + return self.sp.EncodeAsPieces(line) + + def tokens2text(self, tokens: Iterable[str]) -> str: + self._build_sentence_piece_processor() + return self.sp.DecodePieces(list(tokens)) diff --git a/funasr/text/token_id_converter.py b/funasr/text/token_id_converter.py new file mode 100644 index 000000000..c9a6b2863 --- /dev/null +++ b/funasr/text/token_id_converter.py @@ -0,0 +1,60 @@ +from pathlib import Path +from typing import Dict +from typing import Iterable +from typing import List +from typing import Union + +import numpy as np +from typeguard import check_argument_types + + +class TokenIDConverter: + def __init__( + self, + token_list: Union[Path, str, Iterable[str]], + unk_symbol: str = "", + ): + assert check_argument_types() + + if isinstance(token_list, (Path, str)): + token_list = Path(token_list) + self.token_list_repr = str(token_list) + self.token_list: List[str] = [] + + with token_list.open("r", encoding="utf-8") as f: + for idx, line in enumerate(f): + line = line.rstrip() + self.token_list.append(line) + + else: + self.token_list: List[str] = list(token_list) + self.token_list_repr = "" + for i, t in enumerate(self.token_list): + if i == 3: + break + self.token_list_repr += f"{t}, " + self.token_list_repr += f"... (NVocab={(len(self.token_list))})" + + self.token2id: Dict[str, int] = {} + for i, t in enumerate(self.token_list): + if t in self.token2id: + raise RuntimeError(f'Symbol "{t}" is duplicated') + self.token2id[t] = i + + self.unk_symbol = unk_symbol + if self.unk_symbol not in self.token2id: + raise RuntimeError( + f"Unknown symbol '{unk_symbol}' doesn't exist in the token_list" + ) + self.unk_id = self.token2id[self.unk_symbol] + + def get_num_vocabulary_size(self) -> int: + return len(self.token_list) + + def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]: + if isinstance(integers, np.ndarray) and integers.ndim != 1: + raise ValueError(f"Must be 1 dim ndarray, but got {integers.ndim}") + return [self.token_list[i] for i in integers] + + def tokens2ids(self, tokens: Iterable[str]) -> List[int]: + return [self.token2id.get(i, self.unk_id) for i in tokens] diff --git a/funasr/text/word_tokenizer.py b/funasr/text/word_tokenizer.py new file mode 100644 index 000000000..842734e75 --- /dev/null +++ b/funasr/text/word_tokenizer.py @@ -0,0 +1,58 @@ +from pathlib import Path +from typing import Iterable +from typing import List +from typing import Union +import warnings + +from typeguard import check_argument_types + +from funasr.text.abs_tokenizer import AbsTokenizer + + +class WordTokenizer(AbsTokenizer): + def __init__( + self, + delimiter: str = None, + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + remove_non_linguistic_symbols: bool = False, + ): + assert check_argument_types() + self.delimiter = delimiter + + if not remove_non_linguistic_symbols and non_linguistic_symbols is not None: + warnings.warn( + "non_linguistic_symbols is only used " + "when remove_non_linguistic_symbols = True" + ) + + if non_linguistic_symbols is None: + self.non_linguistic_symbols = set() + elif isinstance(non_linguistic_symbols, (Path, str)): + non_linguistic_symbols = Path(non_linguistic_symbols) + try: + with non_linguistic_symbols.open("r", encoding="utf-8") as f: + self.non_linguistic_symbols = set(line.rstrip() for line in f) + except FileNotFoundError: + warnings.warn(f"{non_linguistic_symbols} doesn't exist.") + self.non_linguistic_symbols = set() + else: + self.non_linguistic_symbols = set(non_linguistic_symbols) + self.remove_non_linguistic_symbols = remove_non_linguistic_symbols + + def __repr__(self): + return f'{self.__class__.__name__}(delimiter="{self.delimiter}")' + + def text2tokens(self, line: str) -> List[str]: + tokens = [] + for t in line.split(self.delimiter): + if self.remove_non_linguistic_symbols and t in self.non_linguistic_symbols: + continue + tokens.append(t) + return tokens + + def tokens2text(self, tokens: Iterable[str]) -> str: + if self.delimiter is None: + delimiter = " " + else: + delimiter = self.delimiter + return delimiter.join(tokens) diff --git a/funasr/torch_utils/__init__.py b/funasr/torch_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/torch_utils/add_gradient_noise.py b/funasr/torch_utils/add_gradient_noise.py new file mode 100644 index 000000000..654e928ec --- /dev/null +++ b/funasr/torch_utils/add_gradient_noise.py @@ -0,0 +1,31 @@ +import torch + + +def add_gradient_noise( + model: torch.nn.Module, + iteration: int, + duration: float = 100, + eta: float = 1.0, + scale_factor: float = 0.55, +): + """Adds noise from a standard normal distribution to the gradients. + + The standard deviation (`sigma`) is controlled + by the three hyper-parameters below. + `sigma` goes to zero (no noise) with more iterations. + + Args: + model: Model. + iteration: Number of iterations. + duration: {100, 1000}: Number of durations to control + the interval of the `sigma` change. + eta: {0.01, 0.3, 1.0}: The magnitude of `sigma`. + scale_factor: {0.55}: The scale of `sigma`. + """ + interval = (iteration // duration) + 1 + sigma = eta / interval**scale_factor + for param in model.parameters(): + if param.grad is not None: + _shape = param.grad.size() + noise = sigma * torch.randn(_shape).to(param.device) + param.grad += noise diff --git a/funasr/torch_utils/device_funcs.py b/funasr/torch_utils/device_funcs.py new file mode 100644 index 000000000..7919e7d92 --- /dev/null +++ b/funasr/torch_utils/device_funcs.py @@ -0,0 +1,71 @@ +import dataclasses +import warnings + +import numpy as np +import torch + + +def to_device(data, device=None, dtype=None, non_blocking=False, copy=False): + """Change the device of object recursively""" + if isinstance(data, dict): + return { + k: to_device(v, device, dtype, non_blocking, copy) for k, v in data.items() + } + elif dataclasses.is_dataclass(data) and not isinstance(data, type): + return type(data)( + *[ + to_device(v, device, dtype, non_blocking, copy) + for v in dataclasses.astuple(data) + ] + ) + # maybe namedtuple. I don't know the correct way to judge namedtuple. + elif isinstance(data, tuple) and type(data) is not tuple: + return type(data)( + *[to_device(o, device, dtype, non_blocking, copy) for o in data] + ) + elif isinstance(data, (list, tuple)): + return type(data)(to_device(v, device, dtype, non_blocking, copy) for v in data) + elif isinstance(data, np.ndarray): + return to_device(torch.from_numpy(data), device, dtype, non_blocking, copy) + elif isinstance(data, torch.Tensor): + return data.to(device, dtype, non_blocking, copy) + else: + return data + + +def force_gatherable(data, device): + """Change object to gatherable in torch.nn.DataParallel recursively + + The difference from to_device() is changing to torch.Tensor if float or int + value is found. + + The restriction to the returned value in DataParallel: + The object must be + - torch.cuda.Tensor + - 1 or more dimension. 0-dimension-tensor sends warning. + or a list, tuple, dict. + + """ + if isinstance(data, dict): + return {k: force_gatherable(v, device) for k, v in data.items()} + # DataParallel can't handle NamedTuple well + elif isinstance(data, tuple) and type(data) is not tuple: + return type(data)(*[force_gatherable(o, device) for o in data]) + elif isinstance(data, (list, tuple, set)): + return type(data)(force_gatherable(v, device) for v in data) + elif isinstance(data, np.ndarray): + return force_gatherable(torch.from_numpy(data), device) + elif isinstance(data, torch.Tensor): + if data.dim() == 0: + # To 1-dim array + data = data[None] + return data.to(device) + elif isinstance(data, float): + return torch.tensor([data], dtype=torch.float, device=device) + elif isinstance(data, int): + return torch.tensor([data], dtype=torch.long, device=device) + elif data is None: + return None + else: + warnings.warn(f"{type(data)} may not be gatherable by DataParallel") + return data diff --git a/funasr/torch_utils/forward_adaptor.py b/funasr/torch_utils/forward_adaptor.py new file mode 100644 index 000000000..114af7851 --- /dev/null +++ b/funasr/torch_utils/forward_adaptor.py @@ -0,0 +1,33 @@ +import torch +from typeguard import check_argument_types + + +class ForwardAdaptor(torch.nn.Module): + """Wrapped module to parallelize specified method + + torch.nn.DataParallel parallelizes only "forward()" + and, maybe, the method having the other name can't be applied + except for wrapping the module just like this class. + + Examples: + >>> class A(torch.nn.Module): + ... def foo(self, x): + ... ... + >>> model = A() + >>> model = ForwardAdaptor(model, "foo") + >>> model = torch.nn.DataParallel(model, device_ids=[0, 1]) + >>> x = torch.randn(2, 10) + >>> model(x) + """ + + def __init__(self, module: torch.nn.Module, name: str): + assert check_argument_types() + super().__init__() + self.module = module + self.name = name + if not hasattr(module, name): + raise ValueError(f"{module} doesn't have {name}") + + def forward(self, *args, **kwargs): + func = getattr(self.module, self.name) + return func(*args, **kwargs) diff --git a/funasr/torch_utils/initialize.py b/funasr/torch_utils/initialize.py new file mode 100644 index 000000000..2c0e7a435 --- /dev/null +++ b/funasr/torch_utils/initialize.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +"""Initialize modules for espnet2 neural networks.""" + +import math +import torch +from typeguard import check_argument_types + + +def initialize(model: torch.nn.Module, init: str): + """Initialize weights of a neural network module. + + Parameters are initialized using the given method or distribution. + + Custom initialization routines can be implemented into submodules + as function `espnet_initialization_fn` within the custom module. + + Args: + model: Target. + init: Method of initialization. + """ + assert check_argument_types() + + if init == "chainer": + # 1. lecun_normal_init_parameters + for p in model.parameters(): + data = p.data + if data.dim() == 1: + # bias + data.zero_() + elif data.dim() == 2: + # linear weight + n = data.size(1) + stdv = 1.0 / math.sqrt(n) + data.normal_(0, stdv) + elif data.dim() in (3, 4): + # conv weight + n = data.size(1) + for k in data.size()[2:]: + n *= k + stdv = 1.0 / math.sqrt(n) + data.normal_(0, stdv) + else: + raise NotImplementedError + + for mod in model.modules(): + # 2. embed weight ~ Normal(0, 1) + if isinstance(mod, torch.nn.Embedding): + mod.weight.data.normal_(0, 1) + # 3. forget-bias = 1.0 + elif isinstance(mod, torch.nn.RNNCellBase): + n = mod.bias_ih.size(0) + mod.bias_ih.data[n // 4 : n // 2].fill_(1.0) + elif isinstance(mod, torch.nn.RNNBase): + for name, param in mod.named_parameters(): + if "bias" in name: + n = param.size(0) + param.data[n // 4 : n // 2].fill_(1.0) + if hasattr(mod, "espnet_initialization_fn"): + mod.espnet_initialization_fn() + + else: + # weight init + for p in model.parameters(): + if p.dim() > 1: + if init == "xavier_uniform": + torch.nn.init.xavier_uniform_(p.data) + elif init == "xavier_normal": + torch.nn.init.xavier_normal_(p.data) + elif init == "kaiming_uniform": + torch.nn.init.kaiming_uniform_(p.data, nonlinearity="relu") + elif init == "kaiming_normal": + torch.nn.init.kaiming_normal_(p.data, nonlinearity="relu") + else: + raise ValueError("Unknown initialization: " + init) + # bias init + for p in model.parameters(): + if p.dim() == 1: + p.data.zero_() + + # reset some modules with default init + for m in model.modules(): + if isinstance( + m, (torch.nn.Embedding, torch.nn.LayerNorm, torch.nn.GroupNorm) + ): + m.reset_parameters() + if hasattr(m, "espnet_initialization_fn"): + m.espnet_initialization_fn() + + # TODO(xkc): Hacking s3prl_frontend and wav2vec2encoder initialization + if getattr(model, "encoder", None) and getattr( + model.encoder, "reload_pretrained_parameters", None + ): + model.encoder.reload_pretrained_parameters() + if getattr(model, "frontend", None) and getattr( + model.frontend, "reload_pretrained_parameters", None + ): + model.frontend.reload_pretrained_parameters() + if getattr(model, "postencoder", None) and getattr( + model.postencoder, "reload_pretrained_parameters", None + ): + model.postencoder.reload_pretrained_parameters() diff --git a/funasr/torch_utils/load_pretrained_model.py b/funasr/torch_utils/load_pretrained_model.py new file mode 100644 index 000000000..8e3f05e1e --- /dev/null +++ b/funasr/torch_utils/load_pretrained_model.py @@ -0,0 +1,125 @@ +from typing import Any +from typing import Dict +from typing import Union +from io import BytesIO + +import logging +import torch +import torch.nn +import torch.optim + + +def filter_state_dict( + dst_state: Dict[str, Union[float, torch.Tensor]], + src_state: Dict[str, Union[float, torch.Tensor]], +): + """Filter name, size mismatch instances between dicts. + + Args: + dst_state: reference state dict for filtering + src_state: target state dict for filtering + + """ + match_state = {} + for key, value in src_state.items(): + if key in dst_state and (dst_state[key].size() == src_state[key].size()): + match_state[key] = value + else: + if key not in dst_state: + logging.warning( + f"Filter out {key} from pretrained dict" + + " because of name not found in target dict" + ) + else: + logging.warning( + f"Filter out {key} from pretrained dict" + + " because of size mismatch" + + f"({dst_state[key].size()}-{src_state[key].size()})" + ) + return match_state + + +def load_pretrained_model( + init_param: str, + model: torch.nn.Module, + ignore_init_mismatch: bool, + map_location: str = "cpu", + oss_bucket=None, +): + """Load a model state and set it to the model. + + Args: + init_param: ::: + + Examples: + >>> load_pretrained_model("somewhere/model.pth", model) + >>> load_pretrained_model("somewhere/model.pth:decoder:decoder", model) + >>> load_pretrained_model("somewhere/model.pth:decoder:decoder:", model) + >>> load_pretrained_model( + ... "somewhere/model.pth:decoder:decoder:decoder.embed", model + ... ) + >>> load_pretrained_model("somewhere/decoder.pth::decoder", model) + """ + sps = init_param.split(":", 4) + if len(sps) == 4: + path, src_key, dst_key, excludes = sps + elif len(sps) == 3: + path, src_key, dst_key = sps + excludes = None + elif len(sps) == 2: + path, src_key = sps + dst_key, excludes = None, None + else: + (path,) = sps + src_key, dst_key, excludes = None, None, None + if src_key == "": + src_key = None + if dst_key == "": + dst_key = None + + if dst_key is None: + obj = model + else: + + def get_attr(obj: Any, key: str): + """Get an nested attribute. + + >>> class A(torch.nn.Module): + ... def __init__(self): + ... super().__init__() + ... self.linear = torch.nn.Linear(10, 10) + >>> a = A() + >>> assert A.linear.weight is get_attr(A, 'linear.weight') + + """ + if key.strip() == "": + return obj + for k in key.split("."): + obj = getattr(obj, k) + return obj + + obj = get_attr(model, dst_key) + + if oss_bucket is None: + src_state = torch.load(path, map_location=map_location) + else: + buffer = BytesIO(oss_bucket.get_object(path).read()) + src_state = torch.load(buffer, map_location=map_location) + if excludes is not None: + for e in excludes.split(","): + src_state = {k: v for k, v in src_state.items() if not k.startswith(e)} + + if src_key is not None: + src_state = { + k[len(src_key) + 1 :]: v + for k, v in src_state.items() + if k.startswith(src_key) + } + + dst_state = obj.state_dict() + if ignore_init_mismatch: + src_state = filter_state_dict(dst_state, src_state) + + logging.info("Loaded src_state keys: {}".format(src_state.keys())) + dst_state.update(src_state) + obj.load_state_dict(dst_state) diff --git a/funasr/torch_utils/model_summary.py b/funasr/torch_utils/model_summary.py new file mode 100644 index 000000000..8d7f14f8c --- /dev/null +++ b/funasr/torch_utils/model_summary.py @@ -0,0 +1,70 @@ +import humanfriendly +import numpy as np +import torch + + +def get_human_readable_count(number: int) -> str: + """Return human_readable_count + + Originated from: + https://github.com/PyTorchLightning/pytorch-lightning/blob/master/pytorch_lightning/core/memory.py + + Abbreviates an integer number with K, M, B, T for thousands, millions, + billions and trillions, respectively. + Examples: + >>> get_human_readable_count(123) + '123 ' + >>> get_human_readable_count(1234) # (one thousand) + '1 K' + >>> get_human_readable_count(2e6) # (two million) + '2 M' + >>> get_human_readable_count(3e9) # (three billion) + '3 B' + >>> get_human_readable_count(4e12) # (four trillion) + '4 T' + >>> get_human_readable_count(5e15) # (more than trillion) + '5,000 T' + Args: + number: a positive integer number + Return: + A string formatted according to the pattern described above. + """ + assert number >= 0 + labels = [" ", "K", "M", "B", "T"] + num_digits = int(np.floor(np.log10(number)) + 1 if number > 0 else 1) + num_groups = int(np.ceil(num_digits / 3)) + num_groups = min(num_groups, len(labels)) # don't abbreviate beyond trillions + shift = -3 * (num_groups - 1) + number = number * (10**shift) + index = num_groups - 1 + return f"{number:.2f} {labels[index]}" + + +def to_bytes(dtype) -> int: + # torch.float16 -> 16 + return int(str(dtype)[-2:]) // 8 + + +def model_summary(model: torch.nn.Module) -> str: + message = "Model structure:\n" + message += str(model) + tot_params = sum(p.numel() for p in model.parameters()) + num_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + percent_trainable = "{:.1f}".format(num_params * 100.0 / tot_params) + tot_params = get_human_readable_count(tot_params) + num_params = get_human_readable_count(num_params) + message += "\n\nModel summary:\n" + message += f" Class Name: {model.__class__.__name__}\n" + message += f" Total Number of model parameters: {tot_params}\n" + message += ( + f" Number of trainable parameters: {num_params} ({percent_trainable}%)\n" + ) + num_bytes = humanfriendly.format_size( + sum( + p.numel() * to_bytes(p.dtype) for p in model.parameters() if p.requires_grad + ) + ) + message += f" Size: {num_bytes}\n" + dtype = next(iter(model.parameters())).dtype + message += f" Type: {dtype}" + return message diff --git a/funasr/torch_utils/pytorch_version.py b/funasr/torch_utils/pytorch_version.py new file mode 100644 index 000000000..01f17cc74 --- /dev/null +++ b/funasr/torch_utils/pytorch_version.py @@ -0,0 +1,16 @@ +import torch + + +def pytorch_cudnn_version() -> str: + message = ( + f"pytorch.version={torch.__version__}, " + f"cuda.available={torch.cuda.is_available()}, " + ) + + if torch.backends.cudnn.enabled: + message += ( + f"cudnn.version={torch.backends.cudnn.version()}, " + f"cudnn.benchmark={torch.backends.cudnn.benchmark}, " + f"cudnn.deterministic={torch.backends.cudnn.deterministic}" + ) + return message diff --git a/funasr/torch_utils/recursive_op.py b/funasr/torch_utils/recursive_op.py new file mode 100644 index 000000000..286a92daf --- /dev/null +++ b/funasr/torch_utils/recursive_op.py @@ -0,0 +1,47 @@ +"""Torch utility module.""" +import torch + +if torch.distributed.is_available(): + from torch.distributed import ReduceOp + + +def recursive_sum(obj, weight: torch.Tensor, distributed: bool = False): + assert weight.dim() == 1, weight.size() + if isinstance(obj, (tuple, list)): + return type(obj)(recursive_sum(v, weight, distributed) for v in obj) + elif isinstance(obj, dict): + return {k: recursive_sum(v, weight, distributed) for k, v in obj.items()} + elif isinstance(obj, torch.Tensor): + assert obj.size() == weight.size(), (obj.size(), weight.size()) + obj = (obj * weight.type(obj.dtype)).sum() + if distributed: + torch.distributed.all_reduce(obj, op=ReduceOp.SUM) + return obj + elif obj is None: + return None + else: + raise ValueError(type(obj)) + + +def recursive_divide(a, b: torch.Tensor): + if isinstance(a, (tuple, list)): + return type(a)(recursive_divide(v, b) for v in a) + elif isinstance(a, dict): + return {k: recursive_divide(v, b) for k, v in a.items()} + elif isinstance(a, torch.Tensor): + assert a.size() == b.size(), (a.size(), b.size()) + return a / b.type(a.dtype) + elif a is None: + return None + else: + raise ValueError(type(a)) + + +def recursive_average(obj, weight: torch.Tensor, distributed: bool = False): + obj = recursive_sum(obj, weight, distributed) + weight = weight.sum() + if distributed: + torch.distributed.all_reduce(weight, op=ReduceOp.SUM) + # Normalize weight to be sum-to-1 + obj = recursive_divide(obj, weight) + return obj, weight diff --git a/funasr/torch_utils/set_all_random_seed.py b/funasr/torch_utils/set_all_random_seed.py new file mode 100644 index 000000000..ebdca3f53 --- /dev/null +++ b/funasr/torch_utils/set_all_random_seed.py @@ -0,0 +1,10 @@ +import random + +import numpy as np +import torch + + +def set_all_random_seed(seed: int): + random.seed(seed) + np.random.seed(seed) + torch.random.manual_seed(seed) diff --git a/funasr/train/__init__.py b/funasr/train/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/train/abs_espnet_model.py b/funasr/train/abs_espnet_model.py new file mode 100644 index 000000000..cc6a5a2a0 --- /dev/null +++ b/funasr/train/abs_espnet_model.py @@ -0,0 +1,55 @@ +# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +from abc import ABC +from abc import abstractmethod +from typing import Dict +from typing import Tuple + +import torch + + +class AbsESPnetModel(torch.nn.Module, ABC): + """The common abstract class among each tasks + + "ESPnetModel" is referred to a class which inherits torch.nn.Module, + and makes the dnn-models forward as its member field, + a.k.a delegate pattern, + and defines "loss", "stats", and "weight" for the task. + + If you intend to implement new task in ESPNet, + the model must inherit this class. + In other words, the "mediator" objects between + our training system and the your task class are + just only these three values, loss, stats, and weight. + + Example: + >>> from funasr.tasks.abs_task import AbsTask + >>> class YourESPnetModel(AbsESPnetModel): + ... def forward(self, input, input_lengths): + ... ... + ... return loss, stats, weight + >>> class YourTask(AbsTask): + ... @classmethod + ... def build_model(cls, args: argparse.Namespace) -> YourESPnetModel: + """ + + def __init__(self): + super().__init__() + self.num_updates = 0 + + @abstractmethod + def forward( + self, **batch: torch.Tensor + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + raise NotImplementedError + + @abstractmethod + def collect_feats(self, **batch: torch.Tensor) -> Dict[str, torch.Tensor]: + raise NotImplementedError + + def set_num_updates(self, num_updates): + self.num_updates = num_updates + + def get_num_updates(self): + return self.num_updates diff --git a/funasr/train/class_choices.py b/funasr/train/class_choices.py new file mode 100644 index 000000000..658d29166 --- /dev/null +++ b/funasr/train/class_choices.py @@ -0,0 +1,95 @@ +from typing import Mapping +from typing import Optional +from typing import Tuple + +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.utils.nested_dict_action import NestedDictAction +from funasr.utils.types import str_or_none + + +class ClassChoices: + """Helper class to manage the options for variable objects and its configuration. + + Example: + + >>> class A: + ... def __init__(self, foo=3): pass + >>> class B: + ... def __init__(self, bar="aaaa"): pass + >>> choices = ClassChoices("var", dict(a=A, b=B), default="a") + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> choices.add_arguments(parser) + >>> args = parser.parse_args(["--var", "a", "--var_conf", "foo=4") + >>> args.var + a + >>> args.var_conf + {"foo": 4} + >>> class_obj = choices.get_class(args.var) + >>> a_object = class_obj(**args.var_conf) + + """ + + def __init__( + self, + name: str, + classes: Mapping[str, type], + type_check: type = None, + default: str = None, + optional: bool = False, + ): + assert check_argument_types() + self.name = name + self.base_type = type_check + self.classes = {k.lower(): v for k, v in classes.items()} + if "none" in self.classes or "nil" in self.classes or "null" in self.classes: + raise ValueError('"none", "nil", and "null" are reserved.') + if type_check is not None: + for v in self.classes.values(): + if not issubclass(v, type_check): + raise ValueError(f"must be {type_check.__name__}, but got {v}") + + self.optional = optional + self.default = default + if default is None: + self.optional = True + + def choices(self) -> Tuple[Optional[str], ...]: + retval = tuple(self.classes) + if self.optional: + return retval + (None,) + else: + return retval + + def get_class(self, name: Optional[str]) -> Optional[type]: + assert check_argument_types() + if name is None or (self.optional and name.lower() == ("none", "null", "nil")): + retval = None + elif name.lower() in self.classes: + class_obj = self.classes[name] + assert check_return_type(class_obj) + retval = class_obj + else: + raise ValueError( + f"--{self.name} must be one of {self.choices()}: " + f"--{self.name} {name.lower()}" + ) + + return retval + + def add_arguments(self, parser): + parser.add_argument( + f"--{self.name}", + type=lambda x: str_or_none(x.lower()), + default=self.default, + choices=self.choices(), + help=f"The {self.name} type", + ) + parser.add_argument( + f"--{self.name}_conf", + action=NestedDictAction, + default=dict(), + help=f"The keyword arguments for {self.name}", + ) diff --git a/funasr/train/distributed_utils.py b/funasr/train/distributed_utils.py new file mode 100644 index 000000000..088203a58 --- /dev/null +++ b/funasr/train/distributed_utils.py @@ -0,0 +1,384 @@ +# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import dataclasses +import logging +import os +import socket +from typing import Optional + +import torch +import torch.distributed + + +@dataclasses.dataclass +class DistributedOption: + # Enable distributed Training + distributed: bool = False + # torch.distributed.Backend: "nccl", "mpi", "gloo", or "tcp" + dist_backend: str = "nccl" + # if init_method="env://", + # env values of "MASTER_PORT", "MASTER_ADDR", "WORLD_SIZE", and "RANK" are referred. + dist_init_method: str = "env://" + dist_world_size: Optional[int] = None + dist_rank: Optional[int] = None + local_rank: Optional[int] = None + ngpu: int = 0 + dist_master_addr: Optional[str] = None + dist_master_port: Optional[int] = None + dist_launcher: Optional[str] = None + multiprocessing_distributed: bool = True + + def init_options(self): + if self.distributed: + if self.dist_init_method == "env://": + if get_master_addr(self.dist_master_addr, self.dist_launcher) is None: + raise RuntimeError( + "--dist_master_addr or MASTER_ADDR must be set " + "if --dist_init_method == 'env://'" + ) + if get_master_port(self.dist_master_port) is None: + raise RuntimeError( + "--dist_master_port or MASTER_PORT must be set " + "if --dist_init_port == 'env://'" + ) + + def init_torch_distributed(self, args): + if self.distributed: + # See: + # https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/docs/env.html + os.environ.setdefault("NCCL_DEBUG", "INFO") + + # See: + # https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group + os.environ.setdefault("NCCL_BLOCKING_WAIT", "1") + + torch.distributed.init_process_group(backend='nccl', + init_method=self.dist_init_method, + world_size=args.dist_world_size, + rank=args.dist_rank) + self.dist_rank = torch.distributed.get_rank() + self.dist_world_size = torch.distributed.get_world_size() + self.local_rank = args.local_rank + logging.info("world size: {}, rank: {}, local_rank: {}".format(self.dist_world_size, self.dist_rank, + self.local_rank)) + + def init_options_pai(self): + if self.distributed: + if self.dist_init_method == "env://": + if get_master_addr(self.dist_master_addr, self.dist_launcher) is None: + raise RuntimeError( + "--dist_master_addr or MASTER_ADDR must be set " + "if --dist_init_method == 'env://'" + ) + if get_master_port(self.dist_master_port) is None: + raise RuntimeError( + "--dist_master_port or MASTER_PORT must be set " + "if --dist_init_port == 'env://'" + ) + + self.dist_rank = get_rank(self.dist_rank, self.dist_launcher) + self.dist_world_size = get_world_size( + self.dist_world_size, self.dist_launcher + ) + self.local_rank = get_local_rank(self.local_rank, self.dist_launcher) + + if ( + self.dist_rank is not None + and self.dist_world_size is not None + and self.dist_rank >= self.dist_world_size + ): + raise RuntimeError( + f"RANK >= WORLD_SIZE: {self.dist_rank} >= {self.dist_world_size}" + ) + + if self.dist_init_method == "env://": + self.dist_master_addr = get_master_addr( + self.dist_master_addr, self.dist_launcher + ) + self.dist_master_port = get_master_port(self.dist_master_port) + if ( + self.dist_master_addr is not None + and self.dist_master_port is not None + ): + self.dist_init_method = ( + f"tcp://{self.dist_master_addr}:{self.dist_master_port}" + ) + + def init_torch_distributed_pai(self, args): + if self.distributed: + # See: + # https://docs.nvidia.com/deeplearning/sdk/nccl-developer-guide/docs/env.html + os.environ.setdefault("NCCL_DEBUG", "INFO") + + # See: + # https://pytorch.org/docs/stable/distributed.html#torch.distributed.init_process_group + os.environ.setdefault("NCCL_BLOCKING_WAIT", "1") + + torch.distributed.init_process_group(backend='nccl', init_method='env://') + self.dist_rank = torch.distributed.get_rank() + self.dist_world_size = torch.distributed.get_world_size() + self.local_rank = args.local_rank + logging.info("world size: {}, rank: {}, local_rank: {}".format(self.dist_world_size, self.dist_rank, + self.local_rank)) + + +def resolve_distributed_mode(args): + # Note that args.distributed is set by only this function. + # and ArgumentParser doesn't have such option + + if args.multiprocessing_distributed: + num_nodes = get_num_nodes(args.dist_world_size, args.dist_launcher) + # a. multi-node + if num_nodes > 1: + args.distributed = True + # b. single-node and multi-gpu with multiprocessing_distributed mode + elif args.ngpu > 1: + args.distributed = True + # c. single-node and single-gpu + else: + args.distributed = False + + if args.ngpu <= 1: + # Disable multiprocessing_distributed mode if 1process per node or cpu mode + args.multiprocessing_distributed = False + if args.ngpu == 1: + # If the number of GPUs equals to 1 with multiprocessing_distributed mode, + # LOCAL_RANK is always 0 + args.local_rank = 0 + + if num_nodes > 1 and get_node_rank(args.dist_rank, args.dist_launcher) is None: + raise RuntimeError( + "--dist_rank or RANK must be set " + "if --multiprocessing_distributed == true" + ) + + # Note that RANK, LOCAL_RANK, and WORLD_SIZE is automatically set, + # so we don't need to check here + else: + # d. multiprocess and multi-gpu with external launcher + # e.g. torch.distributed.launch + if get_world_size(args.dist_world_size, args.dist_launcher) > 1: + args.distributed = True + # e. single-process + else: + args.distributed = False + + if args.distributed and args.ngpu > 0: + if get_local_rank(args.local_rank, args.dist_launcher) is None: + raise RuntimeError( + "--local_rank or LOCAL_RANK must be set " + "if --multiprocessing_distributed == false" + ) + if args.distributed: + if get_node_rank(args.dist_rank, args.dist_launcher) is None: + raise RuntimeError( + "--dist_rank or RANK must be set " + "if --multiprocessing_distributed == false" + ) + if args.distributed and args.dist_launcher == "slurm" and not is_in_slurm_step(): + raise RuntimeError("Launch by 'srun' command if --dist_launcher='slurm'") + + +def is_in_slurm_job() -> bool: + return "SLURM_PROCID" in os.environ and "SLURM_NTASKS" in os.environ + + +def is_in_slurm_step() -> bool: + return ( + is_in_slurm_job() + and "SLURM_STEP_NUM_NODES" in os.environ + and "SLURM_STEP_NODELIST" in os.environ + ) + + +def _int_or_none(x: Optional[str]) -> Optional[int]: + if x is None: + return x + return int(x) + + +def free_port(): + """Find free port using bind(). + + There are some interval between finding this port and using it + and the other process might catch the port by that time. + Thus it is not guaranteed that the port is really empty. + + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("", 0)) + return sock.getsockname()[1] + + +def get_rank(prior=None, launcher: str = None) -> Optional[int]: + if prior is None: + if launcher == "slurm": + if not is_in_slurm_step(): + raise RuntimeError("This process seems not to be launched by 'srun'") + prior = os.environ["SLURM_PROCID"] + elif launcher == "mpi": + raise RuntimeError( + "launcher=mpi is used for 'multiprocessing-distributed' mode" + ) + elif launcher is not None: + raise RuntimeError(f"launcher='{launcher}' is not supported") + + if prior is not None: + return int(prior) + else: + # prior is None and RANK is None -> RANK = None + return _int_or_none(os.environ.get("RANK")) + + +def get_world_size(prior=None, launcher: str = None) -> int: + if prior is None: + if launcher == "slurm": + if not is_in_slurm_step(): + raise RuntimeError("This process seems not to be launched by 'srun'") + prior = int(os.environ["SLURM_NTASKS"]) + elif launcher == "mpi": + raise RuntimeError( + "launcher=mpi is used for 'multiprocessing-distributed' mode" + ) + elif launcher is not None: + raise RuntimeError(f"launcher='{launcher}' is not supported") + + if prior is not None: + return int(prior) + else: + # prior is None and WORLD_SIZE is None -> WORLD_SIZE = 1 + return int(os.environ.get("WORLD_SIZE", "1")) + + +def get_local_rank(prior=None, launcher: str = None) -> Optional[int]: + # LOCAL_RANK is same as GPU device id + + if prior is None: + if launcher == "slurm": + if not is_in_slurm_step(): + raise RuntimeError("This process seems not to be launched by 'srun'") + + prior = int(os.environ["SLURM_LOCALID"]) + elif launcher == "mpi": + raise RuntimeError( + "launcher=mpi is used for 'multiprocessing-distributed' mode" + ) + elif launcher is not None: + raise RuntimeError(f"launcher='{launcher}' is not supported") + + if prior is not None: + return int(prior) + + elif "LOCAL_RANK" in os.environ: + return int(os.environ["LOCAL_RANK"]) + + elif "CUDA_VISIBLE_DEVICES" in os.environ: + # There are two possibility: + # - "CUDA_VISIBLE_DEVICES" is set to multiple GPU ids. e.g. "0.1,2" + # => This intends to specify multiple devices to to be used exactly + # and local_rank information is possibly insufficient. + # - "CUDA_VISIBLE_DEVICES" is set to an id. e.g. "1" + # => This could be used for LOCAL_RANK + cvd = os.environ["CUDA_VISIBLE_DEVICES"].split(",") + if len(cvd) == 1 and "LOCAL_RANK" not in os.environ: + # If CUDA_VISIBLE_DEVICES is set and LOCAL_RANK is not set, + # then use it as LOCAL_RANK. + + # Unset CUDA_VISIBLE_DEVICES + # because the other device must be visible to communicate + return int(os.environ.pop("CUDA_VISIBLE_DEVICES")) + else: + return None + else: + return None + + +def get_master_addr(prior=None, launcher: str = None) -> Optional[str]: + if prior is None: + if launcher == "slurm": + if not is_in_slurm_step(): + raise RuntimeError("This process seems not to be launched by 'srun'") + + # e.g nodelist = foo[1-10],bar[3-8] or foo4,bar[2-10] + nodelist = os.environ["SLURM_STEP_NODELIST"] + prior = nodelist.split(",")[0].split("-")[0].replace("[", "") + + if prior is not None: + return str(prior) + else: + return os.environ.get("MASTER_ADDR") + + +def get_master_port(prior=None) -> Optional[int]: + if prior is not None: + return prior + else: + return _int_or_none(os.environ.get("MASTER_PORT")) + + +def get_node_rank(prior=None, launcher: str = None) -> Optional[int]: + """Get Node Rank. + + Use for "multiprocessing distributed" mode. + The initial RANK equals to the Node id in this case and + the real Rank is set as (nGPU * NodeID) + LOCAL_RANK in torch.distributed. + + """ + if prior is not None: + return prior + elif launcher == "slurm": + if not is_in_slurm_step(): + raise RuntimeError("This process seems not to be launched by 'srun'") + + # Assume ntasks_per_node == 1 + if os.environ["SLURM_STEP_NUM_NODES"] != os.environ["SLURM_NTASKS"]: + raise RuntimeError( + "Run with --ntasks_per_node=1 if mutliprocessing_distributed=true" + ) + return int(os.environ["SLURM_NODEID"]) + elif launcher == "mpi": + # Use mpi4py only for initialization and not using for communication + from mpi4py import MPI + + comm = MPI.COMM_WORLD + # Assume ntasks_per_node == 1 (We can't check whether it is or not) + return comm.Get_rank() + elif launcher is not None: + raise RuntimeError(f"launcher='{launcher}' is not supported") + else: + return _int_or_none(os.environ.get("RANK")) + + +def get_num_nodes(prior=None, launcher: str = None) -> Optional[int]: + """Get the number of nodes. + + Use for "multiprocessing distributed" mode. + RANK equals to the Node id in this case and + the real Rank is set as (nGPU * NodeID) + LOCAL_RANK in torch.distributed. + + """ + if prior is not None: + return prior + elif launcher == "slurm": + if not is_in_slurm_step(): + raise RuntimeError("This process seems not to be launched by 'srun'") + + # Assume ntasks_per_node == 1 + if os.environ["SLURM_STEP_NUM_NODES"] != os.environ["SLURM_NTASKS"]: + raise RuntimeError( + "Run with --ntasks_per_node=1 if mutliprocessing_distributed=true" + ) + return int(os.environ["SLURM_STEP_NUM_NODES"]) + elif launcher == "mpi": + # Use mpi4py only for initialization and not using for communication + from mpi4py import MPI + + comm = MPI.COMM_WORLD + # Assume ntasks_per_node == 1 (We can't check whether it is or not) + return comm.Get_size() + elif launcher is not None: + raise RuntimeError(f"launcher='{launcher}' is not supported") + else: + # prior is None -> NUM_NODES = 1 + return int(os.environ.get("WORLD_SIZE", 1)) diff --git a/funasr/train/reporter.py b/funasr/train/reporter.py new file mode 100644 index 000000000..2921fef28 --- /dev/null +++ b/funasr/train/reporter.py @@ -0,0 +1,540 @@ +"""Reporter module.""" +import dataclasses +import datetime +import logging +import time +import warnings +from collections import defaultdict +from contextlib import contextmanager +from distutils.version import LooseVersion +from typing import ContextManager +from typing import Dict +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import humanfriendly +import numpy as np +import torch +from typeguard import check_argument_types +from typeguard import check_return_type + +Num = Union[float, int, complex, torch.Tensor, np.ndarray] + +_reserved = {"time", "total_count"} + + +def to_reported_value(v: Num, weight: Num = None) -> "ReportedValue": + assert check_argument_types() + if isinstance(v, (torch.Tensor, np.ndarray)): + if np.prod(v.shape) != 1: + raise ValueError(f"v must be 0 or 1 dimension: {len(v.shape)}") + v = v.item() + + if isinstance(weight, (torch.Tensor, np.ndarray)): + if np.prod(weight.shape) != 1: + raise ValueError(f"weight must be 0 or 1 dimension: {len(weight.shape)}") + weight = weight.item() + + if weight is not None: + retval = WeightedAverage(v, weight) + else: + retval = Average(v) + assert check_return_type(retval) + return retval + + +def aggregate(values: Sequence["ReportedValue"]) -> Num: + assert check_argument_types() + + for v in values: + if not isinstance(v, type(values[0])): + raise ValueError( + f"Can't use different Reported type together: " + f"{type(v)} != {type(values[0])}" + ) + + if len(values) == 0: + warnings.warn("No stats found") + retval = np.nan + + elif isinstance(values[0], Average): + retval = np.nanmean([v.value for v in values]) + + elif isinstance(values[0], WeightedAverage): + # Excludes non finite values + invalid_indices = set() + for i, v in enumerate(values): + if not np.isfinite(v.value) or not np.isfinite(v.weight): + invalid_indices.add(i) + values = [v for i, v in enumerate(values) if i not in invalid_indices] + + if len(values) != 0: + # Calc weighed average. Weights are changed to sum-to-1. + sum_weights = sum(v.weight for i, v in enumerate(values)) + sum_value = sum(v.value * v.weight for i, v in enumerate(values)) + if sum_weights == 0: + warnings.warn("weight is zero") + retval = np.nan + else: + retval = sum_value / sum_weights + else: + warnings.warn("No valid stats found") + retval = np.nan + + else: + raise NotImplementedError(f"type={type(values[0])}") + assert check_return_type(retval) + return retval + + +def wandb_get_prefix(key: str): + if key.startswith("valid"): + return "valid/" + if key.startswith("train"): + return "train/" + if key.startswith("attn"): + return "attn/" + return "metrics/" + + +class ReportedValue: + pass + + +@dataclasses.dataclass(frozen=True) +class Average(ReportedValue): + value: Num + + +@dataclasses.dataclass(frozen=True) +class WeightedAverage(ReportedValue): + value: Tuple[Num, Num] + weight: Num + + +class SubReporter: + """This class is used in Reporter. + + See the docstring of Reporter for the usage. + """ + + def __init__(self, key: str, epoch: int, total_count: int): + assert check_argument_types() + self.key = key + self.epoch = epoch + self.start_time = time.perf_counter() + self.stats = defaultdict(list) + self._finished = False + self.total_count = total_count + self.count = 0 + self._seen_keys_in_the_step = set() + + def get_total_count(self) -> int: + """Returns the number of iterations over all epochs.""" + return self.total_count + + def get_epoch(self) -> int: + return self.epoch + + def next(self): + """Close up this step and reset state for the next step""" + for key, stats_list in self.stats.items(): + if key not in self._seen_keys_in_the_step: + # Fill nan value if the key is not registered in this step + if isinstance(stats_list[0], WeightedAverage): + stats_list.append(to_reported_value(np.nan, 0)) + elif isinstance(stats_list[0], Average): + stats_list.append(to_reported_value(np.nan)) + else: + raise NotImplementedError(f"type={type(stats_list[0])}") + + assert len(stats_list) == self.count, (len(stats_list), self.count) + + self._seen_keys_in_the_step = set() + + def register( + self, + stats: Dict[str, Optional[Union[Num, Dict[str, Num]]]], + weight: Num = None, + ) -> None: + assert check_argument_types() + if self._finished: + raise RuntimeError("Already finished") + if len(self._seen_keys_in_the_step) == 0: + # Increment count as the first register in this step + self.total_count += 1 + self.count += 1 + + for key2, v in stats.items(): + if key2 in _reserved: + raise RuntimeError(f"{key2} is reserved.") + if key2 in self._seen_keys_in_the_step: + raise RuntimeError(f"{key2} is registered twice.") + if v is None: + v = np.nan + r = to_reported_value(v, weight) + + if key2 not in self.stats: + # If it's the first time to register the key, + # append nan values in front of the the value + # to make it same length to the other stats + # e.g. + # stat A: [0.4, 0.3, 0.5] + # stat B: [nan, nan, 0.2] + nan = to_reported_value(np.nan, None if weight is None else 0) + self.stats[key2].extend( + r if i == self.count - 1 else nan for i in range(self.count) + ) + else: + self.stats[key2].append(r) + self._seen_keys_in_the_step.add(key2) + + def log_message(self, start: int = None, end: int = None, num_updates: int = None) -> str: + if self._finished: + raise RuntimeError("Already finished") + if start is None: + start = 0 + if start < 0: + start = self.count + start + if end is None: + end = self.count + + if self.count == 0 or start == end: + return "" + + message = f"{self.epoch}epoch:{self.key}:" f"{start + 1}-{end}batch:" + if num_updates is not None: + message += f"{num_updates}num_updates: " + + for idx, (key2, stats_list) in enumerate(self.stats.items()): + assert len(stats_list) == self.count, (len(stats_list), self.count) + # values: List[ReportValue] + values = stats_list[start:end] + if idx != 0 and idx != len(stats_list): + message += ", " + + v = aggregate(values) + if abs(v) > 1.0e3: + message += f"{key2}={v:.3e}" + elif abs(v) > 1.0e-3: + message += f"{key2}={v:.3f}" + else: + message += f"{key2}={v:.3e}" + return message + + def tensorboard_add_scalar(self, summary_writer, start: int = None): + if start is None: + start = 0 + if start < 0: + start = self.count + start + + for key2, stats_list in self.stats.items(): + assert len(stats_list) == self.count, (len(stats_list), self.count) + # values: List[ReportValue] + values = stats_list[start:] + v = aggregate(values) + summary_writer.add_scalar(f"{key2}", v, self.total_count) + + def wandb_log(self, start: int = None): + import wandb + + if start is None: + start = 0 + if start < 0: + start = self.count + start + + d = {} + for key2, stats_list in self.stats.items(): + assert len(stats_list) == self.count, (len(stats_list), self.count) + # values: List[ReportValue] + values = stats_list[start:] + v = aggregate(values) + d[wandb_get_prefix(key2) + key2] = v + d["iteration"] = self.total_count + wandb.log(d) + + def finished(self) -> None: + self._finished = True + + @contextmanager + def measure_time(self, name: str): + start = time.perf_counter() + yield start + t = time.perf_counter() - start + self.register({name: t}) + + def measure_iter_time(self, iterable, name: str): + iterator = iter(iterable) + while True: + try: + start = time.perf_counter() + retval = next(iterator) + t = time.perf_counter() - start + self.register({name: t}) + yield retval + except StopIteration: + break + + +class Reporter: + """Reporter class. + + Examples: + + >>> reporter = Reporter() + >>> with reporter.observe('train') as sub_reporter: + ... for batch in iterator: + ... stats = dict(loss=0.2) + ... sub_reporter.register(stats) + + """ + + def __init__(self, epoch: int = 0): + assert check_argument_types() + if epoch < 0: + raise ValueError(f"epoch must be 0 or more: {epoch}") + self.epoch = epoch + # stats: Dict[int, Dict[str, Dict[str, float]]] + # e.g. self.stats[epoch]['train']['loss'] + self.stats = {} + + def get_epoch(self) -> int: + return self.epoch + + def set_epoch(self, epoch: int) -> None: + if epoch < 0: + raise ValueError(f"epoch must be 0 or more: {epoch}") + self.epoch = epoch + + @contextmanager + def observe(self, key: str, epoch: int = None) -> ContextManager[SubReporter]: + sub_reporter = self.start_epoch(key, epoch) + yield sub_reporter + # Receive the stats from sub_reporter + self.finish_epoch(sub_reporter) + + def start_epoch(self, key: str, epoch: int = None) -> SubReporter: + if epoch is not None: + if epoch < 0: + raise ValueError(f"epoch must be 0 or more: {epoch}") + self.epoch = epoch + + if self.epoch - 1 not in self.stats or key not in self.stats[self.epoch - 1]: + # If the previous epoch doesn't exist for some reason, + # maybe due to bug, this case also indicates 0-count. + if self.epoch - 1 != 0: + warnings.warn( + f"The stats of the previous epoch={self.epoch - 1}" + f"doesn't exist." + ) + total_count = 0 + else: + total_count = self.stats[self.epoch - 1][key]["total_count"] + + sub_reporter = SubReporter(key, self.epoch, total_count) + # Clear the stats for the next epoch if it exists + self.stats.pop(epoch, None) + return sub_reporter + + def finish_epoch(self, sub_reporter: SubReporter) -> None: + if self.epoch != sub_reporter.epoch: + raise RuntimeError( + f"Don't change epoch during observation: " + f"{self.epoch} != {sub_reporter.epoch}" + ) + + # Calc mean of current stats and set it as previous epochs stats + stats = {} + for key2, values in sub_reporter.stats.items(): + v = aggregate(values) + stats[key2] = v + + stats["time"] = datetime.timedelta( + seconds=time.perf_counter() - sub_reporter.start_time + ) + stats["total_count"] = sub_reporter.total_count + if LooseVersion(torch.__version__) >= LooseVersion("1.4.0"): + if torch.cuda.is_initialized(): + stats["gpu_max_cached_mem_GB"] = ( + torch.cuda.max_memory_reserved() / 2 ** 30 + ) + else: + if torch.cuda.is_available() and torch.cuda.max_memory_cached() > 0: + stats["gpu_cached_mem_GB"] = torch.cuda.max_memory_cached() / 2 ** 30 + + self.stats.setdefault(self.epoch, {})[sub_reporter.key] = stats + sub_reporter.finished() + + def sort_epochs_and_values( + self, key: str, key2: str, mode: str + ) -> List[Tuple[int, float]]: + """Return the epoch which resulted the best value. + + Example: + >>> val = reporter.sort_epochs_and_values('eval', 'loss', 'min') + >>> e_1best, v_1best = val[0] + >>> e_2best, v_2best = val[1] + """ + if mode not in ("min", "max"): + raise ValueError(f"mode must min or max: {mode}") + if not self.has(key, key2): + raise KeyError(f"{key}.{key2} is not found: {self.get_all_keys()}") + + # iterate from the last epoch + values = [(e, self.stats[e][key][key2]) for e in self.stats] + + if mode == "min": + values = sorted(values, key=lambda x: x[1]) + else: + values = sorted(values, key=lambda x: -x[1]) + return values + + def sort_epochs(self, key: str, key2: str, mode: str) -> List[int]: + return [e for e, v in self.sort_epochs_and_values(key, key2, mode)] + + def sort_values(self, key: str, key2: str, mode: str) -> List[float]: + return [v for e, v in self.sort_epochs_and_values(key, key2, mode)] + + def get_best_epoch(self, key: str, key2: str, mode: str, nbest: int = 0) -> int: + return self.sort_epochs(key, key2, mode)[nbest] + + def check_early_stopping( + self, + patience: int, + key1: str, + key2: str, + mode: str, + epoch: int = None, + logger=None, + ) -> bool: + if logger is None: + logger = logging + if epoch is None: + epoch = self.get_epoch() + + best_epoch = self.get_best_epoch(key1, key2, mode) + if epoch - best_epoch > patience: + logger.info( + f"[Early stopping] {key1}.{key2} has not been " + f"improved {epoch - best_epoch} epochs continuously. " + f"The training was stopped at {epoch}epoch" + ) + return True + else: + return False + + def has(self, key: str, key2: str, epoch: int = None) -> bool: + if epoch is None: + epoch = self.get_epoch() + return ( + epoch in self.stats + and key in self.stats[epoch] + and key2 in self.stats[epoch][key] + ) + + def log_message(self, epoch: int = None) -> str: + if epoch is None: + epoch = self.get_epoch() + + message = "" + for key, d in self.stats[epoch].items(): + _message = "" + for key2, v in d.items(): + if v is not None: + if len(_message) != 0: + _message += ", " + if isinstance(v, float): + if abs(v) > 1.0e3: + _message += f"{key2}={v:.3e}" + elif abs(v) > 1.0e-3: + _message += f"{key2}={v:.3f}" + else: + _message += f"{key2}={v:.3e}" + elif isinstance(v, datetime.timedelta): + _v = humanfriendly.format_timespan(v) + _message += f"{key2}={_v}" + else: + _message += f"{key2}={v}" + if len(_message) != 0: + if len(message) == 0: + message += f"{epoch}epoch results: " + else: + message += ", " + message += f"[{key}] {_message}" + return message + + def get_value(self, key: str, key2: str, epoch: int = None): + if not self.has(key, key2): + raise KeyError(f"{key}.{key2} is not found in stats: {self.get_all_keys()}") + if epoch is None: + epoch = self.get_epoch() + return self.stats[epoch][key][key2] + + def get_keys(self, epoch: int = None) -> Tuple[str, ...]: + """Returns keys1 e.g. train,eval.""" + if epoch is None: + epoch = self.get_epoch() + return tuple(self.stats[epoch]) + + def get_keys2(self, key: str, epoch: int = None) -> Tuple[str, ...]: + """Returns keys2 e.g. loss,acc.""" + if epoch is None: + epoch = self.get_epoch() + d = self.stats[epoch][key] + keys2 = tuple(k for k in d if k not in ("time", "total_count")) + return keys2 + + def get_all_keys(self, epoch: int = None) -> Tuple[Tuple[str, str], ...]: + if epoch is None: + epoch = self.get_epoch() + all_keys = [] + for key in self.stats[epoch]: + for key2 in self.stats[epoch][key]: + all_keys.append((key, key2)) + return tuple(all_keys) + + def tensorboard_add_scalar( + self, summary_writer, epoch: int = None, key1: str = None + ): + if epoch is None: + epoch = self.get_epoch() + total_count = self.stats[epoch]["train"]["total_count"] + if key1 == "train": + summary_writer.add_scalar("iter_epoch", epoch, total_count) + + if key1 is not None: + key1_iterator = tuple([key1]) + else: + key1_iterator = self.get_keys(epoch) + + for key1 in key1_iterator: + for key2 in self.get_keys2(key1): + summary_writer.add_scalar( + f"{key2}", self.stats[epoch][key1][key2], total_count + ) + + def wandb_log(self, epoch: int = None): + import wandb + + if epoch is None: + epoch = self.get_epoch() + + d = {} + for key1 in self.get_keys(epoch): + for key2 in self.stats[epoch][key1]: + if key2 in ("time", "total_count"): + continue + key = f"{key1}_{key2}_epoch" + d[wandb_get_prefix(key) + key] = self.stats[epoch][key1][key2] + d["epoch"] = epoch + wandb.log(d) + + def state_dict(self): + return {"stats": self.stats, "epoch": self.epoch} + + def load_state_dict(self, state_dict: dict): + self.epoch = state_dict["epoch"] + self.stats = state_dict["stats"] diff --git a/funasr/train/trainer.py b/funasr/train/trainer.py new file mode 100644 index 000000000..50bce477a --- /dev/null +++ b/funasr/train/trainer.py @@ -0,0 +1,814 @@ +# Copyright ESPnet (https://github.com/espnet/espnet). All Rights Reserved. +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +"""Trainer module.""" +import argparse +from contextlib import contextmanager +import dataclasses +from dataclasses import is_dataclass +from distutils.version import LooseVersion +import logging +from pathlib import Path +import time +from typing import Dict +from typing import Iterable +from typing import List +from typing import Optional +from typing import Sequence +from typing import Tuple +from typing import Union + +import humanfriendly +import oss2 +from io import BytesIO +import os +import numpy as np +import torch +import torch.nn +import torch.optim +from typeguard import check_argument_types + +from funasr.iterators.abs_iter_factory import AbsIterFactory +from funasr.main_funcs.average_nbest_models import average_nbest_models +from funasr.main_funcs.calculate_all_attentions import calculate_all_attentions +from funasr.schedulers.abs_scheduler import AbsBatchStepScheduler +from funasr.schedulers.abs_scheduler import AbsEpochStepScheduler +from funasr.schedulers.abs_scheduler import AbsScheduler +from funasr.schedulers.abs_scheduler import AbsValEpochStepScheduler +from funasr.torch_utils.add_gradient_noise import add_gradient_noise +from funasr.torch_utils.device_funcs import to_device +from funasr.torch_utils.recursive_op import recursive_average +from funasr.torch_utils.set_all_random_seed import set_all_random_seed +from funasr.train.abs_espnet_model import AbsESPnetModel +from funasr.train.distributed_utils import DistributedOption +from funasr.train.reporter import Reporter +from funasr.train.reporter import SubReporter +from funasr.utils.build_dataclass import build_dataclass + +if torch.distributed.is_available(): + from torch.distributed import ReduceOp + +if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"): + from torch.cuda.amp import autocast + from torch.cuda.amp import GradScaler +else: + # Nothing to do if torch<1.6.0 + @contextmanager + def autocast(enabled=True): + yield + + GradScaler = None + +try: + import fairscale +except ImportError: + fairscale = None + + +@dataclasses.dataclass +class TrainerOptions: + ngpu: int + resume: bool + use_amp: bool + train_dtype: str + grad_noise: bool + accum_grad: int + grad_clip: float + grad_clip_type: float + log_interval: Optional[int] + no_forward_run: bool + use_tensorboard: bool + use_wandb: bool + output_dir: Union[Path, str] + max_epoch: int + max_update: int + seed: int + sharded_ddp: bool + patience: Optional[int] + keep_nbest_models: Union[int, List[int]] + nbest_averaging_interval: int + early_stopping_criterion: Sequence[str] + best_model_criterion: Sequence[Sequence[str]] + val_scheduler_criterion: Sequence[str] + unused_parameters: bool + wandb_model_log_interval: int + use_pai: bool + oss_bucket: Union[oss2.Bucket, None] + + +class Trainer: + """Trainer having a optimizer. + + If you'd like to use multiple optimizers, then inherit this class + and override the methods if necessary - at least "train_one_epoch()" + + >>> class TwoOptimizerTrainer(Trainer): + ... @classmethod + ... def add_arguments(cls, parser): + ... ... + ... + ... @classmethod + ... def train_one_epoch(cls, model, optimizers, ...): + ... loss1 = model.model1(...) + ... loss1.backward() + ... optimizers[0].step() + ... + ... loss2 = model.model2(...) + ... loss2.backward() + ... optimizers[1].step() + + """ + + def __init__(self): + raise RuntimeError("This class can't be instantiated.") + + @classmethod + def build_options(cls, args: argparse.Namespace) -> TrainerOptions: + """Build options consumed by train(), eval()""" + assert check_argument_types() + return build_dataclass(TrainerOptions, args) + + @classmethod + def add_arguments(cls, parser: argparse.ArgumentParser): + """Reserved for future development of another Trainer""" + pass + + @staticmethod + def resume( + checkpoint: Union[str, Path], + model: torch.nn.Module, + reporter: Reporter, + optimizers: Sequence[torch.optim.Optimizer], + schedulers: Sequence[Optional[AbsScheduler]], + scaler: Optional[GradScaler], + ngpu: int = 0, + ): + states = torch.load( + checkpoint, + map_location=f"cuda:{torch.cuda.current_device()}" if ngpu > 0 else "cpu", + ) + model.load_state_dict(states["model"]) + reporter.load_state_dict(states["reporter"]) + for optimizer, state in zip(optimizers, states["optimizers"]): + optimizer.load_state_dict(state) + for scheduler, state in zip(schedulers, states["schedulers"]): + if scheduler is not None: + scheduler.load_state_dict(state) + if scaler is not None: + if states["scaler"] is None: + logging.warning("scaler state is not found") + else: + scaler.load_state_dict(states["scaler"]) + + logging.info(f"The training was resumed using {checkpoint}") + + @classmethod + def run( + cls, + model: AbsESPnetModel, + optimizers: Sequence[torch.optim.Optimizer], + schedulers: Sequence[Optional[AbsScheduler]], + train_iter_factory: AbsIterFactory, + valid_iter_factory: AbsIterFactory, + trainer_options, + distributed_option: DistributedOption, + ) -> None: + """Perform training. This method performs the main process of training.""" + assert check_argument_types() + # NOTE(kamo): Don't check the type more strictly as far trainer_options + assert is_dataclass(trainer_options), type(trainer_options) + assert len(optimizers) == len(schedulers), (len(optimizers), len(schedulers)) + + if isinstance(trainer_options.keep_nbest_models, int): + keep_nbest_models = [trainer_options.keep_nbest_models] + else: + if len(trainer_options.keep_nbest_models) == 0: + logging.warning("No keep_nbest_models is given. Change to [1]") + trainer_options.keep_nbest_models = [1] + keep_nbest_models = trainer_options.keep_nbest_models + + output_dir = Path(trainer_options.output_dir) + reporter = Reporter() + if trainer_options.use_amp: + if LooseVersion(torch.__version__) < LooseVersion("1.6.0"): + raise RuntimeError( + "Require torch>=1.6.0 for Automatic Mixed Precision" + ) + if trainer_options.sharded_ddp: + if fairscale is None: + raise RuntimeError( + "Requiring fairscale. Do 'pip install fairscale'" + ) + scaler = fairscale.optim.grad_scaler.ShardedGradScaler() + else: + scaler = GradScaler() + else: + scaler = None + + if trainer_options.resume and (output_dir / "checkpoint.pth").exists(): + cls.resume( + checkpoint=output_dir / "checkpoint.pth", + model=model, + optimizers=optimizers, + schedulers=schedulers, + reporter=reporter, + scaler=scaler, + ngpu=trainer_options.ngpu, + ) + + start_epoch = reporter.get_epoch() + 1 + if start_epoch == trainer_options.max_epoch + 1: + logging.warning( + f"The training has already reached at max_epoch: {start_epoch}" + ) + + if distributed_option.distributed: + if trainer_options.sharded_ddp: + dp_model = fairscale.nn.data_parallel.ShardedDataParallel( + module=model, + sharded_optimizer=optimizers, + ) + else: + dp_model = torch.nn.parallel.DistributedDataParallel( + model, find_unused_parameters=trainer_options.unused_parameters) + elif distributed_option.ngpu > 1: + dp_model = torch.nn.parallel.DataParallel( + model, + device_ids=list(range(distributed_option.ngpu)), + ) + else: + # NOTE(kamo): DataParallel also should work with ngpu=1, + # but for debuggability it's better to keep this block. + dp_model = model + + if trainer_options.use_tensorboard and ( + not distributed_option.distributed or distributed_option.dist_rank == 0 + ): + from torch.utils.tensorboard import SummaryWriter + if trainer_options.use_pai: + train_summary_writer = SummaryWriter( + os.path.join(trainer_options.output_dir, "tensorboard/train") + ) + valid_summary_writer = SummaryWriter( + os.path.join(trainer_options.output_dir, "tensorboard/valid") + ) + else: + train_summary_writer = SummaryWriter( + str(output_dir / "tensorboard" / "train") + ) + valid_summary_writer = SummaryWriter( + str(output_dir / "tensorboard" / "valid") + ) + else: + train_summary_writer = None + + start_time = time.perf_counter() + for iepoch in range(start_epoch, trainer_options.max_epoch + 1): + if iepoch != start_epoch: + logging.info( + "{}/{}epoch started. Estimated time to finish: {}".format( + iepoch, + trainer_options.max_epoch, + humanfriendly.format_timespan( + (time.perf_counter() - start_time) + / (iepoch - start_epoch) + * (trainer_options.max_epoch - iepoch + 1) + ), + ) + ) + else: + logging.info(f"{iepoch}/{trainer_options.max_epoch}epoch started") + set_all_random_seed(trainer_options.seed + iepoch) + + reporter.set_epoch(iepoch) + # 1. Train and validation for one-epoch + with reporter.observe("train") as sub_reporter: + all_steps_are_invalid, max_update_stop = cls.train_one_epoch( + model=dp_model, + optimizers=optimizers, + schedulers=schedulers, + iterator=train_iter_factory.build_iter(iepoch), + reporter=sub_reporter, + scaler=scaler, + summary_writer=train_summary_writer, + options=trainer_options, + distributed_option=distributed_option, + ) + + with reporter.observe("valid") as sub_reporter: + cls.validate_one_epoch( + model=dp_model, + iterator=valid_iter_factory.build_iter(iepoch), + reporter=sub_reporter, + options=trainer_options, + distributed_option=distributed_option, + ) + + # 2. LR Scheduler step + for scheduler in schedulers: + if isinstance(scheduler, AbsValEpochStepScheduler): + scheduler.step( + reporter.get_value(*trainer_options.val_scheduler_criterion) + ) + elif isinstance(scheduler, AbsEpochStepScheduler): + scheduler.step() + if trainer_options.sharded_ddp: + for optimizer in optimizers: + if isinstance(optimizer, fairscale.optim.oss.OSS): + optimizer.consolidate_state_dict() + + if not distributed_option.distributed or distributed_option.dist_rank == 0: + # 3. Report the results + logging.info(reporter.log_message()) + if train_summary_writer is not None: + reporter.tensorboard_add_scalar(train_summary_writer, key1="train") + reporter.tensorboard_add_scalar(valid_summary_writer, key1="valid") + if trainer_options.use_wandb: + reporter.wandb_log() + + # save tensorboard on oss + if trainer_options.use_pai and train_summary_writer is not None: + def write_tensorboard_summary(summary_writer_path, oss_bucket): + file_list = [] + for root, dirs, files in os.walk(summary_writer_path, topdown=False): + for name in files: + file_full_path = os.path.join(root, name) + file_list.append(file_full_path) + + for file_full_path in file_list: + with open(file_full_path, "rb") as f: + oss_bucket.put_object(file_full_path, f) + + write_tensorboard_summary(os.path.join(trainer_options.output_dir, "tensorboard/train"), trainer_options.oss_bucket) + write_tensorboard_summary(os.path.join(trainer_options.output_dir, "tensorboard/valid"), trainer_options.oss_bucket) + + + # 4. Save/Update the checkpoint + if trainer_options.use_pai: + buffer = BytesIO() + torch.save( + { + "model": model.state_dict(), + "reporter": reporter.state_dict(), + "optimizers": [o.state_dict() for o in optimizers], + "schedulers": [ + s.state_dict() if s is not None else None + for s in schedulers + ], + "scaler": scaler.state_dict() if scaler is not None else None, + "ema_model": model.encoder.ema.model.state_dict() + if hasattr(model.encoder, "ema") and model.encoder.ema is not None else None, + }, + buffer, + ) + trainer_options.oss_bucket.put_object(os.path.join(trainer_options.output_dir, "checkpoint.pth"), buffer.getvalue()) + else: + torch.save( + { + "model": model.state_dict(), + "reporter": reporter.state_dict(), + "optimizers": [o.state_dict() for o in optimizers], + "schedulers": [ + s.state_dict() if s is not None else None + for s in schedulers + ], + "scaler": scaler.state_dict() if scaler is not None else None, + }, + output_dir / "checkpoint.pth", + ) + + # 5. Save and log the model and update the link to the best model + if trainer_options.use_pai: + buffer = BytesIO() + torch.save(model.state_dict(), buffer) + trainer_options.oss_bucket.put_object(os.path.join(trainer_options.output_dir, + f"{iepoch}epoch.pth"),buffer.getvalue()) + else: + torch.save(model.state_dict(), output_dir / f"{iepoch}epoch.pth") + + # Creates a sym link latest.pth -> {iepoch}epoch.pth + if trainer_options.use_pai: + p = os.path.join(trainer_options.output_dir, "latest.pth") + if trainer_options.oss_bucket.object_exists(p): + trainer_options.oss_bucket.delete_object(p) + trainer_options.oss_bucket.copy_object(trainer_options.oss_bucket.bucket_name, + os.path.join(trainer_options.output_dir, f"{iepoch}epoch.pth"), p) + else: + p = output_dir / "latest.pth" + if p.is_symlink() or p.exists(): + p.unlink() + p.symlink_to(f"{iepoch}epoch.pth") + + _improved = [] + for _phase, k, _mode in trainer_options.best_model_criterion: + # e.g. _phase, k, _mode = "train", "loss", "min" + if reporter.has(_phase, k): + best_epoch = reporter.get_best_epoch(_phase, k, _mode) + # Creates sym links if it's the best result + if best_epoch == iepoch: + if trainer_options.use_pai: + p = os.path.join(trainer_options.output_dir, f"{_phase}.{k}.best.pth") + if trainer_options.oss_bucket.object_exists(p): + trainer_options.oss_bucket.delete_object(p) + trainer_options.oss_bucket.copy_object(trainer_options.oss_bucket.bucket_name, + os.path.join(trainer_options.output_dir, f"{iepoch}epoch.pth"),p) + else: + p = output_dir / f"{_phase}.{k}.best.pth" + if p.is_symlink() or p.exists(): + p.unlink() + p.symlink_to(f"{iepoch}epoch.pth") + _improved.append(f"{_phase}.{k}") + if len(_improved) == 0: + logging.info("There are no improvements in this epoch") + else: + logging.info( + "The best model has been updated: " + ", ".join(_improved) + ) + + log_model = ( + trainer_options.wandb_model_log_interval > 0 + and iepoch % trainer_options.wandb_model_log_interval == 0 + ) + if log_model and trainer_options.use_wandb: + import wandb + + logging.info("Logging Model on this epoch :::::") + artifact = wandb.Artifact( + name=f"model_{wandb.run.id}", + type="model", + metadata={"improved": _improved}, + ) + artifact.add_file(str(output_dir / f"{iepoch}epoch.pth")) + aliases = [ + f"epoch-{iepoch}", + "best" if best_epoch == iepoch else "", + ] + wandb.log_artifact(artifact, aliases=aliases) + + # 6. Remove the model files excluding n-best epoch and latest epoch + _removed = [] + # Get the union set of the n-best among multiple criterion + nbests = set().union( + *[ + set(reporter.sort_epochs(ph, k, m)[: max(keep_nbest_models)]) + for ph, k, m in trainer_options.best_model_criterion + if reporter.has(ph, k) + ] + ) + + # Generated n-best averaged model + if ( + trainer_options.nbest_averaging_interval > 0 + and iepoch % trainer_options.nbest_averaging_interval == 0 + ): + average_nbest_models( + reporter=reporter, + output_dir=output_dir, + best_model_criterion=trainer_options.best_model_criterion, + nbest=keep_nbest_models, + suffix=f"till{iepoch}epoch", + oss_bucket=trainer_options.oss_bucket, + pai_output_dir=trainer_options.output_dir, + ) + + for e in range(1, iepoch): + if trainer_options.use_pai: + p = os.path.join(trainer_options.output_dir, f"{e}epoch.pth") + if trainer_options.oss_bucket.object_exists(p) and e not in nbests: + trainer_options.oss_bucket.delete_object(p) + _removed.append(str(p)) + else: + p = output_dir / f"{e}epoch.pth" + if p.exists() and e not in nbests: + p.unlink() + _removed.append(str(p)) + if len(_removed) != 0: + logging.info("The model files were removed: " + ", ".join(_removed)) + + # 7. If any updating haven't happened, stops the training + if all_steps_are_invalid: + logging.warning( + f"The gradients at all steps are invalid in this epoch. " + f"Something seems wrong. This training was stopped at {iepoch}epoch" + ) + break + + if max_update_stop: + logging.info( + f"Stopping training due to " + f"num_updates: {trainer_options.num_updates} >= max_update: {trainer_options.max_update}" + ) + break + + # 8. Check early stopping + if trainer_options.patience is not None: + if reporter.check_early_stopping( + trainer_options.patience, *trainer_options.early_stopping_criterion + ): + break + + else: + logging.info( + f"The training was finished at {trainer_options.max_epoch} epochs " + ) + + # Generated n-best averaged model + if not distributed_option.distributed or distributed_option.dist_rank == 0: + average_nbest_models( + reporter=reporter, + output_dir=output_dir, + best_model_criterion=trainer_options.best_model_criterion, + nbest=keep_nbest_models, + oss_bucket=trainer_options.oss_bucket, + pai_output_dir=trainer_options.output_dir, + ) + + @classmethod + def train_one_epoch( + cls, + model: torch.nn.Module, + iterator: Iterable[Tuple[List[str], Dict[str, torch.Tensor]]], + optimizers: Sequence[torch.optim.Optimizer], + schedulers: Sequence[Optional[AbsScheduler]], + scaler: Optional[GradScaler], + reporter: SubReporter, + summary_writer, + options: TrainerOptions, + distributed_option: DistributedOption, + ) -> Tuple[bool, bool]: + assert check_argument_types() + + grad_noise = options.grad_noise + accum_grad = options.accum_grad + grad_clip = options.grad_clip + grad_clip_type = options.grad_clip_type + log_interval = options.log_interval + no_forward_run = options.no_forward_run + ngpu = options.ngpu + use_wandb = options.use_wandb + distributed = distributed_option.distributed + + if log_interval is None: + try: + log_interval = max(len(iterator) // 20, 10) + except TypeError: + log_interval = 100 + + model.train() + all_steps_are_invalid = True + max_update_stop = False + # [For distributed] Because iteration counts are not always equals between + # processes, send stop-flag to the other processes if iterator is finished + iterator_stop = torch.tensor(0).to("cuda" if ngpu > 0 else "cpu") + + start_time = time.perf_counter() + for iiter, (_, batch) in enumerate( + reporter.measure_iter_time(iterator, "iter_time"), 1 + ): + assert isinstance(batch, dict), type(batch) + + if distributed: + torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM) + if iterator_stop > 0: + break + + batch = to_device(batch, "cuda" if ngpu > 0 else "cpu") + if no_forward_run: + all_steps_are_invalid = False + continue + + with autocast(scaler is not None): + with reporter.measure_time("forward_time"): + retval = model(**batch) + + # Note(kamo): + # Supporting two patterns for the returned value from the model + # a. dict type + if isinstance(retval, dict): + loss = retval["loss"] + stats = retval["stats"] + weight = retval["weight"] + optim_idx = retval.get("optim_idx") + if optim_idx is not None and not isinstance(optim_idx, int): + if not isinstance(optim_idx, torch.Tensor): + raise RuntimeError( + "optim_idx must be int or 1dim torch.Tensor, " + f"but got {type(optim_idx)}" + ) + if optim_idx.dim() >= 2: + raise RuntimeError( + "optim_idx must be int or 1dim torch.Tensor, " + f"but got {optim_idx.dim()}dim tensor" + ) + if optim_idx.dim() == 1: + for v in optim_idx: + if v != optim_idx[0]: + raise RuntimeError( + "optim_idx must be 1dim tensor " + "having same values for all entries" + ) + optim_idx = optim_idx[0].item() + else: + optim_idx = optim_idx.item() + + # b. tuple or list type + else: + loss, stats, weight = retval + optim_idx = None + + stats = {k: v for k, v in stats.items() if v is not None} + if ngpu > 1 or distributed: + # Apply weighted averaging for loss and stats + loss = (loss * weight.type(loss.dtype)).sum() + + # if distributed, this method can also apply all_reduce() + stats, weight = recursive_average(stats, weight, distributed) + + # Now weight is summation over all workers + loss /= weight + if distributed: + # NOTE(kamo): Multiply world_size because DistributedDataParallel + # automatically normalizes the gradient by world_size. + loss *= torch.distributed.get_world_size() + + loss /= accum_grad + + reporter.register(stats, weight) + + with reporter.measure_time("backward_time"): + if scaler is not None: + # Scales loss. Calls backward() on scaled loss + # to create scaled gradients. + # Backward passes under autocast are not recommended. + # Backward ops run in the same dtype autocast chose + # for corresponding forward ops. + scaler.scale(loss).backward() + else: + loss.backward() + + if iiter % accum_grad == 0: + if scaler is not None: + # Unscales the gradients of optimizer's assigned params in-place + for iopt, optimizer in enumerate(optimizers): + if optim_idx is not None and iopt != optim_idx: + continue + scaler.unscale_(optimizer) + + # gradient noise injection + if grad_noise: + add_gradient_noise( + model, + reporter.get_total_count(), + duration=100, + eta=1.0, + scale_factor=0.55, + ) + + # compute the gradient norm to check if it is normal or not + grad_norm = torch.nn.utils.clip_grad_norm_( + model.parameters(), + max_norm=grad_clip, + norm_type=grad_clip_type, + ) + # PyTorch<=1.4, clip_grad_norm_ returns float value + if not isinstance(grad_norm, torch.Tensor): + grad_norm = torch.tensor(grad_norm) + + if not torch.isfinite(grad_norm): + logging.warning( + f"The grad norm is {grad_norm}. Skipping updating the model." + ) + + # Must invoke scaler.update() if unscale_() is used in the iteration + # to avoid the following error: + # RuntimeError: unscale_() has already been called + # on this optimizer since the last update(). + # Note that if the gradient has inf/nan values, + # scaler.step skips optimizer.step(). + if scaler is not None: + for iopt, optimizer in enumerate(optimizers): + if optim_idx is not None and iopt != optim_idx: + continue + scaler.step(optimizer) + scaler.update() + + else: + all_steps_are_invalid = False + with reporter.measure_time("optim_step_time"): + for iopt, (optimizer, scheduler) in enumerate( + zip(optimizers, schedulers) + ): + if optim_idx is not None and iopt != optim_idx: + continue + if scaler is not None: + # scaler.step() first unscales the gradients of + # the optimizer's assigned params. + scaler.step(optimizer) + # Updates the scale for next iteration. + scaler.update() + else: + optimizer.step() + if isinstance(scheduler, AbsBatchStepScheduler): + scheduler.step() + for iopt, optimizer in enumerate(optimizers): + if optim_idx is not None and iopt != optim_idx: + continue + optimizer.zero_grad() + + # Register lr and train/load time[sec/step], + # where step refers to accum_grad * mini-batch + reporter.register( + dict( + { + f"optim{i}_lr{j}": pg["lr"] + for i, optimizer in enumerate(optimizers) + for j, pg in enumerate(optimizer.param_groups) + if "lr" in pg + }, + train_time=time.perf_counter() - start_time, + ), + ) + start_time = time.perf_counter() + + # update num_updates + if distributed: + if hasattr(model.module, "num_updates"): + model.module.set_num_updates(model.module.get_num_updates() + 1) + options.num_updates = model.module.get_num_updates() + if model.module.get_num_updates() >= options.max_update: + max_update_stop = True + else: + if hasattr(model, "num_updates"): + model.set_num_updates(model.get_num_updates() + 1) + options.num_updates = model.get_num_updates() + if model.get_num_updates() >= options.max_update: + max_update_stop = True + + # NOTE(kamo): Call log_message() after next() + reporter.next() + if iiter % log_interval == 0: + num_updates = options.num_updates if hasattr(options, "num_updates") else None + logging.info(reporter.log_message(-log_interval, num_updates=num_updates)) + if summary_writer is not None: + reporter.tensorboard_add_scalar(summary_writer, -log_interval) + if use_wandb: + reporter.wandb_log() + + if max_update_stop: + break + + else: + if distributed: + iterator_stop.fill_(1) + torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM) + return all_steps_are_invalid, max_update_stop + + @classmethod + @torch.no_grad() + def validate_one_epoch( + cls, + model: torch.nn.Module, + iterator: Iterable[Dict[str, torch.Tensor]], + reporter: SubReporter, + options: TrainerOptions, + distributed_option: DistributedOption, + ) -> None: + assert check_argument_types() + ngpu = options.ngpu + no_forward_run = options.no_forward_run + distributed = distributed_option.distributed + + model.eval() + + # [For distributed] Because iteration counts are not always equals between + # processes, send stop-flag to the other processes if iterator is finished + iterator_stop = torch.tensor(0).to("cuda" if ngpu > 0 else "cpu") + for (_, batch) in iterator: + assert isinstance(batch, dict), type(batch) + if distributed: + torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM) + if iterator_stop > 0: + break + + batch = to_device(batch, "cuda" if ngpu > 0 else "cpu") + if no_forward_run: + continue + + retval = model(**batch) + if isinstance(retval, dict): + stats = retval["stats"] + weight = retval["weight"] + else: + _, stats, weight = retval + if ngpu > 1 or distributed: + # Apply weighted averaging for stats. + # if distributed, this method can also apply all_reduce() + stats, weight = recursive_average(stats, weight, distributed) + + reporter.register(stats, weight) + reporter.next() + + else: + if distributed: + iterator_stop.fill_(1) + torch.distributed.all_reduce(iterator_stop, ReduceOp.SUM) \ No newline at end of file diff --git a/funasr/utils/__init__.py b/funasr/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/utils/build_dataclass.py b/funasr/utils/build_dataclass.py new file mode 100644 index 000000000..6675c99a0 --- /dev/null +++ b/funasr/utils/build_dataclass.py @@ -0,0 +1,17 @@ +import argparse +import dataclasses + +from typeguard import check_type + + +def build_dataclass(dataclass, args: argparse.Namespace): + """Helper function to build dataclass from 'args'.""" + kwargs = {} + for field in dataclasses.fields(dataclass): + if not hasattr(args, field.name): + raise ValueError( + f"args doesn't have {field.name}. You need to set it to ArgumentsParser" + ) + check_type(field.name, getattr(args, field.name), field.type) + kwargs[field.name] = getattr(args, field.name) + return dataclass(**kwargs) diff --git a/funasr/utils/cli_utils.py b/funasr/utils/cli_utils.py new file mode 100644 index 000000000..c4a4cd15b --- /dev/null +++ b/funasr/utils/cli_utils.py @@ -0,0 +1,65 @@ +from collections.abc import Sequence +from distutils.util import strtobool as dist_strtobool +import sys + +import numpy + + +def strtobool(x): + # distutils.util.strtobool returns integer, but it's confusing, + return bool(dist_strtobool(x)) + + +def get_commandline_args(): + extra_chars = [ + " ", + ";", + "&", + "(", + ")", + "|", + "^", + "<", + ">", + "?", + "*", + "[", + "]", + "$", + "`", + '"', + "\\", + "!", + "{", + "}", + ] + + # Escape the extra characters for shell + argv = [ + arg.replace("'", "'\\''") + if all(char not in arg for char in extra_chars) + else "'" + arg.replace("'", "'\\''") + "'" + for arg in sys.argv + ] + + return sys.executable + " " + " ".join(argv) + + +def is_scipy_wav_style(value): + # If Tuple[int, numpy.ndarray] or not + return ( + isinstance(value, Sequence) + and len(value) == 2 + and isinstance(value[0], int) + and isinstance(value[1], numpy.ndarray) + ) + + +def assert_scipy_wav_style(value): + assert is_scipy_wav_style( + value + ), "Must be Tuple[int, numpy.ndarray], but got {}".format( + type(value) + if not isinstance(value, Sequence) + else "{}[{}]".format(type(value), ", ".join(str(type(v)) for v in value)) + ) diff --git a/funasr/utils/config_argparse.py b/funasr/utils/config_argparse.py new file mode 100644 index 000000000..c9d7197a7 --- /dev/null +++ b/funasr/utils/config_argparse.py @@ -0,0 +1,47 @@ +import argparse +from pathlib import Path + +import yaml + + +class ArgumentParser(argparse.ArgumentParser): + """Simple implementation of ArgumentParser supporting config file + + This class is originated from https://github.com/bw2/ConfigArgParse, + but this class is lack of some features that it has. + + - Not supporting multiple config files + - Automatically adding "--config" as an option. + - Not supporting any formats other than yaml + - Not checking argument type + + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.add_argument("--config", help="Give config file in yaml format") + + def parse_known_args(self, args=None, namespace=None): + # Once parsing for setting from "--config" + _args, _ = super().parse_known_args(args, namespace) + if _args.config is not None: + if not Path(_args.config).exists(): + self.error(f"No such file: {_args.config}") + + with open(_args.config, "r", encoding="utf-8") as f: + d = yaml.safe_load(f) + if not isinstance(d, dict): + self.error("Config file has non dict value: {_args.config}") + + for key in d: + for action in self._actions: + if key == action.dest: + break + else: + self.error(f"unrecognized arguments: {key} (from {_args.config})") + + # NOTE(kamo): Ignore "--config" from a config file + # NOTE(kamo): Unlike "configargparse", this module doesn't check type. + # i.e. We can set any type value regardless of argument type. + self.set_defaults(**d) + return super().parse_known_args(args, namespace) diff --git a/funasr/utils/get_default_kwargs.py b/funasr/utils/get_default_kwargs.py new file mode 100644 index 000000000..0f11e8af4 --- /dev/null +++ b/funasr/utils/get_default_kwargs.py @@ -0,0 +1,57 @@ +import inspect + + +class Invalid: + """Marker object for not serializable-object""" + + +def get_default_kwargs(func): + """Get the default values of the input function. + + Examples: + >>> def func(a, b=3): pass + >>> get_default_kwargs(func) + {'b': 3} + + """ + + def yaml_serializable(value): + # isinstance(x, tuple) includes namedtuple, so type is used here + if type(value) is tuple: + return yaml_serializable(list(value)) + elif isinstance(value, set): + return yaml_serializable(list(value)) + elif isinstance(value, dict): + if not all(isinstance(k, str) for k in value): + return Invalid + retval = {} + for k, v in value.items(): + v2 = yaml_serializable(v) + # Register only valid object + if v2 not in (Invalid, inspect.Parameter.empty): + retval[k] = v2 + return retval + elif isinstance(value, list): + retval = [] + for v in value: + v2 = yaml_serializable(v) + # If any elements in the list are invalid, + # the list also becomes invalid + if v2 is Invalid: + return Invalid + else: + retval.append(v2) + return retval + elif value in (inspect.Parameter.empty, None): + return value + elif isinstance(value, (float, int, complex, bool, str, bytes)): + return value + else: + return Invalid + + # params: An ordered mapping of inspect.Parameter + params = inspect.signature(func).parameters + data = {p.name: p.default for p in params.values()} + # Remove not yaml-serializable object + data = yaml_serializable(data) + return data diff --git a/funasr/utils/griffin_lim.py b/funasr/utils/griffin_lim.py new file mode 100644 index 000000000..c1536d51b --- /dev/null +++ b/funasr/utils/griffin_lim.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +"""Griffin-Lim related modules.""" + +# Copyright 2019 Tomoki Hayashi +# Apache 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import logging + +from distutils.version import LooseVersion +from functools import partial +from typeguard import check_argument_types +from typing import Optional + +import librosa +import numpy as np +import torch + +EPS = 1e-10 + + +def logmel2linear( + lmspc: np.ndarray, + fs: int, + n_fft: int, + n_mels: int, + fmin: int = None, + fmax: int = None, +) -> np.ndarray: + """Convert log Mel filterbank to linear spectrogram. + + Args: + lmspc: Log Mel filterbank (T, n_mels). + fs: Sampling frequency. + n_fft: The number of FFT points. + n_mels: The number of mel basis. + f_min: Minimum frequency to analyze. + f_max: Maximum frequency to analyze. + + Returns: + Linear spectrogram (T, n_fft // 2 + 1). + + """ + assert lmspc.shape[1] == n_mels + fmin = 0 if fmin is None else fmin + fmax = fs / 2 if fmax is None else fmax + mspc = np.power(10.0, lmspc) + mel_basis = librosa.filters.mel( + sr=fs, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax + ) + inv_mel_basis = np.linalg.pinv(mel_basis) + return np.maximum(EPS, np.dot(inv_mel_basis, mspc.T).T) + + +def griffin_lim( + spc: np.ndarray, + n_fft: int, + n_shift: int, + win_length: int = None, + window: Optional[str] = "hann", + n_iter: Optional[int] = 32, +) -> np.ndarray: + """Convert linear spectrogram into waveform using Griffin-Lim. + + Args: + spc: Linear spectrogram (T, n_fft // 2 + 1). + n_fft: The number of FFT points. + n_shift: Shift size in points. + win_length: Window length in points. + window: Window function type. + n_iter: The number of iterations. + + Returns: + Reconstructed waveform (N,). + + """ + # assert the size of input linear spectrogram + assert spc.shape[1] == n_fft // 2 + 1 + + if LooseVersion(librosa.__version__) >= LooseVersion("0.7.0"): + # use librosa's fast Grriffin-Lim algorithm + spc = np.abs(spc.T) + y = librosa.griffinlim( + S=spc, + n_iter=n_iter, + hop_length=n_shift, + win_length=win_length, + window=window, + center=True if spc.shape[1] > 1 else False, + ) + else: + # use slower version of Grriffin-Lim algorithm + logging.warning( + "librosa version is old. use slow version of Grriffin-Lim algorithm." + "if you want to use fast Griffin-Lim, please update librosa via " + "`source ./path.sh && pip install librosa==0.7.0`." + ) + cspc = np.abs(spc).astype(np.complex).T + angles = np.exp(2j * np.pi * np.random.rand(*cspc.shape)) + y = librosa.istft(cspc * angles, n_shift, win_length, window=window) + for i in range(n_iter): + angles = np.exp( + 1j + * np.angle(librosa.stft(y, n_fft, n_shift, win_length, window=window)) + ) + y = librosa.istft(cspc * angles, n_shift, win_length, window=window) + + return y + + +# TODO(kan-bayashi): write as torch.nn.Module +class Spectrogram2Waveform(object): + """Spectrogram to waveform conversion module.""" + + def __init__( + self, + n_fft: int, + n_shift: int, + fs: int = None, + n_mels: int = None, + win_length: int = None, + window: Optional[str] = "hann", + fmin: int = None, + fmax: int = None, + griffin_lim_iters: Optional[int] = 8, + ): + """Initialize module. + + Args: + fs: Sampling frequency. + n_fft: The number of FFT points. + n_shift: Shift size in points. + n_mels: The number of mel basis. + win_length: Window length in points. + window: Window function type. + f_min: Minimum frequency to analyze. + f_max: Maximum frequency to analyze. + griffin_lim_iters: The number of iterations. + + """ + assert check_argument_types() + self.fs = fs + self.logmel2linear = ( + partial( + logmel2linear, fs=fs, n_fft=n_fft, n_mels=n_mels, fmin=fmin, fmax=fmax + ) + if n_mels is not None + else None + ) + self.griffin_lim = partial( + griffin_lim, + n_fft=n_fft, + n_shift=n_shift, + win_length=win_length, + window=window, + n_iter=griffin_lim_iters, + ) + self.params = dict( + n_fft=n_fft, + n_shift=n_shift, + win_length=win_length, + window=window, + n_iter=griffin_lim_iters, + ) + if n_mels is not None: + self.params.update(fs=fs, n_mels=n_mels, fmin=fmin, fmax=fmax) + + def __repr__(self): + retval = f"{self.__class__.__name__}(" + for k, v in self.params.items(): + retval += f"{k}={v}, " + retval += ")" + return retval + + def __call__(self, spc: torch.Tensor) -> torch.Tensor: + """Convert spectrogram to waveform. + + Args: + spc: Log Mel filterbank (T_feats, n_mels) + or linear spectrogram (T_feats, n_fft // 2 + 1). + + Returns: + Tensor: Reconstructed waveform (T_wav,). + + """ + device = spc.device + dtype = spc.dtype + spc = spc.cpu().numpy() + if self.logmel2linear is not None: + spc = self.logmel2linear(spc) + wav = self.griffin_lim(spc) + return torch.tensor(wav).to(device=device, dtype=dtype) diff --git a/funasr/utils/nested_dict_action.py b/funasr/utils/nested_dict_action.py new file mode 100644 index 000000000..38ec57b31 --- /dev/null +++ b/funasr/utils/nested_dict_action.py @@ -0,0 +1,106 @@ +import argparse +import copy + +import yaml + + +class NestedDictAction(argparse.Action): + """Action class to append items to dict object. + + Examples: + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--conf', action=NestedDictAction, + ... default={'a': 4}) + >>> parser.parse_args(['--conf', 'a=3', '--conf', 'c=4']) + Namespace(conf={'a': 3, 'c': 4}) + >>> parser.parse_args(['--conf', 'c.d=4']) + Namespace(conf={'a': 4, 'c': {'d': 4}}) + >>> parser.parse_args(['--conf', 'c.d=4', '--conf', 'c=2']) + Namespace(conf={'a': 4, 'c': 2}) + >>> parser.parse_args(['--conf', '{d: 5, e: 9}']) + Namespace(conf={'d': 5, 'e': 9}) + + """ + + _syntax = """Syntax: + {op} = + {op} .= + {op} + {op} +e.g. + {op} a=4 + {op} a.b={{c: true}} + {op} {{"c": True}} + {op} {{a: 34.5}} +""" + + def __init__( + self, + option_strings, + dest, + nargs=None, + default=None, + choices=None, + required=False, + help=None, + metavar=None, + ): + super().__init__( + option_strings=option_strings, + dest=dest, + nargs=nargs, + default=copy.deepcopy(default), + type=None, + choices=choices, + required=required, + help=help, + metavar=metavar, + ) + + def __call__(self, parser, namespace, values, option_strings=None): + # --{option} a.b=3 -> {'a': {'b': 3}} + if "=" in values: + indict = copy.deepcopy(getattr(namespace, self.dest, {})) + key, value = values.split("=", maxsplit=1) + if not value.strip() == "": + value = yaml.load(value, Loader=yaml.Loader) + if not isinstance(indict, dict): + indict = {} + + keys = key.split(".") + d = indict + for idx, k in enumerate(keys): + if idx == len(keys) - 1: + d[k] = value + else: + if not isinstance(d.setdefault(k, {}), dict): + # Remove the existing value and recreates as empty dict + d[k] = {} + d = d[k] + + # Update the value + setattr(namespace, self.dest, indict) + else: + try: + # At the first, try eval(), i.e. Python syntax dict. + # e.g. --{option} "{'a': 3}" -> {'a': 3} + # This is workaround for internal behaviour of configargparse. + value = eval(values, {}, {}) + if not isinstance(value, dict): + syntax = self._syntax.format(op=option_strings) + mes = f"must be interpreted as dict: but got {values}\n{syntax}" + raise argparse.ArgumentTypeError(self, mes) + except Exception: + # and the second, try yaml.load + value = yaml.load(values, Loader=yaml.Loader) + if not isinstance(value, dict): + syntax = self._syntax.format(op=option_strings) + mes = f"must be interpreted as dict: but got {values}\n{syntax}" + raise argparse.ArgumentError(self, mes) + + d = getattr(namespace, self.dest, None) + if isinstance(d, dict): + d.update(value) + else: + # Remove existing params, and overwrite + setattr(namespace, self.dest, value) diff --git a/funasr/utils/sized_dict.py b/funasr/utils/sized_dict.py new file mode 100644 index 000000000..105d8c398 --- /dev/null +++ b/funasr/utils/sized_dict.py @@ -0,0 +1,75 @@ +import collections +import sys + +from torch import multiprocessing + + +def get_size(obj, seen=None): + """Recursively finds size of objects + + Taken from https://github.com/bosswissam/pysize + + """ + + size = sys.getsizeof(obj) + if seen is None: + seen = set() + + obj_id = id(obj) + if obj_id in seen: + return 0 + + # Important mark as seen *before* entering recursion to gracefully handle + # self-referential objects + seen.add(obj_id) + + if isinstance(obj, dict): + size += sum([get_size(v, seen) for v in obj.values()]) + size += sum([get_size(k, seen) for k in obj.keys()]) + elif hasattr(obj, "__dict__"): + size += get_size(obj.__dict__, seen) + elif isinstance(obj, (list, set, tuple)): + size += sum([get_size(i, seen) for i in obj]) + + return size + + +class SizedDict(collections.abc.MutableMapping): + def __init__(self, shared: bool = False, data: dict = None): + if data is None: + data = {} + + if shared: + # NOTE(kamo): Don't set manager as a field because Manager, which includes + # weakref object, causes following error with method="spawn", + # "TypeError: can't pickle weakref objects" + self.cache = multiprocessing.Manager().dict(**data) + else: + self.manager = None + self.cache = dict(**data) + self.size = 0 + + def __setitem__(self, key, value): + if key in self.cache: + self.size -= get_size(self.cache[key]) + else: + self.size += sys.getsizeof(key) + self.size += get_size(value) + self.cache[key] = value + + def __getitem__(self, key): + return self.cache[key] + + def __delitem__(self, key): + self.size -= get_size(self.cache[key]) + self.size -= sys.getsizeof(key) + del self.cache[key] + + def __iter__(self): + return iter(self.cache) + + def __contains__(self, key): + return key in self.cache + + def __len__(self): + return len(self.cache) diff --git a/funasr/utils/types.py b/funasr/utils/types.py new file mode 100644 index 000000000..6b36f9c4b --- /dev/null +++ b/funasr/utils/types.py @@ -0,0 +1,149 @@ +from distutils.util import strtobool +from typing import Optional +from typing import Tuple +from typing import Union + +import humanfriendly + + +def str2bool(value: str) -> bool: + return bool(strtobool(value)) + + +def remove_parenthesis(value: str): + value = value.strip() + if value.startswith("(") and value.endswith(")"): + value = value[1:-1] + elif value.startswith("[") and value.endswith("]"): + value = value[1:-1] + return value + + +def remove_quotes(value: str): + value = value.strip() + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + elif value.startswith("'") and value.endswith("'"): + value = value[1:-1] + return value + + +def int_or_none(value: str) -> Optional[int]: + """int_or_none. + + Examples: + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--foo', type=int_or_none) + >>> parser.parse_args(['--foo', '456']) + Namespace(foo=456) + >>> parser.parse_args(['--foo', 'none']) + Namespace(foo=None) + >>> parser.parse_args(['--foo', 'null']) + Namespace(foo=None) + >>> parser.parse_args(['--foo', 'nil']) + Namespace(foo=None) + + """ + if value.strip().lower() in ("none", "null", "nil"): + return None + return int(value) + + +def float_or_none(value: str) -> Optional[float]: + """float_or_none. + + Examples: + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--foo', type=float_or_none) + >>> parser.parse_args(['--foo', '4.5']) + Namespace(foo=4.5) + >>> parser.parse_args(['--foo', 'none']) + Namespace(foo=None) + >>> parser.parse_args(['--foo', 'null']) + Namespace(foo=None) + >>> parser.parse_args(['--foo', 'nil']) + Namespace(foo=None) + + """ + if value.strip().lower() in ("none", "null", "nil"): + return None + return float(value) + + +def humanfriendly_parse_size_or_none(value) -> Optional[float]: + if value.strip().lower() in ("none", "null", "nil"): + return None + return humanfriendly.parse_size(value) + + +def str_or_int(value: str) -> Union[str, int]: + try: + return int(value) + except ValueError: + return value + + +def str_or_none(value: str) -> Optional[str]: + """str_or_none. + + Examples: + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--foo', type=str_or_none) + >>> parser.parse_args(['--foo', 'aaa']) + Namespace(foo='aaa') + >>> parser.parse_args(['--foo', 'none']) + Namespace(foo=None) + >>> parser.parse_args(['--foo', 'null']) + Namespace(foo=None) + >>> parser.parse_args(['--foo', 'nil']) + Namespace(foo=None) + + """ + if value.strip().lower() in ("none", "null", "nil"): + return None + return value + + +def str2pair_str(value: str) -> Tuple[str, str]: + """str2pair_str. + + Examples: + >>> import argparse + >>> str2pair_str('abc,def ') + ('abc', 'def') + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--foo', type=str2pair_str) + >>> parser.parse_args(['--foo', 'abc,def']) + Namespace(foo=('abc', 'def')) + + """ + value = remove_parenthesis(value) + a, b = value.split(",") + + # Workaround for configargparse issues: + # If the list values are given from yaml file, + # the value givent to type() is shaped as python-list, + # e.g. ['a', 'b', 'c'], + # so we need to remove double quotes from it. + return remove_quotes(a), remove_quotes(b) + + +def str2triple_str(value: str) -> Tuple[str, str, str]: + """str2triple_str. + + Examples: + >>> str2triple_str('abc,def ,ghi') + ('abc', 'def', 'ghi') + """ + value = remove_parenthesis(value) + a, b, c = value.split(",") + + # Workaround for configargparse issues: + # If the list values are given from yaml file, + # the value givent to type() is shaped as python-list, + # e.g. ['a', 'b', 'c'], + # so we need to remove quotes from it. + return remove_quotes(a), remove_quotes(b), remove_quotes(c) diff --git a/funasr/utils/yaml_no_alias_safe_dump.py b/funasr/utils/yaml_no_alias_safe_dump.py new file mode 100644 index 000000000..70a7b0e40 --- /dev/null +++ b/funasr/utils/yaml_no_alias_safe_dump.py @@ -0,0 +1,14 @@ +import yaml + + +class NoAliasSafeDumper(yaml.SafeDumper): + # Disable anchor/alias in yaml because looks ugly + def ignore_aliases(self, data): + return True + + +def yaml_no_alias_safe_dump(data, stream=None, **kwargs): + """Safe-dump in yaml with no anchor/alias""" + return yaml.dump( + data, stream, allow_unicode=True, Dumper=NoAliasSafeDumper, **kwargs + ) diff --git a/funasr/version.txt b/funasr/version.txt new file mode 100644 index 000000000..6e8bf73aa --- /dev/null +++ b/funasr/version.txt @@ -0,0 +1 @@ +0.1.0 diff --git a/image/dingding.jpg b/image/dingding.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fb3ee99287f940da2f93125e7d0f6b5235eed041 GIT binary patch literal 192919 zcmeFYby!=?w=bT)6$+(-I}|5KC=}PCgy@g4g2fvhEoD`8rGLx#>jmI8{Phd~aB}u^*Lf}f+{hU6{GYV{2=A;sT>k_A z7x69Mld*re0|3K(|3&`)R}!H$)WhnQ==bgI>3%ExR$1#?Om6${_`^Tg^51dvKiFMQ zN8y&{%Pr=z{ZH8PKVffASI=9T&VT5wT;2b{Gq+gJ$r1LCtbf8k8WTgE^>uG!quZMv z-~rG9yaveM;{T`NU+w9V3jl~60|0m9|CMH$3IO0k008Q#f2FZ~0RWx`0|2-I*LUvk z{!N*Cx4(bc*Z=?rg#f@~V*r5cCjdZX@*j%+tL(q};(t?DvD-5AxAkR-1&>(At4b7nGq>D1+Dly?_~VgZ%Qg^W&u$-w-3ll+_?6~gp}|4`9(D&6ErNW zg2phhq#QO5VR;<`Lrds;FCSE4_3-HO8avlZ5sB9j>^`TEvWo4e+^WHaTcM1%PvBpF z{iD}^Z_hh_-6gz7@bLZvqTBp8B!Ite6Yk%?Rf*uvW5Qdi+rK3D?lV6i6_C>=AbaPQ z@Q~$W9jmcW)(< z+$90X0-C@5ccK43_L9QFc z5PHwayk6hy2?>(wWiGJoge~n#>SLV==}@)&*I`v#?kH=XSZ%dZlnmc}+ADjjr~xBph=t7j;t4n=yTc;S;N$ zA)oj=;$CI%T*}1u{TS6&?iAGooD9&?uDG21y`CKBoQ6(~+(FubviY>JIn&~U!^c*C zZqS3kFFjZP-~=m+I4C={LbQ{Q~5#9Ki@rf7aEk8GNcQn|v6z``SZ`kz8tp#HC{4vWpp#`&62$M8uU5Y=5IJ3$Fw@AFu+$s(Y6V%?YJC%+w56Ly}9;osIotsm?2Om1t1+4it@vPW1VeD2d`kt@$qii_f<64UZ&CYcSenS(%1on41m`P_bcy}G!^ zH-H##q9;vtdpCesgBt*fq@ME!-xbri@TTTbP^xc5xx+}}D~hwlyX{eG$PKSfx}uEN zU-K29H2a^AvY*x}o3WC5pVByL4&`?(^6|Q__eMnco=+(wUGs58_QI;1%@ZiBJQa%$ zNJ};5?NUy&z;*Z7b_<3sBx5ZlmL{1c3X{)lirt}qJWVfNW7H$n*g^NB+c~zm)HzbT zzE+?>LTu69Hb0zPw4ekNfpVeMfxCp%3hf>>YX+(0m$dVNL~+Fh$cwFE^9Sf}EPH>1 zXSgITwwnKloBT;V-e$=P1qYi zRw4s{+gRvWbE^cvRov~cB+liSUA1oW1!}CwH%zTU(v9QcwxZF^fNmGp{ncA18BV<- z&{mX5I>e`Y{Zngm4%6>L(JRpMl-<-zWybc!@6RiY!uHmzoUDsvNbd@Js7rYn-$YmO z@U+&-a2WR3^jhPI8-XWt0HpScmO7Y;dJO>^Pn|B?QA5 zqCjta0x3Vm4))FieKl2M29JX|YwzBTIpq@;u%XnZO8b&FvwyJkF$?N9Z2B(Wgbd*Z z0*%^BeM<%lkRP9LM*&x$j~*5%czV}Kt!Fk_kVlm|DGA+1uM%1Sr1eYalICQd~JSOuRd2jbL zef#O3rCjYApf!%VaPt62CeO5LRCd5<%ZBa?> zscR3@=vKn%qD`^)^JQ}1`N3Znm?*@j!_-{--C06r%--kX4YqhI_GtUxWPWh9y;7|( zdcgFsH}!suy?A-Apej`jyL=0QmxDw?ujAOkPp8$_82a##eDz*$y^voiJrYKv-FbU< zC{9jQB9Io04|Zl!Y|Djen2Q|?a5)ZQ83eFeg^5x)sV&;YIU|1A1$hYGCB0W72zru2 z8-wTkrX~3mbT}$Z=@Z;(xe9Cw#`orhR#@u8R!{~WR(%vN^Wkofg|HI85C?bf55WOe*tZK!KPpUhtD z{5Y};RRU@n+7bEDiz%7O?&r@?#wLl<_bg{lnh|Ry z(UXz$oBD7R&!P*{sD{s!X*&57eaUk|@u3pwqTtwMM$wgprIE6NJg&AvA6I;4)D{4c z<9hdsoEscnM~hY^1fxeVpu3^!nc5NAyK1@Gym_H?0wvVE zz|jH%J}@@NLwlGj*tq0z4jY`FEry=)$gv#9hQDLRODp#T$mWgqH4+ zXJ1nXW+qQSlc>#3QjC+P1Cl3UAj~BQ1;X9F@3FWoXIwQJuY*O|K@5;+?&pXe-*5!R=32Y>xnp9BoJDGO$DZ5w(@rCdF!Urpdeqz+fwKkI&%AT}$V+4TSxY52 z-)BVJ@g6$nE8jrO*!w2_3T}(B*8yqU5-M?w#bxgeFI^&aqMSI=O%2CE=C+!Es~}Ha zrs?P#fKToXV12RBzepN2ht>+;iiM0cenDC`$cnfpS~`CDez)#fU6zY|^gv zZ{;@8lf+KZ0VS2whpvnz4UT2|2Nnql$U(!og>(7YQrRL&*>cUn&>y8#$-CXJ;_ z3U`-rZw_buu)6_>ulhI4eiS*JhZt0^Zf}99z&C)grq4@y*4ADI)!?c83*F_Kr6`So zrB%$RFiIxP<0=T3$Y@$=iQFXCE8FUqnmU{fI9^*V9CWHa=fzC<8yGtQostf9rgT1^ zhw9%Sut**sjg>@m^6;?3dQ3hX94#!h9zXTZzHYk#tT6q_`A?A1{4a0T;lrTXfOFVY z^0o(P&yC@XPI#%1->2C>$a_~uQLz;Vf8)!TlxnHXddc-|(T()dz&63k4J+BJgZi$qNhKdLwR7Iw!wOiAz zDAdIAz%^Bmgz8JoPUh9h>b88!D-P>_(H}F~8(lvCK?93O_!`1JGiSEn`n|EZmR1M! z#vN5olp3j?_(_d-m^|ytk8rS@H>2GMDA{0cEUZelvJpJkGTiCq3tKY~Vpe9z-|;W$ zug=T^3yn?Gg7cE=auLoD$OVI>tbMk9q+8+|_B@f9S^}PkBeNp6x`%Lg+D0Wh4mUsY z;7o6NbavkK@CKkIIxruVeFJ#s_9ATwoBfcU^G>h%uj?+q8Yk0*UGLe~-u1~-TAEG_ zPhr>nn3Fx#ncayE$t1+rjXK63f8tP0VHTKXT*D3Eb_Cd6%R=veKXbbQ{L=dS25=^J zem(@28s3SUcxs)AbbK(d=64qp1Gk+Bw{!B3yNIY2ZecL+KR_AQlXU- zFS|W+jGcE&(&32~h0^i$zB^V}hUO`Hgfp&qE!Bp=dM$OqwSsxF0X^99TDa$HcNIF( zdNPp1E6==4vIJ}UdVX&AG>-`WNfir4_-YhgZ zKt`=haLNoivTraolz07r75=|YLCVc!Pq=D$LSltFeAJ}m4@M@#*fsTD(U4e*g!{GB ziF)=vS~e{4ojQ&YRK-i7r(p zk5BG{&(a1f;(=J4e2QSX2Y9Ha6_(M%&Yhf27%qEKJeF22Nf!4-l_yWW;_BP-F}tIi zLHwuy4#xJQf;j8GWI)@KM|?eEm3y?7Gins^++hfbo+qT%);;W<@honxw&B&vtb&{b zY}>8kxWSDLGrBYcBPRS!k%x!!atM`}o9)bd(DgoGAl)K5wP0{2D3QNCq;0~TRz!Kj z10tsU{O_fe{JNZ26?Wc;I`8mO_;S*h05iLy(ZV;m#bfk2KuV@xH-K)wp+H7ok_cg2 z?uFSj__B)boTksM{a{o8bPKOz5bDxN)s+jU?5et_v3&@C&0w5vsBc9IrmQ4sEocPm zgzLC)4}}PQY24v26fJdGS$A`x89}6>kuW#0`G>kz}Zm(T9R!jlFl!g*;%Jxf(5kS zOer1IbtMX=*Gl;^89_-LDWH}Af&NlgXEQr=r^MyBgnTBMq5epqMN4p^?;c; z^Zpa`cFOR*f|Zzjg@UT!2Uz`k5hRou#ceEgr9ge8lDs3=VeRLLr;KRu!spDqy35SU zq_*AyQb$NjQhc4IQM-_D3>jm5*Q%S)d;OA>l@BjbaY|xxnOb^sd;~d_v`L3Vu40FJ zg=&%*7q6*u1z(_xnMtwe-^`QgvCd(1Ih7!0k5sl{a!cSg{N3Z=@3zE_kuMc-X2kKe z9i~=;6H*{8kn;N1Llf=|dkh;x$9rp|i2gUL-!fY4)%oiVI*)5o`MQjbIz($))?rSS z$K%hxP3GfCCWn$zJ+)ax$LgOHo1TtDs_$Wvhe8IG#>P?kB_7_@<{2)9{jK8PAqSAj z3eNAzz&o@v5+5q;pOO`QZkem^VTjkr1;UH778*W%Wb@-x(B*9!(yas9t1NDhT+_cP z*wQ6qkCKwEN-X;EMy$x$QzYgHo?wvXW#8v+U65Ait(68~VG1{->wCP=u;|Fc-=D>q znj0Qv=KQ2(rxR5*LO7DB-5Hm}h#gD=K43TICMyM}>9aWcx;jh5@^1G>*qLlS6k0+2 zda^SrJ*AaWOuylSh81Xsn4aqRYC6L!FTO%Pg&tWP9tpu`ZB=gouZK@$k8-=o;B`@W zb0)!ZSIz$0m5ixV%SN)u9ddt3vA$a?o$Sd}+}p0wUPBNTM|*JiS+L5U3kPOKNng|A zLj~W|Nw=>$5rVc|Ov{;>ec$wF9NBbg3(jB#Y{0)tH6(?G_j%LoHwqd654{&zJ2(}^ zoF!B5bPsJjdEJN0We%87dxCq8P){Q*<3+qFPN}J_t8C3t;$fEKQIWrXuEkYGIb<BPsteN z))|M*d-Y%O_2Q148ez`)p??RPu`ccBNj8&C(p(ZbqCfsht9TqeAedkOj`poUl4Two z8MjWt*x}F0YR&GfMrhjNPB*O%PVhY?urU}D+U@7r0h>aW`;7t%UfVvR5eP-wY$p>a zHY|H)N~tay9qX1@J}wRzg+?%@X=0AR#ZFrCx?33m-HZJE_@AP{;)qC0p~#fBx9JCv z4AifaE$g7_m+3PCa@Xs*2E$e>6V`OmFe~FHoX-ON0;A`;;CageGVB3SSzuAjAmiMx z-LB`h>~?r|@Kx(3_Rgj*+C_UxbEn@ayxvs5ATuk5N^^7T^C$rD?CCFobyS1OQ z{O{i1AFXuj)pF&jaCIVLoB}K-YvkNf!h$0_IvMs*JMzSLL}Y32IH`QRXPrFQmtwx% z0G@ATjkV5TO&Gg5>rSx$7vo-(Mz(9fIkO-SFawc-KwhQvU>-clxo>_5#*pwj`wtXhkbk zbg3g~A~R}z!H^tp=DvmSLRM=j^heVhcAxomPg+q}Ax&jj(Xe#u4~}v#S1wu5KPUDN z)>;!B=7v~a-peoEe3tK{Zyt;-s?bwf%PYYU=|GSAH1y&csb__MYz`F4STyTCeyynm z!=-s?aH4jOe;+jb{M9E~yA*67_WjC};VlhGHmRtn21of*A5trFr90s*Juk}q5x^Cd z{2wB@s(c@-Y8Z!E5v<;PADwWl%$5-nV=JWNH9U)w6*H;+wt1Hff}{Ep^E?w{>lB@? z)5@~b#NVZprX}K~symG5XmdGil3U}v+M{7j%D+;(a@um+amW%z(u@gkJ&#M~?{U$_ zy5Yu%vG>a{<)&kAPJAQRB2sL3b%6Bp(0p{^LgX$5zn$C#J5JO z&*kkXncsxm=4<+>*|#N1xe_qynf)Zqp`}yLq*UeBS=z6I9j%5O+D84>Z-cNV;lb^J zEC%E=t)JWN(IH4owiVoCopKXUdgQC8Cc@BBF>xt~Pj< zhUM4rq;HaX-0tTABkq@tXZx(B`uAB6@yu$j{4;H*}+v-LY6yX*(zy?T^1B4h}g{<#VTF2|3)tiiM zZM>u>Pjs1WS2Of>IN0jhDt^~1*LOgLarHm!yZex~1U0RzfGP5u6gSgavtqWT7T|NVaZ{=*R>$~OCP zDjs7?*AqSM^(BpR(B=$7@*p;|mR5q#(;VBTPq;Bws*~TM+5D1GjfjYMaVaj*Amh5b zg{C%u=LS$*oM1vHeq}+1bh-gt)HWp)cHu`b(M^dchj-rdV8dm!6Ap{w>FYCh_Vu@V ztT=)raXcF2+ zUIKMNT?mezlnl~NKbYM;Y&`CG=P?I)O)im+e8x|G%OS5>h>6a=PJMpw~)J z_uBsN>R+juVrcn;-sz{Sj-0W>mOw>c-ixjTa!pWtv5fK%cJgj9p_KqtK3f$dghG|1 zNW2F(QZB!4wvHf;|2)(){x9>Qf}Fy{9>ltW~shHRT4d z80vccfedu&vx}`qRX9gp)yE4STsRxiXUD_YTItW}!k;9n*ZA&`FX_9JU))Ks zaiy&BKrGFPOP{R@6=<-;2p&BKmnC2-YdB!)Lq~HcRdRJ--vXoPgHbb|)n{3dYAb|n zjY(e_^|alRc-Y+~u23W3c{?+87DEDY(#*FW7Faat!FFcsvqm(DOB9Wv_)Ah`Uwb6$ zCar0V)aUV(kMuyNv4LgjpOf_1YH@uwpSpk)Yn;mbG>cuvV0&5#vbZ*)5*72Zr)sO1 zFUGNtuWy6beD_yju7;Dkc7Q{l>g0H=Ui;3L%)^$c<2d2dH_^fGOkJL~yCL~WO#A(Y zKkJFdkg<3hVjotyliQomUTUt1B7x6x)pPnFdEqK4rP~}mY7MioJNW83=UL%ns8wyl zX+?b(&+*o>smsS?Vt@3H4imuwvYyntjLU|TyKZjX8vSQWvz+eQ)##6MEC!-{`Kibl zTTe%0a)W}+_>ARIb(E~YEZ<|y0NV$#4+-&&0p$62Ln-}_EbCyS|Sy}{_doC zu+F7ZU4|VeO`%Y_7XspN;r`3r%1*9G1if3>7hn;Wlk}8;k>*V-YbxA00XHCpS=4Y1 zktmaWu9|g_C@6pwoKy7V#rv@&KwrXmMuP@sNv+AcFsp^M@d-L!xXX%wz`Q)B>D{pV zw!J>hoC{8p$fbr!w0q(<*}8#Z;`Z&n_Y1|vsBwLI_*3`+xT7E`w2RHHjVzU`?A2VI z@;QYxat9qVIdpo@`U4eXFEsA!03}0lTS@@~Q^(mRvGTpR)_0XQBRjmjj|qj-AA73& zWY&O&B&$U?v=M1GSLsw5cT<*$9os(@>xcjSJ8wZ3^y$ZnONQU?UV%+5(pVLqq@DT|b1D+{-S;N>;YPBN=1mD||8C+&Ms`_+Kx1+}+Qqe8Iaz;K5mU@W~ou zpuY?B+fsh!QeC>1VMFd5Zs@}Q4B=6EMSo0Z;(AV#9RhmW@z}4m8>z#gU$wK-(V8Fn zQfgZ#rkOLzY~Gi49D%2B7fDwetd)O^?5MmKZJHty~k;?i9A zhkLQ*!!t3-zxUb)l)A5u4u?)<6vFZMx)%cmA{R5Cb-j^V_*lgwpDHD&&C)lKBQ6%Y z3PWoQi#0(yx#y^@9v2&rG%$aN5HAHv_U4M!*E+=v*$FxQKK62NX-JEG;$!e)%)@UR zqbA}K_gddtSzQZ@u4NA8VfB=o6cf$&RiILAG~p{R&eRc*loZ4OKd6MR7>eXegrQiz zq*s2Smp#T4lyuSF0NPsUoEg>fb7g*vhrF9QIVbdRd$n#^Iy*m*IL_BkD)i;;tN2mn znP0sSXg5E20KKbk3-z*(fN<1O}>`n)kGGd-E%##9x!uSx_{zHO0^7Cw3-{^=rZd&r^v~ zVFBqV50N*PY-2~~j36^du1`CzdP(ZtSyRdM2_hVg5cZ)K`xACGCwAF&X3hEM<-T#d5&fjq`jI4u#DvqK{X!Ak-*j&-98^NN_@Ph`ti6Z*qu!^ z)g&BA!2t%3jZWmysN$(?uuG*E`@;yq{x^}@a3^V@<>aA-immGpF2RQ6PBx3l!PrkU zj(8#)Bt{qdEFh_4(~QCj?q&_srqXb)=S7J>9BfiHPK~P+3Vrk(NEQ`&8XC@)^Y`Z! z6>U(4vylIX51(Am=QcvWE&wTkUY`{9$Jk)(V}8-k8dYZ*^WzN)U!RKJkI{_Q7&Bd5 zw-3SP%a0|{N@Tlx1}COqB0-%E!eZ#nY;t0lu@|);P7=wo(|Wn`Yr{QQaoOMBzPk`B z59bBpjJbo+5MxS_-U+@&G;|CF{+$_fg(ZG}eaip)da&MjbxPN^W(} zU?KCVM2uUzD`#nTv+H$sZO3(HG-G;4X^O}%1O zw7PS0xvlWkX3lNA1d^7JjyI^=SgTcd)izJ0+%90UxsV z=S|%#{tS%=y)^V4<3n|=ExyKNs0klRE;pv=5@@q`)F#}cmE5HSs-y=*+x{@N=-X*+ z9CWYM=}E3(>dh@Q29wG6aNoEoDYZA>v0lVmLXCKyd1Om{F+!Je+~JE za8{1D2GjWgf4BdfGr)g1F3e{e**}`lf-`ULtmAyX=y*` zk(a57aN`|wrZsXh*B%gbk4vaUnVJi<5`%~65(M)OO!!df%CRbbbZ6=?qrI;Rd%sd* z3!^3sspqrHZ33Ag-xNcS(#_6~^&F8E(ui_$RF8*L3x4+dIlCMqt+?baEjV>TNPgVr zCq?mb&kS>^J2E(1j54{*Gu1%*ON9)HQjT#P@2UN)imwEVB!Xx3$RR=<47S5JfEJVR zjyKX{fAkLvi$2eY8c|=0yeVD&2%7}wQ3HMx@r!fLWtORY(k8F3vW91MYyIHlvs(zM?!(f0bBWX zji3ekW_~-qSkl9(iv*m{%Vlh}r?p&R+TD+A_`xqLw8q7S%kyqC+gMrX8K~fW zQAmTgDb@`D;gZ3vb1`+nc zN=Cl=b`&H8FG7Hc@zAhmJ{;!SZ5HoB=3&V?*q?EU^>Jt6PzD|xL&1JY*YClOntCT) zAsUZ_^U6Q8!}dyLy_M_TdImG+v1M<-9a6y`KYq6^?vJ2JIDK}O>K;2;L_pXtlP=u? z?MtWWjqw^tOP{tbaBDf>bhsY0s`k$8tsHo)G9%%TS4g6^r)SO3zQnA#262OO#?*;% z!AYZ6xN6zy3eQ|F4lYK(B++;ixJ4h-;m@pv8DDR##hHT}^(H~xe0P)Q@iT|QBFFyy zS}CUzEO_X6>fXh~3`ZNtqIg=qet?+9~q6xVu2)x~P~9RJq~YY$KaQ zyhD4KZ2m$?EINHHoX*3%%G#!DMq5KwwEyxv|rnhAbQ!C08hIjn9HBABJzLER*mR`|0pt`$X|q<5`M$Z)CW zK!FWrx$L83?={fl=}4D$cKTCMVcZKFJpIoGAbrutqiS{G9Zg3IdJ;;ECS2*lr>{Z@ z>E%N9E!+emkGm-<>s1WudZuOw?ccxm@QL8FyEOxtN6=M0;%eNTjDqPhYF#e;spAk# zm>2Z~Z8o2QiRDQR(QDFC=>_kzbUz;wul#Ub#;H^?@PiF&cLfiK=Gf2A@?u{$Wl5w& z3i*Cr{T+feHNy%vl*SbF&>3-_#i-`by~FjUs~h`51ZI7>xuyji4eGe}EIrndLs6t+ z%t}7qiA#0cOQ>wR5AJ6c$xn$C>ND{0YUFHPQ*(1H+QX+>#`W&v9yRm>T;*utI8vI) z!TjXnWDoh3&O=vHQb*TNyiUBr5D5l{G&YhDlp3K`QSqtb%Ji@kD(JWlqw1}xL!K$F z%=qPym+ccFXh~CS$r%+wu5F;R)%&7n7uWL$Gc_A9ioy+H(Oe+yZn343p3HE+pP2%g z@V#)dlIoc7UN=r!+VMoHT!~?|wy%YW%6fQu->I_P5p|3BOkZGL!aV_emZ$!sRCpZC zFT&y(sY z8mfLF7vEzCuM&|Jp}gd26L?)uH=6)&WGLVF(YsF5VDP|-uMB$7hvcAlinUZi1{-OI zo@iQhd>7-~7w&#~PRSE1j1}5c4bP9rIhTe5(~l>QyYF)uOexfP(SVh=abL!~U%uh0 zo?Y>sS-_?+iz%_3(62 zR6=Zyi7Jmo)KB!hl-zy4s_H5AsLt7NSDu|8vrT>kKMsGegoWj zv3S+2Cud7n7qn(#wspI2maq)D4!RDH8owYqIg!pxtgGHGHg^DFGENOE(c(?u+tU<~ zv4+_gAY?3KW&|B*Lb!i>WOC2YT=q;Uq4{cn`q?9Ll^;!7v

=}QtwJGss!_IDiWIVh;5$H55Xa-wJ zrE^DT+Kpte5~+&buEL5sR4_jyetV*a?7?^`IM|D3SMvuqeac3Bw+ev!aXua(~nK;MbJ)rSX;6LcIYc`wX-oenQ6&feZ-t3ACE< zzb`R=j+_nQs7sJPqqRaMZM|uewyzdiwE4yHVtalnx4n4mz~8 zi1Sk+#3(|kTki#yQ>(T(#W4Bp+O*x*v&QfNu6F{HFkU6~PdI`{^_kWwkrUDFHCK36S6N&kgqq^a*#rJ& zxYVA)TQ@H=Xpmx>=Y2G`Yuq*pHJ(fZ?i3z6{Q>|O7ar)khyLYQSB=&wk{0GUIuSw_ znt;zX6gHe>PpYc}{yvlPxdD7BY2H(~HI4_LU49R>h{%}o{UM`vaQhuhvFrc%ghe=m z{JB}Lj08ZuZGyZ%_d|4+=p!qbpB?lNDv7-Louh8oEvnq2T@F|7QB;N2nwR1nr8Q3H z^g}#FrXh3e`S%KjHQtZ^MSJ@uFYIhzi*U{odC#$H%kbJZv%Jfx=A$Wvpj|zJ=);e- zZgsa{PG&lvZ96l@a0NXAHFXD2NNvf^r}&AIO{CB1Grc3%`sJbTAHwW7Sf;(v*d)s} z-ZVTboAyJ8R>y-x4f0RclcBO42No0q#!rn6Su9s-Q=N7qE{#T$QMf`Qjlxv=j64hC zlg-;hMm1_z*mpTT=IS?qk8yfIdojT`06}qUcUvI0E8NS)Yfsb7Wz{W*x(*R#sOH;a zuA|DRh4GabZOKD?$3ZfB66gicm-?zY|95W=R!f+<;$yBuXTn=PiHA?+Wr8Z$V4!K<+G|h`c)@|yPXw+yl znz-Kp_LWN53ROXb`3(r==DXq`BiUe+I{}e7j2R~n-%q{#xDr^$YTTMkgy$3t<+2@` z@daO)m^jF58K+fU429PvqMfVzM}#FJhc-=a0LouViwD$d+qq?9d<2i1?;a}lx=S|y zrku>i!uJ%;YUG=Frhhm@1h$E_3@f)p6fJuP{o<>Vbzq+h+jk8KG{gswlU_%*yHOaE+SR4wR`A^cD@_lf3Y^K_d?yA`KVOHb;IDF=bTF&)&u97Q9-`ZEpaBYfCwj zsFCL<4g?B_NJ;JZ7Tn+B5?#t4TCq(_0RG*OZ-2~|oAj_mH6K_)b|^yZ;J-QKlo znO0$zfbpx<&Q&@4^eoq14SQF0r^FYihZ)RTb`k zcAc(C4rCd2A;>@-sQjjW6x&g#SZim+W=`fO3mVa6uB%91>@P`=pH0er#`||Pl8ETm zr5`QGv*j=E)xVe#wNvS%XAKjdtqw6@%oGYiA;*U7QMJM+4zuyJsDZDlc^tBdxt}#N z7cmcNtOvVgRjvG$bsCEy284nWjs1?ePK1C2EwXo2ZFbtxlaH^n0+QBJE;6J2VrSeq z*$sMP21}ky_;qR=fJNg?U>KXaX&7ENt-5Y%2#K;Y_ch2kqLbMU{BFgk_uc2Ur3Y(Y zuh=-L;tV?`q!nq8?BG-#jQ^OK!_ZEqw0A%S<)UWYnHA7?^YeHW8k($6-ZWW`{cLJB zZZcWwvrjHdj93U7wt%nNrh0Q|McAm#>x+#`tD1~26;Eg6vG<=ky^O5wb$>PZ`jh4d z9S6UN9gHt@k``}$T}YG&8MZ_!*NNu^c*cCKkzKrRsoQ6sFqRHFpo{g_pt zjGS2v+u6S3=P%m{6W34UtlsB;lmteWRi~HRr|A3M>qrYY{vZSoZ)&bbcY_y|@U;~F zLFP7?$EI!06;y)XDx1hbJ(#FLwNXyxp4}*R4tdYPZu6SWDPxr{VM=LE(oRKWSTQcP z!uw2N#2r$FVvG00EnnUMDC!b^Qf_O+4o9zFEjL6?>fb+oP1DV$sX0KI>NLxaF0W)q zifk3%08;szHg$0JD1=9}lx(2kZwR?&cH?_+pv2*kL*Le-LLPCL*n0VhH$-B)8c6ZqR#v|2L&PHL=ThD-#ho$c;sH4hRXR(L{k|z zmfq|65bV?!yqTm))nGeal-MqeY9a5ymy%3XFp2BM4}B9Wc^~)vuX)^Z(0rA3fJ{X ztOXhx;9{U>%EBVuV&L_jdl97Gp@y@n?Y;buqk3{S=#7&V2_0{>FKu2CW=T5aSGF?@ z>T#petGm)CfsB3j@;z8>wj2o}PX@)sF_qo`65vzk^a#mS1?x!*W^?O8K5M}l+jC`` zFKv44L(u1#^Ls1M;*VQD5u%n(F#ho+PNeIPh>g8F!S= zNLix;e*etQqaSJgA!oA9!Oj*hN$~(0EGoqn$tSWnhGKr8E$#HC7_Amyq8YkRp7Hve zXWB5V(7kpw?77z0Gho+ulPX3r3B{4VwEZpL&(0E+EB92BgoUh?$I>hm`&@WqkCC+; zm>#OOas$&cdaBnSFNNJ z`0L4@l1J+SlzwzQtJ+4N3g0(ehOxc@WYAg;7>d^Cerlzo5}n_(MJ$;O*+4EOEok&T zeNGD4q8b?V3E|ywyiDH*{%)H386@ZTK+5Fv4FGsNZY{XXWGMQB^HI3Q!OMDoHJY{X zJ46$f6~mlRwl8t52lUeJ9fH*K`Q-$ZGvmB*wVaj$3px+lDnmUxQhr-hAAG-}KYO%; zCOD${YQ@w%*Al?9;o$ndYu8(6#(BUn5V%n|Udp9Z_Vv~c@l|(KTK*fSq*#+Z-ABG|V9?ooK^8%CRo#HM6I|4!65bLFjXP@SNXf@GAXK;t(i`aoK5cbI}q3 zK9#>c1SFod^u#yLs%T>(6?^r=A)xqCbgdZTg|Dr0fRlzt$k6Kja<*js5l4rHP0<9B zD@2NwP+?^8o^v>UA$%-_3I$OPpVs9kXwlj-Q-Q-usQ(nd{ix)gz$GDdgwyB*hV z0NH7EP{L9jZDj6Fu8Y#`z*KBD$py*_M!6e~^_UYHA1st!)Hh3R?UJ>nnU&SdKm74t z?E&vMu|7H)XYR5b@04j8M*&Ybt6Lg*EQH&NS*@VE0kj_m z4_MF>+)nmt%@+o23cT&61t}FA3aKS&u?M|DFQc*g@9@vyv08&29M!^l0U_ybz@lPp zeW_`%$YS?)MRola*u-?rMC?}RAxT2BAb)%x~8pPWr%C%dhycI@( zDofrvT<)n3`X6OV>k{s3EUZ;dO1unxVKcSJo1VC8HhIcV5Y

aB;fpE3?Jpw?n@A zg`-6>ox#f6TX&Mq*Z|Iqpq5b^x7Cr>gxoS@^(3peWQ!|dh!<+EChaGrCKSD zb1gvFd)tX1$I5%Qu4(ioV)7TPr|PQ)?YukMt~rcc!k^wZ<6ksPTonIoy^-XcEbn$< zYTuf2%ng!8UX{gzbTt&Ctcj+GV zZ(0#yUt^M6J$5rDfpv5l^=UfRM$C+DaN^bf3Uo)RW&8Z6r`gC<%Sd&l__E8fN_Yw6GWnEFh1CMH$4 zO{p?hBSn~>Z&7e_Md|VBy=$ti%m}E-axy3Dq&-hPv^Kk>vOdZ0MR3^kN=D}On5GTY zi#ALpCKtR6dQ?o*z;nW+W5TOBimLV_P>Rb z%Q+W4exD@NAgWVnj$~dbeg4Ui)ZT%QPoua`NA5zy9p;UPiiK<_>-MH=y|B;F%9DI8 z)c*KMjKITI0pPQ-XpInd6}o7L2y8W;ORTUs|D)2@fQIj|ngPV)X-#l(6o_4bfV=(y zv60eqUOs8qQe&IC{pO=> z06}#Fu=cz>w;yvkCFX7JIQ(|nV6K^)h5T|?ERgh~t2o^a-zVnb>7ji_MpW^)CR1z{ zs>^7Nya=b$nn0UGX}RK~8mmBwGW8S2o6mYU?LK;>5ulx+Pbjik+y;K4<38B*__wOe zUwLkcdfxgMC}$a^(4fV&uBaOTMk3$}>mYg=`t+>!!14yL9h)7*2kHB>5Y~JH_{NX8 zowe<6XW7G!{2M@i&J6(2q*-xYv~B?WVPV;8zdjIL znVt&-oxIj+PD0z<09;%CbT7_#MGO7t#s_JtRpq69iC`Z*vyP2*8qkYZoJ4^h502MK{Wr$m zDypq5{1;_cb_*nX=KZZf00kFyyPed@`J7 zEZ1J!)N2)zv!0Sm!@Um+Tbs&veD5b`|CB6!e#`~~YU?DC!Fa?s4c1xECKLv}XJxpZ z{6o@O6L96wb$_O|x6j)hEVfFD;r2Y85Bgi(BeExDY%!-E6mPz;j(G7iRUiP0dJ~XX zTKI?;J6-u+nX~f2xNOhQ65|$m8m{nYK3$THpwbsQz3D17CE!f;xj9c1*#3E?msps+ zXQ7T)*QSvXc?DR!m~1oK+lX({NbWNqpm?M7b*)Ug5ax#C=6hI2SL&C24ne)fP>GDV znH;enP>_yskQRrV$kT-aF}sjs@bPOR)s%{#RvnE!cpJ5BLKhUksRMS=$?FMtY$ml%IEv?d+Z=Dp2r8QG`}xK5>v*x;eQ8<}4}nQ6K0 zPuAs4JfMX?fPGhbf&?ji9;#WUC8ksRKZrm8x3%dPaKY=SYUvj(= zQ0hGBItd+Wig~*{``-$bxnteD&zbYX3A8kcv;q!7tJe4R3DhIOum= zk!!~;qb|U-BAHf$t>-OteU=V4?+t0~`xv-fXM!dnW#Xd>9<3P5e5^Iw&(#5*g5ze?;(v%} zilpb)lAT3##Iu6RGCHwN9ep2EQ|14z;@O$IF@T+N%8kMix{etxYqDv6@2uuw_u&>O0bM zO5Car67hw!g$3fj9+eF$14;%trH{82Xb2u&fikKnK$mvrke0@(F^yP6htw^ z;O6pOoJO(2QzM~D=%=!EjSn`F;t#ws^;!9uXFt_xbmc0bG7JQBq@|XPQPqD)3cs() za_HfA@nxkCNcp_BnW`-oFt`3_=p%gC&_JD{)sM=Vy8EzW@5iI0KW5eyRLuApl&+~C zeU$u`BxR<0uH_)b?jDrF^$yZgk=@Z^QBb0HEz*2rx){YvnnYzG^WBU-W`U+F=&3LE zBWg@+HMaHdn`-V<2mV4iffC6+XKjsNP)3Gp^_gP*rkW^q+6Q%&*>C0|f2?a9`>U@! zhPPZI_ltAeGEO(3^zmn6JplLX#pui8`HW`kuWzil{X+Iw`Tvm2;A_{DjD8nziAIX@ z^~H#6cs<08*bnXH{QkSwUep$9k8;YrsT{vs&J3T={b2x#i&Hvb@h(>J~>S>x5lg zY4d@fL7gdMT&?}^=4(ss)UsH*1J2cy&rZ(zkemJ5Te4Yqwf!&F=@1NerC7V&DPAA&qMW(M>4d}8` z)^x?UK7R)I(6qG)`x9eGvWqV;_i37FX%qdg5^G-+`f?>sRC{ns z7&r_s+Hik5lx%43JBPnO()N5<2(zZ7N_z2!#8#pxJ)Rw$$R2o;IJ@i*!ev^<4vcPD zvNtAM%^2hHYn(kYu_0Y%9tWc2W;CY)_DJY}R!W&^{>a`HCrJG0hspAE=xu^j*5ehO z&s`uYuFGjbH89*OTDbyRnwjtHhqBWuclBsLpFV%Yktx=1PKl(IP{fErjiBXJ*-(Mf z!Kfv=mAs{lWwvU4JBzw1p~ST5T77lH=@(0~Jx{opd>k}XV--p z*hd15NRt||8)5|;V2@J=b99Od>7|;~=0f`U)i9Em)vlOhO>eENSR-CwNLpPqug6$L zhy0>dOsmO@eGyM9>dVh8LzJ=7kWdt?lrQ`Hmy|8(rVJ6i-q+Zw0je{1$jboG%);Pg zu$0u)q>Gg$wuzI1LKS&0K%N}g>xmtCy!PbBF>unetJb08fQI58>E8j)>l(L6g#olD{6IH!xm(3zF0{zCRVe~|=2Q-Sa$ z-f0%899K*q-(ul36R7v&Tkg(P&YmZeybA**)eom!zJ6ZL4Zd=6eDuw4*{1YHjeSPN zB)jxQy(hiy^A*jf>{-&nTC6FheA|ukuh(Xd~nPyrMJG=7=^*#xiSb$i07QCMvPDP`tFPwHb%8f`(8+=p+FfBE`$`n z!V0o=sQoW>%C7Aba)%#$y(#rRAqxCs%;+_6(caxY=tlvwp1h3OZi-%xXj}30-;;x6 zKxfQ3+{4^lK8AasAN8Sx9~81D&kdZ;oFT83)tE6iu90tdopI`=xbcIqigv=H?_w?-OL{RHz zTY1zc`Zy<_VWM1CBLtwz#yuD`?|fdkX)l{hVK3`1J{;61-8Ez9Jwua^Wq)N8Z--|| zg}CogCd7@)^9pRJJNZI>M)b`9aI*lIV6!1W7f_UW^(uF(TQrE?Es>y^O$(`%jN<(1 zAX8DC#Pr-FgDJ%!elF8Ar(WYkP%z=6AeW2KvQN1qzNOK;M0>Qjk1B5Su~gXI{>6pC z(pVg(^WgKQ@!^4ROORxRIWoA)+~?>>P(qj2HIT57*;~hBIKC^(=_=Q~Kfarbo;hke zDqbNIIv(^d0dmPD;uyRh(LjWDJuqKuhE@xpp*qN>6PbPEL|ZAx&%5l1 zABNX2MYUVIsOQSDpvpA6MowFl2~XXbR7&IzOA%ctumL}-RpI96{i#aiP7K@us)FON zJ`aeI8Lg+^Gi@q8Y;Y%cgr~$5jnZ*HROwG)^1*pdRAAs%u)7NZfBhF5XHM(Q>85As zT)WH3P3Y-vP{r8MNkWyu-PnTF?boHH!myzEFVKe_`LgoAdY?N z5^k(7v(}+OtJk+FF2(+rGXGr>sNZd-QT=Mn7+qaM+mq_3e^mAx)H3!4_iF?R02}JG z>#kcL*aGBXn*%pZ-LLXak>O129K9$H# zhvE-P@29^gE6&($Zuytwna2M~yb^QkR)Q#rWL(-Gl6m{^6a4t@Cn4$Zm=7H;R^~}| z7L^wJSDgEbjyc~hM48q1KUI#VnYrFGXNIdxK-VhwE(ysRYLDB8G|PSD%qQ`)kgfRv zuDrYkfS5P*s7_z!Q0H*fI6QJY>#49vnAy}X_Nj{`DOewfXebhaY`P(Qk%(`@eZ=lr zE6fc(Pdjn%VAmP!I_Af3Zn9dc=xpVY;k~j|q(h_S6A$#)RZCoz ze-{<#Xfx)eP5=$>Qk2?&DuEaSuSIM z_qWp;nW&zS5FdU4W$!Su8skgRxY8T zIP6ZvF({*2@)3`jl&@(|>82)I;ac01^JcrQd;iPoSD+jvS`o$5qZpL zzR-o^p{YQVKabx@sHICy*C<*fQ8R!VAs`9X_o@IoTXGt=PbjACuH*cB8q9F7Q46Ez zT1>HBtL12|h8LAT93*U1_=)XFV`xZJ`IJ)yN-dOEjeFl zUHuv^?|?&V)sn8jU}M;A>tyxb9}+zFx*FX8N8k7oVe=gP$xS%hfVn|I2mSadDfSX@ z`0w_{%ReOD%C{kG*E+DJzsx6cEmjpXTmi(^NG8(9=a%{GBIwxXvi)mHF#YWz6If7s zcZ)zh-%!++0rA!>&Iw)*HM%@%Ta5jwHG2(7hkf58oux|GaZVUnuH5q@@(Q!U!J$>Z zhKe?~{*YMVaTr_&CMR#QbO>+*+xXqIb4vdCaD@m6ef^902fa>Ciec(H{ss};FytSS zNAG{1OYAmPeWo4xLn16_<%;k<$jxb2ia$TOQ8s+ZaAJM5k9;d$N%!W2s#9POg=S+_ zAAASY=@AU~SHWLM1Q@AB{G65;(%p6QeDiz8%MUIXHdR+3ciy)r?IgAF&w0pSIsVp| zTuK#6ltit>5kdFgA2u~It*YL`e)N$=CKU12`pa;}s!HcZzGmNY&B5(4TzfxDwtWaG znxK;XPmAtMBDy+e*1XYtMlq**MPRuG#_E`byVfxe%u{?3mh-w8G3Wve+tUzqudjLQ zMFwPlK>^KAr2~dvm#WQVzo;9k2#`dT_Lu-}jCQvGCR<2B-EU#({;ReoA(e#^=~`Q1 z!f?^%H#(XBvnwpko2F~mH{kBbAOs-P&O#*7sF%vhW%0F*Uhl7MZh^oU9Yq5Rwx{7t zLHVZl|4ZBTCAA4)ROZ*gfy1g4ULY zH#H80#u(!*iESVA+}t_ttEf`X@ngO8UmcRy+48@cw|@lA2l_etYzOcV`J$gj))z$7 z?q)M4?APJ$eI@(;pBubra%Lg*OzqmTL0O(!{xgsYSiEJ&29g&hB3NjlBixR`qS~Xt zGuTWl>f(7{j*E^W<8A7xGeEqQRP8{mckApe$-^iA{@+I}gkn(gaLm2>b|<1^$e58n zV|DEE#o7D2%NgAh&dm4JM~thT;obLrPK@Voq;I#vE@?YyLl$lr+ZICdFPIW-8c%92 z%noj!?05Kv^Hd6 z?C*!P4p7ZSj$fx=5OFI<@3zSR&iV_$5@EiGa2xjqzsGe|D%0#07vSo@Jq9xmMRs>R z)-=q^sSCjcSghu3e0YRABKvXB!drOl@reVjcjc}>JqM?a=4r{v6v!5emCdHofKL4( zDFZYy2*)Og!;n2N=mh~V59Je^?xP+kp%{9mkWY);4Wgv9*JwSKHIZ;Mw^Nv4)70x{ z*#7nJP&yy_{GvDF7M*k*BbP;GBW+=HbB1vrH+jn2`o)d3MHZ-A(|iRJ#WDYo^VYjf zH3h~H#JVR2IEF@L2lc=Lyn!df9H8Ho3-hT>4V6KV05h8^dNt}cndVkuP)_2C2qpOP za4PWZaQcTaRYqQ5l?kSYjh}kW&yyJ96!lz-=Sz!?S-q@wM1bA%TD2lk3dcdGf+53{ zA4*1|?yge8!T5Qvrw5O77Rfz!6UD_{hJ-c3UNnB(e`C5#TAxktew@o~s(7^E((~N* zol)G1-es$BErMF;Vd|2yJzTClp|)ry`c|g=-c6trJ-Qv!c&s+o(8?8;&f6o*b8GBtIOAt=EVZnleN0`*H(4>Q# zhiJz=`<9<~BG0M^&ku8!t!sg=PRDDOS3+4XP5zK{dOTbm7CzWJxIT!_)E#`Y6}GFm zBcD8>vONA~>4IsIlJi2P_Ck_yJF#ERI)n)^X*%6=pH+&xY&#Ai8cnO~y@V4kEkn9O zGCNa475C*AC>P&j+D1minLFm`o7q0(1>oCS_Ai>|`}Y}>ruR*t7ut6Vd-lEw%jHed z`znx)cQrweU2Pf~Q$`8TnX-6xbcd$tFBHD8C3pLItK9`JZbFWs-{>C_kTE;fH3 z%Y!|oIDg@_-pLun`UZaH96{tg#9~Z6#As4Se-pf%2xb|Lxihe&6QSrY;hX_;#b{w7 zFcdTGah;P|r*xkDgAx@I&^YRB5n_ODry&agqeH;ox2`SjqU*Qhs6$vgcWRWx!~riB z?8J>Tr!RLSkv|fzVwb1_c%n-j?W4Z=xcGV)n;}f3WjGX0RSLy}W+tOPGxAmWVN#h`&{G_UB6`b!9dXn{If zOvE#F0r%Yy{KB@eVZgp!V@>tqOCrkoV0_(J#>OFEdDj?iyE<+dX#rU+wkRKZmQQ<~ zG4P~omc#QxeggW*ZuWzjS`)O_9U16TWD%_@H|iOzF%>na@-ICoQu^esi4NF9lZHU zyHBqQr>A#>*f?#4otFvNZvnC1E`zv@zUW8O>6e?v0+ph)G!wWBb4AIOD{z0Q`;eoi z$mp&JbSC?g>K@i&9#+brIE!GIE>j*XbmZ_1tBq|vA?KRN{Ssqtpb}J)VS_1BRqG|F z2zpLz==w~ym;~P2sp@*?#fkP!x253tS7+Q7C+#BEq?!J@FC}lsFDYy|NK(#NDyJ${ zcAQfyK6V5usNtrJ?CwS+CQ*#6<}A&}4aaUYvO&OpCQRs+;efv!x^;369klpq{}GGO zk6$I{5qMq6;yTow0wbL(z%|x0uA^ppl0-z3wCI=%(Rvg4zV!xvy7yd3(+J+|f$!;h zjSriAz?YMtMIme9|Xg!d^ZZO z!%yE?<-mxPpw1>59za*Qn{J_y-tl8Gn~fk?!7oxZma{#*-h7HJH~*egYUbgHI$*Tp zsuI{Zu^365;efeto#40vz``yld6c}>&;c+b4c4~wBkcB*Kkif@k2VKjSjqyqZu;l$ zY3Rdq+~?oqGh&N*!j0rECQlqg>Ox+}pZm@9P?R&aU6t;QAR~L8RcpDt_m%0TnGf#P z>$Du$()9GPIN;4-P6YH;aKT>-8R9+ajy^Ai6V+$8KL7rr45tj#^}IqRzq&?VG+bc! zOTE4|k}m7E-7yDIGkP1Y zcEnj!cddlb^sL5S>WaUgA@Kh;`Eg&f7A*2yq2TMKvD<07-5q_w-QA|=WGPih;>D+n z>OEawY$Nm6`A5y7A1b2>Ebx8ahLDUo4tdAsu=;tfTR47uH;3OCV9*?3Y-uu#;%n?d z2r8~$mcp~%lH6y25$iFrqSwgSn4++1E!;mO6=`0A{u!; zmA!V30qNJa)v?Dx4_N1izFaF5yKwnvN&cgl`_ChqDeZ?jE2UauRyzGI zrVgo-2fC>f`Q}4${dv1U@Eo)Ue$>7#{1-PFT=_T5jU#IR+!Zcn|&A=!12a*sFn z7mHsEEnA}Gifrc7KF82KgO(TvT@RWDeqWg-pnc!hFipouLK!4{)Wa>S- z(<{1v=A5%$(^>U>nYdS@QGAtY#_O(<#9#|3`=n6<${TEyF8I0TjlO1_!dN722wEms zMU=EWU-;$H4KHcD7ViXr_k^bkPmMbwvsM#);*u2wjh?vr*WuQZ@)GDxRA`^G5_H%9%02Tn zAhvxN>t$kZfATxneCPJ_>G9pcbWbYk(+q)=S*2#zIYSZjx(}noTH63`T!C3+*Otmi zynI`*My4(K?u=W&J08)OW7CUU7@iPq->Iq^=A=WazHO@ZK8vg!VMK56?B#WqF*8k< z$}+B!r><#W{hV-I`ZkVmW7zR9yRr|F9DZ7NoLEQLYsNzbXfIX15*KsE6HOXaol40Z zu0R>mv%1Z{Wlr*1ZwdL$-~)Gv`atC?*;NxJxu$L&YPGrnZ{7f@Qynk59h#hcK|3*N zeyd+iF2$r;d{a7G{Q#F9jKHhah;LwcINR9$g`*(3xM1UdpR}Mj7|`{@*fCbDuPnkp zNV4E9qTma~sNRs04jW(&a;!$D(2YYkN!3raqkyZd^gSRx{V?>avGeR)OOpfsy{@;`S}*8k@V^ah4m2n8cqSy7C6_?_pR8(1oGTg9B&o0j~IT0B8VUp_(X>5d9hu0=!d!qh-_1 zwesWQ{rGzcCUor+yC%?bVR2ZiG^({ogju+>)Jkd92!GT7VF_BaViw&wI+lJyo45AZ zOLzV$Q*}<+XVA>cgV?$gpuL&gg8R--b~U=*oQB=>p^^}xgh&fZ=HJgb!RMODWqu(L#vA2$Jp4iUNI)|c0JVkB z8&fYhp`lr*>D<$Hd@9V73m)(ey>k@NMA_qmT^ctJt`1*&=ntl>(4@+4XDGk2wMWEC zDrX41dhj1?Ya$)e%N(@9Ak|P8%h%%mYC^l2Kwd|v{(poh8PhX)XDR3LvTKS_Lu3TH zRf~?-L3vsf5I64ZFQvSZw9*YZOr(v2oUTwEXNr+E=3rEQ2`qQI%Bp#doHBQx|Hek9 zeE)Cre!Nl23roM-+A8;>c>5g0=bW!e;aLhro)r*I?=5x-u;id>vxTLo`of5DqeSh! z2B6cjON?Hujzjdn^t?sv%gl*n3();|EVob;vtj(boe89-{!5#*bNT>59*8n&>+106 zuJPjg$8M}EPXk>-%HPjQGFZ-1i(3!2+OTZA=xHaFAej5qT^_HWInrVCFotk0WB_+p zRo3CS)xlJ1Of>AOSGJ8x;^EuF+DHySK6^lZbvk2ikaKKJuBa4zIKHmL5(-0})wBp9 zr;yM(yboCS<;;S<2QBl}$K!=(gX0Qs#}n&z$<&f~_9?yqv^!U|-R&2A?!yxgM8r`s z1miO8_EUsW9_%+PsAsZpJK+6D*~{do!gu7OFT?GHt~Qr$Y>n9cZqt^VuEGPlws!}& z2FI>7J#Sk>m6YVyJidi=)ZqX6>NeWkpDD4FhEw565x@YJ;~aEcSFrb)Eb=vr#A!!{ z%FEJCpCP$HYLQlI2<tJZ@;nf^Z{JC8`iMSYYkAB`SswelY*K@ys-Dqc{C zs%-1BaZr|j1ef@Fme4@T}c>3QpZw+)88sx#wKiC7% z>*K7jP3SgzHr#aYX%9X@M9{*R15rCe&>R%-?rL$?j}{ay;=3lRL*>;VJ8I?CQBK3o zxcs_Rq<`<4y>rH*O{DZU6rMbmU<}i}i7xdt|DYZOinTo5)P*E9niu?D_;P&kx48BS z-x@0W4@oSY&%jmI{q1w|~|^sFLy-SWkZnW51YDbtk%RCY`#5z({oZ2l#Ud z`w1Ji&6Zku<*e~->reiZIR0RTw^F%y&>xU%Wu5hwNzU-JA)eW$N;Bcr%0O#F2jyV$ z^6BvQOi@&Or4^yh%lo;B5o$+^YWvaBc*ok8 zD2E6P85sd2i1eT4xjw&QlnH2*1zaSI|F$`*0=^^Y2` zC~`Hy@Y^KG(R{ymrqe5+Uf4Clw^{=8i)MgKild`;w*k;YRQl4m{Edts#4*qyfoi$n z#G1T1ac}LvIuq`eC>8r8+i+^j7#y9qIJ-^}!?SXnr zoV91Nems8@shzoDh?Qxnkb!x?eWEqCCYe!cSoixgbt0 z)e-*EgI9CmcD5=Wh$%om4j z^hLAya!|qSS{R{K(9#P3!{5e7_Nw3Wv@S#4dnUqMM)YM*?wTQR*V+$`|5oicaB_%A z2}l$_l|YG|n>R~K#|yTVgPGfma2AnM+cJoEu!e~$oMZID-iio2V`+kazzarm!}`DR znBnCBL}9d0Na}1Qzr^U1J*)0|0iW0TQ&7#Ad?Yn`$pIs*m0gIR7;WDRcUsNlb>Vlpa9?T>Z|^Wyb{bNP7T_CY8Yo%mIqZnuZuPZLljhU%D}gn6Vd7R+Z>{gY z`?!=n?I=kwE#lGkXOdcZBF2m!oYRU@%MnxCUJ4U5H_y(d&cQOX(KGmA%vD0s=*Sb9ett%N#4ZD;23g12U9>w$fHb2hJuKe$}D zO-a!gUA^<&KmT~mWyf9K{llltQE8tUM0T;Dham52RG;Y^8uwR2G|$yyR$QU)QXO$L zkWAkZSEi#A@8Z50Z)dU={vdk2Ei`YIf4uPEe$Qp%?~s&IQP9@!1g}qCxkbptZV^x8 zyDK*n)*p;nU;3UUk3-J6JfI%6Z}nr9gfwQ^)1Bi@!sLE_KeHxaAkx8 zXIEVA2E#i|X*$bZsn*e{68)rQvLW#mq1JP*Sy@>${Y<89$jEm%u^rVpB?mRIlA1oP zvgu!t2h>EY4L~IL`tm+YD@rTk1apA-RYsiQ%@b7;r)7p!sO<8U1pbU#lrU5mda*Up z{AnR{{?6mvn7;{B1oZIpHlF^~5#Yl+^4Ee8q6_R~<}b?f_7~M`p4t6XSsZ`H@_G|5 zH+%^oacn;V5R$!r%4&!2r*zWdb$LDMC(f}%H3Imu9)Ome`LLy z6pAZr7?`G)P_FZf)Af;P_RV~~`X$6>2*mz9b8%KAPvyH)I>F2TOU>d{0?T+e+sA^> zM^s`saH`kRWs7v9ncv;rr8n}(tm{*Yn+P^B2lFM;&E(}QSc6@55TYH=_3))g8J+-qI^>Vcz?ow_hz^`t;Wl9$Zf>dB{-zkyp0CJ8d znQHN_UG0AT{J$PaBMfh>cAFQ^e%%SI@=carhHWmUt#oD4PUz13ym+?zH~Gxp_aHK& z;@o@Erl4|X;lccCA?6W`<7#=qLbK90UY;<3mHO}(6q(=0I6y2IIr?a z#3RL<;fGGWOT*X)qcrtFfR^(Qn)2VmBK<7=KYae4@E?8+)&-9rQ@ywii(i4((ISGq;!i}2-<-cHp=r{*_en>m zAoV}9oSbpn$sxQ*d0?wG8*Fac=jL75*5aC$o4c=JN=&SiP)Pqu;6zDI{Zva8xruu- z#??BSiDy_p0}d*(kkBeXf|DtzT8<{ZgWW0we3rv{yp^xmpjO4m_Bpz9je2k2!AeiH zhuNz%&z(>eojky4cK2nwn$$>bCji9Ped=-gwqSare8_j@ z5UqVL!$^JOyzIhrZs7AdZyWSPi+xM$kU@rBa9~kImb-Uj?db)_R!Kd|Zd0}0cRk)w z{u>I-!WfiZt?}tgRd7y)5vRRsld>;NFX(}hCVWQcsZ3!-xC10}F5PmKqftdSSvNAP zcaskVfFkWDj!@HA1)>A7hoT1^nR{`>q%q>`nt*hrrMxM&>#sISL;hxT!K>{{p)t9U z#tXJrdCL09#p&}2!wGvD4oA~pnujI^&W6bPQk!RD*T^It(C2?h+^7kW&ur)!Q}vCl zy^0wi4VE8A&KIMIU;U-;G0YYB^H(W?(~KKXC@jeQ1=d`z*jda!*~jB{@O%Ro(0|=B z6huzp?&W6n5t>w4z7zILCQIxh-?XIaGSn#jm;*RocqMFI9{wE)ON4o^Hfd*TA&#n6h4+iP(mVb5 z=V4Y2R)LU}A0H$eZq#M4%ZjD_nycCTxYw}#(d+wP6i3*d7|fUv;K%|S zXdp5{+s6CB7N=_Kd)vL}4WWWwqGPl+!r$iQYc{kx!J3V_e$ES;kW3xGg?wD)S?y2` z&*uw$wXB0iq?m4*Jh&MuGaI-=YsFSX^mVo|($vy&lDbbI83yTF!*&dSCdON?mWoU= zm*o-_*!>-&j2E(MO}(@{6S`_GIso%_|2jU&oRa4j|7IZXgSBiyjQA8@Pfwc;6w^YM zRGL|YFgy2NqO{}-Gn{`*vy0xbA>!rlXUkr+!av?GEbXa;j>uqTdD9gXx;K|`Av9sN z@`l2ibc{*YOvbRA70&E#D|0d(c6cAMmYf~;Z1~-%bE2Y=P95fvCR{FNC1uv3XC;sT zUhbh&km``?_R0k~*rxEvmQ2RZ*m#|*m(s!Jt7oyDsf|A;sCf75ZznmfA77b`kyp|s z=}ms75!5H|MedsP+rL_Q65vmiC5iY1Kuz-n>(g(m#upa%Z26k($ch0(OrH)G0G&7B zA0+IR;msSwTf0QMu>U#8kTnMCAub!V{7i86z347?eGm`4wyk_~n%|j!?vzH@K1jh?)aAJknVrsZ7UzD&P+dchkm;w2 zNfDGLxHc8 z`L=V=R1>XfUaURMYVP>ZmNP#C zwRZW`qD6{l2~{^yYOUj2c`XE$0|jO;Z z>wc_Z&9)z`tjpXNkJ#@N1s7*#dh2Yh5fp&QBF$ajQPoOfH3G&2*RkVimX*O1= zqbAgsHJ8|z7!!H{bLdDv(}8&06tba6USF1{Y;%8CHflC3MX;H5R9-Yrv(xuTUf&FF zB<)sg+O5d+-Iif0*321x8X?Fo_vyE?hYY!6uMf zj=W+MU(T~;d)uTud&|P>)91hHUURebI7PYEcM23x>Qe?o`1;j+ne^AwKLs{5fj-MO zyvKUcMb)Y}Fx)=JR=!BGkxzS9T%i+ekYWT;8`tn6^AVUS`yb9HY8C}~wT*5N zp{nP-Yctjuy9{khnF;WWg^<|u&Z(ZM(oZ|P*7jzZl#cb5g;8GWrV16$!IW{Ov4|&h z;u$W!8=xdQG*uOEYf*epkGi2#`w?-3AjnfMLjwXA-r^q_Gxs?OV_wwl85vo=+&T(v zsFv;(ilpL+K1BYo$K{<$YSI~h#nk5w3$1TlHnj`HEtnk>R@yUuH*HxHB|j%&O1er) zy+Tv5RCFCtQV+GEvU>=erEYvdTSmQe*koxO3Mq@QE+S5=D%(2P`YWz<*1d0g4cbn$ zxwM5ZTi}1ozKK-~vSj`DoZ64gS>72ap0{}Yby=Ebety9VCh6x}g~?fMb@PUWrOHta zS-5ac!UbT6Q=nz9rL!tFD?jXfVx^%%3-0Q7P6(H5KsrOGFkk%78!#)o)of@rD)Y(n zh-kIFl1djVl~|2Ad^I0DvAl|-bpTf+mpKlyLpXOcgxdu9VQO2)Mtk%ldYCgy(!pgV z4d_ndj2_>}lT$yW==uga%UWx1V^@$Uk9R%7eIBV>PaIqJ8#OB;g{tmQ@aKsMr;l+| zy~^_u|H$`_;d9=U;RqL))-AK9sR zHoA;_T$|oJqO_*gqBNn7{_m$}$kdUNGmL+~N2gcGp(wU&8>rN{M zL4jg+4k~hS^J|+?%Ko3%5)OZFAJnv;H+t~bNoqQXWIwFp=-$4GQVD2A2yEQ56WTqN z{VNLHAjxBz^Y+u0renP%7mo}AKPdA$R4CDgFP+8!P*1RT;W6O4Cwt&XYm)Zzac;H+ zmD*Ut#wOs=EQ5&K)-L)i)`~MqwyEh^*r?WXUWI9n4;DwKTSJS9auSzvQ`|2)_XhZuq+?Th;aFEnltk<-`+h7yH(Zjxzo1u){6GNsoFFzc z5mfO%N;gq=w?8DAdDE5vf)>Kht)(D?nD#+5jk^w5xn$calzHeFG=n_efxixJOBTl|GJ6)$>FPvsvY3)+Z*Ml`DPVq zCdH?LqU!VjA~@10(95I1Z=auVEo-B9o`_smw&MwYPBIs)o*O?EG$oQ(L&5Oorlh(D zI#7$k6DQ!4-lJHH4=Kzf-WG zApa%5uI(XW)De~E@f(IbIl>yh<6{X&1O4Hj9jSuv1$F-QiWTRo^Bt9HDNQL25|MQM zsH-YBYnKZ2=R?`JB2h-AdcQ$Ob1(0KJ>orvBhr4QbubCCk*149$qgTx(TNS|#q_@K z15Fllz?2=_`t6^`-jf(gYvKj$53d3u1Sisd#d35G>S<=0e;4DyF7&s!0zY)g-7l8g zk>ch01=ameCip`>>PKE|rfknRA1tLL3rsFNfbR1*?v8hg4U438DV`Tq4I>vVMsLr0 zNL!Iup4ko&$DzMWks;lPd`}@uO{h%0R`t_ge*1g7>1b_h=<$D=vt|2xb+|~C74`&)GVoc|@alZf&B9CcP!&VU*m7miKj4tS$ z17`i4t#KrelHZ|LNbKmF2WqP*x@L~w8@e!6y!>n@W2m!_+};;B(-7GHuYaMvY;xfp zeVG=4<*#Wfqh2#VtvdAeV-gZ2$*Ju8Gg)U=JlKkg3d#AA-%=0m2?U)!nnB#d_8aT> z&>^YPhV)W(&_7rEW^K9`Z88adO8;Ae{ns5?8r&iSz>SD8O|*H$ZJeslT?Rl1eA?$a z9W=;?uCcybvkrPs1vGO9}!(T}pVr9&XeE9qvG0NyV>Gzys z=f&uAenR#vBQ=-1ySkC=^*yRmzGh+!o-uyjZ>oCg-}_5->aTidlQ@==kLT_&q<>2q z%%N-UJbnPlv7zxb{O>U8ZGT8Y*e%m%zzXA6MnZmy5lF=;`v(<74K$%yw;jereV@h> zPW}LrS}4~&qlU;eGtJd+kieAG-xc}_kF1uer^JV!!k=kE2&dq8zyJGxw<%?3n%c^1 z9!TnJs9&53o2Lj|Mog2DY0)f;mN+X@*v_udPXm@^ZfS1Z71NH#qDrMDU6kw#{?#;kfCX4jl|A3v8X41t8H_)f}HpDP0r3A4{MY89t{$ ziTlMMbgGFz?5PnCe!2jiVT2eZ>fk@fVMkqzJ6Jx5UH?mR_cyKtZIAx$1bsUhH&oT$ z4K17at_8jD@i#Hg=ZDZjr+p0SnlnMD^-@LGk$1PmZSenb@yieD5h`t}TQCtTCqmYI z>7q)e%cDDMHG6S`GJZ$Y%vquk|ArC!`tA^anUcwYLhtR&U$xmF30kE|9;@(YmamX@1V&biG@U(f!&A{kd(Lz5%}Q|e4% z1+%uKw`$~+*CuthP?b77>w-^ zGI<1Q#Zy8r6tUF_*VJW+C|=Q`5HgrY=$EVkCAQrsG(*lT44sh{SxS;iS>4A^muhQf zuI;S4$BwCC_j&qkh8Y|o>r}gXzY1pth$xSzJ*?QrJW*eGQ=CQLV$!S|r}@fVDi%xg zJ-Aaia6Y-j1L2uaUYE73U#9W1(1^TEC@^N#+0HPG6K_YhCuNu>pKHm?xiRM&5)lj4 zXN1dTqWBD`e7x}woLd>_K6ZIM#jTmnQZe7wr@c5~t+M%z!M=N$Y!j82l#LN_>?`)& z5^@SCfc?CUeDzr_!X=+&;tg>9Onc#b4pprs{%DLA$egVj(~vV?5UDa4Un2HnAekLF z=S5oFQl7~A-|xt(;FT*DBT=zo1~J99jFVIEVFcc+RjUEoJWJr zG>hr#y-2?4w(AIX7r?eGxZuCqrSQt~2&ObXgMiVh!m&prXvOrp5;UQb3Suih+3s z#sW%_B5T}M=b}>Cnl!CZlfyO(Hpg}THvMZww;`vieI#M##EAFdSlt8AyGNdN+G(@e zj0y!h${Wu=??JsyhksjaIrA3~SgQTGeeS!UVb$B?R^D+8w!!UvQlGsJVN=I133XsW z8Y4;r{+C#epm0tM4StLDpOBXgXo+-QS6AV(GuSROun~?)8O>I%dzKqy_;N9}7X0+Q zNOaiAPAYl##Wq8`cobR0Au!+Um$Q1lwy>JwmIugdaSkD%)vAfyXbx>FhDVed+&6g? znZX!1T>2=ke5VRHG&nv z-Gfs=L2!p8Sa2;|f)o{`AXou~1$PK8g#>p9?xgFyeMk2>Ip_Pj``-KZxMTFYe^u=T zwbx#2t~sAI*PPGPoM`G78pOQlnlbem)XJ^dbb`;=h+1h@R3eKgLq>Nvp`C-Enu{&f z{N*dw!&VMZL@)a_v182cE;z2ShEM%Wt`at~ZqUG?vN&%ug+=T?xVty5t{oEzV%Ubf zP~Z4gB_`EYCEjspUaM^JZG(&?LZ{65LA~ADj!H4OM2D@LeD4L=p4eE)&If`)J zZj_ra+gi2&2#O)B*~2DkQiWU|emZ2JM6(PVo~x}2Fy zYWco1=3NU{7l^NVd!=VYbp-nAfHs>?gz6c{-owRG{Z$McQcu zIwvd}Fig)%9jBVb&Zv?%RCGVQjYEYBJJr1CyPb+#z%c7Dwm}TzL{ibdSWJxpm~Dt4 zv80L8x8=~ObI`=jb+r0o=V)qm~_>7%Z-2gY0XET@d0fp%0blGy|%JFW60?61H;S znWr0<0c@H@R`KoYb=#(o73%qYinOF&QoR0N!>OD9Dt9klQ=bj|DX;4~x$w1liHamA z*|Xw=$i{SbRl_TL2^Usk4Kuir_9=>q27Ww_7|o&U^_Sk*fX`|qL8{9o9oJ!VC-4?B znOd+5ANCjI<(1gzbqQc9BOM9a*tiTlm8R=I^ru&Iw;{{-gh#ustpvzT6m^cJ*EYZ| zm|!TP@v`z&4PWz<7)#^>ep|ygM8e^d_Q=8H`XTbcSHowhrj7U9Q;S#N55*Zl1gr6;lGIz-IPI4&f zMfD(ddK%%+d>>1S%OUYKbqYPF7;1P4!&W~=dKGJOD*If=&%7GnB;Fc(49qH3WYD_4 zvM6K7>)#qGvPv1t(AcYm2ti1-vp7!?*O43z_MR4?YG`k9LAB%5>ZnIelj=A5$11P~ zinBWNj-2d8vh0EZq|50w*+to>F=e}HB&bk+cU5Tr@DalcsS9tNXKzb_?$=DP=k4he zR_B~vOARFpRcDO4o{@S;+zMNuvLgovr;pFHQG3ARaOeg7stXXPg0(lp=q=2A=mKhq zL_^~;`UR9ogY8y1!~L~uUc<084m@w0dk@0GzO*D4fhBOd_WonJC`M%Z`6M) zY8>N2p4Ibp4MWtF zHW#qJiSS-LfeRLqPP?Pf*f>kM$;4NK zl7BX_n;>>ER2ol(uk6U$jD~8|9wS>xDp6tNW2g|y{$bNgd)1g6`F?|&7S-NERHpO< z-L|Y~IrrMdKxGZe#sU1LnLCee0xzYwWbxnJ#~xU>kaQKdeJD%ibg-~ZvYXqua-IS3%@|owOG-|cbh5uK z?l90gj}*KT%s#RR7Q)1dH5m@igatmW?w+h3`MeScfH{wJv+I0jO{<5yd=D0&8hdtAb0yz%=Co^LTeaWk5;b)n zsS5O^f1hk$ooXk_B^%QZTuascFkCTh0N#=-q%6qi(1fr}?2o!{h%)4@`1;Cv^u{@y z+vykH=iPO!YAWiwovKPLmGjJ6a;2QU>Zgi-!|Mz&aC1(g7cskDSdLWJ$yLET;By1Q zB&qVqwN%@~m#R zMw}@H}K-$u%32#wsPd#2*HWA>mWC23|79 zNJ9_Bg_;crP}Otq6k@L$o8EHS3GKZ0zGDlQj!Y|=hH4!SQXPe;*Q~{RBS*peCp0L zC8PWFSSN=i8>h;oJxJ&Y6cw$wLL++Fa;ksN0_77-&Z%r>(_N+YI(xUv&a1v##V|`d zc`c8muCn)4>V^Bj>1=tutqi>F2Y9>&Z`v^w&2k<5%rI0EVH6rsY+9NoN41}PAMG`8 zW^s*G4d(2YRNW>HdKgwkA*G?FjP2H1%}09FCYYmNSJ))1fc~KnMLBH?03a_*1i*K$ zNbQJhdgjEPt6J&Iz6Y;{*||2=&ZE=<$%G{eojdiO=;oZoUiczGbf_If@tF>fqKC24 zj+Rp`VS2o5Ig)Qif^cHg*^ z;w}v=@;KJm9$PLK1eFVqnb`RY-nLXGT>%HxPg733&Rpk+=zS9g;uvm4q2#=ey_QaZVFA~J;W;nm#v zmD}9PCXQW4PLTQN!)q7}=q&UTZ4v@1?*^b?Xo%PVJcmqjZb~;jE7rm^0d*X4XF5N4 zbR2ulk#JdmErt?kF|n!i#_dUEJUOcd9Eb3Mth^yNxGR7ve%IBvamtgtv($zg$$wV#t z`Ied_eYEUrwINgcT6nT{XRg}Zy0ia-b0gne==ki(I7f3TLDpBtS;Nozl*v=5HAdkd zxIMWTyU>k)OLFsSI@;2sp~8!Hr~!ba`PlEQm+5%l`q(GUgN!6C8U7T35h;CGv~Z-BE>9MYHeBb z^CkX{jr|9RDGyL>S)}3Rohe^5=PDRn<6)ACxP8muh2~l|wc+6p`YKAtDf;BqCkP=_T*zu!d& zoDkj~o2s&|v2HTZXJnpiP2Iw!U>1xz)2z$GPN_Bo7@Bmo4X`EERzSolHb~xfh!mUU zsn3fnq@i=If9i2MdYBdb+N=c3=Ja7r?A!Fq^y)$q@1*8QD!jx}__c!2B`-&GSSGL! zwq8x;`T@h^gbQX*Bmx)BfBk*#`^V++e53b3RKc){XCKPy^2bhUg`Xa7P2mSE!-i=Y>f^=*E5ky zz3k*t{yL*G%Id4M`}mxWvasLY0O}l}fFBQisysJ0scsHTPkd}RY>+m~z5~z*(ao8< zYZti@!k;C3lK-)Y(siJcv+%2#XF>jgxsJ9jby4Ao3mAvQlYCxOWX8BUC*Hq~M7NMw>my&(5I6sFl zw?_q5X~2tKG5(xBgLoecAO59cHHEQ-as3IuFI}e{Gwy?a{rjxAVY7LL%HudzY`xx- zth7@DDKY8Jj`pFvZ3|DnY&F}c(!U84QB+NUdOBrVdJMJA_}X^}%yO^-;71v=wz&{S zfJSjx$2yM8`bzTB1A$diBRv>@4Rw)Nf+A@LMcSwkq4~KzSKJfGHX*0ULiT#WD}5!q zWmvDcHm5}QJs20zVvOT;tl%Jj)yrg-;SwvPrgsHSG^1r|>3Nmic}xEcB|B*}q7N3U zF5VzM2au?YNXK%Icq+FJ#Mx0bdTvvFeK5-|IP2G9_vy?6)iD~Yij!J1u+9)PrCX>Z zM?Hp8WR{&o-WsSx^dXchBb$$e7i{~Yk=5*wKbq3XC@b=fkR5!uulbvmKCM(TnJs-( zmpqTY18<94J6GfOk`0?XfALT6)A1gi6UeiVKRN`k0=YTd&lVh6V`bgJja z?psews|#D(kNLcqz^g)_x9~aix>T1g4?A9rP3MPtsOe+KPqeAZ8VKuAkY}eA7D!y`8 zCZn<$L@d5ULK(jrF=2NnU2F7EGi7fXn)sbZ-vgU0#pSpGRVd|iF!}cVc}oY}aBkbD zh8bf9zimi!DolS>-ytK&%%`2x%}gUC{V>3X%uFuT$YRrS6FJ35(oUgpSU9x&hYnW? z+{cLR<$3%Fhl492Wm0d9%|5xICo3a0U4zU$IkjkTUY~Q5NZ)X;xz8k5<3mK{4(jYt zXIp2-9>fJd2-~gF;5iFyRe@QBeHehB_CIE+A8VJRlFd|Y$t2!7(1h5C$~T6L+6M?P zm{y*p7ebM6sd2DRLZ1hrz>odmJwfKGGy(eO%oJUxj(M560(JX+lP=9u5xne)>sF=b z24<zsJ@+?ih4DT%M&x{H8(n zOsR(8B03C}iTSWGo=F?OJ^Y9`w@CYGPYsg8QqhN$Vr@OIMhxoJQ2h||kxgjQLdd+q z80CUdO2W1s6=qB+p!9STA%#x+VTlYE;m{T1z5J-B>$NGdsI^exoQDG*%V`)%M{+;5 zYCK=eo1o-V=41u^Z1c^hn2%n;ieU3~q2m$pll_>NG||0^$1&Aaiu9t)<~iRvMCzM4 z`D8wj1Z8EISeLqZ&R|RQ^TXx&eXGYPA#qJeT1DjkW7!8rU_GR7*>8A5n~l^<=mk@3~ymirqz$p23t#8bS~M1%@*5*O4EczV^P=-n6MKR^)6Qo z<7`_L2lSIAm}PgNvhcpoJdoI0tGhAc5p;>p?9wQg$B63qp3RII4LvvEZfX6N1PL!c zGwvB=1Z3Jn6kp=rM#IM{pn^bo6A}3()H_LSflE$_U(8N zKmJY2)Y?Tb5vr2%Vg#xhJg8PTXA5S3g!0zwAJ1wbCmJj&4i+vs1>tz?#C8W;6~i?LkTqpeCzHfMXjNYITSNj8#H_0@JYuKeEMJ^_G-^hAOQF z;;WM?IEyPyWRf}(nR87T)7&9&D{Mtm_QK>r#{TAtCUjfZ*U0Wf2$)Q|lrX&vwEglBXqw3!#~Ed& z7$fw4OAhq9;N!S8FzQF9i!cG?5=)f7UexK3f~{UW%{IeOyPC)+k5u1(2UE=(6%q$X zD7msi#jgR7{*B{^#Y-5hx}gFApEqPTn5u%*YAlyGXHK3#(m>TP_~><&FCU1>DcxZo zl)T}6zF3!uXQSc;sLSkYwn~-9k3_{(9XdV!*wL2dlT>6~!&~=Sz)Fq)s)~~<2Y&_@ zakT*RL^ZAfyd0X%UwAq#Fbg3<4Xr#Zm5S+u<2bn z(S^CFYORHJj3%mT`)l~1q0v0?{I|pU|B2iPLR##SKqf2pj?sLXb|3Tar~8$sHyG3! zsXua1@M$36;K_GrqU?yvqu!M>OM>?&tLpn=mkU>LC?nif8B|`pDoD!W(*UTXi3%C0 zGY}{zCo3|NS+SZ~5gWs&Q3u6Lg`_ z?3*HH&YboBIGXG0=WJu8WH>Xso9Dr_-d3uz5YL!wkQ9jtt;clwVASjsySMsLDbl0C z55a}euUvRY?M&*EF|^8w+7r1yj=|?HXDd~BkEL;0H0+%+Rmrmwg=iy$JR8&I&@fZf zNN@=PuwA^cWHancd_&0?GcZ&!NZ!gvPs}eQPG~1f@xdTD3BHe1HAL{UeL;My6?Z6t zcbRWw4w@A-_mXD64&KX7=sMe%kJM6e;n+*ZcwkavltINSVw5c{TD|afoaytrP@g~p zli1kf%rgV`lH#5s15)IycA7mn%N0MYv*N}qfdb3SAxhTbBc_9;xs9^9+}SGY-a1DD zCRYxPBg|1?B?`7tBz3ICA}}_{(;touBW;o%TF|M-6*|3PKSqT(mUCzIc@>kTRuq&I zDW@wM;-&CXw>4o3kt+A}!jdQMP4*XAIlTHlP?}W6iG_`FG4XQAB`xovxT7F7;l@V8 z8iTaGR-2Pk2$R=h6T+H?RTd$&OfizO6KsgC3uMFl^Er**3ethLBNK|}NK)6Liiz%# zgi`Dxq5C;-RsBQjaSL$W21~UOMA*`N`Xrr%$H*I4L?cjFNw%fio?->SndTSICHB>vnC^xk9GkZrU0HAsvrv zBB(;NMPl^UU~vW!?aM&mC;5>VnEzA0|M{QRB!?#I_7~m&r5HTbvLO4{J#jLt8_?od zS0bGZm3_^SGec0<+(LB-Hu*w7(`>c{eMy+z=tOcMinbX2OvqPy@Jv^t&m-|^*LeSN zW-L$Jy0^7Z@=i7#MNkRlxb>1Cn>(;x-UCKikwtj|5fjW6)6Asnp2Z1coz09Dni)99 zQ03{ZYJAAfG4RFRGl`e{wK1zY?16~LI7D5}aJ)n#iCwLPTqj>dJD8yg-Zx$K#0k;MLRU8l;Z98NTw&a1qxYVeMk7QMDhRW#SpNH$eJ}W1sJ; zK)!Q+He*Z__)29NRtr^oiIXO{AuvF1TxDSDueTa~PPm*Jkpr|-c2F}f)P^P*58;+@ z)P5?|LmUWJZ#VDp8OPwfov z^rJ?mpJ=W)j)L7S+4&8x&ItFEo(@)USJl7qn6fDuFaDynZJp+!m~HlDg7&C=J;~MX z0;l$|n5R}UGe&fJ$NqU`+naOnBOR}eDH2?T8C8>YS55z^g^+>`qCs0(@05A}^=FPk zwMj|+`-q+LA~JoDBVv1WRx7~SQ)4G$9V*NlGTK6Sw=iOHcaCzi`cM5t zy*ML*w_s-nbX_}M2CZ-G+5?XLPNvq@F9VP8dIF~J2DCowLPPfmiRZNq48KgzH-7*F z2KyWrC#;A!gqxTl3+2Wt^(hqx)ewWE4c)3Lol3tqjB zTCb>;%2OXO^S1LsZRLjZ!TJ6bBZ+!t@o zb@$pncXw}7OtVh6>G0z6j!asX^%&z?6lE^mF1i)3#eG#g9#_yLtz~e-sT?Z@W@Av$ zDoO=?cXHYyq1{^ic}%w%!y$$B@Cr|r_K%vP@wXw~04&EW+crBj<1yqC(uyzZbQHL1JMit|@wF?d-ZJciDHuF!L=AX3-p=`XtbK-q`nbg)h zalVNkg3k&>T{G-QYS?lr;RI3a*6H=>+X&BG3lV)M3FdE$lc5^yO(3{MgQ`k6k_X>D= zt)F4_wiC@urM^5RT5PHI?=6$5TEt$Mg8K$m7jSdW)749RZe8m__FL>c0=BfyXv)XV zgio3+Z%Ua=mFiOY3v>B z_t16?Fr;RE9p$hFEq>7_Y|Oz!(f&l{pyXv4|7;Z1=^1F3^jT&OrDjHsBReV(ek$Xh zt8RMD28`7-AmUiJd(ad)c!KvM7H(R_nBOuhU^%YodrehaW#?y{3D-up%^Ip;C~{kr z0~L1ghSo@DmE|3T2w6KB?IAUol1>4f;QC$fI|I#}P7|l{J6QFVgy~dHjhicAqaiIh zBX2L9dZwa;EzgM0@g*nblu-9fWMh>I@I~Enxr|@y3F55ezV;NQJDb@smd3)j$?rhd z)(9`W?<`eCc;2S0!+Z!gXp*z@8T>?KDa1cH@cdP|;HzfEXOg@qlHn48==a)SNMm0a z)Jmz)m3l=Y@S{0roP-5|c~A$%9)C5tMGuxtaFt3!mB#07G8M4-X=vaWm$QLg$WAP* zyF|lz%GlqBA8zWT9PxG4YZqF95SiPtJ)HKbbC4QeaY5AUs3)+E&(Vrw2M=ti4^&Zi zg&c&oFT31uJtXl(oNg~~jV0sN`gln0^26(PH}0pMwHn?>5vpQ5Rw6RB=FyMF85SOb zyl~EZ>s4xPs8MG6u778CI^x2wog_Mo>nNI1Pk%g=FLtjpABX4jfDa8RNM(>Rf={rU zeh-rqR%A$_uTC&G{a%s_Hsz-Hn28~h-G9o|&;jCv*eNkAx&C@zL%T$$L#O0phjx)k zlTRYdpa$3nF_CDa;^$k>@9!*s`6MZ_*s$#aXxVZ3PpEW{_~N=|0_?BGM%%uzZSm!oek*uKA+%i z#_eI?RqiPo(?Q9prM{cV$IC$H`10Mw162K@35>d)sWYeZPE>A2*f=VTJD~JmfAO#J z_?LYAYc>2!JN!!*{>wJ}-?x=$SMRW!%O@haB2n>)e=W)I{|b!x*L(hJ1pX67fHpAM zC5Z^bd>|$LDDj})&noQjrk~YPU)YtF_I9Y1_r|$TpUZz=z`bcf=3wsQ^~<0zSMv1C z1+udiJGd~+0EihKOnW8%Uf#s%$6#IH-U+(&*C!awG2hjvGH&=C>8}0p{!6yfa2H9q zis&Dx0IjE@b!WN#IohAt#z_mjl3YDwdyE9}T928~fv-RHOvliZsm^kCQ09NqIs<@> zV_!spG3J5V_w>JFU?1QUX0a|cx1%NQhuWT}Tl8@qIPhpMoQv8#^K_mR3C9lVQpQoW z89v{wx6HtW&nj+6+{(;`6j!E7phtKH?`kl_s~8=ua~BAFy!H#!grjJxQ#O#W!vZ zYoD2DQCJXQAN z8RRwLpLaew!1plx{_(4q3##V1H1IJ}kl@m#DFP?@<1#Pmta4=|af2?qjlFs)pM4 z?h)7*09okX%V0RKsiiZPh~N=wLZzd9}%veyGgIlK|U~ zzP-Cz^zd2p!NNvq_VefQ71VNEIn{_C1_=wTs->Oh!`#-M#ps@Nq zIUkwB5<3nI1-1V~bNYmNhfYNK9}`x68#J`H%|8xVN3*GRc`&6mtz*3lCGz&q+8-Dn zxF(~xk!dVG#??s_>^W%umRaJkC*Y_Mv$a4}Tc|$f2E>yX9r{wYb&y|2?7sdJ4F?%^ zwC5_J(K1q{V#ka6OAaVB4S&IKkMb;K&38u27s{7 zMsWdEBMcJE!y3o+`cvU=_H)I#=5A92$tb?>1{K6?mV2`6yb_mZ{B_Y3nXc4YKW*`C zz{JHn2oHAY`7bxKRs*)(J12(gUAl`~Oltz3u)e;xBl*isW^Xz~mb1I-zEQG z5U&J^3qy*hqS|tF!6e=J#CS;K`{cO!#mpn6w?^R7@ty-cZ&sK@+rC6U_HJ1<*2hTB zBY=cJoqaFqS06!lk^)l0)g&+A+o=T3cPDn+Xp?U0K`9u=xSs-aU2RsUr6aHIwr|OD zh~KQW-n84&f#+r}h47c}S`aNV6k}-EIB&C7cXzo-n|x9n zj@_Nu=F^@XVk+*3M@3iG<>ov0T4NHCm_ZE+BM4bxYu_3o5Z~E{{u9THbH36qb{;XA z`JAT-su zJI%}=JURXf-KtqpJ~Rc_``I+UTv@*UNFo_Se!Eg=W4r@KzOH)d=DjHDI_Xr<{iB8q zX{Od2)2Vaa&7oBLrasK?i>V@Q7FnQ>Wsa(7=NEH~LOFf3$(F$`A>k&3{nPn-tVDuH zat9Z&_R;hq=VH6E(a5?gK(42;I~SzJ=YalPvy7|Ezdu=0`|7sj5M971()1y%Wk*|^#iV0Xq1{tX?JEyq{R-bBE zWtr!_-;kc4kjD#O#e>0#3)e{5ILdtv<^;-%CP!{h!*A{W@fi0w>fC%oAn`(zN?V5g z!^j>pb%}X$Z5GQ&z#vHK#Q>Q8x+CCZC=3I|VI~30qiSzeZ~s7h zi$SvZkeQ&oBB0;rNs6yu_0`r<#JX}lkP`SPtv}b^9p8GA^P}S8^7bU7cc%wLr|A8J zCLn|rCZ=u%kMl0#7D%VEDpYGIg`6QhC0Tni!2S2ejg!Zt)2F;9i+0^SUS|Ff((Nd9 zI*7UG>9W>!2zulb`NYkg^S8LusoE!6n+D)ydWH#MX0|H=<1QVf48;}pFJeyE&}#pY z;eM#l)pL(2rsvial+47ztDg~d)g&+3fP@fDue_csM%KYVz4!3oXKICG6Kb78Y0g6P zFJOo}J2asTbb#Y+JK!n{M7OAyGg)3f+d|y-BV?uZ!$s@Bu@*PM((9@S(j}1&fZ9nF z16NoH;7792_Wo!*zi~yFC%uCt>%azOVeu{tA{l5*GG#n7`i)u58nYkpBWsDS!VhEg z&hhN7g-lT3nuK~~q<3c8gwxk4_Kcorqfsz2|Jw~NRY&nOAC=#{VIS05u~{Z5do#5M z$sR!|Or88Do>zy`zt}eSS;T@dc3nyFLt~l0YLDWwqiv%qS)p#h?bqw-wCVjkQCSSe z`1g^a5PKq%?5YWlY&|Xbc#!0w*k*7|nmsIE` zy^qI|QeZ{JKq~$D;lMr82DLMg`E)Q7E16ye`w)8#Tg_`61xhVeW#DKeZZ1Z8!m(m5 zeAWsZ(Ps*(bsc?YBJqH=;T)HJ+r2u_-#pNuxy9~A<(I)Fp!tc`H~E9p+LRsf6K#vN zq+508)L)=C*fuarq}n`2Gg;#85-W_n;i-MLUf<4aWsr{gP{fL(-XW~~Q7QKmNq8N@ zJCWvIZj+<;)#IGH%#OU0kvLpy)`Y57{*C;0GkqLHNS{)iw)ZKi<`D z2CLk=;dBYZR95JrJV@r#!e;5QrdU@@PLQ_fs3eZ+r8N0|amArzkgyuP*I(w$s3mEh z=)@A4s4Z4K{-rPNC8*NzE6UKl>Hb0|En3^RuXwY1x(r=X{p?NN7ywu`+&5aJS6%-I zKh}TzEM*X)5&Ut2-%1J00JyuVpAPP`du_68=ly`aI=@(ZicFIO0XIhFQRO&qG@4rN6PYXLcxy+o;r0CHKdj}#JD-ZOag}jC0Oc^n zvuO)QaKz@J5|hptaI00KUu39iza*Zu>OdM0fzBtB-*J8T-C&nXlX9sGHN8!m){ALa zi^5E+j0!aipK%!#zVTXmq%aT<*;nU^B%Ca$XCHIf$yC%P&?`XpF|AHK)v$LEDl`DW z(aqiw7yhcD3`*qSqpVFnSs_A`xMXo{dEN+~VzpjUaQduYNE(~``N!L>EIeB^82or& zZNT|#;#rW=rV1MvkurL1i=q1t%X;wNsft)0L|v3!@QQ@^N1hpu{MLY6gHEX?nd1m} zWE?>RgTftzwA)*#?5%yey|KOXhpti{We?6BZl^kM>3?Ct?<^$@*YF@a9k%IC&0@sN z%*L{PRn*gg8Cyf>nlz{H9>aAc?Jl;wy)jpBF=2|*Cbe*?>?g3F-p^*}QW5UqT&0>fev$8^#r^+?l(BC zihusupy)4)^zV>b?oQ+s`ps7B5k-h~ny=upLs_Iy?^=b2me)K^>$5_4^@oL?R(8f* ziPb`h-dKHJECu6=-i2yyuZxnAAP`LELj`p48MM_ZM{4m>^#-y1d4K?T-c$5L-)6Y> zPV*-k7or%gcJuco@V|eF-2Vl(;x>Ezf^6^g`z6HUa#0@1exZ#vmaEEV;`$Rb`G)qu zyZ*U^B|ip6qbscHwQRU^nXX}m_1Hzjs6FdIo;L5+X~b#`B=GVn^ZssFfPM)3Y_a zgMHVua^WN6!Xl>>!Y{wlxgb|-h59Q~q$WYQLo*Rc61!Y-p%M65(S1S;E6hYA_o)nx zM}o#{M4FCueJNHosA4u7FfU}pK@ibL-RTyqCIY3VW&?k>OBwQ@U#Y$I#Hu-iDyw!U{V4W0f_V6;0@Cb^#akrOHeOWrgaK@T9M!oR$DTwR3~PBiuL3n_ zIG@xzEf7D)9p82@Y4Nv;HIO08S#J2=u}pHtJ>&F>@W3fK$5=}JBTsEW@6lKIj#bT$ zh<#(C)@heG-H=yyV7MvW^!yR9oC7BMkZXhVt(zUEU)+gqT`pkLF3E$PVSI{(13_)P zHo0T3j`kq=l986uNN_dsjU*)^+D7iR(VqnA6B^dB%LlI>Ydn0f4MXeHNWvAPvRa`* z=%HtUH;rX3rH)(0Lze4GSx!$m>I~(^Q|m@*jdb?L(M^5E@e&ZCb`k|w=Y%zjCd+P5 zlO=&@-ejEYZJxmaoj>T5E*p?nH%8;136i)tj5I1yVY4A0cjffDzHk^n30w9s0z?PV zvc;8HYmN4@qyQA%G%VwNBeq2{i{HR_;l%DmpR=CXO&jAB)f$t1!UbQ7-#vh#9lbi< zVsl{%wz9}K`p1l@w52CLoa78h@AwgAre&O<$?o3Mp^&Xn66h6Y0{K=Lj)h{nsCn7j zVfZS%k2{y4cyeK=R;S5qvCa{y2{XRl;qVzn`;7P1?`D0Gy`K3vaVjDwL@IaHVrzXg zMAJJHVOzudD7k<2;3&j)bi6;IKLX-l@kmz)jOx_q=$M}o%R*Jf|FUAj4kQnU+V4oMy^yAZ$Xo)8SwAMG$U zo=;%4NARrAf;Nf;zb_|^7QGzjX8iUC)Vu}d-nxs&uEv810s6g2eJ2!>O{j2hH zSANpVCCK7WH1^Vk=r;>u*Z#zp5Brvj8~S@+PWbO2si)v$Sy|KMT2;b|qX63F5xG(q zKkqaRxrbi2;pszOu_Z2zk%?7={u4R{V=T^HqbX_#eGpn-$HT<=)d28m=PY->x1CO z8t?oR@RG)ip7mBDok|f04Bl=r-BsD%o#f!d+nA2f#>(ndiUtP#){)OOYqY`L#F)mo z8Zn)g>FH|^$X-S^BIZGESO@ecE$(jd)6E202LfhLJb*?GzYr0`_IGf7z)vBGAYJb7K#kX>^`(>FNm zGgd?;Sn(KcT~8eL@Qp)xa_Y1xw~itI7+fk?vd?gK{0~A;qHNgwd#?(6pD1h*?7bx~+OD^IsIn z*zVu#lh8gB%$O#&d^h-p&!8%m0HvpVU)jPO__gK}twOdxqlygC3m%JU{G3eWnkU1Z zi_f9v59qjXCOPje#_$m6s39RxRMM;HLUS9<$$M&?XO zsadg+tq;$HE>ZK1E{A7CHz4%>(s(c@@y#9g&UML&&1(Vuzc@wY+`j(VuH0M9`}F5+ zJj!SjUu3(K3+2lSk68&a;g%ZQ3WhIHRFXGv6zc56e~g!a=n}^&aBFO8N`%mZ!lrXq zmpSuZt&x38(6)C{G~6-4{gdtmO57Ksy=`|F%d$II)VG+7`~Pi?XKy5}SI%JX*zkXX z04JP|JzPo6EF(&QV6E}!iXIgaYO8(B>%GU&+ZX6>g47^oQ3hfao7+cKXOtBQ>jw&c z#r_n*-{B?$<1o;4-Zbxc`ER@;_r=WqYvQH{x%t6(?`UiX+9wui z+HCeVC4mEUDVXGP@K=Y(JKl$XhAZp}9JPOh`{<>c`50XI>r^r~W$VMeUjoTaZzry} zrTzxNzvC~Y^grVCBlm9D%a=Nl4u1G1NVWNoo6~!X`IIs0_@OC*H)T~mf{jb+qdYPdi^t^;&PK|i2s>x{rGp{~TmiUM2YV3Y+360dn#%7+`2QpL-lH#e7*;bk2d?A(Vc1{vzxiz@I64c*w<_h-R{OW z-tA0vQa{XTG7*d4p?>*PLjHDQ0CHQ)<=S(p9O7E;1|`I#Dv}h}a?TP4##I+Eo$<=FE>;iWslJKatvZ_Fybf8%s=n^~@-b%k!`PD7aI_etn`cx_oe1kDkriQL z;#i5AbJODo#ESkoZ`3_dwi3BHzcck4Z6Wx0y0_CI_xB_F+xUMzO0#QXOM#=Og_`*%S6jDpWIzM4q%} z5&;(CwT43iF-hxVD)d!F;Rh^zhW?*y63{rEdX1NA3Nz*T78bgM$sYUep{mZfM&{%` zx}8-u%j$BASA=jLVEK44qzmMFf(jhvbYCsowypa-Fv2ugUpx+uqxzTP%L7VbJ=FX@0c z6XAK6t#y7AX~6C8<8t3LH;DFE1Snwq>;IVw)471W1lGuj>6wNEsT%l=A_<~Z*70d? zDYh`&A%yFe8kqM}Y-=sYqc^YpXIc%)#$SN>1a5_2fJGVf^gYnia{g>#WZ!r+yj#S^ zeKyG%GUsnI>9dj*G2zFZa7OfC`1Z7lBY#fEu;VcXa9tAYn~|crXkI?w3g#88KYu3I zx7)QcJE5>SSMeCi+#KisYhTgJzu~<1{q;pXzco4`*2V^`8w0@Cu2Y=@4_jndeS3|U zTbupS@2*S-7Z#_M*C8^YQ{vFpsxWzi((LXup%pY;HcP!#&KBCLmg`k3 zg+RMk1`D){@q(U0_<}KlG~W*x^OWH!y+vnRrQz>8ktD?POQ!*Z+mw#|kihBNa2PtRmeLo4@rAwx2J>7?2 zD_|93QRJ3@iEneB{I%pdk@RC@X>2!A+`F5C?XCVSE@v;{+2|=i=ksQxMTA@;8G;!d5r7*<7(Y`;yg~IO1$mMpa#n9-Ioh`rkH!o z_pqh&o_18K?)nHmNY}Qvk_oEtzI(YbJbCB+tSvCJaJk?AFN(lZF7|ECw;^h$u}h&W zGkhQ2#!ugRI)(N3f1)uATvtxf=+(Do2|pK4QPj-clXYar-7<_&aGn=_;~kn}<}v+S z9_X${(eEiRQHnjA| ze_dpa-&uXfKL`|TCzh)2GYLe4qZB|u^z zb#P~HUAEIzVd%w&9&y?FvATp!-!|dR<8oKyQEofaS-j7L#e2dGfNw>c2u8&irt5x$ zQ^T*z;J)c6iq#{}x)o=!1xM!8VH0?u#qE(ZbrAJB#*}S&lgE=tkFf$1yr%hRRgb4X zIGv%KNIm~Q?;G?bU9=7-t(0X*vZ z+Z_SvEy=w{yc zc>X8P?k8Lu*}s7{-iLpKU(vI8#tz~&zgG_DINd<+jNP@YAE7j~)&)n^1}Y_o^(R`2 zu{N6F$?p&S-@i~kpvYFMj;dYsdb+hKdYtE>ixPT`q((|oU#a5nj9qv-$p4)OY_?T@ zuDA80(##$8wTXe?@4Jn4qT^maM2l z7RLhu(%36JeHF_b_EH zc8Ykb;JVCSh*?=eP5nzQLC(awF5b$zv1eOYaJY5ogeo8|eKq&KrVGEeNfwqy_qN=lULE0_B5FjpDQR|6~Xv| znc4c$dY(MoUPRwx-+o_`(|^Z&s?@3ao$iDJ z=mtTghGytgN^*#yqy(kAK~jkUh7_brrCaF^32BfJRFLcU#p}Ayq0W8obD#5jJhfRC>Vk9H)1Ufu!< ztszvw!o^Y*J^W3A8V9*y?L69F>|;tN^!jG>aC1KN!uMOB7Cg{>=$QVDgNYsa3l)k8 z9<0c87SM5Iy2B{IDEZR0QeV<50BhCh-MHu%R2H|1xty;cZQJ;`&q9r&M&Gd!M^hWW zo`EgZZg_4!>;z`IE<0*5G)kmn=QgZ`o90=&1NyC@MP|r#q}oFrGYEA^(i};UT8U0{ z)NM~>hzK;MfXSZ>HsXN;Le;lH(Mil3=*3QAWJ_7Q|4x1};w5rQcQQlUzoa`@;%i=a z5Vw+1f^=ozcpE+HhT?;(I`DFOVhfsBbuhwj4E7e~D_MU-?%nF6cXM89b!x{TY{`+W zmwL1dAtcWfOv!544|nuM z$D~5YVLxV$AI?l9s^>$2AAGbFE919Ody503gD2k!+Vqw_bXY$%8O%i#pfog8U5+Bg0!#GK~uUq`189O)~tFd0MC7PxXnx1Ra$?%V*lKt z7*c|z*`ZZv^%*_7p1E}wDRED=QSAv4T%tFa8Y}Ec6;AAYI1dNY7Jun2?yk;?iQ<~_ zZxQF0kxjvsoTG@){8-K{%*h_=E@CS8m<5J|EAUTIhm_qkT$-_kG)5krx&CFn(b9xG z&3t%BqjFj*hC@{XHw3+w`Yqv!q6(nEkE-Z?tywwyefH!LTp#P!_f1sRuIlN`G)DA^ zrQFVfz41pbH&xu8RA{(TNe%CSmF?RmWhI&uRKwWE$Hr!ytgvUW?hUSam3eieX??2% zTp?&T)%(Yfo(4_0(h_~~9nqVf58K-g^VE=V1hWQG#AqlW1*uamO_UBr$lj`2ys4He zxYi@Q+taYC$Um{kJ^-!(!tb?wodbG|FT4;wJdvB>5aRI#j-{y&BT@ma^sRg0-5*)O z#70bUA4-M_U72vID#5ZA$apG^2OY6F?2U>{J6kp5&l_Ky%`d(3I_IPBH)0gr9SB)~ zEPnrJ!2U!io<{|LY`#3a-Wgg3~xSUut_GM3#dCh0}sI4Pm4G#ds`%P(iJFArNiM_1sV zSH&yZm7{2qughnxIm}lE=W$(KL_Age8Zp{{y5hy>?)4}nDK6`rpFl|upxMN}IZ;G5 z=0XkgxX=1_3_aPJE{Y&%x=DK6LB1o8E`V^3P))^uObP;_b$>`EwfuDx&=F*sOI9s* zwAYfgx($XJzJ8*^tGTfCouCZ01OHe7uLhI;Juq!YO<}?|&H{b4V_=dx`X4G^pLVcc}iO4hD%Hu zg(|HKYDlOLSU^0E39DvTNqE<`ZXlfZFq9i;<8fG2vpW3R%Nqnbd#Pk4$Z2%Cz>6mp zbV-31@c<@)>m@g0sAt2I%H>?b(>XHTJ(NqLiW>&_4?8Kp_5BQn3#%Te;p0qnV2# zE$WvOZVY{sUtCp-v;mlSl#RjF=Vb;!P?q=5=`qMq^()zwVlIi~NraFE1XBOPyRM2i zWRY0dO)~M>B6kO$=8&EA0-~c=i-SH55n#e+RCKDi`(S~@bdeNNp<^he>y$778y=2b zw7DdPKITWgc&7fJ1kSES(%qw8ShQBU3GfD<7V}rw(6~4Q=~T(t6@xp}EN|notyJh8 ztr*|$AxI&45Qu9ljy7T%2Gk6d7b7k?ZoS5McEs;kTzFgrNc=S)SA>qcKK`LexHwIt zWhouBBY{ER7G-fnoNl}+{8Bo*pyOU(=Z`q;9!q^{8?cOw9W7I{rGkXTFe8f}B)qPM zz*SRU^(zSNpPpm8gDW2?DCa?gY$mLzcyd1qFqH+r9cBi6V%8TuSzn;B67!pWY>Ejs zhB<}Oftx@}!-ff)in1Jqi_0Cv`}4cq$|K!h`nxBQJ`6hmEAHF7p&8NduZH{lWI%lW zFV^?VP0{h5hI5ZvlSI)5x2qv(j4SCMJEEuT@WHuvuX$D7T?b!z ze?;$?iEBnWAMmwT(`TMgqq>N_^9PtPe1}w|du8uKbM}fn*kJ!7O++!zl|iczsrB(F z$>pJJRSPGIo#GnJV^Tp)AkfqF02Cs;bfqAA8RVGYKR`e9pX^zoZI4VW=d@EkMVTgk zD}R8p%Vk&6ovHalr;6q7YNu%0&A!-PGFUlBNBv#GX!YEb9kxdUx z>F=fK>S~zCpD#{ws^o|_(5}v>)T|-vc{F9R>$mkY}W`|L`-K|Hh*tl|~dUNFAa~2$m&ay>H zf-#)UnMieTn~g;uo$}ssd3{;TFhfm0UbIoNyt319tU~3CjJ zf5+*7(24cq5eQ|yYFDoUAN-avYAzHv5lHcp1Qp(#vH!xB+@l>`i<8QknS8rOYIO2b zco8T` zJ_z;ZtN-p@bj=>YiI-hm1>ulS&_M_>U!lKF9VnGH5us*pQN$|1J83bZZOZH<(lskmld8-cauTx1D11D9 z~JOcvqwpA-+eL!GgDne_!At*;+s+N+MNSDxiN3cV`tr<;WR@u zS#|R_So*hkCt#f8BOR@m)?T4xRZ`CzJ8j|GO?CHmm!y6jW{)hGMDMEMB@KwB#L0glR9X$*T4R1w0oZ|{l6Od|?c(xJm6S6GxG&3bT2X+n#Z zyXrQG#TnR}V-RSitVX3;E;?H$1nLYkqV>f?Qk7K^GBKRAlG_jvjO|1ZHc(k0Q{TS} zX?bM-B*Ce2%xQQ^cBJwt4A7OG2U7})MwLocnH6|OJWe+BrX6sdoWWjTNZ}>Hk1)N_ zlz)__PX8?YwMm5Z%klm^_)|9*y|D6EdzaV}u2mjjYVSoPED7##+f6B%RG?$*?NIMn z(u()Qi=b1hy;q$hk@#@n$n!FfqpL6b9&|~?zm#K>WZETp@N*-=`jhF}*;`MLiE7(~ zDsP5sE56&}Cz#nNaf-q&QyG&6?Q*OuCt`G?N06qI?A83nMpjyhMO?&GWC==(i+!~cBdzB+qQcj z8#>3ir^d> zU35h=V*6Jd@f8s2qlTaOFR=AGUBgDb`isfu51H=m(O)I2j@=~z@{R}t(YAdxlN%t+ zXNSv*H$)s5zwiNhN7*{Bi1l2TPboNVUsStonvUb1{sAH%Q;DPi@f>cY^HY55&UrZ% z@QZ2tShO^KM%?bfTA~t1E%s0K`6Dk=DKT%*$X0{ztj`E%nz1waUVhv6mxF$p$T?v`2$n4tQXkfe6t&dDqJH8{8d1>6E-|y0zdpFYgFmDg(Zr@_b z4XM#dLPD;NxyG9Y{%0@HV>Vc4$C@7nYOs5BCq)z=|9rVBz-`x7jEC(Js)@4!aR6w{ zoi9aSdg)5qRBH&3B!Wegl(@O_8x4Wwd@15DIzw^KRUImmecK8&KJ0+C7E5S6;PPiK66(aCv5!tbXI`kMHmf*=D?=K$R>yKfMfwkpjGR zrlq0#uo#M;Ayc^;k_ixc<%5W4p*I3*`YaqeJFa?uEK7k~N?-MC=${MIv@DWMGbocPMJLo0Z=hu{#W^Qi(ptqjsxvYfhu8FP`u?LG{%k!3Pbx9T*>VGK{zp&wqgQgU zc!{ni+jDN!Gt()%xEuMh91w~mx|sG0hGFC)i$Grr#4k^6&hM)jUFUC8!Yv(IE!FXedjmkMXT#UA~h z6zTc!97k3tuakb2IJNT3geSVCjc`mCb8(AU^{HA^;M{uavN8?vuSsMVN;+`rK-0GW zr1T^_vg)_C-E@1;Tz-)ih7aUGUPf&4a*tZHse+y6-k@l zb87Lg#}jb6L+WC%mm}<1<`2oj1fC@8Xd9THoeI#-1vBP1e#)yJ5xXNb>!O@94CE-MdbKSP@4$zBmHEzM@a8;Yh|EM2gPz)Nl|KtKuBnUaEACw}%6 zGX=W#PJ+i;ngP}K3-U|K^J488^ZI7fZoN&pTXW1?kffQz;N|pk)M`j=2}xV{x>YUj z0gbGn#uBZ`#DJp<@k2SzE;==1m5Y}d-k1azZ6nS6a7_>47cJkKQ>je@!z?aX6QmB@ zsHk+5sNzE|-;S4k;pP~O&X*ya5zh8@9TVKOQ8WHn`0Q%Ata2#yB#Cyh;)nwc$P}DC z=b5yeE-11RylmG3N{3i>E+#M^Uwvg^i`GkZBwph4#bnUa>tL`=r6bc{Bj~ zrCie3=u85TxVPsC7fk(BE;Bad{P{+hxEX9u5?MEQn?Uo?L9vHzP2$MM__}9#KzA&_ z;oEMWE=sCgMn(G-I8s-c=bA%dHS7|67}WK$elni1S+n+*?XMO`ub2j=X4wNb}o>bvN$CB$$P%>+7>~f!gjq(!e(*zqZt6 zAgb!|`50;)AJ;V4Sy8`@FxM5g>ib##W(k(iD@WizQLzk~=;Sx6b)(Ji|3}s}LSf0O7iuHf0i3#@q0=U^yIM zy{G!{W~XyrL$#UESNa)frUQ3^`5QjI;jU$?ShMi{dd7R999BPKc<}Yly+Xu2rA@1d zkgi=7dELCK#s!ZW$4u)NrPHRBxMXks;RECqgnss9kj~;gLWwwzwnv68;)Lfzn0vo2 zdb>=V$7ub6F>NIN;2l7$z-<^a@}Q$D8W1a3lh>TCW&1tbqgG8PvrS>mVzM9ON=M#D zHmBXTFLD>&!&_Eb+WYU^s$MxrdHIafLy+HXIbOBN-Tn>PH&+7bx1~qEAIT3YJEM;$ z0&9#zwHp0KYLo-j!gA9RhxxxIlmYsgbBC(g96Y|H5B#_2e%>=F#6VUD=VLfYm1;FT%{{KbFEoP;y{S(C zN!^?R!zg&j`V@K9t(aSzTzbJ=cWlj62YTl?ves&0y70ytx_Na{rH5l_+Rg_0BhWn% z%74CkRG`?wE@ZlUWq-uV!(y>Whr_TxAqgV|TdOda;l({&d2LAOSQ&H;6b=g2OCm|0MdPhG^6VpUD`0a)vs z=GuO7^Z%6})xtbUP|fMi+C<_>QEL^$%Z6rRo&v>>&RCuMq%pR0nw2#Yn~+!@WGwrp z#-Wg{r%;~GJL73#UULIFPfIu3XyCe! z!DPk?UBc?h+e`#LqMCZhwjKk^k~mSn^cLkR%QWz)S0eP?;dd1M7`2T!6;$Ey?(}QZ zrm%{4wx`N_L>y6)ub&fG2pP3|gziz?5!wzr(%FB+Fy9l}{5>5to6V_Zl$9mwEoF7} zmj##keNDP=4*Oq-rgDi>NJI2ij1uq$4MJ--WVCZM;xV!PEL?S_uN8DHx;*+t%ky;Q z*R$emZ@cTyJ|jF8Dg@@o1;}<4k`^)$n6V3i^{QB6ZjGr27Q(DVSHL;gR zL491sg}_4p-iW>&5@z5HI(w&dNp*Kdu!oWWL$YeU`$}mt3~g-cnV-b2xk>K}u+9-ntW<^y3=tFOO*LBY_; z^2;~Ed0C*Z(_F?mm^UOn8p)jtO&TbCGmi5ounz-({f$3?{r&f#S*(i*BO)IFX}k=d z1;tHc=slny{S!Cl#9(jlBCXU}!W`a%k0*2br59)$<}sC@ho_JUP0CWfZ$m0}F263HfxF3Av4mO*hrB zs*6MG^8K=TI7vaAAq_Ml&jqX6eU}861|FB6xmerYKZ;c-)}!U2rm$+zdY_p57xo_Y zZoch?_-{WP`llb3&R!h)FYG-Tz}~a|!rn8zLH%CmCy_kyQOwq>sx=bsBKTkfFy#k?G-}@)> zGa-<#m+Y+ncYYMG-(#KZ_s?FLgxPip0Q-HG11$FClLhE4=Xmj`C=!|&!q4q@olPY> zbSE_RmQ$bc!gV|saouFI>>gS?nRLn-)tma)?pZ+~EUU@M$%BG%&+-Y%yHBj5WWF8h z>}K0Mj%Y9_$d(?dQw>?(F)#wWS- z{XEmXrhkf|qdX1QG0K84^?ux+e9DzWBpg<%TvzM6eV;+nH=UjMAychw_ek29qgLjngQq zc%sv4RP4nyl+R9|!pov=OkSFXbt*dbM`swc+X&FLVLCUh87~)V&w0 zigF|JB;iKzc177(=GYHf%%(G&f@CX@7llfT<$~A1tMe|rGW!9MpRHxi;p#Jy=#hDvu0t+c+K+^5ZkyW z1`3(RuVb|b^g`SctQzP$R3|gZUYZ6Jl$HDeqC`Z{xXP>_5C%#{Sjp?bP~mqg@#4u& zs8NLzYBb5fR4!d(j6n=^GjEHH7F+$G*xJEhLebgfu_3g3iXOt?9wK?Sq+|PwDTIi4 zexJ@tVD5fl=MGr08*-qo6C(zG7OoaJ(Zb`*UuCaI)&vkfGCSkVi8eI3D{} zU}GPnh$Vxs;R|q-Q68Zs!oqk1a#t0pYQa0Io|6WZ!xZ$>wQ%G-L;_`jJ+e>Z7oRNE zTL2n8qk$8^VeX+1JFjM!yfs@wK5OF21i3ibBI|}tHidxT1W5vu6(eU?KbdT@>Pyq+ zWT17`s^jZRcPbU1)7+~&8pd2-+z&-<%y+)oU32U1MNc!CUpsbURZCyXj}}_p_tr1{rmau z*Q^I{pLCbR4SyLcI|lDXP`?hyl}ycX$UjnQ2!?uJnL4EJ zx-wB1KT`0yXh%VHglOGMV9xcJ*2Bx#r6cMO`r_VkdgEqWuf z?!Yj@*&>lg_Dk2Kz2#`FrH4(PUgc!W!muh11vUuu0RR6lNV)-$ zwEDlLu~4pl&f80c5%nwQ*@r*ImqZ3iR6Ift?!1#JrwW#hQhOeRTI$rk^Pq{#=4Ag7qjAnkm{C{dv`+&RF;eF#p6B= zF|2DqkVV3Ovu5Cm=I^#R?2V#xpSUhj#PV}k1&4llYcgR3G zll7a%YihXyz?Y1}mP%Dg69h>0(3o7_CvtGx0~E0vT$IX+r>_Jcolumykb~@ zAba`Pd|F52SL7t!3K>*Lq-W=<)|axD*6HLZy-8WE7d*5E7KDs>ewMJhiIB2dYhOV7 z>TmFN)Pi1FG5R#_A>n&Zwl7<@YugUc*J79-s*GS+#cq3SI{U*8gc0J1#2UOBa>P(VP|nF9WF{Mv?pbNn7kr`^sa5*m*& z3Qx<%1f2eJ1jE1N)GEnQuo$&PL;QgVFtUq(_*w{!U25!0!`muH&Uxa6QN}d3grCRe z%;KaVnpH*Red6oU3xS6>?>EQFO#9DumaRoJ9NC?#6aCk7?;m%1!9Gj3w4=XPC;(h^ zSn087qS3k^*~St%K2f^e9^U1C*6rd{4A^Xm5ZH=enc$0iJ%> zXP~>7XpF_M-T}g`6Z2N6s#DtBtm^&s@-~ekJ2Qn(iu0Um%kC<=!A))&bJ3(O3ZrC( z6Q7fd=*L^>PiLIEa$fd4{l$DjcPGw>Z#S$ZC~+ih|I|DIFu6hAsL-@~ zM-){a_FDZ@UUPTqN6D%yZFv1DN?lSoa~kNBp1Z$|i`CL6ohjcg z1QjB+l=iZ8Zdut>vtJ?}zc^xUTc^|=BtCglTt81{o4I}aa2<*8_^~d!nsY*h4;+DR zpq$4>lenwOza9*xp(W{Z&fRIcPT)aOF;CuAa)d`8$IOhg&u4{gO*R zPDS+I+}eQSdE(h>9E;i#q4qruc3l5BEaLz_aNAkR+Q_HHbA>(g50HJ*pg2{;b=uA5 z7&4&-@Olbq{Hy+|Y9>J-nO4>%`=*v<;OeEBO>F)1^PKdf>M+LYLk`3HbYfon+^Mmo z1ub7o7xt$I%O7Qg3K8Dij0`G!DYwa~-KJ`oZV`J^th* z7N==ZO{L){Ee(yM=MO0cR~v=>gR7+#PsqeC-#$Qi_)X0R;gdDBAB2i8pFg>7aI&QQ z2Xx2*(EV}uQLBqhWt!!)*!_7d>e3Z_B1l=qPdU;5wdG&&y1<*B$F6%o6rfj|KMBJQ z^lDr~MeLPCd_Xi`@XxA2#^Sdyat9jCtnwice$*Y`r^=G7Z>&)B=Ltd@;OT zQ)Ggjk=48+sqF4OkK-FQBuv*=M@?Au`^{T(^_86up^K1O`*KUkjv9kFrf9`CZ&`i+ zv(@*0TLSDyB>ax{H~Uy}PC#$>RTHf*y8*EDxyBH_IarRCbW#-ofR|%N9nZY(4a$9)fm)i&2NA~;?geDrT z48oeV*tWk~tyHcD3#BC2;z`rHv`i)YL=W9#4vEf04mdi`SZEOFCqMdcBrib!RAKEJ zR6tD+;=&d{Ml;CBn`-*eO|u{!NMqfeGOMb({jrwSesoa%H%lFQk)<9pbRLuIJwM?E zm6w66_q=M{7t2Y-OWny#CPI_9kWt2w6-REpe}FcjBsd!~A-YZTJ9I@~KFn{oX4VjD zlQb4h0vBJQ8x6iKd^oc3&ILeE;%o^!zBr}(DTy2^b1{^5(g*O+`5Vh=^Iznj>~1IvF9{YSNo;}z)g;!_7fCi|xIft!ex%I{psR{CfnK^eQjogIX5Tj0J2C_4P|8R?Q1KxVe9c6bn*`sLTRj-l(6HpE2 zjW2nA(7Dcg4RaLGwXfAellE1qwtV>AX2rUqCC7j14Xl*`xo#ZET8njs;xIkh<7 zsWCo_WX(RX6r>`Rqd9KNRMtjlRA@dlyr6)LB+UcU)Fw1V;91^IRxDkrXC78=ua*q9 zc{{0h(`+ifYTc4)wm128YAX%$(|$(f9sj+;78?G|n ztd{#Yi#OM9&DejhdU*d1$(?Wqx#Rk#rC(D~Mpz4TGR^m~g-t@-lN+(BzA(KlThzX5 zv^3?{o%aZPv(l2mo(GQxy}#?=d}uInM@Ii&9V%=?%)R$@C}0Xq^Npw z+gg5R_3=Pxw$!WaWoQmf2nM*=}C8c8=c1&zym(nAj8BVZCAJVyrq9=-&0G?U?PIR!bO2r1SIX!*q zWB_+$4TA-$rj>vAV6i|W5$O|))x+6Q=0&_CiYnq+;g>3Ol&4uf;~yH)(?Soq-rwE1*-atBZ_( z4LlDfZ8iDb#bRw2|4mZ>4#)L}d-##f_UhoiGZB7tBdHEyL5=^}&xDuGF`A>gh_WgU zz>*nprAJ?u-;7N)R0F8wn$h!gJ~QOp`Y^$5V>WcyM4M}=84W>OBK_cRFHAzu!_!#e z&(IMg?-D_|;s4&Sm#$JWUFCYk(@9U1A`AjB2Z@}e#6f`?LZQWRDcQ5{!Pn1Iij{}b zM?e|3u7}z&>boD8>0F@ZHeBGLFA~!5nNh`zMu=)Fq$43~#QkdV$$fECumdhxXnt)KA* zxi{|0i4TW8jItypAT~1bOL5vCQ|7Mzj&1Qs4O8#<(Y9!w0ie>b!asB6aUDOh&xW$` zd-9z^4@yo=#M2qlrrFgryiE2cD4nAU(Gx44_rBRF_CZ0Kt}T&<*#>~%tLIR0@)?pS zpeaE1=gjKXt0Sc~bJSdK_}1wc_VrxjIQ;dWs8Y8OlU<9MZ=X~!*Eo!z!(GYe)HK4+ zR&y{)keLWc!Xt&qmQcm003zgaib#%;IPs56Yu<-_AN%X&hb$I?`X31ldp=9-BEPMJ z+*Dz1;{PH^^;F%EmcB$TbD$Y!9P=VXHSz&LN+G0G2SR&C4r%DDtN0vr9eiGYq1BAn z#%8wUiFHMLNBIOpZ}-QToI(XmOzi;%z6u^*zRpxA1q>z@tt_j?fP+(bZ{Hz)u0Fr7h-mRc57d03cG0H@eiM< z2w*9P@`$$5ssyay?L%(4ZWlu1NBlJ%nMw3MF#!B!TMl>Q=7`HVSphT&Vcv3Gv+V?& zRPy#QYCW#u`)zL&-n_?KsoS&-*txApf*I6S>T5W3ti%-aPFk5$Ii(N$XR`7n9e+8!1kOwOHxinYx` zhd)4fRp7l}dmsNrvZ+Lu{*7eA(U$nEDySYsWXO2C2A9avk*Cg2D%u_ru4zG=W+%OlZmQYHN=nJ@9ie>pSUL7# zfhd`@JK9lnLF>@v{cRGhg;`HzJ4J}1M%FwlO>{6~@J%gi#pC;^;yb4W*^`KPsA9$7-gT0}}=uy>tuiy?r_xshe zX*#1Vq@dG{hv0(uAc6CyfO^g}uB%{od3czFatXFCL}1!4?_$8)T-uh}u7n2Ac7>1Y$3WQ@jV&V)+g+y&+ zKmy;qV^pCqpMJJV&lb;wx3#Tf*UDkS!DARSK^{#d_I27YP#-rN&MNeRi%q^DGq7gu z>aR2OZQ#{Kj>T@3GE#bY5!G;LYJ@X2bFIlb-SSWR7G3l@J4;w8FcR&ta z(X3*<-kF4g%<`j_&U>cQ2yNdO(^lnMeHVSc9#=HM>QdEMvf~)I+7X zw}{v`89+jE#5Ni7-mzSw=$*?0pF= zT#{iGGce0*Bgf|qz#S~yqV*1!i!t2D!}dxm?qWDyZN6b!-E9lgxUNtNc42Rw?ZtBFLZ4+aVhH*cg+V zz&lP#iI=|SQHy&1aa7z{@h93LiyyD%$R`T$5~y6oQI7}2T(k{M2~8_b#6x6N?Z{7q z&7r;410wG|y2oG}F=`bGpA+ZlmGtl48-HV-u-eiC&TP3vmgTRhb6nX`;1h zlrQ|LIg*|Im*z+l`k$I3Zy%lah_}M39$5@ew(>`rtNn&g$s*V{(Q0ZNMDH8kO6u?K z?w^XM$~%vrzS&Ww{TqHdR)F&PjsD}|lt)xbY^`qFJ{*%JR89~mWVR|E6*r1jwkjz{ zCsAnNs%L$B>>B=*{mmd&1w=00YAgUt>WquDdw8r-ig9?jO4QQ!Qhwj0i`+v3pdR#z z)(M!_rjv?iimkM!I{^_f3Mr&yq+={=xvp;^G2!_QQT}!2)aZV?CRTlkgu^G)c$c)V z2F8td!6CR$LY3kON*49nkt=lk?QedZmLvH359A1D|KEr&pXq*?+Zaqu92TG%_Y4PY zOE;eTK6``Nt)%tto&MRhy^!%Koso_~N_!i<*d}}yi2|P(Mn%(w6!pa!r}8D-pP|;a zaI^T_1z2kTLrPD3sZXuVtF#G|thPa&#-D?((unu-NlV850P1;y+d8%rPFw zmUYO8ubP-Q!k<)>4Z3TrktDe)uNcL_mke!Znz(;(I#xc7aC!rB2xla*?ylM0Opskl zA?Jqb^T)J5fwy%-H<6o4%x>_%_HE<_bb0S9!Z_ynehy7EtAI{SXjybHQv$Scy(01U zX_gk#sp^`9soi}*2MaeviiW+zn2gay_JAm^v|8FkezmwgMx7wr(e$Z&SV!GqObTQ_ zx{!lTEk>3zexGp6N$e%cg<9S8y4j{fAq%NRaL9fR4z zFA*(d4;CL8W~Hc)5E#1XdM0R0Aa$M}dHkonTHt>OzjiqbPm2LW{V=9Q%MQ)Xky$Yj z*S#)0KuClp4I}6=2Qi*neo&qhGNztgdSWt2u|#A9CpUU=hbh9u^3oqLYflh#}a>!iRrA4{^wKZICo6z=kT(D9gE!m)lD+>FOv)rn(F zO?xwQ`Q9CBjl^o*5(s_~uthS4ayMdDbTXfwdJwnvS^33a7+f(2r%>|qMuL6sNH^7Z zOJo{vqXpRfg*Er{91QtXBdd~URF;zTuuu~}`<(K=#qiMH4WvPII#0PcU}&a{_2PFV z%Bq?7N9#Bup*i4wRY1cc*_y$3;f|>v z9z;V&TXF&xuM4Z_kDYyXv67dpJ>O|>=q7?8exst4S|@ZjI$tJV#HegFI#1EGIaeMb z&|iy5v4dH<6HYV1%AUzu)3s5tW+iJ+h^LPnzLnfG@swkjZ}bQ=5`KG2jcZQ|_o~n0 zkvmJGJR+L(5uN4ZP)XxkoU~hTVvPBWtipoz!W(hb0&GxmZb-PYz3C(x+x$93HI~Ck|klVJ_q_UmG)bokH_m~^A;44x8 zP%5Hfd=L{P23+s_OF!2^_C{Z|?O6^%Hqt+w+M93;D?bvjT}#0G_y%pkD?u+5fCfR1 z4Oy#Ljkdf%Z{Nbl&kq+CJ5VK_f6Hdx3}z$P0NNm06m12&m$17U8q=30cd#D^9^mzg zmMjm%Ta+W>;%&|Y${=qF{KM(`?S@seNB4EMf99Vs+z;#B=RF5|8$8PAziFP}+up3d z&R!|q_L+TK#xEK(Qo!ZOKlhpIJF~ThR1e!BI<~iZuw+7QVGu3OcVWzE_05PIJSOIO z;K?}SL?1IaVLhEYbv2JhdR3MjPOd83uL~#>N-En0smmJ%1if?Ajo5+Gcftlnu_$$@ zpXVn#k(-;g=V*yCxz8EjAgtwo>tPX0QTTDvUnWr^=a&BM7rZ+@g7r4<=>HcUhOF?c{f%*@8iZ7c~ zQ$H~)+Uql-B|%aHo~R^@>Dm;l+5PBb_n@w?u}$PO9;xnj3f_0QZo)X|Q>Ldc6oebF z5IDc*H&R`QPWYRWzWEm=y{C_-)^^X+ZxU^Z>Jqg?B_N%97Prs9s+JJ8uQJ+VyCs}h ztn;Ci3?LmJMA;qw9e!iuw|(lNIu+!L`#dzQ-sZ(;+fkXlyvgYwZyJxn`I-XbJmeN|o^J!W99*XU;-BO3z*Q=R%)vQm z^FC>-yV+(0)uMuNFVX(Z4_kbxNr#o-g!v%+Tdw&virP63_hVW9)4&}P~f+J^PXlRD8X|@o71Ou;x?sORrPvT zEjVh>T#pRj5l5ywPMmB|W+8Mst~o0!wSWIAk?CS0?6r$nH~s}}RNce#j~z@2PvbiQ z@@;Kc09}R{&u{jF4q6#R-+(UKlUbc7NPc%j?4K|*`G3L8>V`MM9(uv{XaSg6iRCr( zNk_XgZ0h6ov#7S)GPY>v4{>Z?d;f>si5!4JwP1@~HLrN8jF4o@Kvfixk6_yhMzMnq z9@y#8IHt@UrVZW&*%5~28X9D{;2}m0&P9eCwY~ZRVU)-ohWT$U*;_kepnH9plf0E| zfEPLD%Fzm$vDhNw=Dgq2cUboPp-|`ul}>eis*;jWj8sg~MmenO=5lhFn!O0&6IWA0 z&jfv{`n4I7&1|RL#bWZjh7-{bl=EzN5jGw9L+4ikMM0**e3e99)5TQn`_7Ld6X%at zKmQ4}di)8sx?cSo)H=w@Xzz*af0@*%NYP~N@edF(jO6Xag+D;s#^b&8$zE@&s5Pzl znq8%-dGCK%G&p>!;!~?&F}p40pmsn8iMsm;XCDhQ|Hg8PZuC8afUL5H;6t_=jte~Y z6;OcJ*Nm@ec6*2BYtq>_0zbQ=;%909*yQ}4T$VOpv?;H2<0n*ULt4r&hp&omWq`=i zq2`)JFEfH-0K)ZVy_55h88);UPA(vY?dwvhuuCWJb|4U5VgPi_@Dq%54hhMzr(!>f zB7GKFbHI~Mb)f$1_26@8(Ev=M5S|v2A^UQ^_)V?%YD ztPd`B*KdpP^WU_>YFuf3YyGi-w*@9e*x`2YVwUWiKs?9h25iJTu_tvAI2PXA*VZX^_iF<-|wR{E-WZH>?_RU zjeV8#vpy2XzuL%lM{)x$;NAgyFcxF-yPFped@gkiH)?fm!iAVNP zb$<2Rz$5#?O}{*;-n{%95|$)~w&|=2OL(rPnpaWIv^0|V6lic-pEL;v?LtvNfXW$* zUhFw9G<#NLAEmM#uoG0j1D_yplHI(aH>46DeAr)VdflahLHwh=!&ri|K6UP=Dq*^1 zil{^;UyJL4*bb43+T)ZnCBQKUBgsi|R|tNck;3(O)lfGHG}%KneRj6C z7pJ~s<}0iD^=SH65Y;Gdidz@8W~m7yI`l$3yOI|yj`|3ut*B_AKg#lyqwZ!~BjH&= zk7hK5a1s?DA(IM96bR>7C)t|0Rnr8Gu6&q<%J%=S4L+OB$_n~TpTqa=RP$qeE$<}# zi#_a6X33;{X_iQ*Bl9Z)uN}%oW8v;Izg)sTScmY{2*!J7ju1^gS524I(29&vn@q`MfVd>;!U}6^&nXg!*81zeWvUt2{aK3FbnZqT;ZT`>Q$z$*Sc)Sti!Im4A|4niYl2EmX zp4regnL1T!G?Ji-2U(n03^woj>Tfo zm)C~Rzj$7C0q1r$ic{J-6T&}-;5J`l^wu{!_hQRjGJX56i9!nipDDr9FBl=d?T%*T zdaUDNXl&krsH^9mU#?9-d??Iq*)GM2pRH!rMn+8oi2)r~ze(44!ZCgGJGf`zduAD0 zmlq>AGS_dx_Filh$-n64!+2?-jXYuK_50wRiETR5S1onx+`kBsO0ghRzuv}vLuZJFRTsy%#RzC2pB|6U$h+VC93dV% zoC@>*Q=*eWKfSH$=)X&);^(d4Yq*~sSpr=mh7P^9)ZK?N2KV=m^3 z2fiYb$y4{_0PT-r8*$_(#9w*}+OZV1-Gvn+qW?Slx}FIqzyGXqw}SxxXp!$OOK?5h zvkH4j5j&B@B>TU*5sN?O_vE7eZDGVFh>DC{H8>4Ao8~1$fGSU~sBM zY~cd_kMvWzE^KM?PhkwSgik3a!WfACf&BLuyDO4Ov5`kt6C5V+wGaHrV67XLkU{KP zsHIc!$d3Uf%6RV@ikvAq|3B)^Ix5QT?fZkMC?O(^fYOb$v`8~_H%K$2Lkfr(Al=;( z0}kB{D&0uO07^?sC?LvtZ`5-R3_R->=Y7_@-uZ8ad$DHjz3=O~_TJa`^G%2(tV&3^ z4_>EViFR%o>SE20)tDH_h^0V$3Gol7&qB{jJ99;qg?mlj-_D$tIdvKglMykI5$T z9XXLn zBI6hukp`|n^i_+ow;|L5}OTU7gpn)0-v6NuWUVm`Vm15!`UUm-zl`bk{xYV2$`s!i*yz?YxSo+u9!E)KAD$$4D`K?t zJ6D#n0wpn>85^ruTMZ|L1wy6{{n~g~q+AwM-qRqJ;a(KLt^$urd3YE|8?G*^7*ffr{Y7|2JdDN;^#0xibll|nfGFvuiv%^$s!Td0Ns53`buGw%!+3O?v zdo+a>`w+SGBykRf&IfwFp-$$aeHD?9@W@IjXyfH(6Ta08t9S*rsNG|uFr^gzHcF&6 z*6Ua^6bvg=N%@i^qbP=1ghlpLB@sD9A6_Bq z{6y|tdCSJE1sffB?VkfhFKSu>|&;H;(Pq z`+wT2zdmWNK0dNCns>@hokS6|p9G;-=WS!Vnm*J_se8w}#SFb429KqY&Jk&T2AFq` zk~uF9J;%~?6CXkpaKOfG?Q2%6R}Cp9^CtK5`NK5Mx)a;aafQyqt$j7>rl0F#H@<6;fR?M;a9qBBXmM%e2VD+8hG@-_t&!z zu`ZC+|F)Z#$D7mM_~&Lze>;+Z~p z%v~m-C%+rW|OV%!i=$d(B zYDEV17OH{cIo$Vi4%45U!vg?6;KfN=hzN<|xs;i4-v35GU1xh1eF%VFpZ?rW`Pauc z$Dp(Y16#=e`>akYxtm($->h*PIZfGJwB_mV1Qq@7;+W39a10oQ1ZM$(JB zrh&Zt+yo8M?zp5%G)~-L0sIIn>nJ?^YpmO=&gJjvfBqN{@pFfh0q0F&%h@RO>qF3K zkDmmb=#b!_UfkRQ=zsl6b{}OInLmGJ{*T%F?y;f?)C@Omgom-|aBOeT(KmQUTeJ*fW!9m>7{-3vMJb-UP2a8j z_6zlCWp19;Ah?UJ?ncdq`{-5B|y>rNs34SJOt5*S$kK!a(E`F(Ft(QcC{x}FICSJLNXJ0eF4h$aXH5H^rvw=ODK zN=F-OLK%W&9ubwcXqv>%{$rAW^$C(d)Nhi&_Q59*>~kFeD%n|D^%4v>y4R};l|*5i zqHf;%BYi3PZ^?5&H7ynLwap6 zt51|ZP+#UcTV3uIL&_k4rzm{&B(;IE?z>hM%p^T11I%hW(JX5D%${&6urs4At)h`eikl2G7l0juz3sT>zmHyy%3+4%R3Ea?ATq#23xlVZhpN$&B zW8l}H@o{SR^4_FRG~xwY57p{#314i|mK0*^!q$eE9Em{_Gf$FemaDOJf8$AL|HhNp zM6sTr7KB}nd<#^hv=iXHLZBkGwP1g=%cz~0#stHLTBa39wZ%tMN4J{*R2uFn(MqQF zY5{h-DRnD(&Me*ul@)DSmTb3U#Qj;InTjrr129fF;n|ezyrF(l6DC5@lV_EfrTD-)uCGL%u(Vn3!kJxO3&vizfsYiMPR zcpu!+ZH;V@f+1dvGV#L4RDw61l<*I!qa3#=jIE;>9V{58Zd42sW>F5qdlgkNs?dc& z10Hpu0}WNR5__z4U$i6$Netvt${)7AubxJj$OranlW5~6nrPKXmo1lEh!KP{Piuq- zmxEtu5ry$7?Lvko-F5KN2R8T-md=&2eIl3cJiYYNO?dL==RZKdDNiva8h=xsLRe2y zp4LOoqdbW)b)Ru+C6}0%S~k?vPd|*s{g%hf-e}@ez&3NMz){}5d7RNBYA(`dl&Yu8 zlo*HEjcVdT?@R^^>H~ z5>%!?XJ^aawGsLY0E@)05TZyFB9=cyVRWMeb+@~M&&ihA0)|TR89`1I@lC0eKr!I)M;18rg ze2K?KcH|nqjjZf8yxy9hO|4B7D%63N-sId2E~r&bfM(S%Nu$*(kOUr@xuw06G&?O!NENW=w1fwgPi!ykC{|>mFNWmJn@4vlw&tAu#*Bs)LS}qkz zhxcQ0eBFI8IBq=kv90q&&|on4CclW#%EKYUsRHV*oy&gYqJbWnes76jazfJq4K+Nj z#=N@&`%?)0iji4=h&Vm#X7U;RFN)+f8L5}CSZ8|p5 z_^u`O@_JR<_L|p4{{?W@;(=^6)Fy*Q=6Vv3M`-sAzIh5mmyA`lSL?9i7A7U9`l3m9 z2@NgJXf|XjLmbwVrz(39wJk3m?}wPx*9Y0L+V4%6w(Ktv3OC{$K5Z@Po#O|NCt~(| zgamI;kto$x4De(yr9b3Wrry<$zZC1aH|6~PDF_c3v#rS4J58e=?B+Lz0IRJ{&5%s@ zHk@`voNsfFHSerfJ7!6B$}uH-!0!4;PsuDNWCSgH$oar1?vyh^AnSs2-5gayXCB1) zbIrP}RGZ#-6Jm!jk?0nF7p<&C9pzbXecw!=u=7%Fuesy%9T$g}2Tb$#sbZ@!?K%AK zkJZrcLgLY^q)dXPH`f)$>V-N(1m^^#&df(z-+$qO5Y)Ym#=sO*z?#xi9CgdivJ1UU zWtsX0dEX|>GtE6(792P2TcH&$J0TouKwlb(ZUoIbY$YmRxnFHTD#>#%-qmm6iE<&) z3YNJ{)L35&cmfA$Q&kh?l=D+} zz|k^vTG}zs04_3=PhjF%p+`^i4)N+p-`f-J<24N8x)sTk?|so;atn?5))h72PAFW& z^AZ>HH%%SewZh=a8R>k6_JI2v@*^9M{Is$WUkDaHIAJJ(`sQ%7;EIqu(^4N`&Y~_< zy*j&?M@XWd*7H3jKgAo;cI6=TouR%158;C`8Se{u%9C%VY%1yEuF)psuz9A<#DdB$d_qk>OoqzF%K@wx;do%CPwmt!TR>1 zvMbG3DC3406`D&jZUuz}jUsf;wg=PF!Tuo#}?b1BSYd&vSI zA*5|y`EZ_k+FjUR#l){+dn57lujFWcwF~??dFEE}W3+)IyRS!Y2)P+A@>EX=|D+F? zeiiJfB)U+oO4&wXdlEh{^7-p`|J>>#w>;?U4WALo&)JpO;#PA4j_K_3^I=IP)u?Bt z__-ms7nw-#G=c2Nfls4>baCbUi1KR2Hr1Lu)SN)Ni^OigG=z(%m@{Kp;a)k(1kR1= z>BMX$qnvA>ZqQK|MD57&_BNGrb#3O4YDVurUm$M9=@)A2j_X;P{F~1*fz_Mb94pS= z?Eg)0b#l}x_ouWR?@wtttG|k)U(NOgYGKPcsN^U^gih+4z7g^f4ZZMPl0lUu{mEc5G-3SrQ>OYj4O) z-yF`HNqbp|>uj|$JrNzA91|Sj9dZ?2rFx`_@J(ljCRg_pWo^qSXp1OgUVfv0CQk^Q zc_LA7E1mN*A~YTtTL+re!H7-dC*+E|1MeDJNDU-;S#g)iBq0Ewk`=k1Rl|Jrud2bt zJl-u<-&tg};PmEA(ETdqM~b}eo@LG%gBQhZw?X-L&Vxtc{0~ei(53tXraGRoLBB;^ z#8~U>>>bZ6Il~aB<1WRmTA4M@s@@0&E?E7HqTGC&L2eKTUu zJ$)zPGjp{a?^0;B!nd2bq(ZKx1V;5ztxvu@T9L|wVtQ}g+SBBS{W@9QN43C%-&;SB zBu#CXxx231>NJ%lIuI5+k<1{o&bvSuuXp`la!<}4Op!^QI*U+dK%+H$o|ks8A$O2n zU-O01)xqTu;n%#=e$NA_8+t47^p6|ne<;w+FMM`u zB)&&(E@uI6SOP!{36dpv_yA`lQh?v}hW_YH)Ui>H0l#}($Q(k{kz~u zQu)zXCEES-U{VmXG#!Jpdt{J4V#p{C*^#t?f7+y-Lr;!7e}I z5%}!qG}o#A1*qvzOdpMU7iJTTsre$?j^2Mlsa75t0jg#zaPKHZh_ucm(TA~HfZC5km_vY?4>eby zNo8A&!qM7M^wq8c)~d7+<&bYuybSFPv#? z!3UspsgpY#>FwJ#_RrW1NYy@7UY~g^A28BzJ)UWx&OXHG^Ra<-oDm$CpFW6_JK>|K zsse9wb8g*iJKNwLMuc6%%IJs)(#F)EJK^?pVPCooN>)1|le z2L1CZj9uu|5qU)=fh7p$5`-;`VX50b6zUrTC$*g!5TY=GN^Z1%Z~d4LTZkX8zHfz? zB1yhZn(XKpfG>duBN?)YfH6Mr;(cMm(Pj7Q<~Cs5H`7#;Z2vX%05CSJ*oS`yTq{q# z2h2{-UUX@=nkasg-&}B}?vuk;%Pu};Tyln{fOr+tpkTHn8Vj`&eGVF(n6hoxSkc5j zH5uxt%=wrbMhg+@FQ=Sg<6x9mAf(RW?I+)M990#I=d`i)%2!$Ivi?M8+=R+s`A|ZB z$6MY&r7SE>wB#l(zYNjj==Di3EMO?gFJnzk#|M0^D2K?cQtvX>)4Li-cm@#i`^)&Z z&s>D!`_%;;ea*hxPnnj2gxXD$0wx{IK$8uQW^{9W$1q5%Any}(|Kcxoj+VB>xNQtj z$P7fG<{*V}_RBL8GlDbAU;9Tw<#2E++b^7>0+5T7ES)@uKq5kzIxqbAhy79nGu=U_ z9hWu{{oLcexQDaZ5@Wth^LZNOpS?Q_qUYR((yzg17&$TdC@my#d?icuu7II!ByrKM zN!pi5%ocQQN2heYwKe%V^M6>sb!_`;yQR7hSdpJo8wu}GEa;`h8mLNdw@RnXAE5Kh zCfvAZ%IE#kYCAgf#ZV4KJ}~uLPE7rwmK3e{vrw>Y#N8qMeP8s~KW*xO47O<`yxrn8ogQ+)HkqO=DQB%}B!BHdOrr1zk^8c1R0 z=T`T?V_sL){GqA5K(~o@#*E7$qVIxvh7%g`6tnXUA3mN zTD%m?i4+BS| ztF zpQk=({|ZXRgFgaXO`e!M)EL!AeP&)yWi%-37{ULB>)_dDRJ6|MCpDjSh8PL55n#p`4BwLW8Hy3iBDar3HaV zQO=$aW!zd@uQ`nIy}KwbUGc%ZKSL_ofke&!#!}wvpKAZ!`D3*|M|$V&=jg1B{?xh5 z%;{?-UCb2-2-pc5D<3}oJ+YZiYO7az9P>*UbZygy?V_ap#l%HSALf_c=?wJS4!^O@K6Pa&*Vihj8oVs5>I3h%;6Q*B5Q>^;dHTc zLXY0XZD@t_XTqsJ>&c?{wMW48YmJwtf+p7%LUEU!>03xsPe%Gre35=U`f*tEWFqO@^SPh+Asri(A(a^dfd z`R~3D6;YKMrvdyXlZJPv(H^-_5vrUYTe*jKFYBDYhAvbEDgTt`t@-FLAOnul( z<>4_*X8p+RQ7apvrmL$SVIY7gAf2SlXB@5OEz~67(^*)JPOyRn;w4!5Fk-UJ1p#a5 znpY;9r)pmN@Lpzj=hwx@soYwFyY=X?sFzfLzFJje*% z5jJWHNG3EUEsQAE8EwR+an}r9Qt{)@_I};XHt(N|CrL)q^N@m}jnv9meHv;?1c?Qx z>xRzOtGBL^4Xt{Wc^&(6S02}PB_CLr8eP4gw0yoLdr*t`_FWI?jUJSZa7FX69mTe( z4XN`L22BG*KxlDO> zi6!n(QtR56TJ~pjR%0=W`#X-P{VK&Aj!Ut4WwYz|FA*AvN2tdT0Gk?Sz}lk>#;@t~ z6`&TR`V#AETzr#+M|~Z?zXY*9}oE~61WNufU96X!BsH;xau|WA0S{?3tt$6`QYmx zpiZxaQFX}&FMxh=#phkZo^Q5O51faUF1-;s4!QsK3+2E*G${O~ls|LyOo0$-D0h|c zy*CZK)DTk>Q;4*~o~Ey_`VyV!*vOa<->M!dL5irj%oRRxWFU3zW@XZ8?{TScB<<}3 zV7YMu$dvYZW}u<+TkX;xAlz>gY;e%!U7|Umk9Do^egeuKqEto}6AF&(%(yu!v87}D z0$ZaBgOeiBqrb^VM!(5Ndiq1zd6?%IeEk{}8@o{ptz`wY7x2b#;W&O`f##-~*P)RrJ=` zyM!L%zrf;kW&(I{_zTI&d5o&uq&1?dr528g&`T+3CpNTYJL7q+G2lIJNao?4ck z8Wy+`wu3Er99R{H%H5HOS6D4Zr>yWpx?1pB3+df5{^z_N3?D2vUE~W)`wP}l^HzXXv-0EegJy#Cj2w4QF^y&;W*mab&pBBYL#d$LX}TMNEjw-}v2Zuf$HFz7qJCJGo)(3J#Qp&NGo%)X z?WlJ`xmvdLiK$MYii@o=O}r*Wj&wfjrOtpGIpB1?&koKEp|lK8sW3OqF$f*Lb4#55 z$OoWvaYdsZwaHq&!iyMswf}o?1b35=$(qMS1SSe=V(;R<7lrst(O8lPy(F38{5D)* z*BO=CU9fc|S~aq1ux8#i-w&6(*eU`y!YF1P=(IcQ3ME2bduk>f5BB)FRVOL=_L>xl z4G&g(M2nl|#T*%;=L@efD@5UpqyZyoR#9ttH_*_i+9gvEp=wZwJqsVtBxF?Eth^@P z3Iq~jA~i~S7dOmL*{qd8VY+#?_HaiZVu1GT3llqjr; zCg>4tifPhVqjP~QoHOSQrM#L>e1G%|$6eZ;4FxFO1h}j@UPYe1Q+LeJY&h;XK%<=3 zD#L#H=Ho5>&&E*nL&CMn^sBCwU5W*%^vy&lrx#q>{m4*S)E)<6^QL_$lg?yq^i~0r z_`Z8oUe31Q${jTDXE?E0Qa-1Szd(kz0ve2dR$c`x+spR6hjqucVvr^Oaq3B0ry9n2 z)1Kv77g}f48`dq#CKW-&h}+u2f}x5ZdoXT6=NpB%&862QHM?{AY*dzSLLGoZ8yO8M z6)P&mav4)bIp>!5rJXe^sZT-2x2?iGANAuwdN^@wxAiWER$GAnKz%EXTOw)ON&HL% zm4m(T{L&52%v?E{XzGC=SiR3zeeI{Gm|a9_Pf>>*OEPnhXS=@)1`ab&JE%xN6j(&n zWz=L-t|`FS=cmq{>MV^1TM_KdfuflXQZK7^-FzQ%4^M3!Y`vqB<>Zug(F(|Y=;Z8l z%s1_6SBbKU4%8qAo5H);=6Z^1S!2mJkC|!y ze=^g;e>2lKA)wP)IfJo)Pv=XjV;cd>Xjhd0nmJ}q039(Od;mXkOOh+WRL+gfh86|e zp1YNW+mEA8a{dasfxj^>epz*{qH%M8s-9W)b%=}RzD?l?*#JKXRLd}i6^g!&RArXTOpl^fQua>L zf2r+Uws|(Izq^((N{1P)ka(i0L z(Z(dTqDrW$vhme5`)!HW#P9Kpmn0yGSPIiYwL`gV5jItw=D{Z;0`1aiw{eu`M^eXp z6B_G3@g~?R(a{D{ZJbwt!c@A8`F~{T#oJoX&q<`{-RkCfcA(#r8u{Sp&gxFP8+v=YuxLPmaW#;wHopN-Mh@k>>-y}>QJVm@xb!bp1xmKKY&vi$*fL zd9t~dCo;^y`YQ!wi&5JSbk&EnTmCltG>sKcD6T#?A`Hlt ziuW7`LlIq*&ke>WcxfcT4^1{?<${wnZQE7Ksk9NE==kW06!*=vYLST*e){K#QQl(p zI#oU{H}TV8d;AMy!9L_*`G&ksTBuWQmJH5nZDI06Z+LRN`3dietLQM5g*w8Uk{jzs z<@=bdxemdSn(t>UdSB-IEr1>-4GRv2ajy!j6IKOP2PStIB}sTLbLs-NA^LbcuUr}K zx==mZnhJI`pPu)Z0#DB8CA!;%%+W*4wJieAID#vmi`xNdmELx+ebi+}yoR8?|Anwt zDKC&ULLqIF{09*ZwsZQxQgpNx`sQG)MD~JECwtx)>uD_mI~Q zNoSKYJpd(aXeob>o?}5xA<*~KXrT=lOjM9W9tJeWJ%7+A*sfCO2qivTgCAD0YtTsv z&$SWjr;>W;z)27>EL@p;D?%q)KlxnGDa=Sw|0PM~pP8#`pat_3T3l1g>JzGPy+QGJ}!s_nn)eti#0R^V;v|i6q(RNta(W*D2UJCIB&wW{0P|3RWqQc9Jj2Ujrv4^;&x8i+U*= zcH{H^096$P+*A^XO031Hd$iG7TWaqho&mFS(2%dcD+P_>DrGKhIXvTa-y3_Lb2W(j z@P7l86K@Ox@f)f*qZ!>z$Cr%IXc~h?z>XI zH`ntX9K>_qGW_x7I9AVu8c>rT5EUE~hbmm|F7)X4B*~kEWIM5_@Hp#NiBc)WBRp}e z?0J1IMUVbKPD;9mNdD^WxX4sBU-!M%LwNGz1M9Yx!qBZB(Ih{fqM|JS7aW`K&sGzW z97cS@p8`|h+Nb`Fo4T>h&~IMEgLx90WqUy+#J9{p{g@C*OK!CeXVIJP&CX8l$dDOz znW$LsaguM%;RTv$1xo@Ox(a5XZbxEEeIrM1l#O0VSs$?c0mnyFjGaQ9%nuZ3N(~KTnueT5c#h`XM<`QG8#_u%>z&$U1a^+CI69Ns90^s^x=*8@EeVXge>zI(*Q0$a@!eod7w5U`VMOmZtT8#uPo_eLB zJQmQT`QV)L;D3mp^-Hw3f>N)Akk`;eeV(4QL8~{A2wz)!L&4ehV|_dGO~ov0+#JEL zTZ)f-OM~Q%(5@TB+&dHYZi|3q`iPUgn-jI0ckI6sUIf=8W(70Cb#Fb`E|_azbA{v! zlGFEErOb8QO!x(0l0E@o+TN^TKb}mWi-0ds#i7Bjwf`dLn0{)C2f~>5h_4%yb)e9( z#9z^>25fWxCHA+SIKo%xJ6)PCb9)#jnmnf*VvM3bhr(mCQ`(_T0+NyF&2`u|TJY z3sw5gci+C${O{Wm-hY7j|J25)1&b@oS-)NF->od#D5CavRq~S^JB#$xmdH==V;6~S2#xa0q zfNf*m*%!u_G^aMK47t!q?vdq>3p!o`D-OLIlY zu2*chtG2r`U8!HI$g~-$ZJpMcD;PKIyd^j*ZF*sPR;Q;x?>s>Bg10npaMY0($Rs2Y+b3n5xOtAt6vLSh_59D|9IE4 zVaJyiY4asNKK9NX%_tHQGVN`jjbM4lFPFjZd8(#EIILtx)1=o_)PY8c<~@4)&Iq#w z4TKL(>q$|>E6jse-(%`iXJ8xw(CZxn49^_Mh5F+$geT-bD!FufCzx>tSg-6J$Q<3l zSNi_eC(IvXRnJJ(gs*p1g(s#S>@fnT5yIr?AVZdbYg?Z!=p}Ez#j@bFa7g6N_3s^P z;ie;A&2ntx=rA?5rI%$$<`YqCQ-OK2F@&+my~;ux-q0j3@*d5WRf*Ko&(^n9sOjQc zm1V_@t48uG{ttpMU|jk{nWz3?)KJi~f&-Ha<~F}MMUsDjLP;sE5=U^=H?r?)#gx-@ zkSE5f{|PFx13*QQzd%JDb;v)}{4MC(_N-e)xcuq|`6z@Xmw_0%|Kf7VRWWKE4Qt4+ zr|#3wrw$Nk?`$Y_L%$AFM#_FtV2rMb9ax3Ddj-OX7j+}N8O!@p+}zJ)M;j+Swv~wH zDo&1hXDp+1jL1;&%bdT%ZEq|9vuqny>8YU+Y-TVXiGx%LFVs} zu?1gA2^$TWwoN#0dJ)vecJl6+unW_REC)_OT`**+ZQAYZ+^Wg;e?)iG>;0&HdDrT) z1sdfjzZR^yoG^p+P=x?840|Cqb8Z_XDawZ`h@onJ#^EaNx8W~`nzODOB$`93mPg;3 zkIFrt4DXjd09}s!@h>nCfjj`p^d9oRN{_|uDGRXw72VReiPRMD%7=l8DF;e*6~j4c zWbi_zHswB9ivp+7sSYnqLMz~S35KFjwfv!mSX~7y5Xf>cn9uSX zroM4=A#cG@&M+JMuW$xt=K0NjLO`NeR~de?pDdU`SQqw~VlKU|{Y9$DR~`HGH>!qg zEBw!>Ch}=T=PCujsu6*|FaOP|QQy5?8PolVBNbR zT1~??%1U-`$`o5#WeWbf9Ha52t3+r0mf#{xOR`s_RG?q8@?4^jgX;$B7x-c+yMI;z| zMB$mqaBSz=HobHChY*@6Cg;ON!mDnStsf$_rC857+(5nl0PyBK*cR`7z4cIhZ?O&LUwDv=0zOK?ZMa^6*zICjZN(dS zAsRSVt`UAU!;k<@z?^1aI;3JshXM5O)f(LBBfVM#>1!2@AAh(8ucWj|nez29n|uei ze&62=3N|-$Y3F!ZRe~m^@?Hj8&&1)9f)8BVSQJau!J#mXX4=fQOt4P*#U)mm z&>oD}!1nGheoM*a%#e;j1~|J%XU=nUO^B3Q?gnd?wzEx5#YprmMdUEvk8wg=<^Bnk z=i0wusnHQ8_?$U@4eeyV z3nuk>Muv|OK*7`+FNV&Mg*Yyl%6_Z;|J{=O_&U@U9!)?*-&?)3ZXZ!sBW03^>cf$R z;}ym}7B*TzE0=%i$+Mce(wMr8_i>-beD=TOe`UEnADeqSQ7y`dyPHt(kq z+=%RgVdk$bmjo5>oy1!Z zI{t~bV475d{e`z6_>FLQ{q)~3C_d|d+n>yKe&@!|2j{}#!xSCKh5Iw%L^>>DBNDIE zab8~eQ&JBLEvJSkz~nPK)))u-7-(K5i{_Wr5PJ>*o&Sq4;nb63erAf0UTSWO8ZW6+ zaOT&9^WD*Bv*YvcqW&B&S}lj)`#4K9T8-id*Rva@^8oeDw#p{}S4T7_Az_Y%b~Q(S zgt*68{lZcQQybF$%X;jP0QZz&iYkKVf_C1b>2E zYAS-kx`u6j!geW_`{Qg>zy4yZXa_4y*LJv}JQt`lYLXSoT{rWJSEV!Jq^enHL{mTRsy&_N#ymT^)s4fhCG;Z4eUkk4L$=L~1T4R5beiF#Vc_47@X6{X;;(eOT3b!m#(4apFx6 zedZ|Qc!lM5)CO^k^{k^u7RQ>wth=F9c~Mm0-WbZ2jkR!D{eEAbL3<(PLNvUrfDNu1 z0%d1F4U<2dC;0Tjs{RR1(my79)Z7sP5$4VKcZdypt^%bFLc6zj4f8MVM1}^RuwXZw zxPC$c?> zgfiMH7)9Ui!h}_-OaUvI6=(k84-F-4`IJ$wKqZ^F6jxM z&CnOvH2JpU(bbiFcLxA8Sj5q{>WWzcdU*(Xj+J##Sz-bFu9gM&Q2uq$1*0m_t*~y= zTeNOD5Wiwp?#@+58)F(mcd76ZFb>Q@7RTAuMH*J{sVNasneuw{MS!Y=D~Ub^eGRbi z(O4OA7WhxH4Vsoc>oFk2#?b(wV7&E7t@N(U0GNEXUo0s!l#@~VZQt=Y^`Wa*8%T7F z`wOq+sz9Q6=OD7L^_eS@Qb3^UB)XXJYNbb}k_3TDU&5{Ey^Wr;h8t%m!L48rlXV#q zRk$jI!vnqS_kh^`H6V)FCsWy(%qL#(S!aRArtmjqZB*`Nh_OSJ0yv5(>Yd(FXQqt0 zC}kk^l}I{U>Ibk#P|B^tojWrnVP@n&~*V zeR#qv_76K~fzWp89IY>4T3~UWmBCZs`r{qy7Z%q-D0FC|c4#LbV0+31l8%o*5wh^w zYwrv@8uBZj3)1-RgDj6W{w9f@1v#AP(+B#1iC#hDN9$yA9xL+v0Wv5O z*PK+KY8l5b4#39WXhHx--3deYXG4}~H=4scrnX$oKhBp+foJDmuQ*MSRZgh;#G0C1 zOpwXx`OgA75-d6m*fSI z$C>|>bB{65`{?Dvx#k2P+QB@EP4He62dlDiV&0xtM3(2cbx5mm0p@be)yW!gGPV^H zRLdSbsUjA#T#MSxRf|#4!{T$coOplPsc|nU?#WLu)iuTbE zcHM1)g2i`>#jP1NgxWV7QznIQ0)y6j{aWrufE_y1iB9+SfHsei1;M4lbDV{bBGvX& zY>(<6_!U+qNd=ZLnc$H#P5D+H&2hhe(L&ZMKjlvqSv$MY^RB}xB)X+Td1$#tz`iUO zgSDrym76sE78dItpaXsLUF*wRqli!0?I53zz^nuP%MQW|L8RxIkd>Z8w0=rhH{bNx zW~RTHNxUS*6cL272@f|(6u8rtw|R6xta582oS+UG@FA?+l?R(Ugq!SrP5U(7*O^a2 zdKPw0k_>ebsxj)JXQ#r)YZ8SounIyg^20xb)n63wp=N#?IYRGOZF+ zoV8A?wJ4GJm(HKM*LHom#+jDn)y6YQ2k^2YS5wfM0H<7CS3ymKlaS*B;U|vv(UXw} z9jh&hjj2l8Y#;?XIu9l8O}DBP6CwBU{aXzRRADAbZ@;Yr$J6w{@pPun^^c!hvum&K z#Sk`n$h>_X`pFN<9C=6{eBV4;XIx9*^vNm+z=k2CVuqy_lpm(P?@Box zu>De^lUda{Wpi7tua*Tm)~)_iwj3*SMz$PlW%xqpx9Rv18AQof(eN1 z4-j&=IYY#O9Dy!^_6qf`yC-P&r8FJocsP!>bR$2>30_Z8)W~nwB^6-vN{R1LahM3n z=Q&c&)|>Paqhs9Oo9{ivS9ly%4fnF-e+AAhn+#_X&B%a*Ba-1!s|NAk8fs&%W{;om>;m_yGIN;awW$l(_=xOcjVd&dOKN!Sg zH$8Q?$&J&dzJD$cURnjz`@g9%vz(aWq((rLkjWrPbH0yQmyM=bZykmc(k{EJ<;d_i zc2~Tndb59$+yHoXECyE$k$FtpFn=l3y@$Hj-t2((=y0`6a|$n=1p&16a2OB`?HQAA zH|{G!c&{t}z_~^f;bqeQnt32smucW5=qWjJ;LQo`OrN=zmc+Y7n9Yi#14OpxWivt64TFH_crrZ7p9qiNPlm^hli{)NRCv_KIve*h&`z5( z{JU}g>`0y^QYn50v5WkpyxQK1E7nQzd0-RRNgc75EV3u_YzTP#obV~hgwclFn@idX z)GZ~hJwofF)#QBM_XL^uHpVUp*!E9zjC}l<{YpLxte~inf4_BC98a0%tLHO`W96>> z^Q}ws3lH5I`RS^O0S1q)NjxF`D*3~9C2h1fv@T8?)HM_#)QbH}28*@xupZgN`@IW<9k_s3Q37Xzb_JmpUCP%32jTwP#*k=_zBXWuy@PhFJWE4TD$q~VjoW-VkaOMHZD1Cif z{cv{{Sv9YFhy@?eP=@)Od+Lx{5=7f{G8Ys~?D%s+K=zuK68>gHxK=#OOtMaQni{enjM??<-r zNn~1R`SiVG+%@7TB7%yZ!BUA-OF zuzb|Q#a_%_nd{=_<^pZGkr}*hD{qM|^iz~GBW!HFho8EWJaz2mdA`;llIQ?7hmS`_W!Rk?Hs2G`w!93d4AXaJ&#B^3 z`Hj}9c1&CdaX1-5j2~~t7kj&z_!q1a6)!3!HpWK_kSBAfCgYX2ez5)TKX}AuTJ*h``!c3a#WG}4^Z(1 zNgMqshWKq|O${TUbux23T{2fGTXNYI1l975^DqL2 z7}en75=jf_YOnbj+_GN-00^$R;D$LYj@813)q;F$y?dwIJFxkYsV7M9QTqFim9`ez zq~UN}wd_1h0F8~m*>k}p=V>mhh&={EUux~^>heJ2-jty&9K{oFx z<7mKcHmc2UOS14`a%Z)x4tt2;=;NaP$f87D4aIj4tEQ>jZG%H27e|&V77y}JD(+E> zKbxs*x?V#iH?QM*(cB6tPa>aKfE>_txpa)l_vaEl z=*YBZnZL`^%6Dk}0ulI_5w!RRh+5}aGKwUtlSX{e(E6vYdT{Ag+OD81%MT~CKO)#X z9mD{+hAZFlH@Jp)MV%wFgO~)M@Akrt%uf0VKS%Nt7L0Rm{Ra}8kEcypFMLRktP2ee z9<7sTa^;lbXmxGU`YXPXD|w5isB@+IfA0K0)V*a`RqfulJ240;X#@o64rwGsy1Tn; z0+W%L*av)}cs^&a~;Heb|9$Q;+Wa{SNpCww1^ zqlbhLk9gi8TN(?yDw9dZ$U9K-&20SRni(4)n?2^}?ZlGcx3tM&c+KQzE<;uZo`zhd zeN zGD*^M#cUz<7l^3DeW-pV2zPUB{{HJqc(V?y;NjeZrx@UDMt#N)tSgEHxi{SWgYWpZzOwtaj=2w4~ziu z57=)euVm|!_SFc?vA1!y){{CX>l3a1Mb@+YChJ?k7iSvYGN0oVaQ|;QD6zbJ&v{D>W9YyGrtSW2qko9TZI9(x*9@s7gO3vjI z59IIw7_YvDkzmMr`Q|9hn>OM{t26p$vKZDx0Na^LZxn@zrT0hr zM>#&xQq_!>tGi4-T&ekyNVJkY3>0mB6h){Fo6;502S6>(r7||S_y;hjLA9T|i^580 zwq?+RsgBwW<{rLb=}sb6wUFw;4_LPexgT1Y^Y(sgv|d6%637LBHs#9v%_c3$}N;Zs_!vcIINP(@z&%L(8TmEh3xal^iW2y zJz2-CWYI1P6^Ki+VvgDuJ>BEkTY^7ftvT~IZ_%kVB7Dp#Qb{eh>Q@fKp;yaa!M-p+ zZVvDr4zwJ~anJ3rj4?Hw?lf!`X#r8Q7u%l;^d3o-^!+zW%<--)tuXAB_QFv*igqAC zT^_@nQrMJsu%=Cd@3l=+Rsd(83{9VH#VNR9cGSzE_PWw_-~VCv53G%x^~9*|wS5oc zwi9M(418FsBGMITznrU*K!%KWF3QqprG<*JTxx}r#_4Dxr4-H+%zMuh%(rvc;b3pF zv=eeHHp8Dh$y^z>y1n0~a56nRxFPg(O<|9M-j6oXi8Cb-{ZDA^+;)i*+$CZao1k32F-E~NBCP|iFGt#;rW6TzSR_lfMnw&Xa0UuY zaKLHqJgyfpQ-Mnasu0N&`tbp^REZSI!xZzWjpXuu(ZbX<%qF%_6D4XNHB2QB>;`^5 zu}L0onrfa~7?!=`$)?$niYg7Uc&1g~cVQJy_itjyiBdzvrfrzZ~NvGPCI;Eg7j*7}2OwQT}hu5J3%Su!f4at_ilY@h& zMY=M-F5OmQqI}PCMkGJs%WlR!&fechmOZ4aqBDu=AWkw-{N!C@oUi0DUfs-2r#1UQ zG!Th9C{!YUd7C>+s${&Wak~nQ6n5$NAIwE>&rh)bkI%!Eieg<84$*!)R@OXY09~P} zPAAl{k|OosODblTe;b-u!ib3BNb%GS%tT0ME~V~z>|@^qH#KeN6SexkKv$$;!#87H zeI4PA0c}eVhEZ@#3~u?FPItiu75A%NYN)2fAG+Tf_HR?pC|RXrj|!A=Xq03)yu;dB z+OoWp1N_LJ2(`+E@FQ-3@2>#F9=qyk3h3XS+O7lc!yc^pG*a<8tXS``88k!E#HJie za;%RrA{@C+k=*sAq;)aUI0dvA;8_Lrq%C~P0XYPM=>xt6n}Dz$%QWCm;WC#fy=*WH|b zz1&eYRjt7_n?O`yNeS)HPf3?N4OTmJ@P{sqCF4EL$KA@!3PU6KBADoL`&$*Jtg#Xh z&xC?k4{?H7y5pL=O_)OiBWo%vs>swVb-0ss=tG@<|Ks0@CjgP@yw0m>fsBQIU)gFKc!k^Ak7> za8^}R48Q_FLRg~Rp5K2g4N+*wA4bnxRM5Kh=ze_M;>mrmxY%=o2b{B3n4&la#b+S% zFBw}x!|!PT|Utn=j+<&sGDF!P#ydE%AtqzmcITz;F8EdG!xL(FND>vWsn z7&wU52Y9o0Ch_Kw#8t)kB9BL@sje6y)Vgz4bFnxhTm<>c?!^5EeH8~gb$9e&+~+Kh zFVie=b|WF;S1Kk^V>->}&D;ckY8+PvtS@abAx_K=>=P??4}`sOQ*T+sHrN@{ddjDm zJi*@;X(ATa&j0$g6>xNkyXvQ&vBAik+0wUjBjNJ3jzWi}^YDhc=Ce~#H5zUsb}EN7 zWHC(L7a>=Wd@b12;;uWNAS0T9{ak2fZeY4c+kJ4@#ZE1*X1Gu(L|jxkGM_uBnDqli z?RB(6n{vy8rYt#ho<@-q5;SA}$`kPjhurc{C?2;BPLc)(p3TKc@3#l|@=YvIds@9L zK(K2llZvY)Qp*xa$|)#}4wsEfFw^jYk?su!|9b?43Ysb&YDwQR4L z+;M)ejfW?qXIF(l_&pGgEe{Y(0LOi7r_*feO)pvPwfSX8m7qXxIv<`sddDyJ7{5b zloP!z!CT-;kR41_%*jV8Nn_htC~#u&hZ8G zSH$Q2N3O@W;v`i_w?#rF1~zo0q$C%Z4IcaAYsSvra=OxmJO*V1?+M%|;&+m!{n%VL zjqctQL=Kc}2%CqU&aFwl0JrNt>|`($-dvy&p%WXIm)RtGDm`mnLJN6TFovJliuRxl zG`x2U?&|mfOB$vloxy>0uUswOE6DcEx7X${aquo9_p*ODOsbNH?5(rS2|Bsq(ODDQ zcfC;27>p1w%I#yMGt%b4yG_KA_fL2IYHC0C^38O9D}GVqKo zk73MOnx0_+wn^YXLewX)_xTli7bJ8S?d<|1+U#@Udj2rf?M~K{qV5WA?eptg@ zOZIv1TdU7hgF?Wqko?qyN$$P4sv#|<*58W+&XV_xiGFrOXnEVR)e@Asr^&g;m4Gts)!zn zX?y{uF}(Y4SjJ=Z-28b8tKa-hqafc&O*`bLV?4fY zj@~Cz%+v{|=vJb8(_^m0#Qi_Nx!TqdD?N|)9YZ3o*uahlc8NbM>|i|+LRdd;fEusd zn^$l1e|MU^)}?{z*lBL~kZKabH+Saq9<$8lTNur%TGupRQQdCv{aI^ygg~~$yt)Ii z_22W-=!JO+kURz(fPfmNJIqL*(B*4FtK1FzL!l)N7c~tJP+LT=sp5*&!pT{)eu`Vk z9JBD^YxKlQ-qDYiM7Su#-;rV$%!`WW=Q+wBDu&YRe&+qsbyk718~&#y{+++sBj7UM zj>8;!1HbwrPA3>JfmPVrVtCA?2+VHVh458(nRwhRaT`N?BSFHTW=)Q@sMc{1PpI7Y z9(aZU2>}PC`D=YNB12Pm7e=doD0W|)q4aqQ44?6U8{x9TNa^&|6qUSbov~du9@xRG z<3oQ4R-mwhimr~UMbw{L6EfU%fBLk{|8$XG*W*D3@_4#j-tfn~FGa!LZBHKmK-Ta9bdGDN^;y);v zC4iE>t*lu@fn_M*qzv#hVPE8h0rP~E;%`rSz`UpfRA)y~SIj@GexzD6!)CN+_77K6 zVap4%O9 zdc<o+xWK|14SV%DVzv0T>-I4aHb?G?}blKE$-sH9siL>VD z6eWHipZX#;4PRrH)zXexmiq_OCt7=JtB0O8-jsDrQj=CDuEl)Olc|QT#HK#S>e`BT zFBhJMn2RwakCM$oBp1u}$XDkV=H`4d_QQ{Dmd ztD-1d#$kcS>E_DT(;M($z&n5BVy_%7m=^3AM{Mu;}3eqDtnHe-JS=~Gq|+$ilvKX7g~?$oBo@Nchj=`nj)AhY1Bc9$AUB* z2k+;|zUeCxs(%TZjADlLVu?QstLJSr=_q5$lg73L2h9WrM;1|fDAl=l$Z_#vNAPb< zP-PmUa00jZy&s^4AvcnS6o|9qj!yZPGJuc_fQ7bhM89LIgnhBF^x%t5r6^|?#}}D( zMdE+@1#S0lZu|flyE=L&fjq`r6KFHk-g}rM#qr zZbonc8cR#Sn9N3B^EjGQG-=_!w2T`&nou#*Kp z&5FkGs>5FCu0@hrdvyqJ*+(Ge)vwRd%bn4R8$>Llv!F)XeVa!#QhHI##QOnli%A4h zEH4*@;#>gs`QSSa$}?Tj~~7hsTDrn8Spy*87j)V#(xd zOyaPajW{a=BsbvJOCsa8Q|#_L2VoBPDHW=``%;o496gx=_EUI0AnBGpJjcN5vjhfC z3+p$81Pl(Fk;R1Gg9HLjCBoW%a#?Qm1n2(uG_x5ai}Aq=N; zqLu2y#I#C^iZ(@JI;00z*Sf-t8~%5c|I@()7G?FyVL=jJ?r5E&v5%gX2yI*oj=9mx zde4{Iv`>41NU0PtAQ#Cql&yYk zv-4Is)2)G}P@xAmrrN}KRi;PjZPPh_&*!EM{=%ft@Ag}BAPdzT96Qw(!HNoi^$^Sg zIEWGUmv6~M@3EfwMW9TRhk+7R5syc@vT!fFayxtRHiAvJWUexIZcCYbblUR1DtbSn zCSrI?bteudTk=jdxux}ZUb9uC@^Bl&O>x8PYz28B*mlw(~qsnS`S@> zr})Zarm9Whi?dR2I_Jw*wDmAF^n^UX_L3mTvq-(>wVRgtTBUOHMjp8My1o4rslBH{ zLjb`EyUYVJ&z&DFfB7_DZ#NS@@!whHU-%jXjvZTDk4Z4Fz7G#!C01`#Mxk%e_ zkg%f6_TC?r6au!jj+YSm=uY0FZuDfZ{$WmvG;{9r<_k>J15|1+=yIO6()4NaZZ@7$ zg zR1X-;lRBcN+{{JoddOpvibbPN`>K8@x?8ExTQE1AOeQ(*WMZK9Zn?0vgsL(ir_Ek) zmTYL_DNV=8yj*Zd^NxzpoB1Bf`}YUtiMjDXm@7J^wVa(~imusi_f0Yg)SSwJY>D55 zQVl>Tg`5*g5P(n$f)Yx$B|VDleYN#M(i!RW{dtwtLLd)>iM$S>1h44-TRmoCGlP%TzqJOT>&MS*>^ z+}0@GjODA=#64kd0J0aF#?y)9ua(U;+4dOX%EnGLeJni?kJH%l;8}M2Md|s%xSTYM zp@avX=@YKKlNF>8kxpfD986Atdwl&pvcgeq&l`@Sy>kBwj1DUnn zl-#)hBmzwR|KhVZY`=ZAukQFs$+<+b*+Si;JfB4zZ(y;grEwNM+7-!{?c$NNida5i z85ELkE9wZLH9{PZam&`pEyO9v>73WinKK~+ftWShup<2mPp2~6r)nxwQl{|%a!*(t zO74jjJ9xS!C`s!0LNC7VO7De>KNt_7dj+NpH7RQslc_-i-UxFh`3)LkVri`-wERXi z-m|KB66LDM)k@jL=tZ+oh)?ahK<}e%j#h>w#+n~}&W*0p>*DBfnG&;Erx z@~qTtA!diW#bTuVnA*a|kJE@^BL|G8=&>IGsTMDDR%OwM>FhFAr7j0T=%qBxROU)= zoTalFUW5>FS3WRz&{OYQ1BRgTd-??L%58i}QcdDx$baYBm*hlZgso9D(6UN2cQ}_) z)5X)vfPL;;Sy4+Q*5Ey1;@tE~D!=wYMhBpWI&UvP4+YNS<$c3aU}hVY9nIg78KSZJ z?T|G)7-jzyiz9iTL~YFtkJ^mlw6H`nS79Y2hbhCUViqpOAC#T>LLaMxla<`uzv2&W!Ppu-XQ6V-J zJVCyJS3y2h8sSt@hQQ7!v8r{Oj#(tz2b-bKI1FaMyrXabUon};>0Z|ml))RmL44Oh zURR7~CW5W5(XwxvC^^39mwwB-VOX3)9B=AZTvc*PS|>XS?K(m2WL?#{Oq^fmeb<+j z+C~cK4khYkRp^za`wN`t=1h?&kzfeGj&?%YF}Q>&(PH(rivC6d5~=*gV}A z8fq^`J7!fUB|d7Wu$n62NhoJ@gU@K6WzPp-wz|>h85xGkL~tPbhByQ`#<+YS-~~YP zAkZ}#wd|+YWQGX=$+haMyWeoDZuVdeWfydJreD^Y-x?0aKfd|t6X$o`9>L_)CY7JG zN2s6Rufookb3(}NBK)~7oTzbv`jmC@lG+&IYO+>;XEaoHVx*B!!w@lEyTKl+HPyx3FW z)Jx@SiBUE>(aCM1nVCNF$p>N>(#XIo^i~ov#WSev73!QmuvAMJo#CyFP`ewQ>mVzu z5*uwQ!`@5iJUKe6G@OodRV0}sCD#@!KL&i5FVBHTE;d&cA6s4`QJ3$Q?_kj1{P_5_a-ka$4mP=@@RHz zrR7B0f@x(XgV6vKV4-1rCvz(@K~pwcB2}sA#nW)ToZ(#W4MH1%33O@vtRlt7@ zyDq0UBcOr1U{ljR?DVE(xGak!YV~ie!AG@HLUI zfMMaK%rmPP1QrINwI?Ha%8(5Utsv>TO>oBKk~x1mYKxay+ftovo^pxq=_Z5wC<9y1 z@yh!Y?)R(UNT0QN>VSP=21+Ea-g|1sK4Z57Q*orzOrW|EZE6<>x>n5SLCJ!`$kE_} z-?|ZPDL4y-KkK}I`wt+jLZb;+M_S+=VZ<+82bOkx@L@;Ld(-yHEx3A;wJCCx$5(u^ z`0`eAOIuXI>#<;;Nu*EBEjnRU68MRxdY_CAQYWmVITMG!CDk3mo3FDqIyW6Ef6c4K zhun{oh;Nn6sU6ik{jA5e;IOgIrY9(M294~#sh^JByjr?%up~g9U}tAc7zCIu6COV2 z=@>f4@ro~o@cMV2hw$bDA-t%6LU@;;A-oTOxBNd43YEC+4lnBDz-Cs^jP4_sN}Kv< z-JWO3Qq3o9+UFtN4=kV|-EOEkA?OB+(;31Ei%JGl0Xo-*H@i) z?NhjYZc?nBNmU8P3AncHgJC(`zs9XCqIha+b2rB0tzI|W3h1HF;TuC^d$TpbddSr9 z<#Y-?Sr=hoZZ7wm9Tn?oR+px-xxEX2iukEpM_D7}p07kGOSe{I_HfC~LbN3-y#i)i z&Ju}wF6q%6CCNz*({u+A>hr6d)$0~T^qZ;p4Qy@bR5n0_oFBANR&@m0$LLN*`p#QL zj&NoP%az!W?;MdFWDkvAACn`mEC|>I!b_f|n@55e?SLYV>fB^;*2Tq6E<6m6ZC0b7 zjzMV|h-b6%hAEv+5t2EKyLa55QY5lxxPr57#neTXtuiG)H?ktr^zAl$pbNiDw4wb4 zx&n&If31AtZ0uLc;@RdN7JT+)e>~LUEh!(1M^S?GsjbfE!OqtDsRm`YTOs4A35DE) zNmboiN|?Z7Kt>KfA~#>SsSiLyqeLDe&}N0GXfv+S^bvn_iDGah8;DOBIqly3ewIHu zLgi>@Cp%Wq(V5SiE)k#Epgh2`!b`Ad8JD0n4+5&YHJSw6#F7mwU5+YWbnm67B;pFi4n zgex1qqj#YG^o?B_Mt^N>c}T~=uTz=$w?xaIriBIV@h9)in@eIRDlP8i<-HOymcnzH z6&p8ol7la#0Mx_}Dt4hI-gj@Z$Z4s2?;@TBeGrH{bge!5R$rh%T0OY&o9w18Uze?W zhtr#Y%Y$rp0qm~zt6i1V9e6!p93t+-RKexLU*}G)es3-XBhZe*do8B>{*|bo$DCXego~N__|vZ@f@F+)Ud*`XzyxC6zR~rPq^=Gndd!t6QM*!4 z`J|rBfyBWLyd^yE-^&Xn(L^Vwrf9tm`Rq@VDZRC6_#4b)2fM9Warcb#=g`9S}X{IS&S6u5)hBnskL3hH%l-AOr-}su&$l7s7BqU}gq^r6xVx-=P zTu%Ne?sK?s=4Z&Ar=_}loz`6&`nCL?I%5Gniz^}Rn}6W)v#^t%b_W#4coN;rmZRxc zP=3|o(TNL(?zG=~e0`*C>Kn*{nWXI%@>}j4w-Z&Z4Q@WU4mH*Dv~Stln+1AmFbm?~ zc{;g{-p=tUB2a|l0TiKR2N2+%`0(>#$`U9v){Jsy(Pu(96e$1&@a<^RlLdQxSD2cI zx2IR^3dMK?x)}pMQ%h34ExQoEr@hzLb>PG^n3nXuQLH%scEP5WI$rjJokN*ZLUR#W zZ^ZGII#JskZ}`g%wtM$`3abV>7&V%Oz1Tgjxh%5=d*;(JwbjWiM>h=}n~D}{bb7hh zdX6D?!(brT3qNLbkFQf|3K=EmBT?=bu3VjimK+3d2+TNaC#t2DI@62WkRDs2m& zi#D^&UC6^^nL`$;=fJi!vC+b>cD`Sk8n6v6yPUAcm#5m6l%5h;+VuIWqY!EC+0F&O zVCveFNL49rv5Rhu!|i2Ni9l*IulxC)JXILy#7dxA3T;-KNU0h@dJeTKTh-KWMj7n~ zUdV|~gIn-a*yvkZ_iy#UT7(IGolxj!>H1L5n-_RTXA?izaxod3_%yhNfU)ZaTV4Iz zCCfOx#^#n*j8m`9;lY(h5)q_!PnE~slReBqZUmECu{Ka+sbb2D&V*~KFFl(QmjK=P zzhT5EKY?XF-1ub_084&w zY{L_Cl;&*Rgs=PK8t}BgF}rK z#RcTzkdN}V#QqbVE}ukoW=-$BX=J!L zdv(*}d-w{Tf+*qnoeIBC^=wct%y_^;&(TNow=VB6=4?YZ<*w1fJemf=Jb{N!@VZSG zM{myWI4@^+ZhA7X2WiycL5|WiH$z?MRrgw?r)BU2PbT{-!XTPLb|urD2DAbsw+ub0tCA6E#etMFleFot zR>xFa+~{^$1{E+go{&)>E~~1Skbe;eEwu8-^hEv{0i!kvcb9?q_O9 z0-Y0oeo2R6Gd)BSjW>4-on?-JX^ui0)vO^B@Sl!%jlZg`luWO(R9sX`NiE1QBH2?k*m4PM>Y zmus@G65|UaS@YCi+58B<{*ukp03ttjY1mmG%1(N`?O_wxs_s%_vNx9j`tBgtvG^?;bN0z z=6Z_KiLt=&_r$%mPbs`ru43-F=ryFVN@*O9j)t5E=&(+8Of~Le*04_(;h~DVh<3r{LT_;HzBW z*RBv9H=PZx`LZc(FO>#$J|LJda-%)<30tgyQJk~9gPZnPbfCyyVSfOJyj7fdo)0}y zGI})+MZ|A-D*6vRb;IDg@$mJPSZg7)tdF7}`95wOk4%QiU|`gCzXONt@30(9`~lvX zXcnfI;^%Rynksp@Xc+&--66km_uvDS90g7DyI@~U@c8FlKeR8-Z)XLed6jd^d3zws z@7F{b_jqdoVIFa;EyEC_4viqw1D7Y$!E)Qdj@m_~EQ3*!RN7skayANhc<|!Xh$?4; zj|sJ!Q`|(EtTZy-OrUNeGfF){demo^8z%_2P5LY1r=yay|<=Z+l?7wY7 zMg5O$4JmfVVc!8-rsiA0PaqQx)S3tL;!T);L%+!H7o5_D&f)9zK#b%0~_Uhcr$gs-djj!tuF@ND%lKOu+7@PMRYT!Q|c> zFq$KksEmAae^~2`gHAnfe$i`oJ~Od4VmqObe1TDaMhqdD9yxZb$XT z7jmhQb|soiCQ3Y0gz<3KdSo85wo7&Mutu-zNp|Szo@guZv}@X3EY&M{aOq4*0lqRF z*xp_}uggs{zPhQ)zb)k3`}5)k41XN$Wx#go_`8)72qI^{4GJq&r)bXwL@e3pHz9^m z^A91`7|Pye4Yq69EQH@#xpHeixdtfz|94_LNLM`~z}Nnl@gU)F5Yuw7bqvPCQba}*MYhWDRRq47E3Im@lm$J9q5j~s^)*r&5NX#^!%iG^uwINQ_(7~UZf0VB>_9&qpEjw zY6nx8JVte>+#N=_$G5Z`s;Tc;QbemV<@n5i1w20R=R8y3aQyzF{a3+LpaWhVQfeBS;+fWH z0=^q5<)Qc`;uUm8CGNSlYd_AN0x9j9R?nD1 z3sn5>*XN7mJfZ*j@_p6HNSdi%4tM($e)1-mJ*l{579h&$e^0z{5$Ytmm9z0A0OIKX zzQ)+VFoKh>a5yINAy)qk0W`(_Ij;{*GZ16X7eQ|AhDXD_vO`H`Bwf%M54PFbZK%Gf zt8mSf@*k4ZFMH>&Q=9z{xxmnOaC^cHKi(XDo$q5aV#Urmwn|Lf7MP%9;?JOGACscg z6B*Wdu(0EV&BESeukatV5 zM(bVvb`bC{Lkzl9j83X^fcXf`fvI$8^|>ErWqUh2wOn*EJUR+{*64LN3rT}`H#JVmfCS=ZWYiF%ggnOIaFc+ zqzJT1>8(8!Mg1*v)*FPdNpP1#9&dKHT!?}73H;8W)mBmWHkg{8*+;#>Uyp>u6I2D>V@4|8_R*CHu$K@#$Z`QoWPY_8qpfNH z!s8bVFs_+?KpfSV$ank~=Ko2UuxCEBM_pb6O*)x$^w8{$ek`NVhWk7*ZvE)Ly=;zu zfxe;J=Bc1&9>T@)D*?;Mo3~{A0xNvU++>OR@E1LTDd-6UUBx=WKOW#K_tr#>(@IEZ zn$?p#o~LVvoTqDV1S`b+N!J!Y)IN!g`N-QUX!H#|=jX!q*>ff<6-2m-x707*Ri_h6 zq&}mR6B+i`yZDef@$S{VgI`iPAg@Z*%8ICQl|=D5>L7eVh(+jLYaSP8e*1QgosE^X zkd)6L6MYPUl8Hs+U56Mt6oyEtroQc(mIzp9f-}j*XU|@IK^M(1@=H{X7;De;mpcM0 zcKTL`d`iD%#93T!IOPC(*2%h0|@pX@uF46{O+W)@~LKD!l~|ImzKiHN_kp z1yfWky*8K6u|X&s7yIh}p`yzLOFN~-6hNR_J*A}LBElb(gd-H1}LX}Ngw%BolhwWZ~fP2oyRyxMmeH%DwR z0@8eue{~kM@q&Nv&I~V71BWrnJ@$~bCg1hxEjfPMaP*Q%ve{2@#Rx<}ao@@Ylf%vRRw``k@sG~>v%To3x zS;C8LX&1=<&YcH3#uI4ASh0e}&j0QhJr_C#@OZW0Q6`pAx-Q4uQ}a-+W}UTv zlRVO@vrBxXU$*c55{W*`b6HC>GfP`Jqgn)UbNy$tj+Ku&Z-P>esO{9AVo|dqZjIw1Q zTEl2S&FQAJwXP7zw&arcgP)$rya_LU;t^T9W){|(Xt4>E%GW3&6ARCe%8yii@uftW z0#?w`d~3t=N9AZA_HI#B_S5v;Y1j;Eh<@*R`7rIKXvB&py0P;n248Ndspqb5xwnro zL~aA$;g1+AHACu;7fT)L#j^gF7pw4(7i%l)ntdmHkRaN9`pp+bEid*3OR~sP1x7z- z9|C@(zVy+jU;ma9g~?XX9JXpd9H-^~qP3WabFG_b4*dh@QrfK;%Ik}0zXAde)>E(C6H2*aCSBks0{yvce^))mX%$in?ow9|4k$^ zblj}#xhU#Do$vy79Pd1wAEkrw;!(=KgnM4lctAg!Qbxg8FeK;5`GtRb&f!emEZLYy zY6966I1py>Flio0Rb^7!TENqfk`AVPp*I<0B+5iHE31})=`0zlq$!%JI;EF&u~VoZ z#~~b|raxDAG+&(-4Epiw%h|m#5aY`}gx4>Se%!sI^;Ad^5=TlqQ6 zBmhHOf8mgg(gq>K0XUDAR3lfiBA2h0G@+Am?ijA@ZTJGL!XLrYLak6?9DV7tLvPr) zlRn}>{Vgn$ViDr0nA4DgU8TvdBZL}&zoqcp-;({0za?+@(q#d=T_Vn5uF*~l-mG)& z!%zw>oZp=%tDpB}Y3Xj*Q2Z0Bs!BHYT2uc**zN-mT|LVG>OP#-AdtFp3~!kE_b0Cx zb^TW)_}a*%an^J&3n+|`@43fzt5QCR&>Ck{N{M-i^*UmeTprw2$X!(%o1A3JzFAx` zyC99@b@NWz#g#PF3H4tNf6&#)|LOw(=^0EB1<*SZnAumS0FZv}uJQL+g>EI{5s1?c z3NhS$hgfOi#`xg~kIFEIX)X2AZQLq)BGs?=O9RM!sIX=Cd|+W)@fQ%IJxvJ1Dacm* zyK{HjT?6=pCarJYR^mj>3<-NjA794^VDdu3DW`uqV|6(B%kJAblT9~BMe1`qp6JJu zzoB^~u-r95t3O+mZ8^#G`Bbz1QI5KXtd%~jw?n+lL5h#P&bkNpojZp!!Jyaw4Q~hR zY1N;70ORSU$>6)-GdG|L;Q{>R11l_HQjZG{w8$)k?opbd$#fAEh-vpKnep=hq1m`% zNdlF{C)-od*z84_^$&}A>59=v3i5Ihl2l*CBNaOU_ioP2vs!Jt1uH+eLE2gI%Gsm3 zQ?}xXu1A}N-}4A;sGNaE{DuXBb;XkYR49l2KENY>jI@^Cw8qysMox}9e2|!#edr_p zl6s>tDf#DR#$Mc|-yU-NJPSJ39=X*xx0W|09}V}8{|dnGRFl2m`7x-eJ|{`9`n3uF zKZ)n1efzev)bS zqiC>f3-N`cnwM$ZmFkJUzc>wuR62&tbZ4AGvxGnFC8fU-Gby}#oGDIW_O+Cfl42+B z&4V^(hCcAl%WF_QIQD->JkWjj+6ZD zL-8gnJhepS;k@ZSmnDCf?tFYI#@Fn$+0YPVIXipVv0Rq%PfmBnlGUXi=SG?v&P1CY ztIZX?MSLFytNOpWw=c$&Y_^pa#b_^wceR;FxU$@SB5xS{H;mo zu@a$e(2G%ul&m;pOjRkv z?P-byfZSUG$bHGLtDhC7q$_P(zieq3ny;3oM)iBN_j zv@cu9P*mIb>+m#m*tvc8yF3x$)1du3i}RZCM}X~p!DPtqijmCW1uTOkwoQ)*|Y@~ai^AkkNOT(JNQ#$BtEV$cld z;cPOk2tEc!J@^r{kgp~hCrPbv681b8(XU|ykdXG=dNx#*>|iB+>sia_a)`QBsZ@f| z^SBI^ZZwG8C_hk?(KVA|R?0J_s}2-p7_t@r1zOTZl|1V(l6lNxo@zp? zUSAh{cC2zXk6(n9l?GV}E*%QxFHs2r_u(cgc^nZ>?Z)d+t_Bf4mo>f4Yn6@$2dn4E zWPe({k2oI6JH=46*nir2%OY-|;Pk8m zz>VKlt4oX(>y8{(Iyg&}zDZm+xvXY@TjhrpX8d;!leyZvSqN9^K7e=#%aiWvJ;U|g zn&8vnUw7}lLKXOq+k(Qka6FRcPSOfOe(nVVu}b~tE)1aj6)v0_)@gCwjR)_>m$HMv zCs$=1u+9!dp-=t=@~{yn>Ed}AFbkAW_*3#1ZqU3gvoMyOf9kpaeQ#&Mo-g~17Bv@B z+;o(dC9?$ZQP@EH#`RTfc8ga_+aAEj>)fjsBBJJy_a#o-oOGRKhTL<+GmfL8ZC786 z;y8EYEUWE69Xb0ejRZRb%>$UBG4(x~oPa97Qvr}n_wc~p5m`Vqyj>f~GwbL#6_WAe z$D0%(8P!KA<|2^$HIqC3aqvg{HzIqWlh9*e5{mw35|UG2Mh^OP@3*sv)9xQn%N-_L z8({d+52cWKxL&%!TO>68x^_W5M|v)lUNJ*<^lR_AvB<9catxXY(v@kPeY@;tDzz4e zPpS)^`O(|OIgGA_l`Xl^vS&n(X}8rSpCmA%K(ppk1qJTUl_Ero|IV6M>7c?^t<{&! zzRro^Wzq^Q09}^OO+L4H)#5Roj#jMwv3UKNe0p-2(2*JtDU75t(5)yO#O$P&m4xLs z8gHO*9`x7rW`l0(x4=rXXncB~E-`8><22V*vRVnqo=kKp4jB0xPx<`?a)_ewvyx6? z6;OVzGciBY{+#=ZtMCN5xvHu+83*cF?Qm@FVE|u6(W~-Rb* zk(2vfEWXs<&kwY3ys&!2E~Yg%kL9$n4=EV@jFZP+8vR;^Vz9LqI#0x@Q>25ePheMG zPg=Y`yq;oBHFhESTGaLJS8wo35M0^PGo<$jrKWtHTY?+vVf$w>Xs~aI^%e-;N5?)- z(N?h?9ke2VHuwTrvhP<0dR_=*J^*Gi z*l)_xe*;qK@^i&oB>qYJwNn3^`>u=IN#tCJo`3Nxa};$Q*J$dN6-p*d^&WSzA?I() zgpA$-PUYX}S%1xjb}uN-%sne{4^q9G9@!{<<$7#f?_Rsw%J;9Vj7$)LQsvVtv&R*O zjAHl$5Xb42?pOcSBk-3mgy?x%=Wk!y3cRoQOoO2PJvzd9vwF;qfvqp@Zu4U7DX!om zTMc!QMXRiU&mL@jOnqAoTjHXZ$`ZWTHKBLV|LOzS?_Vy22kf*kweoQPomkQR>W+B4t;QA9;3vM;w)2Pa-EFy@f%;8PY zVxN`Rfb}0I!c#)}<#Q%G9?E1JqAUJkvNz|#3!zLlMkK&w$C@IZGuazL*W?7p>>%qT z@W*}(X$jY@?n%OX&B`uEi`1j&46&fM`OY4oRM4bo0@by=Eq?&cboxivGCQ1P*HTpl zpsa9udJ6LJKi_v^5pCKqV*m+pfw&)Ot9~hbt)w82W{}#8)$1U!lt?I@ZZnRUB#xeR z3{h5fiPr5%R7GysEO2IgGki^ouX-dr`6B#?nO++WLU?MliS7lDNW}w8^Ai1gxX%XWT@7*)z^*43S^J7iCu*wlN3ucp+TVX?^bl;5GR0mdjdW4IvC8L>#ftl^D(PMNb^;Q6uQ~SA z#Ku9?Pd&Pf?=O)_G4UqrT5~%cQzo;)jPB7VB}!86_@Dqv@wcy~ykl&^|^$`>;@%{Xy_0miAY?Ny%Bo;_5{j7^Jl#SIcDk zN+FagkBy@2Xp*8-d?ZWW%qCdY%;3Q)A84+#Ux4%+=?|VAp4_+bqg|eQU71VLI-TZ< z7Z**c(MX@3$kc7WUXyJJ`Cd{H1A#ZUnAsfAN7JZlOw>`O&g8HJ*Y)0iVpBQ93u7r# z`*r!zz-`LedgC9@ya@^ID;INZvib%|g~IR}n>ZSm59aRcd_d|AWOlGE*NM+@d7x8} zy(Y1xqA0~EXQ{&1h_ba^n_5dBdpp08v4D`HbiYJ-`tz0_&j*__sITPyenfD8=?whc zBV3`+U$l@&H_M>Hl4^PeF{rRa_3MMi;F6j^8XK6JG-5H(WlT;+z$+^q`E5;@V6x3u!D7NzPYgjmru26{#o zK05LAAZszNBE~uYa-ya+dOov5H0IgEcxp!aR#)%m9eX6w-y7$C{L~r0VFBbo zU$X1SowZ5*NCA^9X{1e-0TuyVV5K?w;;sE?OP;p)$t3kr(dKbk`|dn@S;6GU*-^lm z)sM5HeY1BURN?B%^DF+QPEdEnJ+FLZ2F`l4-3pY9Qihi!<5ieX zrts%qDrZ=!X@tL4nNKeIEa{q{4aVKUe{G++mumm@)M8US7tw!i74Dsm%u}e<$jYe< zT=Cm##KT9$RLXc;q4%P?3O5~`ff!9&b)@o8HdjZsR-wrDZQ`pcfTWK(dNeE8QZ87c z59G8n{RO(JH8W`iO-Fe%6QkgDqM0XVJoUR{nygTZ)Jdk~mykCrU~2S)Qr^79~ax1w7g`{+WSl z2PT>dT}+ZzktBBlfBVKp)Gkdhnj(q7MqqkNT~pibxp!$4a#;q1-nnClFNMt3xmwtY?WR4bh1y9`D`-9OESlhR%GW zy*X3Lvt1yF?w%3k&MDqjZ6=ayZJ&H;8qJL=K-O@n5tvB^leHd-)AkW_+ec3Z>xNek zF_=1UYd$Wb`6LRdWq3L{<*kMnPCTcBDMX}i5?AIXG;4)h6jh6#+Tn}mP>Doc(K5dW z!@KgHHia$7)iuDpNCO4Yn?SaQ#M7!`OS5+T8GI_2zivdAo80?b?T~5n~Mh#4f1_ zy0XCYmy&mqF#*SPpb~~kvq;1jp+1^mSBQ|cWAd3-ol;M}ycq>uv7udYI{n)XpM(IT zdHK(OR9bGG(TRX~TXq)v_FN=tiP=Il>U0b4P$@2rg`*$OGUXjg+a0B#14~G@-vejh zZN}7eMyb+N3yX39$NclUQ_gWPHl?6JH~SYXvj}UhoX=%ne|uZDQwBtq-<13XGw5K! zj5i%gAP{d~!1}BxSJ^X8WpMh48wwqDQf@aTN`$3mNgiP0$qp3HI>~k7$dIa;8>)Zu zuxm~wP?kA^yHM?mzH~_*ycg)^>`^6R^HxpkhcnlZCO|64GsGbDke={#EpPxT4P*7~ zuJp(%IClzm<2=I7F%3ZCknQPZ^^S9kshRzjzWeJU{|?f$hPmO|Vcja00@n@Zw59mX zLlJu$Gy}XYtFNs!YfW#uF#A+)`UDaXeH?rmARM6G7@heuQOA8vM_29y_{;CB5#?7ekZl-vHlKgbqD zK#(r!?rso~P8m7|qy+>72?0@*Zs`Vzp*sc`5Q9$X4(SFZRh0dGQ1{+2aL(E1bIvz@ z*Y)EMFZY16nP)xgz1FkVy6+d2P_e9!R_hLR?oM@8xI+!QKq`})f|=FZX3QdM?1k7t zS{D;0R!Xs2hqKewR7 zlMP%2Bq3_ z(7#f^12Wn#(YCig^)#5!SjVHU^ws3zYtd64(MT$Mow#Q40%0YtaHT|mks-wI?yoEL zD;50bmHL?q{^ylyzwk;KM_jbD{^rIho6X;key8yc8W&8?H!S0V;W_8!rchgtLoo8< zAcQYLm}i39xpGNac1}H(A@NdzKp(ELG+?Mu8JT>K-Mhtp@WpC>V3Eihv*FSCJqr^I zpeg_Bf#{3|IyF%9Yiu6q*GEIjLhtIpZ(%iU)WneflziktBh}cWBUiH8sSoi7|4;|4 zpH>G*fX;p{<-c?G%d;hM7I4r^c$$y6(R60Hh$Zj#FgBHhZ@AF@Ao#n-sDBXrjC=Hg zFBMj-qg_~N2S*qg;bf4-;dPsjNB{i%*QMR>3r}M9o5wk79d70~w(+E)JyvTEX$>RO z+s|ZNU<>K^obF-BVck5|O>0|0F3fMS=sSN{&2aev!4eZ)@L1Ahh{1@7CCyL!s zCD0nq!NU-RD$KSc(kq}#`^j8FjP50`+zE?3Y=p+9o}DN;=$uZJ9M-K2tqC!-F>R&? z^fr9cqD37$>?&RIhnjj(N~ z0ihLuJWG&=Rf2nf6nSy2!@}LAwcoE@(L+BEaA!G(cE<2UCz!AF&?)D2ac2=5N5j_i zU!Sfq;Ajv5WX`nX{+JMV6Cg?#MeM2SFdN23{S{>h)} zEagAcSr|+3m<$j|N#KehH?g<(!h1gDKQiw_q%!`>yk{@oYR9XZB6?`czdJ1=c;2he3fsLERrk%Dw-T2-p5XDerzy)hzfz=F{?8eH(O4H!pJMpKK zUpj5o;G%R)Um3`&*{fuehE6)ede`kMQv1x`fG(d8d+&4mbW!zA;Vf-Mn^xn0_c3}T z?@8TU2%2Acw7YREd3k^JhTp7!+)l*U76*1P`EY#A#DF7dQlKK1OjbcR^&;P$B4rM4 zXm5dSiQvCzQ25B{_V3FiQNUN>>bi8Vit9>j=PZ}5WTln0c{`Z-1sA2uodmvCh+7q< zx8}4OAOCtto{PTz7JSQn1i9}_q9*d30Lw`?$+|z>Br}${045`TR^FgzYW(<>?)~>F zqhGNPne1(xxsck2M<;95b>(9XU+id%CpWUwPGCT*)o&V}9-Q5dFB(t&#m7of;V^i_ zlSU+Gf&ZthgdtAhFImZWB8g_UQ&!4YTfr3tbs_rfd_@IrAZpQX%DJ*U`g!TEnby6& zP$7b^HiE63Xc<%WW0i{LAL9$t^>_O5 zdH{+gZ`u*ry?AeaT_cz8nF%U0d8lwwxT0jV9;8*@_8M&G%L?7anRA!zj!sGPWaZcW z?mS%dhAVKwreH6J2yE#krlP6rNoF1WU4?>EA$KwuwVWGIw4GQZL{Ut`*(D_}xu|rNo)Qa*ac#R{F7G*`(MuZfcz!*-Cpy zId;Kkc@_!=R>+Q|i38~^-34;>{T$kwcIZW%6M>b%tkJ+E~@|gCNIpqiN)_4>t_MmmYZ>=pJ`Q@wTuKhAf3VH#LRL&vlWw1?nb<)x_VSa zp$A2^Wea1{^M-Ev&G$2xH*VDGTx4|Yy~=^eVt2ngYHap?{5g#C4UNbTosWY@-NXhJ zMGc1S4{HU^PiqCuS!)F(;c|@U;gpk;7l^~H-T%W_V`hL>pt)#!H4*ZRI&~|*i&41l z%=aDcteaKTY>(#gUjwlJeR3NzYms#4hc~is)%@h}Q3G&Hf_0X}B(C88p>f1)`B>eyB4wm# z$z~p2=#Xtn?f3zO2^jZ<7LI)$EZA4ATIjsBTZyW2?4tJYQcO)We>RgGBlsPK-|uYwNq8r7b<)I@P~kx;r_W!s9R0hz>b9=bDs zT^XASm>*D_ceMp_EQ08u*e~lE&fORB45yEX1A4OacLon)AFw(Gp;(FLv*UgxZVbn< z#!FHu^?$`OeW$HfI>@eK>R2J+(7afLIH=}?a(7%kN4fWJuum|#^Xe|X{nmJoB4%>z z3qRJ}@oV^N8K*bs(~icU`5&wR2Z?l5xoulOhl6cp-6?VN!+^;4aY!E4rd+V0V6#1- zKuSVQY$XX>ftcA}^?0x+!_HGJOoKNq&(z7>I(5!N^8l54EW#|&POFK6ck4x!|Cl0YuoF}XdPT1L&j=^*wp%KYVw7r zlQJti@Qqz`H$k8)ME|!nGoESw7p*q}ry>x|0JS-`HS-R_)Aw0-OZ0D+9e*JTlh~&4 zbO|~dF}YH0D=a-3=xVAy74Z1F@5j{s2b9no*RRn<0--Lsto;T0zbShC$L6QCth15? z0PsM(t1J;k00ZAZSCG!>q5cnd$C2*V$s|KeA(_ip+ABh3wz@{Bn`qQ)y<1<)xZj$x zlB!L>doHm!&NjwJ<1Rkq@<6Mg3LB~vdQ-UyyLT7Q{IfFnAB;e@1M-9K*qqj8G+jNCVF5p3qj)!*=_BT3!3{UL!yB7!6Tfs&6f3^2f@LhnUivNJ6ef! zzXm)B73#I**48gz{|*ONgyK7&MwHrtREC2_TmY_Q(o&Lj+Y4n+|G# z^Shb|?)873nt;HD=K`&y7}>W|&J7F}Xs6?VXzXA720BETJawJ>kWamkKw{_KO5~h4 zEA0cwEV#q9cS>h={Z)Jg=9)OqVL>Zc%rwA^A>elqyf*qZY_Z3Uf~mCFc`JJE7L7AQ zj&&GM_9iz$v`@p}s)DLjD`S1fhu|kW{q+owWH*eE_FgBwdR@}ot?&%Q`}V{8_TyQh zs$?!cz@61ld<5%EkR_|VsX}LXlu+bdrxLidll?sUkX+!!xK8f@+h%UtN>WlJu3>Oi zo-_Z^HeAKy+^@Vpt#THyd?Mu;Sj_sao!C@)Tq&Z(Z-V_reJxM*P|1I6<5Ur@HZX`Utd};@B9u5{0{!4{`p!42-WYcv$vkZ<3Q2Z#Oo~>t&UBBLv->C z&*yY7WTax4=d4q#&9;@U9~@Gv7pgipG1FG@$G%L;R~w!&x4Cyn^=Y@z7u5V|5oaS# z(0{%r@peB{n;}I$!9;DLBkqQ^lYF$;%_4hS+WjE*N}sIw>;b$R&QM(2!{OMG#wTY` zG70jsNX{1WE-Sr0^9{97#|&T~$v2meq%3oLr{%1O&iE(uEv6=gqZbvaBkvz8uLam% zyMjlG8Bsl)Nh9pu^78|N zNlTjdG};-0#Q57z$CztWRlNzu!*L)ai%@GGcaVwE&vSE~xPQ~@Y-a6z^W{%#JdLiN zW)9W5*V_58+teqqrox6LkXySpRZNTJ`(K)fYO(eRp`kulfMh(Uf z;v!jXzwkvrjKd=UAaBg6{V%iKMaLx@UrvN9 zP1t7Z>pfeTS8X_LuIR!>*t^x=Gga%L`q;nyJ1Az@UZ&LkhG#?2V{dbZ2vrJOrSZU@ z>CshbcP$(Qy7GJTB31DkbxBT8Zwp~xh7;CX%E89w837~h9J>&8W^ml&kR;}uxf|RtH3*%^z*iikTwU` zo>dy&kR{i`H*8Xk&3EPHoU_k-(t!bBL@F!gE z_$7vxkF3VzI&(H~7?~SZxMV_%2zd(k?G(JKmWd?n)Hz2DxdD>Oes-{lQ($2IcD2mf1|UxgUPfQsaFg&0(bTTZH5^|Vzhsb>p``G}F{2*laONb<3%Srm_OD{aW8 zBhl`s=oJ>HlB-_3Q(_VL82>gB$I z9ET{SL5!w%#{T?eSklYnhjroyr}{HFJFBbglyu@^O4SsY z8MRaxMk=N#V<(IcTl%t>U18A)rqA6Qs6!2wxg_&iFcpR%uA{h+Q znIqMStb9mqz0O&t5QS36r)g^OQ?3UuECl5i*>{#WMTQgA&XOusLMH`QmHJ?+*4`p@ zgeY@qWYj3MZ*^9ol>vD}MV|JoXh#^3uE107@CSFH=qo8JDDWmPwHktIkR;Ve>ig^IXyeUzw5(XnSA3|RC+jT#+s)usx z>;^#Gg`2$s*XEwL95T$-HI%hYh!j4TJjyx3>uqUjnJyYzTpVk~Q-~C(s^WXexZlH$ z{zEbl@xnTWA;FlI&@FDl)*(Urt{s1*(FtgjhAy;CGtZ823%j3Ry>({BHl@hs9JVA+qVe3`HEc=C)z2y8Sa@ zvkDwa!yP>dq-O+0iGZM}tD(yO6DQe!R=7Yn_~R4=m>e}a)vQ&z3#Ce*Ytm}PU0Rez z)1A9c{g&9@)zv?l+-!F(tT=cct+!f$Nvhx!cp2qZlG zXCS>bN^NGhqYP2wSSX7g>#HEHVhyPl_ZYwoBt^g>sL+#VY>ro?sOKi4pRT-|&-h#s z?5K#wxzo_4Cr1Mgks~u@;)k*$+@o-_l$2X0bua5$j6(&1Ema}91D$mav$T^9o^k`h2Zg*d!@F*0%Gy&dL|=H{>4|95WZBfG{aw_B z;aCfc*o7sQ2TsmTMno#UwY68lq2My>igrL`W*>7CxP@d zOora0Cm~$WUYP7vB6p95R4d@w$ks@pp;2P^A&gTB{2`26?9JXC!jY$A5~A~ocQcDI zu7+5-fD-{*2?E$d&i7)!)3AKK!-`zzEs-Tdn_4wqZ|D)#+-IqwMHcz1cq~+9JXsN; z-42*d`8q}NazWxv&AA#c*RQ7gCppht{z%^6kV{hg?e*MKje3n^ikO6p<~qG z=vhsf9gO;IZ7Zj7vMNe@xAKER;~;LcfgAH}9>%3a4+D$(?rGfGOckJzuzB7b93s;{ z9wD}l&!p}E5n7sYx^r%l9CIwBx1aW~m4I!evql8f0;4;KfmkT7A3y4o3~!>Dk{joJ zdL%4+z7{sV0PcF3g)3JL15I%8D{Bi>O`3v$*smOHD;H_h)B1*U%?1B#bRuZ{9C1&> zQNHr%235HD77CnKoU_3aBXOf^r++q*k=%j8F|6R3+y4^HlE7cA&( zShsnXc4|I;8+bpf;mH<3;Qf9)K9GlV0`FJXa8ukE<42{Nilxp+e1It3QK$R!0Rq{g z?4m`PW6Pra6y?>;}{TpjDZpS(u*pDfKxecw_OAME} zV3e)9tm!YZ#4_eqCy7Ka2`{q7in(&7Qy*HAFP2u8K*rhbX$cmrh(K_puPQMn&7vbB zXn@z~LfhGZK(G;ZsN8%UG)kPjp3^bsova=AZ*>$s8IOu1&>Gmbvh)e%!HUb*o~-Zw z=5iTTc2egn9m|oXQ>bwB6oridgGT;3T9M+>NQ1zX()x0z;Z4oQPs2BIFAQ%#2fBj1 zC~rK4>)ojQCg$itPC5}{zO5Z$_5k+H=RL)tHfG$xq;%ixCtvgur*s@NQZ+H2LWfxn ztJF$p^FfY+i4B^dH^#!rWjzz$77ihoE}7x1QEVsR|CZQs)0JiMiZ~!y#^N?>1xyp` z`0p0Rt~yh05)!J+Nts}*Q)IEf>b@seJM{D(AEib?4|cy(rJ!H2#PPCrEYW$|c0lp5d0cUE~-`AkRaRI%O77$=^^>xDrMK8=vaKIgw0& zb(zst`x$Ac19pN0f!zb%Iu6+SJyi4yGX?A(HW3o`@VEqSiIcV5pI+Hv*8nzt!tWO1 z8BC5%)jGW@dc}_ zdR)Tb1{mMbHN>m<@~YPb%h%`>j;R8jT7tgzcC1~A;ZdIZ4ucPMGKltPe+Nxwbx@5* zE_H=R&&2V$@;gw338(Z{$&C$Z8ea*${nU1kF0X!0l{No^7l>XP^5y0?or4N=)9=1- zNouzyt}haJr`_Et@vLWjJC$9zWs``hKIuk(``+W*qc!}6G#L(da?bALk-mck8x7m{ z5NX+$n3h#XON3}{)Gaeny0XRLLXBN25%=YTO*sPVA3az=?<>&oOPhfNtBbbOJ-@^SFH-Ic(#KDi{^{GgU!GDS{Ax_O`` zg2-FAbrrqjp&kyqv(Y%F96s|V|AN8@;|iQMFHHCdy#`AHA9|kD0gekGkjK|En@a*@ zvK%`5+taWNiGId-PrN)A7yn3@1VIw{)V(fm=`7>rM2< z(H<0~1xVhFm!8Y-TQYJ1C%fz%QpUkt!qC(rlZlQZ2LMES!V#}XZrEi;(pNGH*gccD zr*#Qb_pfo^JHH)=SPMyslR$n4?ZCTNhB4<$s{2%5+^a&Qvt=ynEnIMMY%OuSUymAG zXFjwK*a?%q@9Eu97xww2?LR)s6w3Dqr?s;J+Fp>q=+CzA|H{(Y@PUTNUf@(YRN9#Z{t62#@{TbNVp_Xt)zbJJ3u~``!MEy!-n0qCUp3*R4mm{&MV_&-NI7Nk z$IIV{QTl0PxkP{Y3@;9BQ0W6POp&WO*F>wU%Wa!a%4F?rvbiNj$@dNq$KqTdH04%` z-tX+5qY*{s-X0Gmi%9CPiGC3Uw|nD_Szp0cig`x{0S8|H^bkBVb&iu)Ly!K{pjXz! z;^AOnHuyMcyC5txu^GFy1nYQ_UhUdo9KYG)>~FONwG{#190s?1^JRMS3$936lOEZp zvnem*xS1mvU?XwEx7&XQ&9zp=>r-s7Um1GuG8)cUNp^%9uBk;~A<900-m}CY_oMGLO=TUD{ctaM zGBF%vW=ffty?QI4lH91C(#xXa0AWIjmrwDxP>UPWx6369z34Q9arOK2H%iDYFJMk| z)?n9XX72f)X(>80&P`_ST$XS&ID})y-#C^yOjPfEB;~PBp`nRT4e1_*m zGRGHLUMXmry+AcOY4yMVbv~VAD(c~b;OVZ_gAtp@R*S_VG^00cdg7&@OMEe@&?QRU zfpJ8?8BaAtmrz>|vAJ*XO>zYNI<{k7KelsT5a-&M`NKU#xDDxz=&ATonTm|KNrGB# z54J{9Ps%!*D1HtUvwO*4+{lczxn8ZY9V;BD`lF1tuXIe^Pfa8yq*ez1WOb|3`DZ+> z?!m7BAMhWj()jS_``X7w4cR>G!+{iz<$Av0%-(O9q)i1x+@z1E^gD7aY9vP-I z_lp_tO*-m6mul9r8qUk`39@l>VxB85q&kc^${Qin7GLZn&;R~p=ra&nk6wuQA|gvm z$t!uPGGC-h$}D%tf@n~Xy`i$vcu{p<5*I(XKquBuFGer-2nQwRs=A&tj8qbfk8UaI zwrynxp=V{ogxxY!)c-#+M66zv02_iQx0ggW(6MtvFdEM*S>BAR{cNI#n5;g`2`e%yy= zPIUo7N||JG;z+$YafXd-nqkaXLpcPgD#2oLdV3lP~lalixSx zJP3~+S^+-szqIlJ=ey6#>sb=`7qhIgs4dxjiw4D#bYN5rPl4MbW*L7xMlTyxQ<)hr zyK+_9Z~WF$R(=0H+t>ujt#V+OVRA1guKNo(@of`~lzDb0w5mePwBEPAZ`o7PN7Z8M z(Y_KtA?qj}>fA7zRGtX>*X4@&E|o~19@Wdd)EWyWzqO7B3U%1A!k;C}ok`D7o0(d8 zvgX`8^uE@yHUBjL#B&(GeN4CWnx2QM`jgGrz8T~=@_)|uA-AKaQ!*q`nIZVPUvy?{U8tYE1BujrmO{ADEP)Sa8o!gFiOleMNpMu6o27ZK&X%;W{JM0^pkzac5yCDa2gMD$vu{ z|58zqu;IusT8jF%z3K#uw>;XQkY7vBTW>;`T znv5%ljY667^AZv|B-ew#45-m~!G(WxLKIreU!AbmyIbBZt9Qbd+jq*G12$2+s**n3 zqeGDKiy2=_ZIwNN#!bH|w0!%#9TNJJ8m}owl?C_8V;Pt_URLqV(>0OARCjbNNvE4O zx2;67o4t5t(?`t#I(v9??qT}FSMn_nbDs#sD$m#KHwc;~{O_Yk*HdxH9QQbgSExK#1NUW|Tbf()lV_6Kd1<23LtquP% z)(olt5L~Dk!(O=MaquRRnrGjsD})|QxQa1e4*7f$0nCd&S^T4+0a~oCrApN_vz%-h zz{TiHpFVbNk>TO4Gb6e^sEZ8f0cib;U)5g9ksK^6Bc{)*71Hi~zQqk2=K-*I*KLex ziigJxBBuC*g_gqp^9g3fT>Q~=(LGVGn8KinSMI7)8YJ9Ox-wF$DcQ7IOvR?C?BJ5( zTqV5Dx(tssxS+oL4+nV_g!+b!myI-Z)S2hljW%UxAZYD@4 zJ{?vq8y*(et!RGfq^TQo+?vFL50eJURp^-;>(|!@5fZx83cerOy82;m31%}j_{-cvmTq#&+`=zJYU1Npk!&4QP?`a3)8=Z9Kos*Ns1|@V*SwFq zmhY>N)qe-E!}Fr@S$%1!`Q_<3 z8ei~kbOc9M!-{MBgumTis66VR-iwj@;Md<>v#wfNO26@A-t_nXARDQzt=-nGJhFjF z@&d>1Bi0Y9mX#*oQUCy>Qh3Ru9KsIEP^I@Ml)S#pRkg*&RJgt?;dGEdEa8QgSZCIT zgs+ENq~1rXnR_oyII(|DXJB%rv?|c6ZF&L%@C}-|oKZ?a-y7i`EYI zb&O|8{-Wf|oGYbf_6)~t;dMi?ZN765uAXFmeBwMC{d&Iguu-=&2!7>)^AjB6RLj95 zDBo_j;1K&7yScKN72XL(lV9xj(~O@bgP7#_f)BF&Vyse_81A1Aj<0D{O0S04vO+;E zyeLt?-04sD{q1SG8`SM{_Pg(b9;@=PGi4;LmM2D`M9W!}iPG@jCNz)k=&+@h+KDj$ zi&4H(Ojla%2fPqLr@at+F%4D7-V!LGL<14_*%cuVL5XY=-APg~Beb9n{7i-Jpe=<6 z4PZW9?$cf@%yj_>JBYFP<{n|KLHLyPaEjbmQ9$UXU8@{Gi6fgou55XC&=(SPf{FL~ z*Z#o7Y0TKzY8|u7qgY>2E#y@R`DsKca>q_7egE@LJva5`F6%4STpGNWonx#l?`=cy zTG`~03}-%jJ5vmN!GuTQWw2 z>O1d`yDigT!U6qR{9;8T7HKLb z4XFskintsqG25&s6?EMB=nSm{nDTwCt=wYRef7=KL0q{P9m{Ui*j!vlUu)Rk46~WA z>gn0Q0Hgz1t8{>LAX7_7T(m2X23hn;Q@5q4>`<+tbo*GK_*wzjDq3*fN--D03QJBy zs22ducN#nlaKlYc{d*IveZLHY?M1a#w zJ1zmdq6tG{l(+S;r_PHAlzbycR=qJNQeB56BX)I)KS6d*zx; z_kAgQ8y8_|s<-i^EQ{Ivgs>1Zb)Es3xQ|$e{`i#AEmA5g+n@T4Rji--jYM5QzhSv@ z?s2a@_^h!A(0AJ1Q6jQ&}rc@w$)l+hf5%*lsiC$@{(jgdcvf8GuSZ83^SnP^YGVdl21FG8L&(e=qkN^h|gozu}cR6_*rVqCN*CHXOt{2bMbU4Jh8&L!EZ zt$IMWMaWB;U~{QEL!ZN^+5BoB)S}syPA*@f>Did1*~kr|`tcXH^0yfilbl;L>ez}7 zAwVVnKJY}*u=GK{Uq?-Vsb}g|_r`Z4 zV0S$W!eaBb8yze}@Ju1NkBLeeox7Lgw+nHcqM6fEq7Ur!qCX}ddHt{coP03&bl`W~ zEq=JjjhCN}wVHJ$ih=I?{zKa));o#)_!@&F#8v_juw7}7P44A;XSWgm8x)TqGD5C0 zATdISUItxq5E2=GR^rj!v`b<${Qk}N9S1URvL)2zdnIrtDScQ9b+13F3!m7ie|>2K z?zqZt1W!*#cz3MWBGT}j#FxBA))8sQbBHRCT90g0j2Kvb9x_9E!HD^!?i_1{NMm=Q zNWA3xib$I!g@1m&<5(j)djoeeq&;MxzpN`bXg^%gjFHuQCQ9UlNW^Z%VOZHMc?j77 zFG0V*Tzke`Jm4r*bTupt%B4*^Lh_5s`35yikfK!w~|Ul8}FCa-M=*af&21y`lA`=&Ybw} zN8ecvmP>*Y50m3PcgCMk>Cr{+(odSb_f!fd8ykBfX8FP++VA!en0h4ilj#mY@zUG* zPc4~nLS52##gk&!3qm#<0-6P|sndxzilffEn%~=lr3|%X z=QmU8=R;xySvOH)7Xm%3+@UNQPMSh$%M_kV1}F*1=T}9!Q2zG;{TJUt2^Zbpa6;x_ zpJTrV^G%7q^aF)d+X{!#-0W&5Nri_`BeOlnY{FVhiZNE$duZi>7x2KgD+X%qodC4$_(ZfT{=K?qkC}ls zZ^*()2aNmC0slaP8x#x%Usv9^L#H`zCl`~h-nXX^MJ9b1_W<_^h;lpncJ%~WrYPr* z#k=+2cBd!UWq0@~Osrx&Qi3}?G>g>)?JXZGPP%#X>=IAGr`@uga`+P+Zp6vOzf*={ zyu=N-fcFjRckQjWhc6CvS=2v?IXY2&{6}Vu2iN=mtz>z}yN%d|!0q(a&0TNa(5Tk#G>pVk~k$>@lNtZzGm)}9t zB{$^u!)RWl(i|UrE;gEQTvm#Rj@K4GRB%9#fJY7R$rAS;B%muu8d+=f4~)CD#+yIz zSTPisa$H^FF}$hpy>@+W3EAl7_0YFz#@y4x4;lwCeS3CFT*e+U&c`u~H5Jrs+`lpI zq$&_oiI-FJ3AW_ND;Dfw%`Hs7H0U#uzRC|zAq)IhlBH#f`1fzhvBUzRbi1^qaXW51XP)}8^5_*@M zkxo>dkxnq9_`Ui62c+`@(*J=ja8@h~94b^YIJ49J@+zm}hhyVPtd6bw-;z*2JvEv# z$`m%P@%#93d_aXJg!4+z+#H8zwt?Z+!O<1jt$vRmLLZW!LLcR`LLae&%hBd1LZ3rK z=pGg1$o81Xt!^_@eD9{u<=3*5Ru+D1tyl(kzxeG_5FZWA=aKb2k$iObP@5Y#O8KV6A6I;}m;86a$7nef3$18kKr~6w?4y({lb+9_y zqGp`_fIB+)&yY$+5Va{ih|^K8HoUV3X3A#}|IJii<|M-&k3Z?&v2u>i?dh0pMqX|% z-b=Y!7rS#5;5O%|dbSv?quWb@jyUgJvvfxrc^t_i!>$lj5#S7R?ejtLrWFgIw4 zSv3?$QJ`E)#mNk4ye5;-CuQ7MIC2Rm(V-=q4zd(`%P+xAOW*lWs}`tAg0AXC$9=zp zIEhV%XYg-&Av@o_Akh7P{$s`$ZHo)%{Kl+NhdK)_&`2$_?otoOXVJ=?$79yWlafwe zysD8ayxuLk+ug9MA~>^wOeXbDUl$g+v)2u&aOJ`U;S1&x zUI1q2xpS)1q1^R8OMhRQd5KJ2EM8|>#VHOD`q#bhx4dbjzVD`)&O&R#kWkL+BHR=E z&Go99y9G?kjap`Shy9L2>!gBIQ=EDz=lIyzjI%WgJo8*w?(Mz?GLbwW2<(1HySRSfz39X??|f(jAPihD0c?Np zs@xl&&P@62BfKU^=O>L9^JPgNIfcvV7^u~iz2>T6v2wF;vAd%OJCw8iAfl)q@*EfT}>w~6rQ1p^x+0w@Dr?+5Tm*Tc)v=j}*)ed@C4pj?0 z!S*xrHK|(dtgkcO>7@LF+jl(6?GF}Sv_NI4Bv9N=WQt|MTXe7dfU%`k4VQRLlRK*& zTc>`OMHuQE4X$FdTG+Jn6-dp^Kx30{XQgfIq%FXNuPc^E?>zN7h}oAIdcO!DcVD4% zIUhk#{3`7IE6_!Z|8qjWt({j&Lskoc{YFB=o=*S@7Edp4OO2&ba7mi_k)|;nLz!ac zK$ER$)bn8V7qSs7_y8XOyax|wgt{1k+mv3}Uajen_Bb}Aet5FQ}6IZE(F6$agMQX;zs<}*< z$4=N*LIv%E(B}s&{kFcg&D&p#SVbpD+fe@t9@|i3)|y@!T=801PfEtlXM{2iXNmeU zsj^lC0LSQHUy-IvD(j2vFAbaT)Ri1ZCe}M0zt$d5dUJg4?mS7)W;geI38ZJ25p*tW z_?Ho-Ob{(qYa1A+&_^m&hm$3bzGP>|2vfn%J4$}eU*#dZ6J#)0cH20gbeKRd+7HJ$ zc1D7^3E9~mvGMTo7E?Yb5&Cob1s6358sVB$l_Y3$)46Mhe(DmxmN&~R7{WOin1Uxm zoY?(&p8`B*hNRDe4CUy0J>0Q>HbeRzlGH;n*L+9 zxk_t}QuMMkn?rR=6GKgM3=VHg5pVEddx^$xv+u$WD}bZ6=zr?uzRKeQC2uc9@id1*1j3{O?52}s6t3mVW;@1 zdFcpga_vlgi)7;z7LP%h1(L#D!S*rR;(Oc0R|Y;T3wfA|YEc%Y!|%K%iXqZCewGJr zw$@jxBP$ZdzZ#t!ozW#aQJnF*(cf8kW|QJ1dfpaL%K#(YnIzPovDo5=REq}bafa%0 zJ1ugl3tz7C&n4pAG!ydiv)+lx@*d2mDgyYw=2QINU~_7o{4oA)EdKR||4H zh*vZO^uKj@8FN@_*;cBKpW@-VxI37h2zGuebT@KcF3E#yXI z^I2sXW;Lzi;<{A<8jDiQg*V3Ph%_d|4hg(!uB$f{$PsJ;Ad>}r!5GkF*0(OXvRFej;=|Ez8(Y!z))QhxwO+^3TiSgWY8Resa z|IOwd+MREfLh+`xCX)Dc@{G^hQ!}_}^R#gto%5nzakAMv()tE&)%Z%qX7>-`IyaNr z9%#WvCeIIpf4@)t?anvz?fhaI)gw=gVhC%hTngQ8f~cP*>tpdDRj`}>Ky;AQ$NYB+ zswQ!=Ile4p$yjAVv=LSvgURMuk!l+7SVHs4FCRb%+c}|L4`iaPGBzmu;@AgM%ygx=(2V@yK*0LKP}W*Z~! zVpD~^64D+hwzDdlT9dIqjnYa6=gP3nGc;wvd3BuPT}8@9rTPLk_))rO=abbWkF;O`)6C0ZWs%KL$(C}kx|Ju$-yb4#@%dd@v| zigNahM>u#zZ>=NSD;F)N?-IN@`St$(7jP?{O@0PW#S?H`j>CChin8!D&LD&f1yCuT z-9IsFZfE@1KU+epkF+v0Xe0a&gbJ)X=O5zk{`}4U<+G2h+NlJ}Fvwipho(9F1CB9! zDV%Nl*ffav9Tdv5xy1mzAN%DLtLDX*ax%`%CclmU=og&Pq?1W!E?wxR&CTpCA4XXo zctX)rbG`#JIZ<;CuGqL;4W*rnA}j7!6nz&`T2N9`muGt}sJ&l5;#1Jh6YANzgvM^ijnu(IFaJ4{$7 z@cA#?|17U*$vhuv3&O4!wq=jD125#oW(=kz=>!`8mM>Stez_wECtu-Zor(&S`?*P6 zv>R%Sr(dLl_M|R!%Ju-_Tn(;TLU46pG5kjYqU-;Fl^|eQC^Hj8;h0AafR3>{V6{5% z#^;F8{nCyN=Es><^R7fOxhl(T6yCB~{=#k^537Q340yw3DFw$IOLbj;LSD=yxQL30o~g1K6=Eed;kggJ>FJw2eKrS~5H(w!7C=Mr`J9=5)IE(xi%uuZ<$vhj+yjs^2Hf3w2{7nYK z10nXp3&?r{q2ur9l;a^2e)4Vt|5DZ|si#C1MD_d<0+mf9{xfME`1PqpX*{@ zw=Z9taAq-L1|q@F6w;u6bVMyElX|B}qmVb(EGD6JK$(mJU?#MY? zC1vcB1zwRd&g&iC!6%V#rWb-_#k%@0P;N*u z>f2puYMy`}4x?uFRl|84m^R}^V2b7!9P5n1o{$o>h+2U($A^L9{hHad9g$Wt;OTcq z?1gVmhMa4l^FxSQ0O!Q=1Xgi2bOCh;NEA8U{df& zie>|+#5}|Gj%cD$wQbE8tQ;XoPO{@&FD#FNr!IaZpHj&sm0m(-b!R6V<{|d~i@mdqYHMxRb*i+*io3fOcM8Q5Jh&HkXmG92;#S~YJ*;w-+=?LD<8Ee9UK(Z*$~~2JljHI?SNW(nf8z-T zPSb^5Fxq0Cq6bnB2PTh_gK60kSeI&NfVF^2>??&zP~p*LOzK0-s>3LHBnMY62TB51 zw`~vd`KdIu=K;t{ezGhE|5&;H`gEB0VJiihAQ{G{0b`EP8$@vOLzRA~IJw9t+F3%v zon(}>l{X_fd0r9s_DL09pvfvs(TA)Tj0y;IJ{N`5l$V`vVEH4xphLN|0jFnWO|7@Qmovtd$eJ@hg{__CXr z+VhRv)`l+9JDk(skC6Y$<9}pv@i#;V$rvW#?*iQZ+_#^_lg+>B3@iMRE9d-;BjCn; zQ_D7>d{7dmH|;dGTv<*{z)uP!Nz$R`0014DTSMOyloBCsdVhY_0jsQA&0p(#CrAlOV$|_m_tSZH!zme5=C+yOVS7 z&3@^%2#p6l=^9a>b7mIRmL<2-duDsmXS)Z2ppj_c{G1I$GJ5)JXtg@5jMB_4+)T-U z#7`Na3K?*{W9B748v93Vw*lGr*zN&-;koNI>ja~(ZC|BKdsveAC)90SZ~=sGgm6Ac zx7x*<^%IVmX&qTHEGqTxpnR{(+}K#%?6Ub(D2A&iwK;DOb_0yH;BK7oz)1)u7bvJW z#tMEUU~JiF`q#ZFp9b*U{FQ;v{xxP@?(Jhhr>B>YnsXMG0P)Q<2-mY#Hbym1U$a;2 zr~Z-oMl)_vIZ_|@)7}Iq<`v&R$&shGIxb^ipx;ZXpzgq6?{|!f`HyYb|0b{NEB_Zh zm*eO7<{Q^GZ{X$Wo$v9SrPX9Em+dM2U2G6oAlSgTIYw;f+H}iyhz&|T2IQ^Sf%9Cl zIZj-?*iQT9av?Ydbyx!aT_C!u2mqM+^sC(kHDAmfPDIh@zCZdS8-Bb;y)Wm==YSeT zkA`^UHNr9?&s}v~t#Qtli{{WZBQtcI?uU#Eo-9b|r$y5r!aIn6c8Z_^{G_!zU7ic6 z*k#)GlKvgH9a^avqHQ})fqersl!cM0iT;4wy8MdUTK@J9IAFy7*PRa|w!l&@CvnIX zPqXNKm~(1J5MwzC92Z|+$h}UpSEFxuSoMg?b(e`eHRe5)b4^387NR6JUpEW7p~Xah zIKT3UrA1$AitazO65{7viUkl_?%VFpqeRAQH5SmllFLU$ z_n=1kh*14i3IY%)dAY2nIs4{+u#E9%5&L_D=_e&9LMSEg#w0vzeJLF{AI3q&!^dha;KwE!;6B9SH-=1$= zl&`r(WV14t|IloR?5%X}M?>kqj=S0DyMDK9aR1Y?!S_$gMlWS!1(k$wJ$w&%weoqX zU{STun;5`^7uQ;kIL$Rgxz^ivu28KOI6VhLk4-&LtrqbhU#(G%J5Q|5m1n(td5;$7 znm&31FTi9<^ilnJDB2&qy@~etH%H=MlrbdN0XP1Qock0b&uJ83@+Xd;;9J+}E&FDD zTCAkuQv?OVUS+NE&4hSQtRUY*njOPG#;`8$`xr`XX{|1JU@oJMgdarR&lz6<0OT+J zJ#)fu3^P+Z0FsqM&j#`Xk&!Z{d^pYRLd@j@l(Y2s&9z7Y?n~jscGb& z736*kJ_^{cC-UH*i}Q`8yU<=Iut4e3JxQcd4q`vU-`{jbdt(-1Bs4f9L224z)xQ@_ zy9&&+rhISyrDb*sWe9q|NQU8E(xkHWfuDN!RBID_vK%`eqT$xc3Z;*;Tr*f-etV|w zr7Gm(U*n!Tc2nls!rcF`1_L4UJC4BSwi4~pRq${Y?CMbV)i;2!au@N>x{T+1dVdy} z|H99IB{}t{%cWeskvJmy+z!Q{mKX=LTpjh&jq5EfZ!$5@Uq&zrPfi=t_6tqnbF^#N zUS)4uDabSHCM=>nkPk;=c#SH9dgw~USxVPQ^NV+xo5T5viM}(p20CTGpk#z`9|40e ze<7s)wdmVlskHAeb(U^Rq=nYQbw_`Kp5r?-ZiWK+vJ;`&#Sc4QK&9d*GaQ3{05caH z{{%Aye)D~4VfnexU*W!MAzVN6wfc{K8Z?ydNJi|yXPG9r;ZJVT%rN1f++=BZ-XGlL zhd;PUPXQD+ss7+*{NK4rl>8Z1TiWrPhLGO`@@hIq%j1`tc9lCGeC;&zvlssk%ifM$ z(sCfd)buuh*INPGB;zwTl6#Ra!^jMx6d(K&9PRW<@>dR>c>L!uR6r_@(o8)EoiYuc zup5U{_70+}@~-HWk$z{nP}z9#1eZv$ts0cS0K_gZ`lpk?0p%ockN?9-z-hr{{_718 zz-Y69@MHb(DkS>L&+~%Cz7!O%*TpKM57@|#mfLvvQxcHWaMGn-3JS$X^x}|6RB|#Q zDmnSi_vGZZKa!J2oEAd_e`Fm{Axq@9I(tIsZv+VxfKHH%kE}>YQ8QM%8A=tQg;A6T zN>E-HlOJJGad=^O8*x8WDos^O8XsZViFChLw|!p7CjTp0{^_fB{R?}ZbEBzm0HCt5 z$TO3Y&3q+l%0M%k+f$f>SQ`w;gfL>AaFc+|L8{1W{a+J&H_bzTgUL3h=|w{ z248T4PnU|ID1MCi<^*GoIfg(G(wgyCK}0Lr>+_?}Z!;AtvesD|Ml7!VOM%kzKo1tFy8Mm^M85s zVU!K6Pf+phR5Z!jwn^2-jPQ+WhfD(^OZmtT1Guc#TKDa&J+z^to1)UTg0D*p0>YGK zl2R=(=;0}jb(4rGvq!(dSFUAn{Y;r%AJr+_Cxgh?UQmul+6WA7AO9mD z+(29Iq0B%6Jy?{gl{XuPv+7u1_|JrJzV8X)Gf2m6z;E7-GNRG^!v7T+*^}z|e23ir zAsWeOqUIL<=A}aa3|6E=N~ISs zoy?Mpmy(g8{ypSFWm?T*r?06GLqV<<7z$)IQvCNE#kDXrY?JUD$G@+Aa}eZ`4!hL0 zpz9^U_ugKB><-2*zL+TUn4%`IYXu8Ghly#T|J-c>C@woZEax|n73ksLQd-xoyZsN^ zSb2&(2DKOody)FcF0m?Og@wG7w(}DMjypjC^H6deo1*VMr2b*>FFKI0H)6kCL_C>c zRZDb^DL9g0iIxd75l8^xhl`{Sij7qze*>%^3DLfkN;aQMX>E-6Pc*-?sUB(fPI3A0^_+xC~18EP{||tCWD5pp+!+(|~JljC)pZ zx9N&nn4oXb*?m#>8MWLKma_R&dCKJJkX*YuzG)W8QR&Ma6RTxWZl{B-ThzUdQr)+T z*gWhv?~)TFn)HPd^My)S_R+_6jNQGz~^+#QUHNj-Sp*N z=-Ou0U1@rK?TMJ^7+wY-epJsVr44%-Zv)3IPPGBZ$pmej;)XJ7LQ~R-2lD;DR93^G zd{Z%(_4;FV7k!nuMx|3E_`bKT)72BRh;

w$2Zfg)W@>Sf1G1ar-&E5)di_P1;#_ zmWsts7IHOZS_Yoz7`)L1TxV_=-M`E_5dP3OpSLVLEzrQ4^ae<85$KBIig}Xg!l5wesd5L1T{XKMLQ}$8;GpZ_`6SthKu;0 zKB4fP7c=@vig~-3P-Cf_%)2c7xQ4dzT)C_$hpvIWfz)%U${Z? zo8HI&C^s#z;SBxI!~N6W(82RhJWDee5@LPata2D$b9b}{V9;2c=vDeA79IuoQqvS( zf)YW#KzmdzKVF`)0eW^4nfnC!FS9kJ?2j-khfc1368?y?O8Y-hd15Uhi@&4t2A7=y zpE(2hm=utvCxzZZ;WjbCvd8oZ`pB8dt*MfCfWK0oWJ7;dv;%&Q)RtSIyYKYNjQLwY zu)mF!Gg=J|X=-k!2T!?~ z4%Q%tHSOPbvE(#Iat6j5f$ESY?)D<6vbW_fGO^HB)dQnNrJDa7g8ieNAq@Qd`u2}V zC(V!P=qw-rCLN-Y?7}|{fpk@zpV>*{9Ws`TW)`ik23^)i((&xPZ90#OcFDvZ#wp5Z?crJ>s9e|%G+Mjvbc zh@BVRy!DUR`PbVk4SS)kJXc#`RrRj=7jRf<8G+>a86L;T1(Dzo6Y zkh)O3coQk07Oa)Q8Ro3}yG{zpo3lS!Il`=zq4)BSgx&t+?)w&2QA})65`#j~mk9%u z;i1@ZitOi5@2$CE{i}C?UoJuZ2M%Y`Ox61Df(&drhxiAatECBHIHt@>x7Fa)=fc=o5(eih;dj5f zq(u@+;`p|E29|LxvXp~&Jv+}?edEo?Fe*MLCQ7XERjx#{P-HG0+^O8P4Uc@|B(1#! z?ieXzJpwi=YqDu54iveuA{p)gu4x(l4>I_iH2@537fmB}Iy%)woA}I1d?`oP@I2k( zKtvH(8*O5jWxS}S#DO(96Re0S5~yeD#&cO@S%(}1`IpsMt+(1v8&Xx`)LAv+2j8{~ zxvg*h_`8?od$7FP%%8#Xd7PDh6Kjxk{vpMavEFb zx<7SOLhR?U+sn!6Wq7gBd#hYJ!iPn;T52IVeb?#x;LrSxkzhh*W!rnjs4mW$w6Y>W zy=VL#e?2oK|CnIaRYTWVkL>s~`~@_?j}cwvW4f6*X*A(ga7(8~;!r%e7DVCrd$Hy1 z1^>=V()#$fT|juQKb9kkU&>nj)Bk}P4Ym6}w~YLTN@Er7$&FX8NM{gOpxSO@ewS>| zBA-u8Dwvq?)dfK&5rFhnXux6SGmQ zVGNb2{B&$6*@sy@?F`82Leza!mDTuIHJs&qRq)lwT@%$~@?X2s^Zu&WRiU(u3QNF; zD;aNbCq71nB^bn*i;sC(oXXzp#pz*AZwzndoCZ=jKQ2Y62Tx#xwvde#AN!?V3RqW}Ls?bepJ&%0=yGXqUC z{cNJ+zwL_QpCj2m^5Ar5y6CMPAKXCM`YCYl8J%UHn~}V;Af?^Y3=JAM_{4~ZH!jOD ze^l=sfc!y=7=KMc=nahP+)cy=3klIgqOETqsa-BxrhCo`>XFiOd}3|3#vl6FPksp@ zv!nGv3&gq`cPvf|R^aP!e%l22p7a`hJt|jX$@&S(On*P_3;PF{%Hnba71v~@S(Xom zBVDSHg<#=Ab7`E!Zr{sXQL^jnQ9G6-5i{o!Dr01pz}e%dSNsVGBVp!naJF90=d?7b6xq4s6$DWsmC?dI`S-Ew8@*4Kp2vCHLn#_f zGo&)WnK9%c8Dru)pE|2Qacp#b1r$eYDmb_gxREM=RIMHJc~U*p;wUG2I@&QxDSWBf{{ikajz)RLQeU?4C#jsyi^F z1C=Z+^=B^bkE8YfVuN#>6T)N^c4r#MWW(Kq!hrQ&A8%=Sq$z$+q>c4}cf_-+BR0lj zRaIb*wp4aRq=)#BC{)Ba8C#N;T4BQ&hWzBwdFz8v0@8hka2&$ zoQvyj4MI`q`)#q{pt|@2YSZU(g3{=|hm}j~p7k57RDKTCQI|p6bTWaht2z!#DYGQz zSx$70GOCjM+3$It2Iql*x04-DPbC4=XswWs>f_c|N_$k70l3egsRTu{Vvv_>q+rPIa#Ovdr4Z7&&iRxRD!o+(^q<*mx<*KYs=LBhT5VvkPtmEuzg^g*27X6whZ zh)_Yvf!_Jly_e92v}mp|rIg12No=3xce#bbXDPE1r#C5nI)}180;I5*R~Glp-=9lR zS{WMU&#&>Qu%^nnklzOG2O;XqJ`T`)y0fqxP0{muoym$wWnGk7p}g^vvJ;2h1?LMF z24DnN+S(c5+8~)@0b;XNNN?R(AeZ8D({vNP#s7SXX0Oiu&Tfxxb&4tmYwHK|6frq9 zd7FmU;LI0nGb{-n6u9l0IPP8AjDIPAQ5lqr{2iA;m&adKp=apxwnxl0JX}LbzFSr; zp>y5IOA|vFeWC_z7T$1yp)uW+n!St+QJ(|~LAk??iy5V-lyFar;5jTCrxwswN_7xb} z{=vLkkLvAhpXVO&;;_YoFX-}+0}u6A6@?oVdCshtZ(yaT(!Q>5gl$9P=;@;*|Fchp z`aj!%W`j64F8F;k+RBx83dAGLH@Q!!7!@u5goLS`%x=E{Lx0wI}JXTG1@@yr#cslb}4F{37<1zKW=U$VXbx1;KRfBo+W z{O<_-?+Eka(fIPazS1e7PzsWrXCYi7#6EKc*CQ6HpsKqNCmC3Eq3pW}e2IdRY z%^;nJI`8|qv!shnw-S!C=>NupNJE>|->egvdTb8xR4L$pTFzL5#jqQky8iu*3 z6gj`teOS83oBH;`o};`Ubng}B;5UH$g0(M$VpY2F+*F^-j?=icfnwTYXxcJYJh#vQ zp-!Uo-QbSZY0}zkVN+(6w+L>pVCm*eR+tUDOtCb2P$Y%2FT`LrV{LB4ewchd>6)d$SU$u-OG4XB_L-)d+lsN8Pd$O@6+ z)6)>grjpPb)zcV7L6#`b!0sZretRvD$0{}pr_<6iMW((|5#2!y>YwAEPiv^-pbqVN zw>D+=+9kT#HtTD1-Z#Lrrf+}}qU_1;V_^v$Q__X%`vOfM!~vfj75B?tQN6)1e*z0@ z2M6oSjyb#qnexaeW<^Uk{o42?O6t*C!Z*emY%!SfMKBYbGQuNA1y@L4LuY~BP_q#G z&aNvgi6h1wFCjxwqgt8yk=t&N!5~lPz}WSE4wf}T4f`1z3Jgj6scm13ll+;v8bs+- zHRs5aG>O-abyUIwPnvmCKjsE~E#1vGI@vntf4JS%sFjP+T}J@EWH&d?#}f@&^=m5b z)V{na@c_wG?crNxz@K;$T;WxEBsxQSu5tsfM%al{H+mbZcX=M`RW)`?MKRyD>J3HQ zE&qIQwklrSASmi#5(H>&No31|e|~MXSXwBV7OXYi0=-LL4jksZv>$ftN(Z5o2wU3m zFwVLy?S!*Ubulsev-s78rwU~?*1~aYjZHk2Dd33DGb2;09{z~F> zEYqZ=Wo4ZX?T#f|jra+Xn1ayUc~b~iNf1mw{=%Sg{)0fdS+mRxdk4c%HHH|Q^|>52 z_j`6D&wzo$Sz`(8;O@teX=_2bR?=fu6>is;8TVS}X8lAg z6;=PxLk3<-#c57*(#TX^a+t+BlZBYB^wp65$Antko+(!<<1yo!2TlODvo=bPMV}3`@GdD6bn{4~i}NR5v~Wa>r^aFfw(CbLtbgy5q>; zF>0hod*&%-svH<8G_WtH+mjMSJ3{*;qzPD=*BWNiImcqF^uj*R-9FFX?$|anZB(R= zMIQl~WK)tl4kFh|EM%?sd6zxKO7h8M z2kfLY@}A99W77!FrrXfGCOrfy-9$XGxbE|<6l<}%0U_VR7ZlLZB;9i24KWwabvoZC zkv$RZF{}Ap73{8>(2T^-LCedSI=%*w&=0eDV-L74LyGDb+@~hD=GsSXTPOPZ#J!9s zjU^00EOU&BN4CyRNcsHK+KCH4MRq`;)yUR20P_PK_a31g*KYvg!Ww8C+R}NZntNpR zqC$Y6PUrn@{^+n&?!i}>8byY@d7kK{vi-Gdm(oBb zWm4lgrIm0vFFf76Vm#8OPlKLi&*E^`%*t!Pk(Ui*JS*IN+-pDP(0?PlV>?$ljlsM9B>ef9?@vhEv^b z>2c>`*)~zA#afE&Bz;eKN``rH3U#+K0S~uae0NT*qXw_I zQZfR{FXicRTxZZcx>=v=G3S@jv^gC8K%BMMt}J%qL1z}lg4QRZ5~~YK_yVCrvKXfdpjDhpOQqg~%tfl!T~Yghpqx34yjCAKJ8VT;D1rO& zxxm%=t2tM?uPuVBHn^p`MB(*=S(SHPJhJ@cre8w4^uv_gz^ei+#@&`%U|sVR7G;IR z&x0q$d}&F!*~i7a5z$xkQ*Iwey4an!p>+-?@ofR6FiUP?%}nZE(#*L6E8Ex z-iveQ0NGkX%v^fN9arr2Ny#7Dr6QSzM{hsd{b=7srei5uUzt*G2Da29o9Hw*OXi+@ zs4{n*W9eEn*I@!i5qAp*`C9p1Wty_C+clNqnc2l1c4JXy)(ELwSi`Eka^76lqROOa zvrmZ|Xz#0GpBA1iV{Jmi{C1o%=ce9SQtq`rvSC4_gL^3nscEiIK1rH~cj1|3!#1e} zfdy@7uY!q4q`<%@S8$Gh#9hucFV;r3CCVX(;h6?O22LcoQ&Xfp+xGt0s_{3#jAenz z+34Wc%Pvi)a<6h>`fmW9A)u(fDxAlDf_9pk)&H06Iu;Ayt=|Au{VHz{plM%DLua22 zVjtuPP659G0!->BvI2Jl#72En4hY%2UQ}4jW?b(n*8@peD3aAFGdRQqx)uR}c6A%G zY6S^}TNjruPUaw6r_CuUMOaNNM%$%hc@9Iv_ zZ-9Q&geK&(^|wnP??UcB0WepP6t6?IX6B z3DZ~Uky2iG5qkDjpF<{2l)D#g0=`BSLM>d98>4;VX213-B_q9xG@B#+}#_YD;D5>HpqQ&_0v z3a7%vXtifFHT9F1*kLC4nf$s(T>HYwRuM*bdxZwa-1Ekib-Nr$Sa@mWQ}BU2wT^mm zw3YGg7ppEC92Mg{LN1<`nvS}$W`}|I)}qEYG#`X!G$Kigo$ka?V5dBZS%@HyyZLB8 z^%WUWEX1#gAo6je69=cM7IPo^3O4aXLt+jSF4}pnnE}4>WMk*%V5Bz8*|=@O0>Os_ zU5M{ab;v>&D3y_lM;l-*u+yYR)Ty{ZZDM%t+b*wEN&qYidx)=DBSXV3imkteRWMBG zAO;C3)wZsdTlOC0;;wO*d_i+we9o3HYOfMYL@y6s^K>g2l0rzs)Rf>u@*z>pN|;7e&!Hk4 z%X462Fn_&nQ^rgG+BrL*K*IU__F<#{@p3EvT*ILs-n!G>xQ**MSHd;VBCv)YV;UiM zBD)Rvq)Ds7n1}Qr?~4!GQ}oy#qXCs%EUfed@caycP& zrO&78nO3fzl)WgKh;3GkY%wt=_Y2p5 zo|}JSvuC@wP@T?cwr|XkF3AhpEt(T^kH}imL)}1AHg6-J6YK6X+16w)2m1O28t>rb zxYD!-ziDhaf{CTh_)kEWIzcS-V@;kbQ`i!)yAf%Q{8lh`-vb>0fK?au{pz~FS=hl{ z&kq8I+WTWgzBsi>7Ck@%+uf#fVhyGzK?cMMoyA?FW*UNJ#hT9iJ`F+3&KrfMyIZ9N z)`Q0F9HPs%!C#`&L4IRRK5gvTENoD>D;>?I&o(9L8r2?8RXaMm2+_LI$etfXyrk=T zGHKC01C|b zW|V>W_*qBWPk7EJ(U~fmCPsdVT@^RQV4Kg3m(ocal~PIjx-rMn%-qJA_A}cYCB*W1Vp70RK45^n6u*4%H z-L5{Qia_nCGps4m1!y!s**fa1EOfkl#L;-Y_r75fcJzvvHFtysF+S{8S&Vlc_d%U9 zK}?;o>8r$*&aXpIbAzfI7|h6fx`FDOgRmqwD*;@lspW6J` z5Cf_Do@AhK{exmUnhF9@%_hI)Tlz}oOnf!-(3lawNcUbGMzL|e)_18anhCe?qYekf zq|TzYUl!c65$Ignb?r&9ck$HUK}U(kODr^O`sE%N@C4LUG%lYahP8;1thycay4hw0 zTG%0DfIitNh%1M|GP@Jg!IF zU|CyEwe@t3?Q>hv7p&M-3JzTu~LmSL}NJ=^G%2xAIGjUQ5?Ez@pYx z);9p5VfayE9~N?npnF*7%)v^b?*3qMxwSP;5LBGlM*4z>-#by zoKc$a^_4~h!w4u|mcI~W;FI66V{BWDt&%8cV9Q~k!{qHdxRJ|dGG;Om+aj8^AJxv+ zXHwy${bDuI#-SE|i{0&cp83dA~>92hg0 zgLxwN%HkY3fe@zSTk@-73XQ;uK_5Sa9uf8z0@+u&4S`nfo(Xz_U>MlJB7n|qQ{!{x zc$$r0#8H35jLQXYfj#%q~1z zn@*Z7%`q!|@rz|0o|ZXbdqlp3^p@ZUAUT1>igw@A=|OFc#39|9Dlu_x9icDuA*M;a6{@4^ICKW0p9U zlOS{?O8rz;*lgaSm}oWaE3~|cCuh`-E7UnDjrZ6n?oGGhHj#A(lcj!!$eOzoeT=Un zB7Q5Kx=Je53S%w?W|ubG(61+$6xI!1%uk}>*@#Pj>km=UDZ|JxFKt@u2r`>-#Nbxn zsk-XG_l$*`M&8l~b$%K21I4s=$E_*HO6A@xv24V|osrwxEEBNZwBPshdg#xt1s@s| z)!)pUyC`AkdTKGdIy(}wk3llVxQ>x>yEFN7;?h0WloerVF_2J#R7qJSDO(2RL@vlY z;E1D$W4ANeY<2zUex&faks{maZkh5Yj={ZMHVvU#ksQ#<^v95Gd^+R#hJXn={g_8u z4b@|;O19RoA)ZF2s9XKj74oEC41{c7GqFocf~q(Y`hXeHA|Fs;1IlJ;i5 zdX=rrc)`+YcWB(u@|FI)y3Xhp8BQQl88JILuW$ERWme4H0`5_l?k(6ffiR?O`Ml|j zCmnN$(Rdq;KCa)CdnQl)!u(n;hqb!4ExKZQ7|Gu21UTIZJSB!ryj{W<;W2d!Q?{u9 z=ySX=PMJ6cV%^?U_Gir6ewYt=-*O%2(iylU0gPhjJIf3^WIZH5dhol+`lkS~)V6tVt-zr|H6mq?XWv8PE5aqFm;& z-Cyit-DJB)Y93|PU#Wq?iNhRPruUj|e|mYMtG4=NCvwwtF7{RQ$^?I%YE~Y`(OmP4 z(Ph1=JIu%L8{k8^PieU&EOp+HJ>5y!wQO zQ)TQ^2=cyLLy0;tD$Fo{DtvmN6{Lva%D_^DJKxwgAs6Rz>?SeUMI=OeAU83ivb$sD zI31s&CHEOZJe`l>9vxRN8SeGmd_W8;&8p;#E-zftuAyLG`3T0YYF%hofv8zdPC0p9 zxa}mYwb5lsL}dLGeG-1M6*s!N%(3b()J7p}(mg8)xR?0fTTK7+zpi67e0lcKJ@V=3 z{wLYScug*Y;pnNaXu4Co53r66Qe>rXAAfoN4RGe&|Mrvd*f+qI4XMou@*AMQ@Yxr{ z%)%FOv(?!L0v?Yl^(%3m;79aHla#4>J&*WIrG{7o4wpvXnV`|0W3hwTRTk+9=7A@7CbwR2 z7sqDGDpMaC#U#75V>X3zYDnArpC4_v^{tEBrd$@hZzS-$HM3kxX3X9M+opQ}BFye1qw ziXQxE&T#rtVAK4Nj6AM3-@%mQ3tgC|X4Cw;s=iA8o+2&NAd^K^hp2Z4Us4<+JPN3T zurJE%MS)>OuUGXr*msv)N~dshBRfBCE!r(sraZ-}VDiZ<@9 zm)A~N%M*=s>#!mFVKO=1QbD4xRA<#%hBERE5e5}ZAUn=xls?IovazPz+;6^;>Az7Pe4`k83xk&HzSod z{i+q8?RSkY)U@3>h7yno?tBBVApMeNnusY!cz0Ar91T5gJS>myF@G}MA|Mh-uB{W* z`PgOP**viyqj^Y=r59^Fm(YN=X)rXhJTzBZH|o$6reRFxoZ|4*5qVXah8DYc$dTfU z^7*}gyzdpkNr$FiM#cSJy?c%LLB&+(N9M;z=L#pKjU&O~-pRs9D=HguiRtSFs_|t1av z6*c#RBu=6wMBUND@Qk zpJL)JB(bvTW85TSYa#ju*}Q8cjEu#;tJZ5l(dNSLe0C>&3@)(f`P}71<^2p3y+!xV z%Mk0>QfC3lg~Yjx5JWPkosh9@Ns~ZPnX@%=90%FS7H{2g z<8h6-BbOCduebODt>x^EILJ6htp`<7b+!(DlI$!AX{r9!ij((M+8wuIQ#!N7`foqy^u~mUU55CX|>Key2&W~cw4DBZC zSPfw|?Ko>61DM}-fc+Q_Vm?%=7~O4pozB@hW4wKqL?O|<{`ozFsDzI|^;e<})lc4E zo*ZTpoO{1DIm5P3boGerUE@f8242-;X=l2xBsF80K+fLx>V-7EBWYY5ljg-17(Irc zN&L|*@|q>Z7Sv1EYlJj!UeSC5Sg(COZwk3U&~LwZ79cIt62@@VM}yl=d!RfoIB`@j z!AP68j1HRL@ivJ)VC`E_UBteW5Ih|mm!JIbsbD|&OIF(D+7{imP0jSJ#jEHm)sv$O z{lS8(?JKrZ)L)a;n1=}!YnJ`fCHDnwhW5K=dCC-Cnp^PbiQQw5<)_dSj;rm)d)TWL z!z^}O>d4fj`*G(@EEeCZyW^yKawd-KO`S(X$?WP0?FJI=q20@+eRjTisb-BC83{!= z?oHcKPkem%dGJ{rU{gk@Bwpqcdz6gSR(~>g`zG(+`c0VY5>YjV_cE#@(mz%*_sR4z zq{Lu3?z4KjlKC=g!S-aowl2_aUcEuQzByO2ZaS^Aax&Uyx8goTpXn8}zE%2CPxw$(UnajU*(pC9J zVYQcIp=V_5f8}&@ch%1JjZ2G?2>tG3EJY4$*Sd6Z+%D3FsjnHk1TrO?_K^9nk@!!| z8I*W8e1aeNc`JrD?QixUtrgu8@15xhsBeb%ciB%JFKK0k=r9il-F|P=jd(m@`ufnf zO{TvAjd8iqz-~w4fwfc_kTbl2hhMjLqW9HC+K4{P)QU~hyC$1uhv$-=53Z|q=-aN9 zc$e;@bt${I6t%Ww_nr@oPEjl%b5dhQ<4)vy|2B2R?0|RP{oT2`CKjjKn&mJSM>unG z2YUs4PE}@UIAA~A1S9g;&7mV1);L(M#jBBK9!BnqtI*om*M_xMc_V%E?D37t3vB!FD2zfaA_dh9+`f9LHJBA7C4FD zqOLB7y(7N#VY|q6--m%Tf+@1-6LeDGK$!`oyHkGaYS-NSY-!Sz$~BW`^R}1K0adlA z`*v^3Wx-b^nO<_rXT|pH!4yv>hj0d;bP{LL5L6(>k);ls1u6HOz5z%RwXd%o1da^H zGpfBOYg5LKHKH~yg1wjLb;Lwzw0R)`GJNH@wuDm$JsmSdCTG4DVmC^YhrvXx(1_J9 zJ~|=9qr3P-OyUZ9F06zrSkdYe(Y@y7mG-*i+9DJFWf%lviS3aVQcW#gIb=*-Q%nbP z(-8|VDvV1Fszf_rJI!pAj!0=2;n0J%?CN2)_Jdht{HDA7p{x!p=M=9Ag7f{HbuF?} zdC3wAYBN{EKrZPR8lBZ)`IPf(6dFrp!R~-MrbyL6Uxf_LoNCLYnalo%Y&LPMGVi+U z+!HcL9{BsXg$WD39`GgjYyjDC;26Q|nKkvcKxi9ka-8j*8}~sr%c)){+ubWa&xbL%JGh&*%u>KKT;ie~hro`}_^k}wip<$jpmL9vVrqQn z0z`JsyAJ5l=n`5CIIFVF-Q0f4_%n=mBTAk#SNdv@iP(Z{NrZ9hih|Q@ z>qK!+z@QIn1~6lNpV2Ks*5$nikb`Gk?OP)I>wP9kqOtk)$a!r$k-b)TF{hPr$yiDiwUvGOw8nGA%35j;GV1=m+>Cxc;*-$X~fG7 zSQKg*JbGp7gJ6M8*8wCmvzIifi*Q1Yot=mKZbj7c>^wMdq-xSek-0Q1rij3dqW{!D zj>)d;vH=u`(Y8@IpHrGKO+#CAFWP=D%idEmzJG+7n@QP}b&SNlKu?K7ebE_y(|>D@L6t3^v@Oqp~`t$H$xdC&fx%UP1+bZmdT<-(u(g`HSp6Rp5A~Ca&tow65Y=MdRgm8eK_VOKr}RKR2LlCavJ<_ptH_3 z=ppTUVusZKd~KP2c2UuAmm4kOFahvOPD$uZDe+h%N=hsjKc&gO>0D<$O>N=X707KS zIrm#Rr}3OQyKGFp5+5hnU~7BGCZ7utMR_n1zOS5D-%yb2;Q_RPIf{j{E;_z7lys41 zOkKnXY`3Rb!hC8F_y*@lpcW}!-NFkEm#cVhPm8B<0EL+I_T5s|tgfV?iIj;*h+qVU z6?d`jISJL0XTc}u7_Za|K5z&`HO&C$KPajJt(fl!!I_;qyeen#KChM3r0q0HH2Foz zDAO5fYA=zi%RxhT+52NtAV&7>>*ikVK;mO0xz#+ySvAF$*_9--7s zMW5BX)jV^?tB@n6N}0i~Hm(#62K@;RGnT8%sc9n^T;)N?IjBXVvj6G5IV?Ktp*`G| zaN2G}=*!r|zJ9Ge8{_DRk;~ig4d0=f1KT-2_lgVbyk^V}f@ttT zU3a{dK>MNSQNP7>=YFZiZEL>%ndIex+wqnWj%>8dqDB+;f))mqj7-@RN^fG%k7)KL z%eLiNsx4it#;k5*D57;;u-xg0z;=!t)p40nh0Y%dGtiM*jCcr6SdzFIt1~nFwL=@Z8E4a5cJj~QL- z#`EUg%9rWnX5{bIymE~z=?X!n;nY)$gU2>B?D%ZsCOh+RjbalX(`9*{We8M~)Z zG_jZA_uS*yUlxYx!)QbFIQs{|yHQ9VSCTYC2W#r_J?z_pZ<@l2p1>e8=_K{Ujn(}2 z`H!6X&Kg$-S2N0t$7e4l1MTDe-FM4bLY3+4tXyDf<;_cysFrA6c7EZu$k!syqTk@SL>LsfmfyB%7~~jgSHmd(~p0BHrJ#!mz}uuDZxOT!C)SY(BeL zLF*_5x?mHd8pxO4YIhlF^Rln%gH&PZ;0o! z&LqxZgUIRpzOhuUc!w3L0Bkt$a(yMmC6!+!4ko5nvyoD0z!&5Z56 zEfbR?5Lq7sOBgYdpYulxvfy{XIcsm92V7>tXxKFl1ZWKD%7x2ngHu;|ErzfV2yN}| zfT=yx2qeycJ**|!aN^CY$SJ`OQ5EXtqYWFNdWRVUbum16(hI)+72xjQZU7H4emvQ` zd)gSSJ2#-_`9oad*iZG7i9DkcVw2betqH7PwY0woN>`$BIzN8J$Y0cjD9%j1C2wsP^T)-lOhS4$#AA#ou8hk2 zZU55S6#paJcj{?Xf+b>;`*QuXBh1lhz>YYo(rkKb@QEx=`F266`X zPiRmClHPhH+-&lXY_TsX)N7hcOx2`%@rmQYRQ-8|Nwjn4NI!+DOSF(#NG-k3U!PIf zy(a~$m`N*m?;5S&?IT`%kzI{+?KBwFbd+a~ur@U>cE$V09b|<6ZFj`wo0mzkW%pARrjo(;TaQ<6 zGgC;&u-N&cal%o#3ER5c+Z%uj-bouzx#Px^4_&CfL@(Lp%~DSXou=6A!I?vEKflxt zTGfH+fb(+K#=&=g3KfQ>3$i>ht`Z+T=56?F7i>>t&Yxii`vYztKK(~V&s%X-T+E^S zU+WL9TpKOgzmACsS#M#sR(6F?ngg;s5 z9jHgZUD@H4S+le7Ert8DzMOJw{tq3#e6Jx2@IveJ+#e)jK46^M4F0pz=HJnNrC20R6OPnXS=J@e5GBri`dCI{PLh%e$ zaSwmEm@cEM;!)|Um&kod6etVB8)K%fRE1MQ4Bb*=@2dx}Ibp_U$xCdTSaf*uhxpqtcvk(HUNB}yhFmzPQ@753+W*#HLD0TXc-LHx4{K1G+;atI z1xEJ9%Grumlg{s&4#A3j~Y(EI0!cAa@L``@Pg*-Eo(#e;`9ZqE?-5s|L3pB%F*o1}_OTRF4iq%Pa?rZU6T=d6#+ zg#D^~OSuy-a;(CiMNxbz^D?A z9QzH=v}Dl#hW~|$vM!;@J=s)G9DI^?4=39TfH_?#2n-yr+)1~O;(=8}_}^W~Tu&bfVa_#S<3Om98eILivC@-k|}^KB+G>VLdbayX&7R^J!>M$+vce!uSEl5{9=d8Fvmi*8@I zq`o&hrbE7_j<06LgJR%icw0HBZ0N!_f@SsSm$A9{8*IV_7xt~S*E>3R zPm}UoAGT_hRHML?{*~R0RrQ!s;z{QM;9If2m(A%r@6v`IG~V%yC@<7oYEVJC5q z8tb6;(&hvulXkpe!|4F1d1dwGn9HfB2$HC*rUQD)72nc{d5y5xGCQf4G(CWj%O#Ot z(}n%nLuD-(?T@mYWk==8FS97Kg$R=MC9o*u-VsnP+*G17qd@gy53SnCZW!pJ*1S*m zynD8)qsvomxKr%-RmP3GMQWnai#dE>%p^}vB{#o5oK-2hQ>E$_>OP~n;Y*(-+A=g} zxew9I0P^3Jt1c^BCj@Ve4!X18Aln@g%>^vep)R!n?=H%GmRdb7Pw)-!Ry3zQB zaFUBe`nMzRtF>^Zg>+I$m45NVboT6pbKZpwJzv8p3nLFGZ^2RtdO*+Kvp#pXiO6R? zs1P?wss=WK_wnGfim+Z4ZCc%FJV}uV(fMelSgxjFs@Ow~jhS?Gf>1hU@D!7>6O1_3+G}H6wM8K5?Cj6ZE z|6zJzw~C|nhW6^u6#S3(q<@eQ%@{(dYt2-GZPdkoO7CuXbba5^D2t1+k~g7np6W@G z7e-1i!NOQm@HL#W7ihi_69@alDSxIF*&Ta!0B2X;5axl9(<%rxRdfe%J&^oguNzEk zHkG2)>d?>MTc{_|Cq{8jqDpoYT&gS9sd3Fzh=jDwcjA`VhL>qCG*#PEdoC#v=YAZs10|@`Sw#@9{o-plOE- zMpl<1HfPQ9l)Uo#Wsk`zWyd7&^X$E$Kb0ixS;aD7@=11jCH?gCdXAtX0vH2r{r*S{2iU^J#U4n=%nqS zdR#Kp&9gPczNa|XQ~tHsWDXS%r^W=O>-%vqr@i(T)nYT{ip*ibJ@~bzjS~(CY{NVl zD=8fjS9rPOD4LR%eqd_eSme&rqO2!JGwzs+++1^)5*Mbeo7)J|W3^|&yaGc^XRS9y zfSAUaW7P2OAYl5oRQ*lIjs1AC?nqWksRF8~GUHu)_0b+w*NLyumPAdM46~GAw0OJi zljzFa_-0EC6-L&ov6JDN1^sD_I==iL+12D>;il`mABPb-=I{Qx2aX-f25Cs19G-X$ zR5xY0dK-lAJeBM}@nfo}ce6iGLssYrb~vOeF7BVP$)2nXqcbM+^m15~VhTl_St;i2 zDt#-D5b5jojrVSCn-k|OlUExUtZS{aUwdtzSiEZ~Uhs}9%rlq${>dA~cqg_)$H6Qp zBZLJf5E*bsalw5txJ+9(HK<;nt@6m1R;?}XX78cj+M>W+Kb&^wJ|#L!)ZlB8*|>+J zE)B%JBTzb(9!*iQb}r2@S-T9f_R8CC_SzyZ;jgflkt@h3H(slP?`E zOxqq#wh|pCopyV2d>M`nT5S9bI5*Qt;UF*4)T8f`m zN$lR;P(%VvHi{%4b$PqWt?*BvLlPnC!_#g$Ib;;dD_re1im4y@0@5H1eqt&te2n4@ z6yga=X4S{+FZ`Eherjt||H6f7dU#7xeR@?jp;F0{T+$h(b60c!fmQh}nivNc{R9T^ z=;lSwh^3=WUqRZ8ch+I89B*)=^;Q=myr1O*ASeiV>!@Q%~9NF#EEN-s-@8tED< z%Zt`jQ%EoO0TX>i|Hm>_tZ`=2=-C|z-m|p9u?)G6JR&3GOynj;7CwAS%o_~Y5tmvx z#1acep1wwjnf@cI5cN`$k`y>w496Nv55DUDhHPg>z6AL50W?a&v}~{)G<>|CKnG}Q z3-Fm9RLyM4z`Q5Iuf{JV;Hr0QPjKST)KvMyj4pW}<$jvlzyzLcYDilwGdk%m~|lU)C1M;1rsoMv9a*8b!h2%rd1Z(rEw6n{}(C2@sT0j z3?dGj!nG=U@rfMtRQiL_62LNh#BXf&Gp--$S`$U<6ot2A$%%uw=dvn0cAp45Mra>e z3vqJwZcYLq>zLjSK`>;>6JUuyw5GwYwrB!6lE zR5agjBcma2+HdR zWd|eFJD5RpB2?gTN2*(_ej=K8B|3MXvF>Co&>I#F-ds_Va(H_p`P_|^`ITv!(g1X* zUs<$pcebqe2~_hiX) zpCAbj1swaW%I%8(dfMX%+@?=GEJGaUaxj(DH_2s(Q}B6~Ax+m|2-ZHjsJ)eN zg;|+q-r$%%$!nq(*c$9wS zSU+~~r}S1!9-@*_2;lEqE#Nk3{^`xc>+S|8y1*90t%Q6fxdNLPirXz}GKK(+*mzwL z7XYzuDe9qw6o7opZ4wQS^nB5>T>b!FbklTZIcv@fKV^S1OyTVx z9)S`IP9Nxhj4X#G)S{jj)Z>L-el8WZkI^Z7uVgc&*M}TI;E(^2y)gAR(QOXItHIJD zG)*Of&U&sLkH-6n#p#bf&p%q81nh1&$l7a*(<~2@xwf0JCxu8slqb|oySsGevk#Ul;~8{reMuXdA~*XogV@6vU1Mn zHnqKq9(^oivkP=W9fHfUM2!LPjbIkP*SC;v1WTJMkBd7D*+e`J6$S0wTO?`)2c$Xf za-AtewNhG&KkA1uY^r-q~G;pCSb9FVFe)5QwD4F;L$ z3LBU`Ws5m9wHExhZ4{-!#q*C$Qc*dR=(}FN@enRYha7Pm#a&$x!w9;C0-@p?J|hyN zf0W6t(X@ziZY7O4RSr>&EJb&BRN1QNvr=|mYx%@Amv-=S+%*5WXp(ZRGm?CWsk+)h zSTaYtvE9SLcCgutD7fijXX?5%VdK35l4y9cbtDjeKqR6V7U#Vh#5O%%{4&;|1upwP zsaKegZX>+e4``=95QEBMbL_YC{eL;^xRi*?F5 zF9^M_7R~N;1@yi46#!aR&B+l2v2nw*4kq3YhKGKK{@id#{=_XmB~bI3 z@3rlch5j|dM-bE>t3i`Ax}^PU+6=0Xy&%{6b&{*gVh*|E7C!{h$!%Ht{uC#G30 zBC-_r(@d}@wu_JtuZOS3;+JKLZ}CfvA3$Y{#X25E@>-skt@5Wbn{)Zwa^eYUBso5( zd>r<;vkoi3hfyklftDx@fpel?d?`+)Zb$0{Sic!fG{xr25G(~4t1s_S7@O!>OtHhT zt1AXl8#k%rduQg|*(5svh9yzuKzPmi^knA^a$W19uATLBzF*lBSJr)sbA3S|y#ZUx zz!o)6kEhDru%|cVNsgcGci#V6yg0AO$}Zb)r>z682$*gXa7ozTrLobB>4GQk-Z1K- z1!myus3J`gjg?(gp2t8Om0<4gH}R8oQK3#E(}SuIqs`M30x>g zi{iBmu9r3juoyLr_Sq0s0OW~w8Oo*oWlDEXA`aYAen0sXZaVNjZi^V`X}J%do1M7u zw&wU$X{=kv;kWg1Aw;f@V#CEl-AVgsNW^K5w<(ZYg~T!596PlW{G;*pX}DSI+E3wh z`v@p`T>~~>$RrzFe0`%VE+Es4Wkgik8?nqwZq{Ff<49D-O6IZkb-JLGUCstgj%32ZEO(kjbkX-b-m z%FJH&F?sms9EXBN(ADo;I-W_AzoS17H%3)Cz#29; zzU^R3uP#+*b|{zVW^Uj%aqqUjK*Vc?{9X@Rwt&PABQ<+%mvA1+s<(P~^E=D+Jvq*g zdg7@YnEGDEgDuEFK(D1f&9vK16oN&2Yo(qsp#oW}l+WY5ccrrIn%ZexzB;Hzw$;i> zMm-hS319O=PXg1X!P5vgT2Vb;v>43$g_jF|8ZUv3NS|+$B>l0}pt#YtW9+l|+6%c$ z^W-bhGw?RG2Kd|L^c5=3=;8~w57hiJ$6SRW!#-RisjdKomM-wq+M6xk)N>Boc{Tfq z`^93xI47TO{$!lG(B9#p^-F|7;g51EM4jC7py1-u26K7iVC; zv~=2?$-=V@3L^tTumAEX{GqP%R1zJE1*Z*Yr-daC^h?5|y11%mJnX4o0;C-h@i9#B z%l(PS2Zc#F(tXVG{I5Q=1O?}64&FzCwmfPt*D#N2yOxr3L7yI!fxv+`odP_0TW71>DR80xC4|T?cKRGAuH1+Fr5bKAj;jH(eF0P4Bkj4N?-ZY2IFA$heSgmLVGe!u> zhydZai=7!P85w=i>PCsT1)NFAhMR*|9MRQh$ca&Ne*=|`Qd<-ng8OS;4+-n)wmO*? z*gN^ztl6242OiWi`}p4s-^u>K=&<;){PkP zZ0}zEgx1~w_3dglr?@%oBdpd3kC%fTGdR+SS#kB8f@yAX7P045*@b|7)Awpsn*>^*(sXP&>{R&pcZu;D5y5X zH=MyKNj}C?v`-u@j=dB|!yAdk08C>l8hSm2Jo4`bv&yDJDb#kPENz~3B$866U}3e2 zs}G}s^K(Lq_Xr0rZ`+@EI=f5O-TMg8Gtt8uJo@u88n-(JpVxBwAx1yXH%>-Y U{QCbIvHbrV{|_rb^Kbfp0gLX!AOHXW literal 0 HcmV?d00001 diff --git a/image/funasr_logo.jpg b/image/funasr_logo.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a47243e760e2b3938c22482d87bda4890d7fc929 GIT binary patch literal 639403 zcmeFa30%zE|3Cgt$P!5}UAuSXC&J46#fkYUI$v7v}Kyb?nO!LPADzlIKCipY?G z&k6+=7n&}z=MWpU~ZQkr> zrmejxaHgwpUNhaOzS=ZbKW*KaI@*XiO7nAd^YPfMyw1bZd(&){&f5$XWpDS{DlUe0 zI(B}R9$wxHLIOM-L+qX0LVVmz-BnNvWpkPt&DYP@W3#I=&3EIbKr`BGm4U&{;I;5$ zZ58E#CYya`t1PoytZYdQ@K83IsWVeY1$qa#uQzj;XZ3SmcsE<+=Sc!^M~0UHO#ba&JC*y!Qwv1xN4EMHeR%g@vI_50fRH|g|ELp!@aZtCm%b=bho z^S8i^{%8XNowoXUXghcWQiB5AJmznKsi}T>nBNx$`t2dUem$_0D$4F|X6vZ|zOL{@ zZ(mnW5A6+p9-hLn{LJiNOWa*IyUugn>@izK$bMS7x>|brPG8t>x(@K6&gaIK0UoZK zsR2$@>c-hBf6U6?bny6LY(1whO!)2Cc2sxo^;>^v`kT|Rvoj+HcmqqiZk$JT3li?x z!g;e*EKLoqO!Q3Vm|D%TnmgBE{(MtY{W-d(^XFKZ7|t^=G98!^^j)|qaI@F*z|J!GN=4^MW z8;tRl=w>D!`o`-#+;p_ublr5dOiT<6v`k!0*K4iUF*Y??r)#oa-^fr!`Iq*V)Q!{t zSU+rlzL14sRxr;6-hn{ttv?Heqlf>`uQqxs4{(oJAa(ubVAlYTIi5g)f4v`{JGpte zZu0a1AqH0b{7JBv2MG5c?EL3u-oHUM6BEPr26~3BKqn(pEh7^HJ*{>6uBKYL>vi;W zOblI3bxmBqJjT^cD2}sL0$sOwxT|RY^A_{xc;GWJe!UoNAB<&yf@0Xo+@}O;A zI)oU6|`xNqrZYq> zs-SMIS1toUH?xe>uf#C`D>k5CyR3W&+vg?595fd~Ydp(M5f z5uc!Dtpov!LC7GbK^Dj`p|!88tEZQ;(!e z%WFRO8J74`P=H7e8&)KXiHk`igQUg8rNsm#!U%}i5V3)ufo||iY>@ciAwwmG4IeQQ z8eAKL3=$I;A2e8e$PnR_#KPcvWU%y*v6FS@43$~pDxu;ptGDC8iD9a9FBi%=R5GXN zuM5~Ye8f2U@d}D+Q`M(wXc`zA8Jn1zSYd$f)Sudtwq2laddn96EgDWcsPoXEM&7JAdVB&b8~gH*Vf4DlRF#TXygM zgQr#1&z{%3cv;)f*z~UXeM@UwJFBak-P7C0;qrv*g3!w6Wx?;yEBj$xKm@Ttg9nQb zmJqH>Y*4Up;nIVLOx7JbcFqzBSAQ85y&c13=N>q5xp26uz5`QkT|niCaZ?N$)L6o$ z4Xo_XEo|ri)5^Xs?3Z=DLM$OB_WM7BK7TdnyC3oIe*`ed&m%zdLE_N%`v!mJo!|{J zc+h@9Eg~rn@i%F4X#_iV#SIKq3SZA<*B3Ku<*oBtjq&0*MglZ$qFe5dw)2NQ6Kl1p3<$s9JTFcYhrop#o7okO+Z92qeOszYcT0r!^`N)dPtT zNQ6M&ONHp&Ux!DiKvWMTLLdZzW+1!T~rSw;>3RjCyMsrzjGf% z^*|!d|L^2{(Z2sP_FYsD^k+FyDwuzm9j3m9^QQio`s%0bkDfNS1<0<%k1IB1dH9}f z>$9-lmHlaL|NLXMdDJWO7Rm>t>P3xLRwNGaRNgIDAIqP7W80KQ^NEthLsoXYt&-Yv z+WwAW>fud#r%ciz)a%%Et`+r;5Rh*0_5Dgjuy+fV>ZUpMQ0a0HG%ckODe48rLN4s$<&{69q_v zTthZtl};#o?xe7--7$`L0oH92<|zaBvZx7=9cZ`IuO49iy(clx zhRa};+MHk@fJ+pEs~FGOkv1HwoOl49mLVoU9yRo&l5sqR7!0q`hQ;_)0dkVU>=Yow zvPWTEh9s7_`B?a9mZ|_Lwig#5hl@K*=v=NgK3sqt!ir9C>;%X>MnfGTy+%=hyxV{_ z4yD0d=WVeTAW^9j0z^Ohxj)+7Md83IbOcDGCG#k;2P@qpIE%IdHC>r~{5p=MxrrFB@%d?vTz#SdIf71s%~}eyPm=ha_ULEY zSAH^AXn2aL`q~o8CKp+JCiZU2UM)Nu`98ai+$33x@JL4dURIs-Me3nyUr zCnjJGnL_pvAZNelPe%bV90dtLK?3xD`pgkSK6AuE;D`%9aD*<_Kn_(i$BHg-!qHEQ zn0?(waQ35d2yM!<8VPsdw6Jb7f+6z_(?!>J7EyRpQK_=3!u&~S#>)yWUmpJ{8-y(W zKFWLsGS)2i9aob>rwlv+>wy*N7dK__ZwvWix{xor$hhHJl>5`StC(alCUh#P8)UMn zt}uK8dg2<91C#m##@5ZK?<-ovU|z5w`M;p)7v~X!YE$r^7=yF1o>>FC@uTR_nHj`b zp=e?-r}aC8k|10+2jpzt79pkbexQ_4c2MSi0aE_Ck7u+1$-0zIC{u(x_%o@#0}FXV zwy$VO1wLIWx{hPTp%HXX0|t;*ttR5iZ>$=c!=8O!L>SsIlGy_C;}AZWqXJ}e6!xgc zzNeEEN4VPecy1D&yO5M5-wSQz?!$gnmwb@0XVW41|N}tpR$2O55&LQ zXc*POltW?mPkJlK{DiSyt@Q%=m#*LYg~`{9{GQ2mh0I|O`w%KtM&(%MAX3#wu?_)J zLoT2WHil(%QD7Mi?t*W(TQOWRRE5etV()W;1YbwZWD>AheAUBmjz@`&O1j zfIP1v)cOiavWTX`>Eu{V!eeukH|QG|8qW_kTl`uPvER|JHFD$k#p!3y_>U$h()d2W1coF8pJ;J zjA&Em2#`XHn=x$;6k27!1ul0_fb69Pl)})8P%ub9)7B~08vf?`_?+l0v0W#z! ziJ65P_VmkSPhr1F#pp%xoP$#7ef+V9F=7OEA>}=3RVgUuvOZXrYGN;qqcM3xz7hv( zer$erlPifiStb_7E7N4m&rW~ixs`cFqIr0+09kN$hnqi4>Hz&^Rec=#F0~aq25WlR zq0V;XnPZjHBl>~Qx+?j>z^N#y-aPyvd|I!vx7pOS zpBX7Y#DJ#qi!E*stbi#`D7>oA;LBu75o6wn_uM}uKs3?DrQz{(w#`}wQ$aX<*tXBZ zTh(37C>%blqBu*^ZF(f}B-a12{r%|f3oUFXo_Td}==6v+X)nW_9Y{68NfU#xVq+XQ z?qo0kLfso%0TK_WUUD{U(l|!@MPQ^sSpwuCh&e@^4olMG4cs`vf=0>$`A1mZ@_(*YIPpvlS*ATy@H=1u46f!fmtds^w0067}b zZ%x7l2zLf{Z%)5vI0zV9rR5bdHeM04n(20m2`KaRWIp*g+?(B$=;(wH^lhEY24UhC+UNJb~c{(RU+aOlLko5k~8Ll;gw!q&z6gXtW6j5M|pjaFh3Ry!KZkk0f?Ef7=#;mxH(c=IBE2ECQW*2T`$icR>owZv5cQ*LH|4p$scC1 zq+cTxnHV~PTvIwppE$Ce!1!D8BQYx~Z)&yx36iY&UQQ?+5A>w~A$FnoO&qxVp=LHG zfsw8aCK>lYIfHxSn^{Nv3d7?6548oqdi8?JuOP6#u5U>>h3Nbi1L`;)r0vT|EKB1m z0FCkBb)S{$=9S?KUB2x30QtVzb3Z=pxhFBt0sG`F-1H48PcmA|!%VyxpV{;qL~;wi zUne%aso<$pMvVDdiTBi_+-loaJGM8~UMYe7NFS|%2cjI9k0})uqKrDOM=hZ6#)9Kh zlv!Gb9~B~)E!!IqEQG@Gd&J;g!fI&@p%5&_il^ehPA62zxMtpJ_#_7_N#HADt!XKQ zlD;;vQR<^Pr5pUnvzGnPQkZBZaYZElzU;ijr4KgU7h~j zVbO-faPT~y98f3Z*35(l)uN3H!}p#nb8?x1S1_0! z4<_V3A7EogwkM_A4tsBD2zF0BQCSA8PuP;Wz^WwIOYnyQxkqFxVqHZkKMP+reh*;( zM{v$yiW9qZVGCULeFoQ4W$9ef2Z+WtXM;6oU|>BiRD7Ub7rsq zVmf61U^@J&z2hI6CZHz-2oRh%23Wj>1*&8QkmV0-8R$SDlO(p^6d;$=@1}Fxoob$B zwmx{nct#(ficlFgXVQiAGj9zqt{Z+|oER&$duvPlN6M(~Px`0gj=`4pfTt@!Vm5Q8 z9=v%b7-$Az!Y$ezo!iWt#^mH!v;bp{MlW?1`RVf+ESdP#D~pxG-E7bYDY#=sA<9$q zq;nhr4O+aOQ}^Uwj(-h2KQ(+Q{frrifGSrQ6uLXXJFvB`hmTR-CGC<$7K{Bf1QGoa zk>Ky+0x{9y%RhPYQ0gS=LE>t^9RxwK3<0wEjuqovek7$U(Hsyuh{Shxi&J{Ye}HN8 zHAq+G2rfzU|EFY+m&ooaN2ba@33o!)hDbfwE-#M9{5;v_ zeBcnNq-Pc~{(tu59W{6UKm>urt zBREU84dtma$&E(I8LgBt=0PXgmI@H}r);VMjc!xr8`^f!qQW)9C$M(;D{h};m&Ltj zF551<#a^D0fW1Fjx-n%&14Gs%F*;yHc$viZa(zj|5usJ|tLc{VE0U)LRwuPsESo$1 zNI=q?Yz?DV7EgD*u^Dqtt68?LoWdir?z$8x{(Vv3HO*%l{)F7vZ8E&cmMYXD*M}b2wL+p467E}BZ(r~Jkhgm1)kfQC8;`6x&~Y#% z?8AafQ3We^yK%+%?Y@up^0nC^<>B#q7}sS0yqy5ZVN%>xT%Wb1uG+Qkg>R5t{jR_-pN7JS$}gAUX0OQrv0@F&l6Sddv=_ z6H~HV!J(a3LAVD}iA_?!GzWhbTnOw*l+f2Z!5@0^kAtV+w*3$kLVpNVf8(-A{ns=$ z-v*JGB~43s$G(Xpfg`79`lCk(LB`L)|;<0@RUxK>{q3Sxe3V54U zO=M!(zw#cFQO{1EG}hJjC&J)p!#}6B(Ch6kU}~ei2Am3BJA|EnFTSn;Q8lEAnCC{P z%F^x;^jcXuIK1Ul?o>v%DnLbdaO*db3`_8jg@M<&I*ua^`2V0B%2J~U1D=`N$NABIbSTP;FBa|OJmGkK`V^F zSZb(McpePQY!FI~*Z1+f_yTR7G?|dIfck)+^A8aA7mog`qzP6$Ep!kR|BR@M{&{Rr z2;s4BOoTHp{$b`)9QV%c7+V!3WQl>HAy`XKx8}=`g%bzf`*Gr-;t;up@Fg>Y75#H! zPeT}o{Jn`s7pT4C$nBmOzXx}V$Agy)g$<>ro7F{O-IImu{5CT>5~8me{V32KhE75E zF}NP&-`A7ht5y2lI{eBXXx0DAt)0Fx9vDBYlJdRnAobxib8u|@peBp7X{-YRnUIRx0z7{7E(M30t$#VN6;@nA=t5?B)bGRga}X-wlMu#Uf!qziV)TnEc(QzjL`WQN5Fn7W!oEw&MgJ=L3^gae zr&xjD|ItbAKc6(V1dDhebLSQhHbyb8T@E%KwZeE)>j7HG~2H zRT%?`l5c@^1nQx^T^YL3VC-`$8k_^jAACtgljVig51K-Y9E=ua4D3m$e-TeOyfS&@)D^ibV(yl>Tl$dJD|302~Rdmq3U+S zT|CY0Ncf>=+;5ZfzhQ_}{imb#Z#Zwi0$@PC+#UV`B+xoGo_MDrEQUA?NNV`G7~}LSZXsrMM21qWB6DH|?1;AFocl4hz^*^=kZ?&l0>gi3l>53aLTmaN$M=GGDdkQc5FSdi(y6Qsh?teI&nDsM!_#`fgA=V&IU`; z5S#j-r7U!>-?qRjT=F}N_s(Khoz*v%p_tN)eD6{HG{NWCufg39bJJ7)N)=FS-fS1n14hP zm3hCBDi;=F8}bt3 zvbzRq7}5^hQztD7cM~AF>x3l$&3TN0lD7(uJ9P(zt+|HOe>ijj+WQG=Blb+<8Dmcp zvI+SnP$kDdmpcdvW5tmJ!W?Vun>3Y0>bYe=e-hkUN9B%vCY=2&s3aHfCoAA}JHh&= z?m*d^by$OKBZ*je4#u3wGlrEP%!NT2T-q;X5&PhHO61B;TqD#Hl%WAu62}MFYPDS{ zKt{6a@VsxIE(1@`>LX~<%Ah*$-{SQD)4Ssh-J?U|2!Ndjm7YzRc8W z+~s}+eEc(u^4@8Ajtxyx$KLOoux9zby2H5-`xmeV%fIOAJ(j;DepVd6%!_iYB7b{0 z@}9Wg%zPA(?O9RYvuYjv+WY2Qw8q;_>;40782u`zxuW7zfVX6gg4}uXGpx%9)b_o* zKx8Ejn4fc?UYfx%VwOQgZAS~E@tDPM>`9e5%sv%(bCwNJQvzO6cngqB0@rCN!gx~n zd?;AXjY7FD*1~MP_BYvj%%>4Mhz0`DB8c&U4JcodB#kkB8|ZDPfr>wjZP`ze!7uki zTycsUZEvq4bC%bU6kH`uowaI`>bPchG;7q`16xmCtJMjLSQ2elRiA1-ZK(bX`xSac zkwKIhwv7iGSGWqbMh^(PM=R6aPTxlj>Hd8@|pE`2NGzezR1q-xX$-Oaw!9vE-- zmvw)4uF>zA(z^3cG9OfXcs&>!+vXf;#%_0rzG-#MjQz}JMRw^{wBzwd1OFiA&>epw z=qzgT7glzV1OV-#1k<0$oc-4)n7~8MV$C8R$nQgYrrgZt-ef%_$dvKei%y7CRO54c zO(=aEu!i_|@w2lZ+7*45@&6tAN;TVwvB`zxT2d)}c=pDem!D=DG6qf9v#&3zStsZE zN!GKB{3jRJb5A6%)Z-2*8GF#La5espIK!JAropKxlq{A?YTQ89ZOAn>dokQ3*Us7d z@%7QC5|TA9Cj>feeLgAk*c#g1togg6Cuxh@%BCjdfcl~LWZdH~<^;3Ig>+ei)6YUR zGD`zGB=y=A=Rcf3^ji9TqvmCXb|$7?M3DP(*&J%lp*Ph-vwOD{F%@m$`Rd2?T?C8v>6t!hI54Gl_Gc(fH zrjf^&iJP=N#_nf7Gq}7xXZyeH+HM(fVbw>@uj>c~sxf}EZfxyn`k%=M{tBAMSFAu5 zNzLJ|+p9e8^M;SsE+fR46(w{@3T^;B}ytrnOK+ROPBaj6kYlG;}~~_m!5rIToaR!v8UL{KjmsR`N*m( z8y2UknHio~8o44TBTMm_my`3fJD9{fFlafXWK0QL&lroV^}FoeI28)pqYDByOqzc^ zV@zPC*QM8k`z6EGa~C8S)TtWg1pAGPv38!R_15b++9dTu_x46^A9+rYL*tAyye+$Tp&Pd(~IH6_`z4jgx)riB(2ytZ_L9P z_o*j2QH{&z-*2cTDeD-m7XwP8lb*Wn6d+enYs#76ksP{B%1F)*GgFH&FRQUh z>Ez*Ysi!|Qp0x?!zwusJ#!m~#d!2AXGybq;%jK;eHWqwR2pR)NR9pI9^Xe!0x7DT5 zD$ae3AZSEW742i60HHkZPg}w}_F%cVqoH)B-MR>BeVeh#x?=vr6P4+qZs{gF7 zmR6-z=>1Qg9!}#wJ+in~c6j~aO?Epn54VgzvTTu|T)9J*cTw`RnO*0MPRQQAf0~z* zdhF!U@RfuFxXL4g8?3DHRS=w|b5_!*ZDqVA6_!K@u{ke*Y4`E2f;utL55=EsV!k+b zCkf6=VsHX2*FrWymo=-Q4T>Xse_Ih{4GA7j$l) zw(!Wu{w9`B!*T_ZOgM=ZXB~^1SzFY3XF#9Zg~7S8H1j@q-j~mSmzfgZ{vw{+qtJyS(hK_V5Wgw2Wy~ z;_fnaAYTN`|6R&!@T)N#qAgKx2xX3v6oq5ES(kUX!@99=7eo&nn@Jr1LLg7W!0h3S z?Xu{`;NmrPV{qv--FSH~ySl26;;i?lX?NfP4LJpCLg^BQ->}GD?Q~Q`Fz4kQCoj&hc_~E{e=ZS~?jC!<#DG_TvwQOW#{OE*`AKb#ZmPW`R1?=!u4*vcr#Uxp`;p{ycHdI)1+!KGvCeD1 zX}Z+K&bAXEA0P5?`-NNF8Lt#w!pL^wH6MHHaOV{j0Vq!rmJtCFsIIv=k&4$3mbfjs zWcn%h;0IKvu7z9sI3SqW*vjbNBf;q&atiGyLj?GEWIQo+4quVpo9_sr7$=PBj&hZA zqRYZOw58v!G!-E0XmAMpDt9TH#Zi1*o#~j^l}Zs>c#p2GJn3pMg=Ox}+q#F3WIbC! zc|@-uhOOR$!%hVsCiPVj{VT$srv&u@Wkz1CqjRI!<`S#uRt}ivhSR3Arsk8sw5D)K zLtXt+0m58=2txE{e^d1MW>qjO9Pi6%grU?U+`vwGN1A+bvRf!rSPZsFOe z#qxBlY&`C7%Oe=s9I3FBp1$oZ60^5?sxnoM<=lE<30B}^ZzOv5rP+_kZDqnfw}Ztj zT<>dIA0L7_R7W4b$(E>7W^saX0GrjVll8*$Uk&5iE;-H~WN${Q>S{PQBn7L$w@-S* zH*MlKB@&pW-Oh)Mp1u(Qv!ovH7&Vxc+V?5L0z`~XT)mb%F%QUdV{7`HQG?$F3{$;? zUkc=-yt_x^=ajXQFTvBFjyY_xE6fi&KqP(4=-^L&84M)5XWByo_Bp>#p3?DYK( zYjs!pCQd=$`{=M2k*{_4&zX3yg}WsGA+Nu#hcjz|T>%@-=&

xnOu(+_pZnrCavX zH3}1JU}P`8Dw)eA@dFpk?QTakLl9#!Ec*E=#0;YocMz>Vs1N^NG5cbBjt{*wes+9z z_)S$v2UtYb^*~a<@M2jJI)ULna#UdO?)37lkr_ABUM+lfxFomo;qisa zv=0{*=V~2%a_B;DlIHVFnXPs2Tb`0y)ef>`LLKXa6cx7UZu4#oXsb>(H&SDGFSi|{ z5m;69JaEGLCe#C}!yENb9ice7c?X8`}O!`sdgTWy|&z?n{ z@bUKu^YAx3)_m;U*jKAxzIkQidh^=N+|=o1Q#L6~oM@>rSbDz?o?KS%e{+q|_2q{v z&qb9qwA&}IaCX%U@^;ElACc7Vc*G%c!9tVAW0xekr@ALD>Z^)9d0KvR&zf*LlSt95 z8UMdR%U^kT|D-b4|1)zD6aDM-B2VLAJGtSXJ&mjX22TSyK7<7JXpP+liIO?Vs-LP$(^XM_YPOCn#ta`?!hQ)`~2O-LEe5xPo_=nzxSf>oqwTRv*+P)OG;j@-WYFDTr9V% z#QNZA%@|hFnzol2&FS{F!FR0e$2!FABOQhMcsNwFt$g@Qb6aNlrXV|moU3m*?wPh> zu4CS)+Gadhe#EOt)^^TQwy0d}WNiz0>+v zK`I9?j$PR49jY5n$mly>Gq;N8-XcksOJ1^_AyYakb;dVBMdv{XxOb z5&MVD4r4qvV^4SWA&=U#>sl<;2b?SW3Xr3u=BB}CCcSJ-*^p93x(>vT1r3XS;ljJ5?tfNkr8VP4jLDF zJ9GE_b_Z`qboH5>jJUg%gq(NT(*Q~$Tog1G^J*u7Hs1=~o_=2k?ws9w(Hkr`L$KT! zkCyIPH=DX*x9?t;XR4G1ou8hB+ce%f=Py5INxYWQ7L}1XK2EmG1cD)Vf6_xH32to~ zdI$R6p$L$nE)0CNH<9CS%$M}1aF)1$$31$%7D|8`BtVrBsWBbfMB`R}H&toO_rOxwG9u+mOLALg8ih25GpivBUd%dhK0 zhB6aO(}G?nu!zI%T`~7=lCCOEn>z7EO^|&Wy~8J+P?4CV=5eg)qX2O|6i=+s_ORv3 zzh>N9(nL`hZM`>-O!wX9Ogek;2umXVZ_Ywx z%om*OwBD^hjpugRwkwk#Zhh$lexM)R!Wk&g>j|q#3fH)nP0U`2l{m9dB4$w)RC$hP z5Gy7XMR!?$n(v+G+cYam%D=Hw=5fJVgIXeUx>&11bGd(``8<}myqfvaZHJtEJk2*v zC9}?#eelXUp5IK6c{r5A?_!kF*#@+|;1Gm>B{=Z{T657D=KgpVkgnr1=@om|l8Y|k`6wg!2 zgH4SbcUgdxZD2G`q3sbMmUtQXqL)!TXZAk%ULmb19I|;Je37viWuAPFa`-D)`XxcR zW7bnRt0BVBe6m*-O+W)=Y5ipz=1>$lInwu3OF{}%TSIyNtXU<8ZkO#2G{7%&vcFJd z9)o!~e=_!@uSuKCRp`lwo$jD=V&1|YRW5|`)rMku@Vz41*}`F!EL3qz$7ELLK3QJ8 ztj~5L@tO8GbH}*LjjNtvZ>)ta8QJAwE2Z!Q;Jq&H*#jVZ!Ld1R@M5F9w%7F|X%q z0x9e`5wJP*-NIw$Fg5`WA`A}C9vHp2{RNfVN5Y3-`x{unjRkEiv~fI-R#OteUG*$m zDR1LS|2RN|=#3aeOC+9Ms_XuQaz=neSeL?8UMSniLM7b7kTsbIHmHdUH>x9Sl{$ey zFW@E=FR4sdM$2oADM!Fo-L#mARoa8mPRP}=_;%$R^FmnpP)V>#KN~k~!nnaboak?c zcWE*%Eu9=)(n)gP{HC$oKe{o?uYgbm4}S@(yKLG*!%}WVg*0;@`9C3yJ82 zJoT;=Pa_}ryC@Ne+mw8ZH<#9dai@1<_{$;+yDqr|p@}(!T47 zc2}*=KO}WL`7x?ZT(1s4kT&bdbEVb5NUvTz09kx`2GFR4cSarv{E708p-16RP`K9U zG%XiE?y3M;@g__y6oB{SRXuq29_&zlMkF|H{{1Q3cuG87VZT!7Le}z%zH92(y*u7g zr)uJ>iAggag$KQ6z@B6m@X`i`28U)A0sMG3o~JPhc)6gxikP~s4PgtAmAX*ZXNYyF z@D9olVJC~iD@hZi{VY^D+jo>b?!97M6Ie#Ce$L`K@+WItbS|aUw6`%ztEjyG_;apk zOD%BIC^=Rx5o44$8xTi;oLZk1rkTGV=AV!WA%i3acX9rSUBrA&JSivmF6@<|{3_g$ zj6PPZ44a<6zhjeTDZiRTG?1!w==T_;F3=k3z44s4r%0S-N1j_>Sm90D?X>_+5yp|) zT-kf9!9wjSz?$r3(?(m~k0pe3Mo>zL&sOv_oafEuPhBYR?%4 zF}#q#T!aNbvxdVa36ODm*%{e#|@0{GI-MnKtTo*V==n%8(5>a#r|zr7Of5+ zO~SVm=>sc{?~UQSl74_&)l~2|`z*9>o}ooxt2M$j!}sS9qc!h_@{<_s^NTD7S|?(~ z3vfT!>OB_(NL`jOT&xo5D?nsm%^jsC418#`0MU>-+khSjSBbB+ZQ2fHJ$o*M>v(%| z0Y2#OYMwr9lfB-U>OzP`VaG_-7AHVP={_v2IsN3&Ex8Vt@j3Zxw^f{`g|smwZe9r_ zFbf>5l4?_pjMthu?R+QBSvAVKue#`dcA|W5xD*$aCwzGLMycBLCXl?t1huUA?O612 z(JlEZyqjF{-zzL^iH^YavSRp%I(uRk>}l&lov=nZ!H4I($3k(J%|W z11fdY4k%WL;E@TLBQVf}bFhSV-hAxI!%;vm6DqNk^pa*#(K+)K@e)-$;c(?GXUoSu z4WA5#Es1`z!oadyuGQ5d4K99EGGCFkyPzhlW#>+xs+W(C;3RImX3^~IzJq8(j)gw9 zt%%N1*^hk;>3}CdJh78F&EQz0#2Rw1y?lS&)-~i;)0v&xXE2@)4Q$yLI#}K#VlRbjkebhxy^=?AB3S9Y_c$;2V>#v)KPlI9LaN0-_`Sr> zyHveu8+`1o>L3B~$%cgQf9n$Qf#{xt`I2MTBCgDBZk(g@ZV$P0Z{=~+hTYUZCvAj!FBf*OnB_xk$D|m{C5@zx^Db1}iF~({Hx6!v zw%Z4XJ@=q5$5Y>@yk&5u(S{oqTA<2g8o;g?0V#rx^A2z|&mK-46gj?50l2u#z*$ls zVvW0PJ1O{XI`cJ29&H_7FNMo=)Kxj96PoHx7=L5{1)q8sT&5EAicVd*_8qKCHTq*I z%PZ(fK8bYkK%;is`S@;L4esQ47XprYuZ581U-#|E4CZTW9HUi?3GzN<5+G{xH4<$#Iou^|Gz$nghkqc7@D2lhIUqw!*xOjcNAgKneR>I{#tAM$J{b=6qPbH*0yYK< z5OXN19ECsauYn>167w|$#va1#7mlq$965f5;gzxPA>nZo9(41^=GwTid}%fh2?Zrz zXuTV@$CpfKYBf>$Bj9%FsbC#;-}}1G$w2QiA-zdb7|0Kaa(ctsG|$I{jo`j9$B^RR)bD$%1P^7d{C#|@73yIYezH(-V6(k z?~SFUOV;kXkvQ$a`L$aap_i)tH+OX9JZpP!(Jaft>$c6)Gt~+HdDIHI8EFxF7bhjQ zZOH<|B)zjPnj)J=VxhKn4tqoM?xSP#6)#R&bxkghhj#F~ z*&+M|1>VS@p%M*u!*@oX4Vk3Q+E(uwVWIQF(8q8Y&h8g#ZiVFJhi>WH z@}M>yADl>eCJ6mFyG8G@so&qNj2KGut7gW_O$KrLsjHgOYB#%B3Eo! z9HqJQT=0htISq=*`PoY&ZK}@mm^W$A?Xl3NNjDnr+?w5s zDb#mHr>LBcD?XpBW7ur+U{zAbgH)}h=85l5EH&s*lW=_5uQ4}(w_*JG@yQ|NH5Eba z4Cjot=4YIS8=?Z0e;vE~OJdOyf@)E9Ci4A7LBIV@RI!32{CncagSN{${uGi@)VcSq zF30Y|%=CtbPkEtT=^J>X9$xhh$y;z+*PPPzcrp5{Vp5I6(BPTx&?_|Kri@86wX`zB z26Nkk<4&DqnP~3IGnzWx)@D}GN%HF~`>MT57zVN@W`opIq)w}NU-PC3y|>|73TMn%`;}ju0CIt zAx9olQ>@6cAJN%ZIAMmXYZpopIoVN#@T8A$w5za*Or*{pSFZTDa;x!dXO4HyBQfh%0F^} z`_0CEi9v3P%g)X1dY`UgCB5tQy3}*C&MmYzYECwJ*y6u6$VMuSeuwn1nwMtk5j!Q= z?&b~W8}mJ)RvkM(MdFib(@vY~w2jR!{btD4YG?Th|3suB_zBz`zbUtjm^4w|Xmb@WawHkKARm$iQHS=zReBOVvC(>Ls& z;PYz2DK#}K-+QckyUYXbLQz?|2krIr^yz0^yK(2D)LCUpXiUSJ>4jBg&72n|OVgKF zEVt@&u-CFbw!o(9^wQLHhvWr!Pn#C`$EI&?(R6gyxNTU&BkqtNVf*M;JolJu1YwP& zXrtgYYRpr)(q>-cyo1it(&zUtip`eX;-7w}`GVc0DV0|jovphQdx7j%_m1>*LaSwI zs!V(d%R15|@TYwx#lQ0?x^tTVa<1|=l)W=cVt_4$%h%D=4%ey7y@&u33WE&ojE{#Pmq zo{3Bpk%{uJjx7CKQ{4k1KZrPdenW^>PyV^|cH>(6(_W8yk3<~}_IC^0_$f+#+qTT5 z3&$j{cW{oHqk5vH@$&4JuaA_UpKE_-L}L1iqu!-AoHC|8)+)V`JMO4m zmZO(LtmmZVnHiPs@lUgNFs|J_+9>1QfW^ox_gYcCH0B_od*f#AmCB%-$KO7^cp>g` zsp~A|rDG*@7W5a4q0cmi;5-qB*{{lo35y;|D|3ij86P?2#1Z>b8m_vI(fTRR9T#Ur zWKEl9du(Athg0vK8M8K@b7JZqc0c#_;)J*HT|H}#)iiaS>r;Lg7a(QQsv9?4iC)9? zCt|PplCqmys_w+AI-jYl>EUlWg_wxP9lQGRh^@a8WnZXx17)|_Jf3z;&e1Tdr;DZ4 zrtRcCOHuR4xqX-O!o6wm+SJ%doq~Q`1%YPXz#Ut{FwI3;EtF?u<9ES{Y_)~)Qk3)Ph*8M3CQ!dXcn_+*&ddkN8 zQ#M5h+|sMxIqaVD^Q807PAoNSyDIDE^1L*ELN}x<=L{mc;1ayyqDBd6m+$v zQ?7kD>y{zgJl8n%a!J75?o-(hA62KWJ$XIC**tX9N`>(5(HHtjCu_rc?tDzXn0|Uy z+1WPZ{ik}XUfo_)uCQ2qUYQ!tbwA#2sjWT?eYP=MxA46}m{N^emeQJ`c~0jZYai>n zqb;AAaGe#Lx+2NBcJUh7#JxGmbT{WI3x9I=I51(xc%_{Z z0d?%EhrZj7L?#9J<|My&3eLM0|6bc5(b6DpBDOn!KQ=diG|QeAr&%9Np3*p?DsxRv zmZ__5caqiycGAbFOwG%&%`wGAT}doCU0Gej^LJ09j>8W-o%po0??lxtc33#I@>KY0 z8oek@mANU){{$z)it*&y%7FeXYCSEqHNo?G|LRnRL101W1g^`NS@V?@_8khOG*QR< zxs3B08tf9qsnX6m*mTY~#JRqyT+XzrH|aEUX}ZST4aY+Cjwzm>>C|pxL~Zr1&RA;B zep5wU`+(<~wsY70*<%9#H@W`*$gBmB0TLZ}{S%n^Umj8kT|ICJ2qhkn;1H1T!0R6z z0y6&}90K|(C*n|s9cg%jBjvU1#49OTw)Xk z)6%zJRoogk!ZF#{M58KIb56dObNW)9EUPU!=_4-fI&Hr=bBLz%WYf^*4RX_LUp|YU z4ofy5R&VZFy=6#P*2@F3m&>LESkLgD5^!#zTaaXeDxNhV(|ybw<(;OFH8mYGR-~SN z9+NaR)%}Q_!LuXYtyOk!w%m9gPwui%XJLz^3>vleGhf73ruxjnNM2gI4a>)=Hh6aHW0{zwiD0H- zE8e@rE&jh5UT3UH8vTJcp}ykdsK)F_NuiSM{UYL7?4la;*d&zSvx?0t7wliQvyZd5=i3R0!0fOHU~3(2<7f&>W=I*8QJ zdyx_aDN>cHRFM*j5a}JHOB3nRA@okD0g`w>_sl)dotZP|oO|ZnGkf3hpF9tc!nd-% zwchpny(P^gG;U8YJb9+6EAxo1QvBm?tPO2RzxeQgj_O<05c=n+mpbFX?bbtp?eMhH zQ28qM+t=NllZTIZ#FLsJ(ENn=)7sj4kL!b+Z-_0NOSTk!cK4G8u@C2nL6UnFwhVct z2RO&A?X~&XP{UKW;GVUyN+SGS>Mz7`&1v|aAWcTmKIfWy^qe}m z>|b7gW$)E@2P#W1eMe7)jQ!pGe!g^)d&;}A6cBGc_tnjuJk)?n+3A${$5EF~QdYBF zqDi5f(AQ`&T>tqS`ne88$EurtQzPTXgUFJmmzk9SH1 zEtS8$e|;vy>uOrLt0PO%#d)$}g|a3pLGAr*9qJxN=_@B_1(bI2Y#s7o4-j+!vNR%M#eIYO z+Mf>p|Iak+KR@9w7Gc0YlQ(_}4E}19Qlf@*CmgelkBmw%N&P2%h1%D!K$?! zlJJ4@P*dvneY%5z{Dw*uD!&9DX9^nMjQSPOX?odWP^>uBR&iW-fat-ka!%f^LUx3l zB>ycKB!NFfEXc+qWnk%OwOTgL3Y!07L?Rc~dOs$=aj71RHpo1%ogLleo}CH5=$H*z zolCz8ap>G>TiESmn}okHb$-7VeZ|?(Au{j{xk1pF=x!2k-O+avo4Q|4^mfQ1KW4uM zFcyaI)L`+ze5pV&HNC9)iC?^|dv#uS`Zo#6hu#8>7P43Ti#3;>_1x{G>gsWx`rssD zV&>PATdZky)ZQFk9^hq;Xveks3P1%oYMbUCX+(X^?3sMkiVq9?SFi0hlK>($t~f_r zb~xtl2PHZxDU%AqbcXdDw9WG2OHQ6`#ktbw9V8qra)=Y~4ozgW+{a@gn`X~-hL5-| z&qy2f?D2s|j!Mjmw#r-YW7k?RYja7?Pd$JSgQmwd()*Kp4bI7*-Ym8rt>KC((Zz*z z@fkp?n6JWMEut!)yTGiobf3{!opy9)Whgs^B@C7R$Sa}&W)meFVZNe8`tN5>n_d&s`X`x8+88x9MRy*ch~Q!3Ynpnhsm3XHPR_Jnx!icb8D7p9PbQpps}?RfuvneOIQi(^ zK{j086vnt1gD9W6IecuO&OgLA8p5rB90aOUKsdKQmH$^FJ+AOvRASykagnR<#Ux_I zI@o%0Nv2SftYoz8#qmwoej&Z0LF~bi?W1PXSw2?TC5opXI$mnCRVa&B5wN{y&nHf6&wI++zHO0?M4xl3O(9d|<7Bwb13h5k- zK#oQ{5)XAe7)L5bE2Jt0^``ZVn&{iIr?Pu2FD4r#dk1oM+_=q|$wx_X_Hvu^{6{9i zQBnheo|P85;8x94{xX|p2|n#?>8dZO5Lf7JlyoI?vr4;$y@X|ZazxzwjgdZ}(1gd~ zj!rpWNuaOd1zx<~otegiD$A~ih^gdFdTI2R4tB@L3v_i7)+UsZ{hI>#`V&As#A&NG;p)f$chrud8^X0MXzU{%F8gUQ|D%upb zb8xUj7otk*ORJJbw!W+}3l$YQ*e{408cZ5=k#c4RL`9rAd+FC}fkfkZC*7V(MjeqJ z1T^@@i$!Rm>}#hyXcpJJtm}(7i{CCI1}df|o)%J(WV3Fvegbfd4+`RbH*@)aCb}a1 znR)XQ+x(-jO>y&gl3Go`r5?%Uz38v&@Mm}ie}itCK?qoMw(Pfsgn7zq%F+aq4GIwE z!I*Ij8SQ=fy8cU_SDN&gm9IU!yT8)>1nWhqE$H#fhn3Mx=^+#>C4~o?#S+Iu(Uy15 zDFzntelT1mCy9!stXyfD<1EI_mMiR>VUw!d zvuyYUAEpp&ArXOlR*3drvhk?gT{vRo%WNWiu3W@&?s9LcNsW0)Z{&^p1TAlmJsKvL4LdW|9atb*{)+f{H z_IZfPmy3palkMz$8dXhfO2(Nf97Vpqry(U!zODju!~4Og@a89sLJ)2GdG4`;?TVW6 z!VNh$Jl^CFv02lJjGy2_afI%~vg-LGFkcXMUCZ64krOtM?%b5klt zZuEuR3{&QcVa5P%JRVWH*@{uhS&IDF8W8!l0CG-FfHag+z5;K7r*Y6xXy&O^b%{a6 zB4Pn1De#q+su{`mnTijOWNC-~4Rh-E_Y{S1bOw`#`s(y;ANC9k@6FAnP3r_(r?`17 zh6ZbHk%I?)g$3vWTJO^b=J87)YTKP+Bi^4#cS$0O)-}YFN{0yNaa5;p1Bv>?0Dqp& zcp<8^R1Cs{>FiYqcSBLtJefo16(=#0&kE=Rx|QMm!Ddy8ke$7DW+NlbjOZA{x@3N@ zc*l~>P6_k)dHd_6ZS9MNl{z$gCZh*ye#f>>ELW<_s-t0p%S#uNy%Sc4&ufRNQkaTS zl3ZA9zvbzobYdIWc2Cy}^HJL@ycs$f&)2Dqusb$SJ}ib$N!=*`A;Umd;ZF-dxr2(- z8P9c;n9oWcaI)QHYy9%mYBVAPCbxq?4TH=_6_d6J-WeOxvy|A)nb995S{^GmvOX{i z(h5nAH>{8fU+}VOzuZk(fC*<4<6CNsd=fQPC^2E~rN8N$JABPEPKA2-Tl=k@#@*#; zfQ{vF4%NA6<5ykWsC*d9s##r=IIiwi0e#i)XLmp2%}2hrw)hj7WK5E9EWH5C6#y1^fhL( zMb|3@te>Nq6S#!W0P-kbtsT#3JaZrXeutMxK}|#d+`T?2uP5)gNtxroUu5($RppV!Vai?_eIBwzCn;WG6Vd+2!S^6n z$0;CMnCG0*6*M;jDK@h_h1BQxkWNVyz}?1F#Eqj~9DqRvcO8-%UVB747FZuiNn$L5 zcl&Vysh<-sl{Rn02)c+WT_ldwfd?rUjqTbsp&#&h585z(Zfjx*Gx9 zVHtf9uNG!x^1#56+N=Btm(hfNdqPd-^V2GKax>nGmn2_`rq_PH`jQ9+nogI!?zf%L z^|_D_c9nkafRwFft6mzeHZvA=hOR5XMsW@UOD_9PZ-XFIipKlC-$~T^x*b{ZV609{ zp`|@do4#QMqq0Hg8xGm7T?sqyLr<06nlzf+v*yBz0K4WgUSwYVg6rzeHp}D1;6jFD z9xSYo{_qDEcz$GD#Y z5IfwRx4pGti4MK(8Zn`>aEN!nWI?`0mXR$68k4l5f& zBh`iv)TY#26{>s}^))kiF68&24u&@8Qf~@oKK&d*cmL+u*K}W%>rQ6_1BYnO`9Hr; zcVTsR`gZuy`;Bad6A4fA6Y)t4M-Mf}$h5@>z*QIi$u*6CCpy^IHcKr7hrT~pzA5Y| z+R&YAkkwAHP~iiv&{c0uI1*qj5!8{~is`6|bC(BT@j{ViUN1@A$>Fv>Dnr$Z3 zoOM{TpsXq^GSJ?BYkRxs!K1r{UM?d7uS?{iT~fW12!~a!#EREJXLw?)h2ASz0*oHw zD~@{&;yY;xoMFz$7SgKmD>R%|Ee+R4@O*`<4L5eSvR$**_0_s2znP4Gq;0u5 zc(s5tScUU7b7m4VeP)cL^#WYc0(`f5QnOW#zLg2eH)m+{klHjLy$s;|8h(vW(MJGw zQ%)`@c*;CrxCe2|zvs5d+hs+r6&|GEb_9OZK;b_kOB2=G7=nN52Dq|#HDmVmzo|GR zI^lc__qyfFJW40a8qKRn$Rl)aZ^n0dLyb+YgTM7ISU6%IX2bs{=I`+M zl+j4-I;0*#x16&G74qRuM_ZBPP9g}Ew&O3Xr61$YTlrKX=Ja-8%~R`atiXis zl$@>UDu%OD!}pXUKb_299;gl;ga&0rw-9fYODC#Y9j8J)x~&9X9KWXZD3Mv%loK6q z+~feK@E8lS5MLd(`dU=55Ug8$r=qOFxy<#htzGC#Qy;Noo0t#b4t-2}W|27wtH{2_ zETyl3LAXBBFw6}3$DOXye7DuA#r=4GW)C4)L|g_mDZ(tUZ8-WykaCx_%9ohKp}fA! z23KiJ$liWQ)>?0{K3c0cn5i#+x^k$#cfZw`eWZ@3(8ibz%aHurO3|%v^3CYn-s5;J z*^)YdpzXpb_^S7+xJ!FCm)+;C4tyAdM*CXG#p{k+;7s)3{g=MH86k7jvO$J3A-T~kFQmHcpt?>UZHeo$k-L`eR#Lpt~v&fl)aS!mSRq5u60NFlpUFv-3V`o9*rTk5+y+>Jtjf2OZ7@V8`BjFN)*RNG&vSfyDn4D zRiR%vziOWqbzVZj+28{a?*~$4U&YVEQig2qsQ_ZpSk+}4%~a+ZclxYGw0bS^YQ`sY zR_*N@NzzMU%9fGbVOrMNQ3}*CO4r72$V#2Pqn{>z0!qddiRIaqyDm$o;!okNjM2k1+;QL zwl}&R-E9RF#D%Hva|(kmf8CeU$%tR7PEd``PO{Km4sO+miR0IdjlhgrCixdwmLUql zG$IlO;|GB(^z+2w!YH0>O& z(pG`)J72C%KMr?U(n-RO*V>F&4=Pd9#|H{feeUk6dHt$OiK&SU?Y!QBVpeos4g{oF z(A62s!h!LbUTR-HguSHpU9&K{fp|ft11Wt z#(-StruRVCgk~DEr-4+ev@Ie=xn%RkySK0hc(ZmIk~#@&t)MPQv}yNA=0?0+3l)o2 z=>^ejnNC>n4Yz#0M~P+m+4su$AJwbCBj~LM3#dPUgezFo20Ya>C$<^@@@+ksr$N}4 z3X5*0y4zlAdk8r67}?y`UGWzv(USGgE8!@=m&mZr*=gm}ztvmcbetqp7m_9&fxMdL>RJ^FAdRZwYo?Fm97-fF+f z&Vu!!UoW@)lOsTZn6z7uwwmYRh#o!GnbkM=0q$zEAiw^EH~(%!`6mp>nV*I*KcUUv z0d4-Wdg;FoZQh+*p4r>aK|2@h%X7@3+f117yd&l#zTEz={fZ<)GzZWLs;M0CFroG- z_lx3Q-Qo)0+W2KZC#MK(deb-R?8LVF%kSub6MR15vT5v@S}q6YXUhrS6kT%CUgf+0 zb`VfR4853xrHju~W(Y|bicXAw9|FO*v#%gqJ43F;L z%$)00$=FL_8vRn+k5X)#D(Pw$JB4e15>sHQvjDP^06Sd&*gDer3R+#m+GOD`w{~6> z%-VLzZDFJ34Y`I2Lo=8w)w&Z>b;YycLA*m*%X@F?s0;;ak23qwqounOy&FDNYq!(9 z7gF3llLAhy1c4JJ)B(ko7$es{p+sS#R$XmMe*CSjA&qk;q*kKx?TO!<2|doIUDodDc0jp-5xiLe zo;J?xbi-TI8DU&$y_^FBLZ(>^XTj~`vJHJd ztg1wzqGo&)b39(Qrr{f}oqhmN)=fa|({omfo&tQWp*Y%h+?P&9FyH>CYf(yPSx!?x zHxh1AYi%XrGPw~7hc+kP2+MMPy}AR16q>!wfy&G9!!JM%Qq9MP^2+++h5&;*69QG7 z)#Y{NHTTc;mJgS8{V08iDMw|A#n7CucK$|5jR+`Auyiwh+495`C~s)Luz?jo-#YT2#cuBY%FY~+tZnM+z8CIuEsJps~Ra84oqwgt~D^Etz9kD zQPHNIQ4oGE_?)Y*@-k*hh$3OnE85nGS+|J@@w)*E@J^?m3G=1SG!bngo_M9yxW2$j zTW}!YVolX9Lv3FpXfs&vxEHTwA*8!m*0tQ^H*O-B{kQ4OPgKoMY{w3Kgiej{{EFwj zWLErBo%{CDd!9MzKim*}MAX+$hz)aPFaEYk^2cx0{G)D{f51o}e>%ea#3+A#KK_UD z%l~tX@`Kl(HnO(VwKTo^1>>VytAeR2uc?TitE#Aa-ulYWp!YImR0Js}-`Pv#GOTW< zqFx`)^C#RPg=yJ-jkD3rqmTw%vLtw+bm6cUb6nZX3OQM^(AiP3MVsXnlTS0_xZ=_s zbX!5NShrKf2{N+PN#TN$@Iwu+B&NFZUn4_Dc6qS@Iqql7Q-IJqAd^i5rJ3>(&paP# z#rMVL%54iKMG6SMxQCNkw@qO@sdX?A9%u0~LdyojgE13pJoXNn_+SEXQ`Boq3Fj-* zuI6S|RVK-7E-S<^KXH%YGc(<-k>+t-hAuoB+T+%!2y$1Z2ntgRk`O&*GTYhKal!F71!ykB6F!y6S&0boG69AjEBnO*WRg$Ak@b3ufZ><*eF$(6t2S$1%!QKv^kh zn(s+GrAx;R&}d};M<7J&(o?rKWUBVg!N+n`NNcm!+TGs>y5Z1X@c{%X6WR+(t@p_( z`e9|W8&>@1i* A`V}{#h-tr?o?tr$YPsI1?P8{q0e^6 zAJg1@r9^fRO(AD7uYP{-Ak``7dM>o9oNZAMPWNry#!53iO)HY{H3W zPAYd2=RIG5Zs{w!bG$+F7&6D};Qa{|hnTMmAqg*mwc0ekKA3GngWJ7xK426U zsqL2P;^P<4C33}}z zL!#?QF($#&-1R%?FgyRw-0>&6mZ5b?{&}XHCHiaQYJ>U5BZKz#h2?s>;;c93jhBh_B!*^J?1C2`Rg4MJ zpmI6x<7o*Qb=o^Ihofp~DhGYPbjsxbhW#-$%yzv!VS6^RRNb>}VkwocOg<;NCC76) zizA0-l%lhxm~m`xG`fn#63kj$Qc~^Crg-Q1C^5?7WMhue$|qreHhdTu*2U0hTVH z8Tsf|ygLdo8DpDgXm8Yvh-*i;ObcGVlRbGEh~KbrL5t>OQ2)xz!E3QXvOt+Yl*Iy} zBhQ=sgkN%nQt_-TR9M5WiD@jQ1N_N)cqR2TV))uNq99xd#1!uTg)i1t$f1Clx zrP1XmJdchUC? zEH*kYqT52HCQf_QY}49-@^bsO)P$hL1Fv%OF~fZ2+Dt@}U;*EzSc=&;$r)E-;6R(~ z6;qhDd;>c|dL*hMB7UAvq9>anEna*0!(#RlJ^}4+EAfN!qQ+(4r2M$+lUT{C`+Ur2 zT?c_Mr6}8>B^X3wq9F#lw>V+O zb^0xdZqO9SM92vqDU+!30E z)R6oY!Xuo>Da=b11N*iDx@IlMGjA0(nV7)E;LfTN-i8t7|a5Wbu=B%4B`EinR0`|9FYJ z88iax|0VlA;0bSUL)DeKOw;ysVsB#b0Wr%kjfRsfDwB!rln5#(#Eni7yr+N)k6L>2 zpvE#@mf%VhA2}io#rCyq14gO+MdDyO#4F<_G2+r(riLftT4$I6Vsr^c@#PGAHQ8%G zog6yNwBs3O2-XSlY9ymn<3qbRfa|gaKzhwK1L11*bSl$p1EmHUXxNMDyjqR~By*#s zBR&3h%)HJ3-0`xPYVWHKpR*R|52mC~BNfN5r!&!vBRnB>oPM;YwrX6&D+|Q%i*Gd1Tk2DZKoA0Wedjnf9YO@jpZQ~II7Gk; z#487n{hW8ehnb_kXSf@JFvr69yK}$3Cudtkh?F_UXv|r4dNs*doPO)mtmK$%R08Ly zU&fii5~-Xg*DSqnKrT5zwLjKONlSUkBG%@yj#!_3kfn%lkjb>hkp&J&9NHz^-suZ1 zGoeGkla5}vU-Z+2)W5Wn-05ZJy*c~hxY79WV^iw+$Tda>D^>#AI#?~ zhrIHOl3q?LKhzSvQhdub=diy(ukA2q`ex8=2-P{**lk zEwWBz_0!miZvRd)Myfv2`J>Tef9L)%*~FHF$pv*+`G(kn+sBo8!hu{3<{modVL~y( z5pW+Cu|k_0j$r)c$pgQ$yA43rVg%~bxVYGc&Ea6{;$R-Xmz}ErK$%5gT=nXlKy~>& zY0H&G;uo!I?f|WuSoU^+s|EH98+NmW*XuaSo1oQule-1#W!KhP9+g#k51qK1tS);W~ykA_y>aQ1JW z4B0ZdUG&$v=A|M08ZkC1;sYlabK>3)s@u8)2rYzvlqukk2*7{m<;(v}Qcm;J^W}F4 zNjN%o-kt3$M!EZcVrCE7wed6=Fj+eDdIfp&{rZV2{*#zI&BBloQU&?uyix`}DIC2j zvPbfH9~`UUPT(d!{_QowjaiL7q?})gY6_$KiO0{(pK^5O))0XZ$y=1s_m3qQf4=A6 zKA!V4wk+~sZlWEDbGr+&4^RW?0I9kX_8bSq$3oZswI*=6hQ0Cb&?!wJiIm(jrz8F% z&)`H3VCb@rLpFUUxyJV6`$PTq1zneOZaSih&Zb&LEq*6CdzS2-y!V&e(W7Zl=Y9O( zAn_{kNA&wMnykO?taApZg!Lgp`r2qtdDA4mlME#+H7>1pfRE)<02>>#9w78q6HE4O z4mkznaR!59*-^!Rk+Xi2*)s-PQ{Yp4F&$*Vz$L*4%57N2%T%g=t zt5zvekr{6uukZQDDY6vG4%xUsw1DWoI245ol=cO{NwH#9IPFXa^LEh!UUfw$28%eN zzJ1>F6?s{Yee|OTnoVc1a#EMtrEMyB(!{LQ30@VbxO>qX^HW)9>+ z5zEe|L^r?d#K5$FBPUa(Y_>iCumoY)raal`;Dbdiz`WSwR=Xvhrs%&pidur~uOcZ8h={9TV-F78QO}tsEU_K(k3jy`(~&p8Kovp2jP>;+^wc+Mlnn=aUmMo6xMdi6)0@ zsN|hZD;>(If;oQ46?h;nR_Dn41R{yZ=E)B?#6<`RK4i6OR00w-(CcJBV%sj`%5xO_ zGE9U*Lha``#`WbIg{Df+>~Vcfp)i%(+M1$T|icsgA0Ov%jrjz2zF05qy)cQf9sq4Joe*m3<|#= zVEAl411O*p6p^l(Vas_NqL&Wc-8~v>1>6J6lXeGGKe(&yl@BwL@yN`R%%=m)eIotn z(r+!Q^GCE}s zm&iF!8@xQzTmQx5L##K=*_gLvvuV-FjtDkWWOQDsXyQy=$6ME7!aQe>7at+?!YGJ!|JJ9$JPTg0iP9ys` z$*s}C08G28>g_p~d|Wga<6c@hv>Yr{d2#U}%eaM#GxICX2c4YAm;SeK=P*hEGU-FO z4vM~@^ir*2oCQ4Rm$EfQ7p;uYRDR#``HlR%M};izg=)i*QunmhhE5)WgroSSvJMPT zHOIC`D96FB0<2lqe+HC(a2S66?H@S=;Gaf2f8%ieQv)H$dXeS+=mKjbnS{}Og<%P{ z8w80+YVn4FZGhO?8fByadti{PsB<3JRRXSHP)Fd9DF{4> z&=i;gK@(uy1sa^*3@viy9c~`*VxIB=Lgj%Z`O6_~jVJuBNW6?0^4QDIfCm5iJISX< z$mTD&Sm15QIyi9f^%=Z>G%n&t=5@bj@^_NC?yG=>P0QLhhQ5REzDEx?kC+_}Q>AI2i{-;c!3Nd?C-Cx8my0nTWt zSRP^+O%!;iaI_Ie;J`BZ8UQ1Mk^;Jo3BW^Q$S{!9-pPSKM}`;NehJ=xT-DKL`({CUjR1j< zkY;GwR;Nb{L2;CfPi3_fYi&@4t`U&%*wF)nC$`Xv`5lK0S=%P($uV+=37gw=nu|;}-r;OX8`& z+miUtYH==h9&FE4KJ_%4T6+qNTy&(r1WqB3tA=Xba7Bl)*vJ;dRA)bKna^ zURcjKRm7%1Ga%>KSsEgpOT_w2acF-hAwAhMDc~R{N_*YGY$aY9R~UNg82Y|aY^{Or zM|?NzQ*$R<>yDXeO*!=n?V9+b`@C*Z_xjPN8nccAJkRRfI`c+41(Ya~6=VYIQU9qWzfTdnG4L5h@t<|Uigo4gvf1k82+4j!7S}v zQoIhfV1>{DqOcNcUf!>7_ytDzA{OHiuRPtqWH6Xq*&;L@rlTg3)$(NZiP=^yr~sfx zm|Qy|$QSGsCCWbdL*_A%ejfWOcfZifM7w+h56&|}=x4_TsQT2Zu*h5qOpPF;M3M4; z3Y=~T43qxCVk6>{JUj+8>;>9YP{WsDDf}HVHY)^i9OtQq zwZELI$OYWq#);HR=;y+6sr1huhpkv2V0)HtLDjh!JLQ??LIEb2e>BHKe>N&|Ld;>9fA%F%9bw^lozyPBz3e+XPuQUIsp_Kr$dn-=>iIMw#50o z8u#2G^#qL?1d0Jg(QE_cG@y8gfdIAwh;tr!ry&n4WfA_nR<-o46SzQ@W_4k!x-QA|+X9wEM6UhRm*@JQM4iGjQe^u?to zlffej=goD!6uE~+d}~jN|W0oPhgk+nAVhkwEvpyr=8#L zveMCQQo(_L&1OPuc}hl18EK66Pi@?dI!+`6Jw_hfB+E!Z)^{v_C&?<^rEP4EUHdPW z0Ks-(%bDZrGt}QnHfVqYIm#dm;KZuGPsC?-T=-7%4LIEo4g)SL$oioJ`Q}D|C;KOu zxa{A!m~#3I_!;~F{5bkJ|2xSM(AeLyY#izv1B^x`(1d>ASGiu^efZk%B$bVagZuXy zhg$y5`aDfDIDBRo?$3)Hdw@j~vnqiul9aIwuZyvP`q}&oDOUVlYkFLfO**j8YhA4z zjv>GXAaoHKd$uq!m9zXxltzM;N{62TzK_{f~iadXUNO~g!q`l>LzbG z!TLnjPsYVT=unPro(&>nX?j8BzIe)8Y?eF4DrZ1>`x^fb0nu zP^ebf(QJ(W=2o?v4L0!Tqj#KToG1&30J3U{Y2%_*p~QAl_TNzu6w6csw9fW!Zg!XM zp1tHvR*5?9Hx9Cs&@;u}ikE%m0OUACWO8lH^OpU{8C5}L%DWBfJa7+f{sUIr&vdov z-r@qTll79)J@O7HUt5-`dbcXS4YB+!)8_w!edK9@9C3@Y)zFs&ufK8%Nv-DG%N zFB#8TdD7WZy`-IPzV8+C!ue9W1um$Uo`O%nfe^b?W}LsD`tB$^tWhdR8bZ5}`(bpR zfP~8`^F+<5w$8OOWl`NPpT8{)AgUkkU`M^=FG!fA*Ib zz+gxH$9^h1s)uPFjnJKy_w>~zbhS<-dG~dcBs91eO7=^3?SF?GU<4#rN0~`QOo8CjeSy=;OZDN;Q8DE z)QPnF7BlTsbn(8JK#}6F^WcpmQrz>VG%j-|j4(wH?-G-qOCBT+&9CY7f)H;L z^hH=Jznv~}7=yOhi?<%i5V3;Gq4$jS8yksC>jE2;FF053EHmRz{N?&_X8_F_axA>L zkv7K9>cKAl4yNg1Ty+5VDSIA&`jk@c{oQBEpD8mVct4ZG;@yEp;n&kZIsI%M(A*+x zfvko{9y!_~G=5(@qxVW#4J^#CBJAi%vVRogt-O#Ut8Jx|i4&zoRHWQR43cDR&$@x< zhiDB0u6_MFvTzdFBJp8?WUYl#`s^=aGG_t0F7cX|V>>(3a=A1z;+AXkNT6?tv;UV5 zPoZ(+5Df$njV;l8#gCq4f4vEs+_>aIzZ0DCM-(Jy<^p8(Xd7!9OPmU}wrz&O{YtEA zwa?~xMEn5<_0+?qFYcW)+*N&BSxwT9seK}5qi68cCl8$a0+{P;77i?4cJpwp&*npo zyKFyX)SzI!<0^VH+CC~S7(j%maa0yFmR?{&4dMl421Rj6qe125_7aciMfP7VL~o7^ z@W)gf9lk42en$H8lKGkV$0#yF-cDvalnjgFioWFJ!7uKd$8B&?M#LTpx4%~|(Y1Uf zwhU90@G()~sKJpApyH%+jij=#ekU=Ch6SvsTo2H)7S^JD?4s<>A&fI`XLB8zz17EG zO+xIv&yYXlCZIG1H)yKOX-uLm2T~`pl4@WP2E{$2TKddfR?-{tfHl@SpwBv)LVPEQ zZ=4N-^n{TM()|CF7 z4ZR&@b)QBWmd(TOwy4Ui1}SOQIvnP_Anb0TPRBgxmM@!M&cdM*)B$T#I<^>JCG_oW zj%NCkIO7P&c%7F3;SFz4}AV|{Lfytgyj#xAv! z9=htxb9NpTR6wCHl-yZUwi}m7yVq{+p0~X-#U)4>H75v?4QU={)d$WJRiHUGhyM4vzt#w zjdzx^opWqF99U##eFbu5ZcRS$*W;A4(7im9d)7~h%sHLTiQPvj6sOK8cT+g9&WbG4 zpz6yVD9s1+q_M474WRYc*8IN4YK^SPWZbr;k=CqqXC^|fkg=URpy@_QlWE@PcNp7? z&px$@5X9C4hvP)S8z%jGt3J)s!jCeV<=pKWLH&yQ_aeRZPWP?cR{Kl>MpYWa_|@=- zBEoqJ)dNQv6ZxP5Mos;g#8#%xUj#mRw7W(Yc$S@E?!(f=eJ3exPT!;tD+u)OyX8M` zn^0KD=Ams@NMVlb0uzHVcSlE}3OP#n-yn2Jp_I9PWGvz=$G%&aP zfdU{x7g8t#xT-;1AdgbGQmiPrc{`;$Z+`GW{EJK1UZLT$f498-U$2KhU+@=4@XSxQ z#-A|jC(QagR*AERl~R(Pl1Cn%bmK>(C!-pV5y{cs3

SMpVdg>ax2$0Iw}@?@ z=DY;j8xh8lOc3$C(CU#7nOExLM9a*e?zUys@3C%+`~x;I1&SZ03{|nkZ>_{r4du&E z?}%st>7(*)#V6c3YmzMWsX3_Z>4`O2_x;P0V>5b_O6~}EM=|f$YG-$1iaumr`n0zq zK!A1a*`ejStu2owZ={J_d>N3+KZIh5Tf3(s{e>>vIZl6A9Jmzjxh}E5#e_(=^0!FR zum0Lo6|I*h4JIaA1Q~DLcRkx(V$Jg-9z-6l_3Bc~^CtRW9;)E>uL*R_fH+NE<(@Mv zqTY-Pb7DPux2_~JBb!<_9nSy zhtZhHUi2!(K6BUJ$}pypZ1!W-%u{zKp=`wu<%wGcwqu=zw8Id1{bWkUhUsk?r6aFM z9sdF>SjIi_s@7MDi>U&7AFBu5FRDly>Ki?H5>NB%uX6O@=Ji)ocRA^a8dlZkU0ay5 z4HLmF>J(w`Hf7xz1!K3z+P0EvY_-OVx0;>v*sW_bh3i}b{?qor?-$aY!FF=NxL)4@ z#PlVKD;rTU*C(SQ>H^4_u3zgh0577A2u8jG4v2Wc+Z~Qp4zY-XLFGIDPw_NS~E|ryHbtHG- zhNglGgACnM`G{Pn?f{yYe_=C zsWc1T>uwaUwzud`vcEec!9Vf-m+D_kR{dPhjT&9u$~IByb!l*r?D-_*p-iA<7N_u) zWn7?@v(QX?K2|X&@dUU~%+z2xvQ36DO+BIS3fECQ#{Dimq3kf_Vf!q_Ohi>U)!CMe4J;u6w6wO2$8A`)KD%D>h7wK@V zoXfatCr(-6fYJe8ORRNQ^TV7PHMzc5Al^bh_rEvZ;$H(A{|n0dk1^2Cbd&$={qp}T zHtlD`)<43a{;x-D{l2g-*f15t$Ed5rkPo}3@vAe_#@xlOS5|$OE#}m`LTt*`0}9+U z@=YD4?%>YtdmhCiDRJ;3p;dl6{kWV0Spq%d@G$69qh;~iAD+jZ5A*71B{)`pmR`R0 z%dp6cmAv4|A}QG=(Nd=lH+vZonZo%=@i9YqJx9C_{SL<>FGdv~s)#BjxLH&~fR#1o1JeDjk-i>9Q=@Mko zm2WxV7k@QFQOcs6%X3M`QZwVSqwm%BOlfs@g2dAr-5Pm=w|#iUa!4sB4X2jVrEY&p zK7l$;rr+cGM!^N)7~#W|+n?*RtXEDraO||8%_hBbv6p>3`fi+MOsR({4=udY%adu? zrM2g=oJ~`lV<}GWi^-w%p-G3gd0|B&=`(A?ZEc29&H@|)oqqK?H5eD2&4hipMqD7Y z9X2AT>BTe|6k27S9^u0ht&cgT^((6L@#B|ml(bEWNA@dh+dv(J&B+Iz$sPPU?lrVi zzGwSLCy$vVJ>!TAOPJ1kKCY?lRIn_O4hNTjjUY`>y#eDZGlo9CWW z3SlbVh3_4oxUf!Q7?5x|`$0c>nVTn+fA+WlK^)@5`b<-ZjtR%z~a zNvION5(*rkY*f*Dv1t0eK`~7oc9E>h543#VtP zhzCF@hWDO2+lVxMi&th}CG;h@t zBW`h}{LP1Ml{2L`qbSeAZoz`n14{QMAp2i3g3^jbGMa7pdFQCq6m zfLX_hFx&iQfwCy*ejYyH#M~iavm4pgeFCBOLR0Eiqpl54WRz;f!rV8bopv{E)13c7 ze1hkXKi9TWVE$%8FGMzyPraP8Jxldfsq?YErWDN-VaGOWjOg@e>iUVpM30l62JYtc z2w00eCOWX!Ro5_-!}V-e_o4=&WN%N;wCCXhzxfe0l{)_8bNofFBVn|*Q?L0K)V38| zeFu=T=5341=m(B;__w(5JBnwmJw(*&0yG5Rr zIZ9$wzmw}5#y!C$Oj9xnAu{=1j*T~!t6uruFXC3|0vBjib6wf$LQa*v&O*Au z6P+3w8O%zTMYMcx6l>r7yowOmtURG~3?sp^eAf{B5BGD!{_XJT;n?zvHzqTz*?J!` zJ*{Z3P`#3yKmuz0Q}>axt9mFkzAGc=%ld!8ft#N8$&?yip>^H^6t-`lsm}_{RGvJ^ zb8eDEVpD`qX<66dx&~M=NB-*PBHOJ?gkHk&@#)-RG9_VdcH; zbjmr5m(15oyvaEbgcPkxAQcdCIRz$&Pj^C9Xpmg#p(d!7G9fGdbm_-Zh>Ln*r({p$ zt_$(Km#@a{c{#V|%$YYl`_IQvT>bseoD^-e9C26jO$)Bp)r}VkkZFPwrY6(wb~S@q zDWsG4&Pyz64V%w|XS%xO$SIL0k9RH^wfFW;K$0t7wKrbU*Hg~vKA6s{ZJ}Zjo{|5W$-_>_4s+O_V!xjPp@dnz{ zmwvr>RH_Q?R;|GteC@A9;p=gAtk>9f!$kkuDdUoDT*E3Q&+#wA^pj3heiG4(+CZ6= zp=kPcp~2rR-B{Tg6Ljzjk&1G?FW$|BC-6E<3%+~3qLv}7pzl5>%AmD-EpfgI;U%=o zC#b$U@#9!IpVcj;hsk@tV0ZBhEPlOghbgTMDHgni5YhrOePfbe8b(fXWe$cv5*E)p z8FcS6d_wwD8C~G*dI&?Dm%8EwD&{%esxALRJOQNoAg*1dU6c&oauK!sRia}gX0IEb z&E19`R(4@1=z>Bmf#v=zd|{|KD=P-WqM!0Co@8`iU$M1$Th?04%WN>2me#lD`Ve#*!0Wohn;B^#d% zNl;d`r!7tHiu?Tw?#In#ZGYTGOG2@4m**!liApK>u}}bj%yPvVbd;R~0LXFWu7cR8 zLPBoouCW>+jbmrz?r)Z+6MNpwE$L9-#DD)&V*?yGlyGQh5A)D&+)jJl{jOp)j2i7( zib*%WMTV=XoTzsi=Tvlh+a8etP>?z)=i1}$(5dQdN8W0=V9n{CqN#nmRn~_7hz{BR zdtJqUMH%L23eTSg*UyW@fBihOWts0BVopx;9P|hx^R2w?g11XTJ6wC?&8rU8%;$n z>1YI{c&BZoriLYUnnMm#R_^VcqaG1qw`?SPFgI+`{*{wef7apSG$&{TUYYOnOmMZT zEoBc{vSQs!wblO6C26Fx??U`?=Go8e=k@smuPGmVk$Z16P)C1XmC}`47dKG1inW4@ zZJ!lGEOj^eg8j~m*VQCMTDlZO&U7Son5n;VO`}i*R^&zfz zuSZSGy}g)6n65D_1Pd`?>g6%?J;ma622%M5yGEm9;h~O1ZjD;?Rh0spFl!^>x|7ikT9;>e@~-+jD1(%ox=hb2N1U^Asex1ja-B(< zh_i5^(AP8e5qAnTgTvJ%m(?Wa^JgSu^5vHRkkvr>&Jl7^-MNI7>Wd#%04PE!Y+ruU zRcELB+k?qmH*c}N@>&am08Cg3o4R~$XIhn~L^HAGvxrmP?D3(jCXS)oGjX!Au>$Yf z@*TkCwAlvgk>5bk8ib1v{T6gS;_OOM>HW!!d&IqO$Wk9)y)rS0h-$mi@(PXXyYqfw z``lhAy;#iXoQd$=b81(DDmz2v4oiO>K2-igE2@k<&};owHr-MGB85Mp*+8*YLP_?` zi>vPyhmT}#*&`d8ej$%|WNExRMK|)HM%Jap%c@Es1<_zyvSFX(vigtkE-y(h2D#lX ze6N$}H>7(P@BtgB?e4oMcPkBCldml-^RIJtyT+=js$%wkSFn)y67_%7I;A*KAs}cfKCQr>1}sG=H(kg9_B)s#@cpn zk)n@wUwQpjQR+zAIg)5&Y~+Mqk>tdXe3sS$mEcUwn;KK0p~%EHVIQ&1lYMTPOXWG{ z!*b9v$O_zr9$$ucC3faUc>TLf`9I**`Cr*X|Lo1xKR8SMOiTICGVqV$zuLxa!_@1! zpB`lEu0z7@rpC)3HMr3*q{gU34EsX8YZX;sgi{Z?ekeaQNojD)1W2b@9ZF421?v5Z2L?W?x`B%IUcZZ3 zoZmnl2Z<%o;H&FD0Jy@9E9AxSAMN`zSH6ejChwPvCOYiXSUWL4l1Zh;L{bJb7&{^ajpR@+bc{V(eW;xJ-q zmbd~Iy^1})Vg2%^Isr@iKU60kRph3s1$PWt+^>xLsAdIr`lXh(5VCM4f~seCUJ30*h8CW<+TnMrabZlE*`K~|2~93G%@SL{RS zp%S)qY%SWWuJlBuY8N*_^zmj;AS8D%&#Q{a7?%6|^TRG2)J*{&8p1i*nN6#Z4n3SmW zlpD0vXq=B8=eSXQBeoV2;nR3hy6t7w;}e!UNrmpHs9CJ zxEt`S?x!i-zZAF1sii8o?XGeUic%~3;E-)_?!?^o!zy$9UW$(OtbB_D`{hm!&a=6C z=Io((?&k<}G#*vV(v65*Gh>D|EXwbwN@!NI5X~z5v`E$Y)&{qH97G$Bdy^i zfpkFPsg$Ves?m>H_U2vbJxXE4oL^oxG!9ftqbc4E14&qJ>UraA=O{(}1SR3mm&;`q zwNto_#IDHn)we>tFG|CyrkhBl(6pzkW^bZ<>8!Ck8ipIvmt`>HFeXeE!VyPY{*W;l z8eKp-Uwg86e)ANgVjn~$m`)oXiu5XNwHdM4_Wr<^?)$-)LZat)EEXi3l0J>;;AQX- za&51Ya|am(Pkp{StF=p`Xr^;H;qZqz0q&yD&de7s^jk>R1y;7zW}D45m4a%PP0QFy z<)<{53VX)j_m!E6r{L;!c{=ob0-z*OexCz|!;@4^6dqAq(K;4r&M-law)A?*# z$HNfftJq;=d2+HI>#Cb(Rc)2>OKg5nSlDO)NE)D>D5(B6f6irvsCrJQid1rKt?T{1 zF-6(!D7(*eqUNRLPe(jsed}uWxk{3BF$V=~%(}ZuBOa-oNHfoBSKY-fnttgbo9+E9 zDQ?wGz>ZZ9caK|^xLD4SXJc)1p!fz$T~=pVRoK;$%Nnf_0DiP}zi8}XugbAv*^;{! zilDaWQ=dtl6y1FQt|RioE<^pz@i=pJdm9O#@Zh9wi?rKGBPq@7=KU+M3r5OQ+Vc~a zUl4wuJAYw^bF&b=VpyB}ZuL~JWvQlMtOrlb?)NW8%0Jv#=@H0S6nF9ICMxQk0vXWp zS3_SXSkyFAu#Y>e#%@T@rUf*a%U}=qX{Np^uFfdNjTPs}xA7z@dV(e}C%l{`1=3n>Q z{T^}lFKO{V^y@!U3jV)#N)=*UwIwmZVUQns1uM57!%9`OwnAQ%v4G>DK-ItsUn8`< zXxon_pq1m(vUns-h?_XQfXdj53F!Bz*zJYolFpQ%s z+gKqsWsYEqKk6|na?ONg%wCUY?DTFt)pU*T>-0j69k|)}$MgAbcqZKU_;%aeT=i5p zt$u9iEA8tLcjozz%5REF3rmZhtJ*qR+wa>G#^?2E>eALNdoDJpvvLMEQ6lozJoLZS zUXK&e%uv_#RdGHlAfz5Q)R8EoEPPqRQX@=Fvh9G1N?Rv7iLi}u|Hi!8dvZ(NW0tJi zMMM1`u8nQ{MeoOsqF)U?Ro&*BaD_X2_M1ZV_N$2@yea@oQxQ6VmDTrk$Lw(J`>dFy ztd*i9m9}jshDrRa$sQ{$hd3R|!Q1JA8nO}TnrX5vwRL8?HmPWxkD+VUklNA!VIY0- zBvDC!Uxe(L!=EbYUAkyt-r){U^klEO`Wqb;YXJ3(aZv*?2L@8W6~QuOFhE>G@Z`3^ zhpde+pJpF;B0)$`vsnx;bkuJtki*Zz8g~n3yNTkP;y>}O0(l5(ubqg;ygCt58}Vof05Z2 zJet>$CyLm59IfznvvGP$G56Vfu-b_kSj9m(~M4bCdiD~Tf6NLJiBTqD&%~QFP)D+W9wMOYR$lY*QT1?D?FE~ zzW*RuDfYGoSRCbXa_AykWNE>jM`wIRxao0Z_DO7&r?iR1C}&x6qx?6|!v*Guy+Rkd{MblUUh#~ zaqHMQR*uV8ZAqq|(48H5&O+)k_p1biEw}>Yl|L+DaSk5m3@r^7N-pqKw)L@hzJHj| z=ixRi$U{l+o{+^W5n)0T&4?L&b$cAcH%~rd;P&_r@@CZG^|P+#L*Gg%d*Vf?j~enO z$NGCTgO5$hcC5?P)=02h@*FUo)eksF6Y5>$VBdp_MMVMkmgez zPgKQ6BJyPR08eh$sP*a!k-YWe$>tw!+X~68`o4RrTdiGW?ZcwA^*ik|x%X0j zi8{xJrFhr2?52LKRzF#5hvOpA{Z9I(Fe@%3KTQ@Cz#i~V@9@alS(X!ZGcQh%6BL?v zp6pCvO-eS1e`uK^=uA!f0b+}4W?Jirm0bU$GWq9I_#fJ$`u`&y`k7Vm|FtlSzp=Ou zY@pJcv!FKV$ApDD#pjV)H!Sk{y8L-IWKj#&gnvvF(q4G|mn0oJzYJX3N|l;o_HQd$tKIR{8Ky$CJ_I^atyGPqSsh!i>!dOoL#rTX7%YyIj2JLnmY zM9kF}$p=vd)~`_WNhkUeA|1uL=eeSUaeQH=YUJ0zOrb{STpj+B4DH}>e6AxcOHa1< zTbe;mmq|@fWm}z~WQlnz655moX)RlkR%=X)GZTrw|SR#oI6&(x+Zm#v&{(#<+H(%@jf5G*{&<3e`0G;VQai1C{NR z>&AUJ|oPdDCP$wP*1U?uj@i+BNT{ZD%*q#MPrF4 z)an!3(79GoPiNBxF5h@M;I@I%ZQw~x*a5&o42UT8^Pn^jRX72AVn|lLXUD*teMC~< z&b)_L2K1QE8%8~FDRT}B3vD&YbFx3B=}j!sYW#Y79v3L_#`@|XZ<}Gb-8D7V2yA@i zu@5ChF9eO>?H{7yp46)SI^dSjy)~jj{3;js_ znDxsr=Sltho{wP`dKOz)OEsEJARzIG!3}ES@S47%rP7{)paox^F&&Mnaqf$@%S;oa zO9Q?tPBQHEuj<+@v{GKmOwbf%3JR31)<4yEc`iq2Mh65p@fO;F6z=rRGQpVgnR)h^ zT;uQR&PKeWx;F#^;I-Sn4Xixg$F+x3U9wO6O%Nvin7GwQ!lnyTgqxL}ff45HEkbqO z&4#bO?lO|f;FhwONFJ8?_#+e30uyqgYBl-azE?W-nR|C;Mq|<8ii8oGof2=KNI|BS zj?ib7Go6y{*osWrWa*KZcUvnbA5R89SW2Sx?vZHdXCSxK;&ftIN&UE$#hfvx8VZV{_|<9yx{ZsVzttD!DM6zB-lC)vY{Kgc)XP z6Q6idhwOOdDcm?t&!{<*^%=X5yG(t*#He=iOD*l;9ggbiKBpybrgvuSkbP0zQG2}x zlTvLud7JQnD7Nn88HzLXe3bKQ9l|YOS{9Kjj}G@={zGPf`X@K)|Ay1^fBIJAXL7-R z^5gtbqAUK^Z}roP`X92QcB0PQW6U5!A?{j(hb=LCsS5g~ty1>}V~De7hnO`V8G6a} z7=BbO2xA@+(11&locE*;+FmhRmuI9f1;b~CcGxRz8Dl#$<8)Nkvu)@8;K{GO9aC+Y zoNZZZg`5sLX) z@m;Ty{Vx;6Q)6e6#957RHnxm?s@tjm^_N%A`EBj<;-L2xC=ZEZzB*SD)i%f@LAiE+ zh>Sx=m>(;rf#Zs$j7X%Qut<2g_YH%V?*61SMOT034*5rHQ97@CCr{o(L_4ik5pJ~h zvmY%t_bTy-r(uk-%Q>^$hA9M6In=UP^b&(kL#tf6V`_z+mW0%AD7JQ z_k;kHg(&}vJUvCtXW41HgR?#z8QH4@P{uG4C#d3(qIvR6kVWb{01=X$X*bgIVAVN4 z<)?e=58$Z!`!fsTaj*OX<0R`7e1UlY!q$5!wZDKi@575-3OQ!+z4>-lO;gI}qt$T? z3kfo)eV=TnG0Ed?MZt3Hb?uE3%-A!ZZkEH6Z`RUNiITYs;g%a6`f1rfG|>uKf|qIJ_2 z=OuPsXwvz{aB=sovvFRI4SeTXm?ZBlJ+fX6v86*-sS3j`GmlNPwo_QERjn~sIvFbP zI^fz)!qCFtUFewcn*?;25#>iO=gyv##MAmS!i8D7#}^6B18Nx=n$Nt%fo|T zcy#{LVm9}esgW`T?;NjsjnS2e(~@tmXJHfe#hshf2^QqNry>z5sV6DGd4fM@UQr`X z`*l0k%8EL5ml$#C-|fu*Pn#ou;1@7I6A1n^rT$*a{-??S{B%hDzWlc_y=WN+(e(F4@?4x(C_p3?_}zWHKA z*`qF}5fu8OZ9-u@X$_yk#ao{28mso9s|@q@`?RduP`<&GtaYu+gYf$g0=4yT-q9Fy(cA^qi0mcpkszTwcBkFQ;kw&|N>CtYJS#pn9x zGqvd^?an`3=GQNb?Kyhas4+}r@0%k>_3SaVxT`sd4Iqa4Nmi{p^Gj0*rZn<^+z;_piimsFAukPAxEj1r24`=B_t0O)pM=i zdJRvDn|g4Kyc9pt(l{JEBS+OI;Gg7@iFz&f$(U>z@7Su<2}JxX1z7-vrF4pXA^c)R~rzf zaf}h^no>cPb_VC|Tq%#4vjY4>^U0ASaXEci%JnY62k9v?r4D1YCijLCEwXf5@@OUG z(BWwBijx+0pHuo8QeDGjcE{xS%-bY-Xns!mUUZ<_qovTm@5NS^(@DX55AVPBdIM!y zwv(nv7JT;V7m-mZz9-FzOk#?Lns3aEV}kqv<@u#oXF;68~ z4j6fl(ChQxe4nxl3D(EBxIZh~e|0!Ge0dENaq=nF3!L!K3v~E$+)P#Jb5Kod?PZE} z&Vz#E1=fYBAs^XyMhb&45Ao06NBF52a{QAQ_6fU9B>B>C5nndr(1XaJeZ&5j<#JrUKn^HjE7XS;=2 zyZF~{a06T85ozBz!j@Ju@S5!J53msc`PH)MJE-@#&sB*Zgr^uYZJ;9jyX-NG3pgqm z>@;iDyw4GHgbiOahndnMQxQ8&&uy+8ksGK^o^TMH=88r%(fYNZo6pc776!%CtCOj6 zn6X&*^XD#LNC(K16X(NGby)b~z!m@Jfyx=wANd=t&t11mOafV5^dP(^XM7057=L^qiUj-3Q zfV0NU4*9nr|yAlZ%3FL zdj?3WUiDB((xlx{8o$bGPn{~QN^J71MQJ$c{wry?_?n#FXGLN0;+(A8Ak`4=(dJh% z9Q3#K@82$%i!|kdL`#^){0&q9M7sdci6do8SGipwFh94R;V*8z%ZZWNgLO zqX7&NZCD4sbOUt*^?Uv;?GoEi(NHS<6duR!Sx@(#fLOUq^EFl;-&fjT-(RvlE? z&ANrkKeGJDN{?^RmYjx89%!wO8NDeT0Z+C;u_nv!%=Mcn?8w2iN&;=4JBY*e(#69L zcvy9{wE+ou2jXJ40qEsv4=Xl!Vf{N)9ZlWOD!-k4(t{;Fdf65PR(4rcvL*)~9$+y; z%W6Z02uK~7b(;V_2{`5KG#%8V@2CKBQ+b|&HqyR}pL1UFDs+L%u`1HcKn ziWNzilgb{twoh*R1?2NB=vwHw5_}Pt4ABp`VUV4Pi0UG4aHU{Q%V&eE+SIQ%;X7O9=yKt*GSW5~x) z#C-J#IzMe@t*0u6_o023Wy8|Ck<^kd?Ub}xO)(a<-9Rl~m9sg&*1dMLVa0}y4`_jp zaw%0vl{;T-QQtNJg#2C8#H%G!d^wVBQgNfqd@>M692E0Bzc$cI_+&wkh0F(N`z4e6;2DvVX0yfKLK6 zo>ET3cO@0IGELM3Jn^AE2>DpU9qPSK3J!vyPnd$!O|!_p4b(9Le1Ms6Y?SZFj7fPMGtEt!ES znQx%ZoGbbjTlx$Gq7D~sfCY>Gp|t|=xQVR1V8KPW5@}P4q0}%!vN-(!X6;2Y1kk$$ z_F#2`n4|{S59nMG&zAAD$LbzsTEm#>bS~_K&eGk@6|mVk__}BPSAOq2c}0V|<)Q1o zfF*fh3eXgSq2nOUmg?p!1UMYKf!f3Q_phLTC=BkM9@O33G3qfyjbwyBZK8$fM_4=x z@Oq052;so*)5TKF&|bKuV?_v?Jkb4{AY}oZqAl(CZ>{a1d&2R zX$f>U0U=|7m?0I<;DIFo(*B(YNP3e7pv-nBlXoFLy0DzaFnThb3r7*aC_=R30|<{J z7O_UJJUhQe*or1MqEUp2D-bLU0~z|<0o6u$a1Oxj)=;0)rp$grTEXrvyPk9k0WCrO z$!gZ)7nlOvZ)1_@O%7w1fY>Dg2_n^OHn041b0Kfn1V=!TCV&{slJ#_$4)_fQA}`6bk{|)5 zc$EGE*&4}*>@)1(-^<||TdU7>SaCYoe(v-y72X4iJyP@_9iw{)A#}Zya%g>ya0F?u zholFCL#d{=Pri%VJbKV0vzL z@8u$$bs&Mv84UGyi#L9N;)3j%C8U(RBMjmDx43*Ekh9V7YpDBs&KzNZJ_uxKt=|s28O_wq#w;C@7oMX6y| z@SPA-6+v?IWAtY4MUSyp0-B@oUAe2UowX+`9QxY;pBiSA$U%b0@%{_b?dv#24tS{s zDA_u>O+(~$FVICha|v)YX7;AuDuiBSM>s-B?DTE!k2X-=A?%=nEPi~ga2>QJtj~cZ z<9h>#n6QGz_Xy!Yy&k9gBl}12vn@||ls}p3M`IFZflk68A2jLl2|J{r+1-;5Z`(oH zHyh|#LCDqLCd>-+gRwVIwc;=r6$}(989g&iRzUppiO9i0!dfr7`#WLeS8lS-oF_*k ze(0wcL8z%=4fJVJ>I^;Jc*oM0#lp#eN~Gi-WChk)7{(DWhd{2uGZQ9mQ~f;rvPkcc z((CCU?}HOnU<0LaZ4u1W4h_$;&bsR@G|WNp^Iw|-Q~wq@h*__mZ=l~@MI4Hcy!Fj;JQ)qY&a1i?^4nj37?jnEzp+l_5Zur~=s~@8bWEZl#`eYz;(IO(mEs zJsDV%`U)>CpWM9#YG`TQz4ptprW=sb%wWkEGk6~Ja-{WV-$<{#Yoc*e~ znDrQn+JYetLRLdi)TYRc_<{Ae8$OL*l%N6uVU)W-mmJmy%i~-bSa*r_K;pgaaBwYV zHdtw)fdXVKYn=r?UDXf(yRzOUKHQE$Q=?hO73gvOd!4>Ygu>iF3632>zURPs>tIRq zOCTKPq`U@L$5^I;?mi*7b|Z9=)&gsR`o}b$aWqw88&Dah2=UMD#o<#8HMTUaGCOE? z_!2tjdkBV_Chdb+I`}m)pOSty4nxg||-} zYN|IrS@e8xEm}C)SW(fjR^t)_k2zw1#+f#R(Bq7`)Z^dJtV^WHFCW_f4kB}CydmaD z!|e1N+@DrKGLY+|q)-}qc^*D!F#ta(BT!T4nV?oxV7ym9#K1r)e*gk>42HfatnZ-S z+5(Xx!bR1OvIXmUkSegc7Fst@_qCBLtGQt9?j^6%53<;T%3M*5%j?Q1U}NYd zVQ9h>h^h*I8qe{AL($oL!U3T1 z6Q*z+&YZsu)$F(@h*+M@1YR0CeyQ|e^vris0|>uW2jagY16sTY-EUfB@2?`kWcElj1m99I<5lc+`_svl@Yxbitlk%>}_7=$>6E1)&nncP3d)K51PysnFz{ahQ1EYiGx_BK7 zk9g#ig%ED_Nn^xoKMt~9bU(r9G#$P|p^HkdOT7;Qs>D!Ve4QPrpSdA4AmO*{Iaung zJ{gCB1L7!fKJd{U*v#QW7>ZfL(Ahc(_3?OIWG^R;R;K>$Ypo+(%qhu5L2x-+D>X8+(69$12Bn>4)2U!Y~lkMGA_}9eexUS zKwfsUeZ`Qh?ttwgP+|=Nyg4HpjtHmpgg}e?fXRoQUW9ruOVfWzRiuOjW&q|Me6WEk z?!!=1qjAeUV4Z{2lYk#X|K3!;fbN^UG@UN~&Qly~;btY$Z0l${7{2>kW7?=wq*R&r zQg#_(#oqLNoV8e!*l@}9T8*#>5e4B0bLJBhu}LV(y3RH%9{;e4Nik@ULE6sFufAf+9i4$0p4;>KE5NkpmdR$$*G-Q=uU z8&j^nao;~EA2a*Kh@(*|&mno{uyeB@v#fj-eIp))T_og7ti~Hj~JKrzviJEH5VFWmdV1G`&o`^h7kF}Dw*AZ(Y6|1*adNMg< z)L}Dc(f-qvGqBBjkwN32W?0;v?3f>E`MPuE zkDf-QZ>^ku99}h?g}Ut;S`#A}B&nuQQ*o^QM4<<}cG20QE_NYbjNd(;2l9*Fg4Er@^e-tp(%+y5nZZ{kQt)PpDtuRFvN znN=Pq?wDvSr<;4l_CVhh@1Z;bq4eGmic>H_|L&_#y$k2h@Bg9g}H=b;`AuzhhE2byh}M)S}cPby%oiSf+>*l%qTs$EPV5u7^#e zHLr`qR$a)6iI`6Rr%O~VeX^4*k95pvS&tUZT0iGXBPnt@%KRwUp zL(evRTbyL6^*m=cW)elYduZR6GaR=_dYsHD%ftZJH*ZzPEMww7irkMq*Wi!QE+OZ z!50^jfW4xzfg0Eaj@7L*LJ+^_$QF*HHU=C4%cm0&Xq<-DON3v5E>^6dH4-bG779Eb zs#D*gkx*od7dGDzYAHjnPGQIgJq&^EBM%${UWxseR;+B-R=g29u#s!qXdk!{4`AXl z;6~V=d&Kl09^ep90HabQJq&?6;n$aP7IuBY=5+(Fd{WOMadn^L@-T28pQ9xJ>#Hdp z8axg1`IMFUth1%Tv>s^SX*wi@DsTZMGC*NX&GPaAdqiDpTv*+_7L=S3tSuBLmGjRq1Gq=H4f?|y+QLkJ_zpYV;uxB`b#n&4v1L$B(nARx4dsE0&`Sw^vcNv` z0KYhG+k%+_Rv;@mZCOqrYbFsqY=8-$#E;51+{27^^&s#Sx`a{(IS4WOjiJ7Uxs$di z+ArQT)n;FZuWEiG^ z+ZCEtI)=n~sRaUL4mVvVC&Y@j@v!6{vH#eQ*pV9kr|*Po!{ z==9^EX6&g1cfyzLGdN|;{L)fI4tgFM)C66zc&&s3-{3Xq)vvhO_#G)L1hO#nlHrMb z$;;u?lY;2%DM-&-uZ%!G2QA=;2@?lCVf7p6=_7k3iYB-x=(@0q>O!=+o44b%Z{nAJXjkRlju zGSHI=h5)CX1&BCe1n|1-pq1w;xB@l+3gB=C(p&?MtT-4ANQ&AvuzYKMH)H%NfqJ@w zb;3-t%g8^dG!i@oE|Iy5fJX7}B!oVWFl|ntF(pJ(~mU?_Wv(o2O{aWVjPUw#FgX zP6D>=T9Hwr^a1CnGljk@#)|2RGag|9)B5cD-6KkTz7HBS8eg`DCus2}fe)vu5VLl4 z*UD7_!4ZTXxIplO(6^fN7?=}~8jf1$$Mu(N-Xao^^Vq3~k1c=`+JJsb!5IM9V#2Yyme(hoaD zo)?Gr;Kmq6(nll#Y1xbDnlXMYZ9aCV0fb4ADt(K;;Tp}93IC-l3RQXGBF2AM>E&Ei zsXcLFT)v{jbu zF?h2=(9U|WmXP)Mb^+=&Bj@zR9j;8zi$8znVVr_863a2EJNq_M?3XNH>!BamYQjF= z#aiBjVLjFIk|ha?*DGE&$aPBxoJ=X`&YRNJC^-}+HxZ`nX0?yMY(aHf?XzrrE#WFU z5x3nnQ*U3L+yS9GH!m4DTXo^Z#(f&JBhL#F-@J+9dNZwfAVHLMd+*!ZBE*`ZOzmLO zl!=6F#EG5#OU8Y~JvI+FFkj&51 z+HZl!A(fc%a%Q8RVykbrK3BFlIJUIM;co^*a$5|Ew$t-S>10RgWX|#eFcAkovd}>! zPS8~sBLZe|w=~$KR8AZzom~TdY!d>3reU#-XxjqU{MPE^M^9+%|FHMv@lfvV|Kq5% zPbFC@Q&B=vNhC}wA(U)U=9EfENFvMJZAi#Y2~(DAX(}XHCVR*pvW~H4AIpr{?t6Zp z&UrfDbH2|x&*_|&-}8JQuh%($XlCyFp3i(f*Y&>M*ZaCIA=@z|#AJ_SD=_?93L$7J zrf$I#GJ)y25}4Qc6#*wMgA2diLMFo-)2|O@_9G`f0R$L05GVNGcMq{SsHkT80IT+F z#sZ>lE|M@=I?_buNFyezQ(@2(wtZUEy(nqPVZIMu!vL$x0>|@y5NN$gQRf#bP|OieD2E>jE-jPP3*Ww0a?hk*awq6G zrg`CT*wj`5yUD(|k@idY6e8^zQ(!!$^F_c zRTG7_G}O1!3q5G2S{Ju|Ucr&X)<{WMm0xh7{^roQOtWkV=c0UyM5uZ&Q(F4PGf>c6 z5W&xsN_@sS{V$+v`Ff2;0FkiXys$8nBV8#C+T7cikxDQvvOs*73I2R0x{`}JIw>}= z&9g(8%p~;q`$;pF(>`PMe&;Q`Q~T(Emx%y@l77tR&}Hz$v62lJhjuYYXq40+a{lIV1>2lQO~OQDCguZa`Ybbu!+ z1-K|jf4DcgPb(oKa~h&|PZp3WLV#me4s8C%TP@6gdOQ##{f`a%_7%BEj6xs3Tp|)&GMe%b#E|?0)Qz{!GB1P6JYW z^epLJb#pujT!=7$AJd7>SE@PLTvR5BUO%;$^scb*?;Uh>3DlB}XifV=qlTc#Zc=t*qOfySe{xJ^zu++9{!BlSk@# zqFZ)5w!Rp`R!i(`4=Wh`NL;g=G{c#%ryilWHn{ZDu8xFe1h-*Y8fb9Vtch-gQMdY{wH(`-B6{En`pEcwH(J2Dzz)epE5UV;7Ko z=K%Cx<_H5HvTVPMD$tOu&bSJVUXnzU?xQ z)H{J07ssfPSxZ5hvCtFblOlN8y22dtbu{C&EvO)n?0_iRT{uJw)GXcj&Ezl)$`MP)))WAa+;q zhM1c{kq|o87M?o@Q3^qHRmt;nr3`axikz+dz zOX$m4nS?#0`wtf?KAQ(W+;o43d(8Ral8XwL1XxWT9efPy&bm^lNab|}9~?i7f7*Mi z$f%!1%Agz4X;v36vR%S^G8MM@pkKRL6t2^@-U!jN@P!_*|u{ZeT_r?zJ68Ji{ z55UYeAHal10D=U2Hk{K)npOwc<{_mYJ28fI0BoaM3$V>m^uq7`eBbbXoKZN}#Z8jM z7-N2*A;*w9GBbuW`izUpNoy-3_4dObK|Ll~067-j`NRVB^j61d%>J>G>^bMxj2+KipjG_UW;Eun)o!62MXaPCSiqg2ILA5`#4nox? zs4dtxf5JBWaUeKD5&S!H!C+6%ZC6y>&cmE!JGTfvWt?&EL%nE<-jqH;pXg)BsJgS$ zII8LiyS&;<&`b}urk&2CV+F$D+cKoGLU)-$HVc=Jxxxfjatr&4Tv`%lBO(a4 z)no*zd|&$??}z`R!T)CwoPX`R&$OPM@jYM9_eefay$avUS?p#z?pk=bdT#I`>abr| zs?n}zJs(E>Pu=BRC`ua%tO*Q}nZnj{QTx%0NnL9hLm@Rqpl&=Q^WG`hx8a)wifiw9 zCsg``&MlXT-?($kT6I%F=Cgt_TKFpxt^h_i7Y{03>hZl+FCr}r zozB1Q%l_K={S*d#r?l&*+kFE|^mi}4uLfSkYZ}|>-&%+q02GjUCcPxPw645^REcTP zB(sPG6-dl80?W#K7}TT|5n?#im4K^Wuj}@A?>O_hUu-7tOi#{V*5Y5EiJ83PXXG6v zA7DWp!YSQZT-0fXw8?nxTjQ+pvoYt>ne(c)+sy80dnMX&rnOD%4x+h=1wA9DRTq+n zv5OrrOP<-AwJI06N^J(B)ie1!kG>kI^p0|_w06`)KO(5yZ|r-VSb9#HWCS|)k^QSj z+5l;@_d@lWZ+Vn?#|iW&fKG5$ zc#`7r`-PoXTlX^cTdq7|( zi07RO30c=K6O_fS?<6m3ao$yctfpq%S@h&aY_nxxMW*#}mDE?IDs6<3a!MRt$2j;J z83|(u-h8@>FaI8HPA}ng#g=z=<6%`7BA1_0fCq#0k~WWO4?G+XwNlDhZ)jRZW+9|b zM**7J$!{q>oJpqqI}}RKR`inV5-7J|iw+Wz(@##c|!@gBcZfb1HY`@7Ok^ zn>|ON;liG0wG_MMkty55H|`NSr>^2puddlw7+<2gUEO1UI~^48 zP0!E_ul28^_Pa!S3g7z#?L`KGwyzk~*ZX3DN++{M>&7fQG} zjI+}vX42GYk4sdtRa1twjQr}uWpwA!4GZiq5AJMU=-LY%(6b&^29r`C%=im~{3|vM z$E*1@y|x{13ycg(G9dQG>qbA$R$FwhAu+Ko)QEl*QYCn^8{#b+T~wnI21MMnX5YDY z#wOv|?w3V7!Vb&i=IpyLM?imZR5oD2<0BI`<)6M{wsL*Nnk358kffUU+17ibHYV=f zp>ZyDd6-WPqj{(Hq!yl}06T5B&I$OoNp7q#7y>rNlTs?S={5NgkrP)9 zr1$N+xF`c-{=w(u>5Xn+(8D9QF@i3)w^r~8JgFdnMx2pP1SLwjNAE1F1Cw_85u z+y57@6?vZ#JcYaW6R$Nk!)tvx3k=Paf6X|YPe$Ymb0`tKBc%7iue%M$^D?`A68D)f zbahrJIVc~)v?>iMtDQS--nepq)GCzjSU`_GVQLrsinAk}(KW}sYsa>z4?XEB9&@@K zX*nh#$0<9eEmSf{%_H>ve^iQt(!l5ZF> zeA%VlREwc#yB-K@?Ol6wGyU7rMxH9^+_*MaDaM_4EdzdQNXZGu$tof}V4mx(N8-?cjP zqt;glHH!xw(U4hoLZ-*fed=A<-EYl?^L1lKPJpTdb}4CE8X6Q#Fk-BnY5`w7x48_oTmZRtLWDe`F3R|H6uuM9_+`HHuhW{}<9>WI zwIeew<`prVhT z{=or>!&gCz^^^PYQszk<$usZO>f0xbipy~Am$eoNtiMduQ_x7~D0gHQdYKWYaK!cr zSb#`V8A|!PjY|Il_M9mg`8Un2^X$g(A?@S|DFz=>vIyjQw99aPO(SBUxJSVg2di~K zQ$=Wt4Tv}_*oU`hVhXuZzMSg+_cYqy(cfWa>P2S!!`JW+JmvBC4EAh#E3o&vhIl86 z&m~dqXv9taU>ETWo7_#|PEU84p73uYUhkNT^CB%$mY{@ri;*(x^`?)#Hx6rb6r+#q zm)kP?y5`8TWqjqyTebp{A=Va^qwM9;v7{$*wPIq|IZC&Yz}fb3W1q$%t2!ICxZb%h z!n5Y4NFQ^qp*v>5F?j({A-7Ap9!6*3Hjdf(_gtC1N&1=h|y~teTx-= zb-{#-+5uS#asqQeImpSEAl%?@E+hREZQP?FYSA;%#+N%ozL}XwGv4GYcoWIg*VDSq zrTKiNI6*fV^X+D5@5deo9Y(cN{6i9-*RH;>?)CxY(1@Pa)~Bl|)p$}JWI)ZSGr}CWdfROF`2`TGN=SF2}3xpZS}=Vt+m-CYcFF{(XayIUj=&X$CJ+*pi9d z?=ue~+C{pS1kyTeRf$c*>+3f|*L zV+8LMnZAaqKzH(RJJ4P6QnQWWfB3}8Llz7Choe?vEhCogENaqIwhg*+)lP{(=c7A0 zJHdgknF-#CZ{&>>o;-bjdF0W`xjhLRvvJcey{q_*EOWMB(l*0kL6ei@MAsQlFIO!Wn7cQNQt>;{&$l}+WGqLdBKIX;pC^$AG(s8S$ z(X(FB*pYux+l%Q4xAkw}38_tqS-ZmVs(tAGW-ssK@*xcWo7C#<$xj)a(s`Q<`J>`j z?Mu+JOWCWCIrdN}Inb{)Y4PRIf$R|x{!l?f>HWiCX7~_??V(Jou6NST)$I2Nt;CyH zXj;PHDUE-B&I$9A=i21_(Q}bzJlA(ow#}5N%y_P^;JGqSnmZOy#lC<$F+fH}tWEGbJJp?bP(#`=weTgq&wTCELt|3Z+Mbpi@i%e2oU%OpS!yMp zP<^Pd?v2$qoK3<_3QUNy5+B@G;@836H#In`zv5HZ}1$NkX(Q0#`=Xp2qtDz@( z3MbRv9^d)YO-0AJ`2`VU3t8Os4+%Y{Lh z7q_@Na?+3kP!A6VN;D%2c5C50b=sY-cg5%@|PE3AtjkmD$$XT8HjC^Vw%+zktF93+QXq z7-tW~`82%>{qZr@({0x{ueaNlE)GV-F3L$=@Mh>pj#NEV6lzI4coKm#N*+mxEwWq4!!Xu)MwiJHEaYQ#Tz;Lc!^iE5I_oUCVA!qviy z_`4T-7(1?kYP@db$=FrP7Q!}49>sbxCxBGKV}Hy_99-M`sAe#|Xe z46Q7$u9Hwv>7=rbq_^{X@Cko^!=jlZj2gZIZ04!{0yAE>Na8Kp`cgLQoWw+FDs-Qe zeg`w|VfT)0oVBsQz z=yfCOWs__B;ywLn7i8F0HMzNMYTjlO%*S<^8ATUm9wsmmCAYk>Cn;>_$?pSYXFTM; z^pKQixPB3p;P=RUH(UdPW;|bg>V37-1M+smy!Ew#p7M*76<9-#0UoA_+sxXlc9zS{ zaw3xP{${)Sqi2n$%jL+bift;sSYJxvH%wLIqE;=BxLtH4J4#q#WuDc=poE=~FLT80 zFEK9%H3MiA$UZ?mFgB1ho+VYGTlRryU1Blx-rW{fw|EAE z$`yD*5;GLnvvuklKFfd2r7<&AR5SkZi~XY#dcO1J>Ee#{HEvrIc1}Jodb~9PnL-BQ0XvCFop+YMb?*0>s9Ic3r1@6xYoS+!D@deA_$JFcj@J8x zY77SXo^I9HalqtusDhs9f#dK%&6Lk-UX8kRl^tv65Mn{&pcB5*9`h{CFUX`eE%=Zu zH+W?lJCzo2`X02X$5>UgTr%1C`0V+{E{9FkLWQI*HAYwp#c^n=+b)Lm^vbkm5zU^h z{VtiQ|A1bmSo*x7anpfLTllRJ>mGYMsV@V}55gieG zegGPZTm8K+ zf>9?V_FR+d^aA~hrc_v36$R&Mc39z47*^KZcZMMu#tRR%_-ASlty0leO$-{Cdz##n_Dd{6hCR z8`IA@x7(e-Ud|A*CM=Q13b*eTDb0_tyi?h3YHc0SrD1vDFd>9_x#E&((4KRxs80W> z1#~y(OE0~O-r8ciEhkS-EX2yJsLyQL>NeF?&oE8!y0dP`NKh~2ak_0;Wy-}GeZvVC zmMmijRv%l-)EyNnfj^(1p4pS!y zOeZeN78*(T-~lChd45g^#l!SsJ3?d;JCh7(dugJ>JnNx{%! ze9DCCO5&t1eeYF4Z2$IAI`7m`x)y_VmW?SO2;;Vf^7!Ptq}E?QeLz@r<*TTwJx791 zmn!4Cv70uM2QzED2IOdoHB=-2XNwwc_m0Uvke8}kjLWKWQGNKE3;TmysSc$ox%T8c92YcgLD?AW;!el)c5P7o)Y^S+yHYo~ zzQYH%TN&N$R3Za)j`(?I@Le@6#;?u_Fu@HzWq7AQsvYaGopP47QO&=!|Dx|E2WE11 zNrAvU1xUeto!jLu9M8mC*`<`1XZo+4uyj>=f6F{tm}0HKe%bKosj<(|kw(t=g z^DYR1jhLcd^c{pEE~*Ub1bt=~jYMN9`rTxNpNLG(CoqzJd8aYy8EIe)0lffQF-P45 z7!40NXKJ3vFH&t!ceg%t_du%iW!H?{m(Pwpx4CfgRkL8!tcNU z3Vr%zfTaXsylMk3H}cw70S-TidU3cYLiNN0bYR{#V#;!sU^8DpeIZ;_?=&K^D8@U{ik;6N|&?OBFG5* z0&shJu!w3<#lTV+{Je8X@5dxq37isl>sEK$AslUiCmmlsNGA97vz1fsd#$`~tb=pC z9|6l`=Ep^5oZmuU|iNGuMkcE7o&&TgU(|XaLO>wnypwJNGEN! zmBn^U+!yuC!xTM{Gh0nvY*UI}M5`Xokw2vVlpbBt!j{MDT9zFJIiAYle%oGm`B>dn z13&53do6TQr<*d?R!@l|C9%pJ;ScmZJ&lyAwr4x`^Kyov^vu{!BBz3jVn_kSUVN_q zuqjkg{qZ*<{bprlfBifDa*$?C>3U!?sy{YK!+a2(!$oOPb@oxB@EeUGokZ2Wttyv}YEW&k?jx57hbVG0SYABhDY9_Z}jX~ojq>fdZ#>!ik zC(Xbeepp>SdgaBK8NNbwuIK{Bfd|{EOcTQGL(!Yw z&09n|nQT+E@$UD%*QTxlB@vzM-g$ z=BV;ZL;S}%r|_sPRAqzqeB2Pd_VMb)t>th|Kic|HySK?<-}Y_Fdc~V; z@^+pI+)`d{XK?L}iu8Hab$QEjwknzg#T!O~e20SY-Xl)FvEudM{|L)R5wDywJfEbt z!S-;3s`7#Hefw~W8P^8{OCuJ=xIfN1Sbgi#`)Bj5sEMz)B}r`#-Ock}QOG?7>2+V4 z7j#MaG$p=(bHqg2`(n9P%cQ)Z-@_F#&kk#rRVoN|nG3bZizf&rzB_{3diTc;(b!1U zImnpogldBDn;w>V^vh}o1PZ6?5bBzv?N~oM6dvKu36&4L(x$cPxzNFPm$a?QSMSqk zkU5)1dLHE_2n)<{y*(3II8@lwYwDC9)Md(LUb0M$aDq0;5`9HxzV(A0vu3zaIambm`d?7E=I zcedc+)0?iMgfG8&Fh`2isavP@dR6!9Xt}W24qe;MTZd4E9_4K`n|Npt7xQentz}$L z5}(L34UGcK(d)(P{95TvMWhNAuAf5Ve7IAX#c=F~T`7dyQ)B^#i%q!Uh1%mS2S;Tb zRg_MtHa)_vbN|qTav8CEzIpX>|2k#K@m=0-YrD_OuC&V;F-4Cy#I@>=D_+??NEvK( z)<2>{cMJBFDydIc@vP{G=epno)m&HA{t??k!YvVWQS|~#M2b39JaVC_{8f5j;FGCT zwe5=>AK{X>E)aJOVQmX_EPn;^LVKcDFzy$sZnirs=;v&hAir>LhNew|(V`RLgC~c? z;g*om^ah>H@@v{8Rv)Eey z)1VvSpkQJXO`#?Q(X;R*je-}FAKMq+gR*pcLCbwH`Y94wtatC9sV{gbxK<*vpF9hC zU+auT4~G#az32e|QO_%c=okRmk2DYUkV^;&Ow2ubWG`Xwe>S4|w*Y)RGd1HguBJ_NQhCs_|oZ&UumK|rD)9)Ye7~)@hSf?lBj$V}N`HOOa zEa-+Zk7+S_`8G4YgcSqb&$bw}*vNY%u-guW1YHg)EQs}ae!MLy!}YPfd>(PHfvb#) z3tnRM?UGX4Ra^Fn9GJEB@s^U?$$`ZpHh$*w6?t|GRjEE$2-V4pY`1%N(zCF5ZO)`|VA9B*fFapK%Q50`!U42JPqs?nuG-4s(x>r$t%Yxn!&15nrpajq$bOkHh!-r#>2+o7pTZ-x>C2K6Wqf zlq_A5ioULlqhMCFGZG5$nDIpo3AZ|5ym;BT{l1Et@fw?LI}dGDDF{l*mJlv+AA+@5 zvz;v{{i~>=PItOD1<#H!jP>_ZawtA&X%w=S=5n#uH+iqN+DM*$U3MOuaV*v9-Oz!f z!FQrop1?&QB+Rpa+D)WvQWs8n0oab}5?oXvekB(bDr(UmoxpGlk0GYI&>Vi|I&f&HiV^Ub{d&sPRJD{l!GRVznTpiulaTD0!c#*E8G&j$Y_76MU7AigWV zG2~2SGG%kQsC$#LBPXe?uSHid4m;u&IV%(@tpLz)-ex~r`iQjJvuCfC916a-hQBq@ zCiU7&3qD?zIoLoX0^w(v=jYASRix-(Dkg?;O-}<1p1#*yZk4E^-{-HQsCNAM^PCIy z&xcdV;YO*-dcHj^X|*r7C?gn;0dyM{^Lw}u^AHpj=;CgZb$OxDQ>1s*&GDcEDiN*) z#B!qZm1<5lkXVxF^;2r3cZG$Yh2A-Z@GDZ!6)FzC?EnpQ0dx+O@gO}O&e_F9onnz# z!D8exk{8y1iCNUdEOA}?k+A2P1TLsZV@zkF8C$=xPiccP%hv(Qe(TH_VW<=u%m(IFxlYg9KdbdOMpw|-n? zbNRaIt`D{Nrntc)H|W|r#do%h^S^ns*rVGm_9?L{$AC)mRK|5aHkviQe`jKspuE&G ztrkiOkzy3rk$kCRa6tb?lw?PgMv3WRqMnC=A>i_F(69pdwayF77}i~>>IL|9H`kB& zJPtTs@r<-Cj)9<S_s>Qb-eo_Jci^wAvP||I9ORJ%ViKOSHm5yecpZA$m~Q=EezmlI z#%+4ZL~~)GGXIkvDfNBtQ{2k;EaaJmo~~b4y)Ak}8MSRsfRU&4Y~KWtbn87G{5KoA zx?8oo;1AhJ;`NyoI-3_2SXGKN3{;wDM)TmSy_%8I8Eo}h*kSx;g)uy=BH8AsOd8rKq@)$Qqfaaay; z$QjG6yW(v^9_kK`>pF?ln>da;+-^MbAQ!5jvGd02l+hE?u$l6b?^SyI75bC8+75RE zmM)vp4||>g#_T)|=ASwQdh*|<*s|vnkXXZ=TgjurO#C_cscsj`9|kCD?xm~=+taE( zZ}uGM;rXZkP-@P~{w^zR)%YgIx-d-i5P>fA4$P*^7(z}dk`T?e^+>@umgQp6Qje^1 z7wj=_s`}FyVmBrM_cVdyEhmJAw9I?cxL_(ri3Vdu@ z9*UU`?3pbX@O%IL=oZel0F0rK1c|fW1daH-9K#i3bgP+L_(2hX=Yol3mtz*}u`)uC zBLYy%?>NYR7odnjPexeVsqCVt7r{;xtj}YPR&2KR`O7UsCRzu}Y*PkwS|6o{*EV^1 z>LhlG8rj*tS6(*me~YuU`}V41fth+YQ*{MBFZL$fF3cP|8g{4Nzf>)rINuxVvMh3a z_3gq^evSGt7-Iq3vZml`&;4keI!G zU+uM3zF-meM50UDc6Q0m?_^YXY+b;Ay`(UD8E&~*eS)48CseXhQU8XMZ%ku0DL6?p zUihWHp*0I3rO*UuXHWhNH+T!t24Jc?V`9!aH{ zG}I7J$2Ug4v))({bU3N$Msf85Jud3ZDCtRuSGNu^ZqQ_Lf_tp>4!MC>kM29jMA7`} zg!*5+^mvhTfmC*Ft=F1>J=jh|_rU&Da*?N;A=N_NdA|AYLZ#j%OX~=%84+ix4m|1> ze;3nYNqX!{(UY=w(mng?USSX)iCKI_H6DXQD##d!VrF$CAHON*WM}Hxo|U`Gddvkq z<+VKXU(T))ON;Vn?e0_~=Lx+9p+`2}3#$=Nxo|#5wcBmqaTWh0c0^hzCH&L|T2y=1 zmW;Tj%%-rA)}r;xhq9Pg?#ym9YYa=n-R7d?X!b%|3=J3iCL4s$)=}FxN4QFD5tN~k zbU;TbA=%pc(io|o#c4IH3A*-6I^cJ<`RBdH!(7ci3zb=fBgHnnT$HsD!)uC*;-Z29 z1$3zmgM=Pu9g2k;z}N`nAagJ2R`uae1RL+vKKfiPDuM{+y~`qf*6<-X0yk*_X4+yv zBEicm66keEY!I8!Ql1BHB3N0Ti*m%k2IhBGRlwzt zEG&sh6F`_4sXtwgkGf|IWaHQ@7pOq@3e!*0i;3itsxUv-4uK%|=i(t^FeH)?1Eb$2 zb5Uz`AAVNQ_-TE<0?GJ)@qRO%oMv#~7vVr2_3b?D8&v6hiPx9~IqM&2E%7BK1D|P2526JbP64O7qqBpq?Ry6r12h7+=6yS*B>1zH_VQ?V?i|*{rJdY^!G% zDok`VZ`V^E<=kg3JYj2VX0W2^`IL!RQwqKQ?sq%KUtCiw+R6r#3Ck`pu7IMAd!(sm z6=*-qkL_Le(iABG1~ajt>W~vV4#7oW&+{|#fX&?vc;vmS(ZQ~3r*8h%Rzj)>;QRxS zEO045zyoNRk@_TjfWY!`C3aXApo4@*?x|QxX)Rt&#ckjp4*y@o+AK%*eCSPJH;yd= zBI~Kz|G|CG`*JMH=OA;qn=uahJ`7f8rNAqcul@#l^Y)NhvB*IZTnUig?U%$ZLc$p5 zt}rmqg|nFkDx1>a>7ME`CGc_GN55k%(Y5IJVu)2H{D#E7N2JNoEul(l;MX8{P*l`` zi`vXZjf}zR@Dd_z3$@_SlGh&%Iu;Pk7l=$zH$mXNBFQZ02+|vD%buUqr9dY14W1Jk%m(9#|9ftJY40d_~hB` zBSmTop_-E06BKaH~Jl;(PWHvU;lZb<^*i5S>g z;s#GrJbrE(ExK=f8aY^6L|_X5kcBd$a8VjK<`y?ou=U&2k)TW9jvYX|>y7cs;G&fJ zn>bh8L3N)QCNstqAqBjq}02081vO`$tz{ z{V`PJF*|}(YyRf9)>TytP6O}}K*+X+2S7ZMavPCk;enI5Mu2^{;6jhqRba{ZL!4kD zb&B=l@3;ea349&fmjXTyu8{>0{U2(J=YLae@z1%JjYHZO_&Tu9ol42W*7tQiI=18i z4`?uq7~PaNP1y9xhBJ1v=83q9O#elERS|x4U)uUrSD*oln&5#Q&o^mf<~UFDBc-

~?Yn8=GE3O)+^h9N}jrXqxXdE|_| zd8@eRvDCiXu2)W9@0z#l+y;*p$%hde(5s#uyeV+z$N5j!TvzN%FYMYXq)9D4X(8ok zS*F8(GxCh^fsSpy!8+S~8n)MKpHQuE7fpm6;0XLj3HE+b(eXuZt-CAdH{jnJi({z) z1LFiqd7sl}$~AVIYj^X!apXA;U`7^hzkl2#Oo`db<8no`Skd(rmPD(lSXzrzt??^M z%WAxV_f0)tD^PN`Txw%!-&Tf#7eQJ6<_mX%1WwFP?{05o~=&mX``~asH6~3f473j6l($+ zGCwR8N#qm~$ER4|E#vZg5c9i5`rr0O=$XECGjQ!o;Mx(y6~DX&6OVg3b`2?h=p5cP zyKR?6qR^67{9i7Vsbuo+&+J{HK?%41;}7cS~otH_$XJ32d&qA2e*02_9^}0g{ie!Y~z-0X1a~ zLh?ZwvHmpw?otjmMYRu1RGIko%Ag*ggSe>kK=iW?XyxS3VOdgLq}zA`5aw8bk1rJL z)t`D^JxvbAiVtyARDX{d#qQ&xevBpw(Q zo;Q`&cBgKZ{lWS9aHem%>EZ1|L-)2J;&nE`D?1Kdu+DZw=?H}d+RMHzI1YgY-=`Z_ zJ}$e#%adb$3Jrncr5KoptJ5@p==4W<1fD6F-ur>aaE6RhVrB%_8w3 zhuCZQ19qzR-%+}Q-3+m4Fr^lc?I+PLUD@~H9}ee-1-I5@U3Yc8bt<}}OL9%Lu=%ld z5yHV0VUp%}VdG`Oy0h$@E6?E2`vmtoAJP}o3pFn1zZqm7c{+7tnaZluQ4ucNX(BFd z98Wl2b=Gk=nWn1ThkG{V&2Ab!ZOvyO3*Si8xOb*zG!E(Gan)F?pdPaGT!jL|dg=vh z$0;|FE{mcz1&s>NNhSaH%?dQBiJbVGi8QweO=Bznu18S{l}b7eEVnuGr`8Vlj}wc* zk9E~8pLKZ%!N69CqN_Cw;lyfo;0jDe!@IQSOlU2_sb_0tODB4lEA^V~6kMZx zaqYQYU+RaqegQ?8hZIri>X#UAJ2%rp`CiY`AFZU)lMf6xyo+5Q6QvYzF~Jbso}{C9 zGuSu6UN6!rzdkA1VnjCI*5{Q%G0=||K5o8p*`?Ay+`3Df|1EmQ{lsli&mTx^XmNi+ zpD+&$#VRRcrsj1D-r31_r;UH{+6tR}7_Z(0{{ja|UG#~Bv#(wWq)Hx6tn1S#MkWNr zFd@Y!6FifKdo$JBFSoKd@O6^?&b^<_s|}y|Y5BjI0Su4a;P^ zjY2h3N86%kPS9$zlvEQ(j1d>LFz?`~_1N1lw9LPSHvjssuTsYK0;$4InC6K>@6X!8 zJ9Kfi*!|VagHKifEC=Wp{`B9*QX(BScy8tSameXY~04<9K1_;)=4Erfupg&H@_|t~?Lz{w+!WR%G z#GRXvk`xHuegHQtI1F+)!45?j`Zj+grcTvW5mAN;S!ttIT+{*oHNSOrOj&w0fTRGH zIvCyndGsgJjD!IY!f%)Z(YNH|4Rp0Y9~XC~+=LFPuzukBKYLT&ssI0-B7gmrwqNqY z9j$THT4b%3OVDY!RFP=AN!EsNy0K}?jt@Qi-l%#UT;qFvt~ds_w%Xy2aMjbCJprdA zZkBWos-=q5Tn<$VaPkdTZ!S&{mM+meVzV=Ht=?vSqeJjL#-j22+Khb4xzYBA8RKRF z0jj0S!-CV zp`VL&>U_?V!|er{m*QHl2Q!bKUEypJnQA6$fk4#KY@;cSl7D3Cg{nl?#15?--Mp9ay)dymQOWAMzZ0UaGI@&Rbbx7tT*< zl;FMdXrSRny?S!U>5Q8c@dO=vtHY7zis^M{>XVcljy@W0EbS^S!QNz57hqRZKeXZ* zO}|gtHdYwX$QPPodB0@A^4G^Go5ZsGc^4|6+n>up)nxy?x&6rO@m#Z2Z?jT~>d*`BFQ0)zv z-2%djW@o~FXzTLSC&c(SqhT#2A)_LFHV56yR>wy3^u`0#wC|St~}hdRnhsB@XlqIsPrASMODl4$HV6v z8Bm>HI&2P3kdGTMxI<0RG4k!~pj(!2mf=L19F0}biAmE1BFKcsre@*a9_hdM7hjLl z$UUGYh{mx6($R1#t_uUmG(@)_uD63&1`Y$;{HKi&38q-@d_+uT_b0miA3_JxUlBT> z*D}Cq7B}zGg>S#|N3KePZyZPD+*OqUO#cNj5S=U8L1X%$5tx5r3l|mTFjR0=^%fig z^^lwwAbkXQFHHn_#1eqY4^f-ocM#3sj-rF?v?PZ#6^DOo#O`rR{8JOHOF1`=C9B5F9F}oKYk`W`pFma z&h#Js%;ow9y3;>42YeN{7Z^_9Y8+DN!iPjQlKk;eq=`GsaMo!#n>j&s;9?1HxpFRYkbq@XX=JFJbJe!u&UdXq`{}8{gXw+Of7o{Q%w@)G@O7-3B(;@#$wDwyG+gJTd z?2@9Qia*JA5&*dkQM);IaH`DakvwJR5?3G7`_7IlZ5NsuYbkG2Up*4zY*Lj!Y9k)N zBW|tCzx%|z9}c$5Wf`NX<^$PROFEIDD>OF)k&vXWhv5yek_AWnth8hcY8@ptHKYRk ztPIX&sYgh~ONF%d-weU0$XIqh2-gCb?+`PO6nv-ilcoR1&yKc%BcYy5H)Vh^-;!D9 zqe(G%Z^A_2abUbp0_^zCs(-^SxDJx7#Jrft!GTJ-2Y_#VxY(;KH-NSGx0Vx2fR=Dk zZ=mDgS|tfU7DdEO@&guT!tgACF6a&Ha=mIJf!h6nBqo}7>FNJ^l3(w`pSdnr3HHGD z92_UYeui8W>)dV9d)el~j~u}*?C=>3d?3z^ZqIQB{3gw!tpV{7-Ti4d{>+U|;6^|T z9B~}dm&yll>2afR-TG^1;{-&g5D5(=^ zQj$9t0Gy*knZaW)RERXWsf_pC7?&tu+1}>tva35rR`>C-JSb>l!&-P^ApDu{NZ1C& z^Q#~4cz!|65IeP&;_t02*9FFsDz8zER8)_E-47IbG{|+@(Q(HYc z)VM{z|BQ3C=GGv4+%e6^(nk#>bFrx~mZ*9vPpQwVhKougmo6aRR9(M?oLgLdo!0yi zbePqil#eT%wzqz5ViqOMvMLjORu7EsH0b$I zaj;ZaO%PeHIAFLXji|A)OjG~EOMCODgkuTL9BW&w1nUannakye|Bt;d4}@~<-=9)P zMW&LJWGW$4q+|(mS`b1AS*MbOBwHrosgM@?TEZmDWNXTvZL)`uealXEW-Mc7%shT~ z=RK#hpVN8Y_xJsN-}m_EbjC8CnR)KdecjjRx~`8!QhSG$q0>jHTsLNh)D8U`Gixv8 zX65*(v6D~bOQeOKccX(ALH^AfW_M~xn;nePQtY-~Rg^v(ueV;6Yu8~geN;_(n_>d+ zF}r3w@9zjoXF`P8#?-NiJ^unU`uh=_4&M=+Y=GdDs$N1Q+lE#^Oq_Aur zGs~R*9PbTd$8tK<*50#^RXjQtk|d{iu3+*m+#Vu! zAW+91!jp2ZuYBL!`=9s+S%vF-1gqBGSwdV|Pz9wZT%SPprT|S`*lGq3(##Gk0if#E-kU`Mr+oKJmgb zA}WqMi#uz^_U@;K{)78;%DZOPj7c2wxVs)IpefjflE2aQu9HyxP})m&?9=l#wEyI# z>hLf`h`U3DzR;>Tljaj6v@KbZ32D+@bVq`|zxgC@R}3+j#85UmoXIkb>_GK8;mhjQ zKdL#f_dL1WW0ikQS5V-~m}_y>2Rz*P&u<+Rh~VCHQ=50#>=be)y8$;ce1}}^3Fn0> z9nSIJ?-{-6~ZZNx)Q-!k%d~IDA!O7d4dtckK9aTH02F=2`ZsX)JO6{~p zUhXSmR&*7;w%^Do@92wYwx>pqc`FWcmh5Er@mZG^um-JxE}Ezit1#a>)^2L4gmu;6 z1MTV1Q_^Sg+G1hXBQFK}Ja5$4FPIzWcJfl(o7g0C!+w)o_C;HYE|np->y|BPxLhjF zRV>kXV;@udlV)&9Zq&!8(Y4OOp6PFH?9V%TF=YBeW>+|qgTBR0uyxhIC|0xLu@^6n zTWX$HQN>VNWPVT2>!)FXCHhZoR5E!a<|=T^Wq?H1qjo_T!L;4Pnl z4=xt;fyRWJz=4t7N@pGg=i)=d-v5HDr{4^u^tgoby%}ukK8leW^tOkA4 z$WMtyO<|SB%n8zzw|QR_n<9zwl&9=M%xIXQ?sLS7nSD0e9a2}FF0(iU-872~ZiU08 z&Aij#O?@e>Y(obJi-)K#=Uh}Qr?fmRD?T;dj8DF%o;B?-Afn#%G&0*J{aQisuQ{=Q z;T-;FPsT`M=6v?&QX#nV6F8hE3Z`NPkdxe?)7u#e+9>`Y%^;B$IA?%fOa`-MMmYym z;nnize}RLR3B^Ak6ul=lf+#@@JcwbwZ~gs|hx-2?M;`w227(<|P@E?Svlh_A-sELX~CK2~ZC^`@_wJ3v28KsulWIxOxG8!4llc zPuM!h=ec;x$6cP-{xsnwkx)%%xLK8hE9TaL@@;Om zTc~X7T|3#>21s+Q1wJK>df5BQlNYZXJ8+30(qwjJ_N*trPFuBIVXYccYmv}rix0<) zlNj&j=SM%f>%#H|gmKW6MLt{6%2-0Eev=K5p1GKRMQPX#;(AxJ7cBidWRapJvQdpu z^p+?aKBcE}aeY&%1DLL0k3DLY7D{4pk=we$VBEO-N`KEtw}+oR_HjqJ}-L?(#W46}O8)3#OxvZ~-4hVNje6(_bvf-(C0&KV~|5Fde{`i&@|}n3>EtL5ILO5WLFVYoK$g z`qE8M3kEO;Xh!z9#$ZYh(%B`A5MB*!C?*$p~K-2|8_vK zx%!9xDWQe*Mg!mdM!9X^4!=sITSWhc@UR`dSV}CR$Ih-JRy2;>{Wsv2zYKK#%YFg9 zoZ7dH+kT4MKwD72qNInSd6>Xd$%G7-5L<~?vKi46$#5ywmZ|aHKeiGiK*a-z%1LjE zIO$t5Yw6QEu8*aA(nuN+jmOVPV;ED*b2TgCHWx?D5N9)#s-2B`>!JipxhX0M3aO;`<6TQT+yaRV{i=(8YA zpoAM?K%pmKB6s}HnSDQ3XGS73hd7vmy#-fx2LhDE_uI(pCVDZ02x{?O98W7rcT_r!=6#u!1*+1S({;5popQ*F=j-fSY%OsQ8a0!?+sG_l-%?NW(L7Zsz zvh7=);*R*nFgJ{5*6FC-{YpbGZ2~yiHk}S+M@7pK>5FqhboB!wMOq{e|AQO`(LC$T zsSktl%Ff&+Hsgg4ce!l9~35bHO1d~-0Sv~UM4VA&@i%u*s+8lh~E;<&{}Ot9Qx124$jU9XpOn^dBNT<#aa3;9VK~#R+_s^cQi=(WT-lDd zD;$U?x4GCgt~QNJoqQJdh|Tm`je&?{gi4OCrzMla$WM3ot9eWT#b_)K0>A?Oz#k%@ zdqo*!tDe?*dUT1N`Jo}d-PUeZeL&SnuS8nF zF@&EreuZE8&||txw29u?gIiq!l2zmO49GeOe>Mq+y^g#T5-V^yLR~T^LfyXL$>RD& zS%!TE^-bUO>oI?>Y`?>M`PHv|*g9aM+T-bO^6G)z;+w5r(c%Yry|=Eb4~EIiEee5; z#dveHdea{$`EN9T?PC`b`aHShbDLHaHmNn#y{i|mJ2Sdg$DU?Yj5%%USulzgY_8v^1|1=L0O&}vluY_2SY7_f;>JZ;SG!DVW?$WB%t7?c*lPC3D090RPbjhL zk;!Xc?99+L0bi)OGWve~Fmr&u&u#ZBQ)yCg(Wj}yIo|tx%pT#N>-NQ&`NV}5RQan0 zsNCYo);kj_BoA(^Qv4no4m zlf0YG+dbm?CO&R}81Y${=t{;^v)VOM(Pi|7JFdzJSbT}IuX(bi&Vy$T;rFm@HaY2e zo*O@%kl^G^3=Eaqa9z3WI`XXJ$o16rqRrU6ok3nE!u_OL{b&Uir^qn~{#=OcfegJun2fbgK8gJdDN%tZNf=qD9w(1%Z>fHAEt z8!*~&QPSvzB&c7xZxX6~GvD_u>ZE%WdIpof8c>6paEyyDq~o@ZI3$R*hQY9oqzYb{ z@Wl`qL)S)M7G=&Toc*J+`}unjACcZ5FVm~8WSaF9R&-!}oZ~jPO+)@DFDR-f;|4dW zV%LiWgQ!A=zzulnCy+3F9zMDZ3IEBn#!vl#E0z=dmLcKyHN+cl>|B>{G&?uT^p5!NW7VGtYD_OU8WJ{|?&g}lV5r%s5$FXqHNcOU;LDfDGhx|ArmX5i8K&X@$ zk6%OD&qkQ!&0eh$tM|!k&A_b4qX8lECe*$Xx;6?wOlC+z8=E(w$4SK63cGPCd2 zvfWa{#@=N+XO^AqC6UO!L236x>3Z*PpPB6-YR>DhdBxllGLnzFrR z2eMR*s#<2{qzVh1VB$-!q>sn`4J+a2;NXuQt)S^H+JSD00r0~ygKK~hPFvoPSdQud zPFSIAQP2b7+YW-}X8A{E{1+^t2e6jH;m;bxUm{dWPkRNd=3G*@$X)}4CY9rKp%Gy1 z6Xq)Jp>Uy@N-yorh+RT6jQYj%xUzW^n1w{&Fyi{O9T~UtX8@Unwm?QQyywC5SV45d z`6a|bISim-FIxhu2NuVaKQsyck3-?n%)#+twO?MvuSztskohC!&Itx^VKNZq*tHF%C9{l;S6D0t*18;~Tzu(H}M~Kub-5p&vT1Wv%287!83fUff8})YOpnCF?IGh8+gtW(OwV2Hy# zc}8J7Vb+Io{I@o44t!Cxc&qE`eiN}W(;a!%>YA!Kn_?#Hdwa3# z*qy!p>7+$}Y%mxaT|~yM({r?YER!fUn8+HhDrrVZ)b;9c|R4S>-HpQ4e{SKBa zAzB&eNsD6KgIu99XiXb7R>@WbIAauB9J2y1;gp+WeI#pqYKqJU*{|sW!zZYe=#^w$yVdi%Xn4S*OZL-wKHYiqJ!kEH~W%u z9ZdamM_T9Kj5^WZ)c)|Mt*E5-6h$EY8HbXv`lUA>IVFq*Ozd0v=+V*8Pn~Dd1MX^H z!EpGnUfocoit{0}-p1ONc5f=bZo&0Im-P6Zbitd^8f%SA;lnhmhS2!6&U#i_A*OZDe!W`t3lGVWTNOW5BoK54(EcBM&j)<%03<;1Jy3O7* zJ3653$*a;!;Jqk~zOOH*3v3@7)pg1)L`LBC(AE28tQzyc)H|*G*lHQKYuh)ns|#qO zB-`gPgE!xfiO%Rav?y>e&b1V7#1{5o8sBI0nAC+^_V3@Us<*>Pz2{B1_4V{(d&*3- zZd%;qZbjQp#wv2U0_l*SrJcgLA*0m>N?}LW)?Deo&S*{3AnPVt%X00AG_U!F8KoK+ zt{}ReS69p;9Z;<=!3Tz$6!LP_d`_)9-YBQuH(mtAj8A%>y@1^E>C8ubx>`*@&FIf;hW;wntAjok{C+L%Be}98z=U~!t z$Ar-u(3k#1Bxv{*2HgJHO8?5{;pdUzbzo#TSipaMX82!Td;bT9hWCqg#=IMh~`Lf+&(0`hy4ZS8>dKOdnH=b`vv@IQKuhTl{88`j7u2YB@@988?2v zd;5MER%oj*?Q$DosbIakdb;AK9OS%Fe1ypE2q8!JRmF3}Cu05ea+-#Z_lRSkoz+0! z>-C6^CDl>AAq-u)nS5e9(qTmj)tG~aV%^zV@k!E-)3>=PQOj4_L~^SCQpBy7c!P?y z6Z|dLH0uRy$KO#-uke$-F7N$}@UiyHUo4O(KFl*@;mzL^uvbS}sJsq2=9VAOeupZL_yTrEtB@h|%jtNrm~rbuyQ%3Q8b+*3nCdUAJ1 zcT=#nX7xNwtU^*%@@2qWEGFj?;+qx}!~PA_;s=B%F@8WpvrZVAIiBGGZ?ffpK0qgz z5Jj2b%*z|gywZoGZ|#Ff&%=QVXG;K;aV*C8RDjXA;py3`s=yN*@fQ0U@_1MI9ky5F zd#jfr{mA<@SL_L-UrAGBhnA1EPDosLm9R=y&&gcxeZFUIZmV_KIHGJub0^}`@%V5p zWZX|s)en39pBEnYv+zgEjGi3G?7B343ZOlIJp$Q-@(2Ph5@4|XhmGVNT`u;sjRXqO z^tA@*zQ4pw^4S!Ckb}BNvvA2FXf(|Qen1z5suZthGrcsyEVtyy$I}Zc4B}z;HX*W` zGRT&IJ!MgXlwWC{U^m7IFj?AuHc0G%j zRV^zGY8%7*&sv(1 z5<5FTXpCtQbCW+@(}@+b8CI_nPdq}i!=CF;w6_sRhx6vqmU1%vJA}$y4m|DOv5qXc zo?kZ$1O(<#H=P`q%H`Agzz9-$EAi_5+l@f9_HCqCX*+5giNg6%+IAza9@`EL$;7h{ zZ-fZc<5SGJv9%HzF?(}V90bFwG&bR-)`}j!rl0%m4<)H_W?r7T8i5@y_xE%`mrbv& zjy1h`BcjM^g;UHg*OwEBXsbyXrw-2u(^BwY}hIzhf zn^>)^nkutDE>l}`^0=K{l+1piwMupkD#R4LHrM&26K5$vyE0R@L(z4{liq@iUQ{&G~S^Xz)b*!p?RH&$Il8XeONdjoCW{*X5$m+$0}3bBX!Iidvp4_j_~YKtM>F5TVQ#e}}km@DIjm z@&9C;)*lya_X}_rW=4)G2A9bY=g5*!00a&dYB13Ky>`PeRRX66drELg4F7xMi~;$8 z?~g?Z9u8R~!tKl@#I`nU!a@n9$M$Kl%W0P(rsw&miE}*}sRd0ffdj)=PORk-w?A{F zs}up9?8UKTai#l`W#3~>!&|H_VXyR$pLxif7;$)UHgxZJjfl#k%8CAv&2Pl^#O$+b z^%X8Z)EG5ocaACBI&5?0s=&@KD?zK}wn7T?iGrL8soIEdD_Ni#$6~!;`RUzMnX6=? z+G2e3b7Ry;nU3%)2fbG6R*UY#b$$*Ck4s_j_j46Rniv%FzE2XZQqR0ys@8SxF(p># zRKJ3gf} z<9Ii?KL(_+WfR^oB3ViGdXUC}17lh)2@&-PnHtk4K1En7gZ%2&kmet9k9d?(9 z>@zdVED<8(8pcQ3y~)Ae=$4n+Z+0W1k!|5?<@hXE7M^p~hxk%fg1rRism-8ut1OhZBc2(@w?tlB%kXvBzE7SbG#C zBT4%x@UJz*Ue%S+PFB?hDUAbrYY#N|Jf9fYo6-141s|)g(W4Pzt$xbu;!DJ^`zFu!2jJ~S)SQWf2R270e1tz`Z2wA zsD2yIQ~_SXMy*k$xSqvOm28Gc5HwM%x!|;Cq$kWV6utdWk&M*+=b2-9G%yxrJP{-e z>~A8e49@fVMMu+Q%AnLOSp+(qm91GuT@`ZsSN(>Szs0xza`53Y_WOeUet~uVcz6B>zb2W>GINUpx}QM>2><~2J_j6A3rs8%>i}e+ zP#wT=7k$LwxB)Ol0^Nn7;s*T#M5(9eSKgL)?nN=eZ143@Bu8;{38E8S0;*g>^r08G zYB4pyY~Fay6ANr_=*V%*+$oB3<=mElyMIzPf3wbhtrhs@k6K6lWkbq;nph^$n5MmoLt>57^Ceg-EHJcc}*|Sd~kJ8>0))CvXAX`T}9$uC+jC`X%3mLQg-|s?9oH- zqD$UwWj3_ZW9N3#PtR5$d&76=w}=J{mp0vs&su3y0LIrkUK=fwJK}rBCx2h{>g3XT zBZgWXalU7Yi8LN&b>4A*zD|chtVK8306c|hZk=~1&9y!@_rr$b8_DZ5*+Y)ie^yT; zFB-VlEg>w8rXRu_8EkTv!`Mb;U`cjSE+X%UQ}OQ>8JK+J{}_36dwvTv%L(S>U8FPY z6eLOOQ9Z{G+W(*f~FYfv7! z4pnf$+6u$@g6_m9ft-2tS)w_?pGbC! zz0jG~kRU*Ntg5`IHg}k=F19J{G1j)cc^cO%41da0v3`7^w@|PN<5yC%}XnP5SJju*$m~Z)C=C6+0BX8(WC^Odtryv@$rWP=D1m_cxO< z-d5Pf%!LM{sVCqHppZ{c17c@6^p$DGQ+98-9JiQ!|Hl^eEzH2{^M)oc`amjr=@w0jG(1M+@5^06NY`EZ z0jIM}VSwyX`8(NVA}bu#Pnff%*n(d7+kUY{Zjh4E^qB86SaV6AOV(@1n6y1jY>>3x zD>m1q+F~s_e$*4Wz9zM5>!*8)b33Cwd1uy0^E0p3%($lCm&n&pi8^ynXze4stZt&Z z@bP%rK)c92(PJu^UDq|+rTwWMf(!rHEx2i{~O#L<4dC#_+@Dv3fj>u8|KpttJmvcX6e2Q zHz*k&t(5hpOUABCmp#N!TS`ym(FpuS*0jjUFZ|%%m8ar@wwZ+q^9f&c3X(Cqtim72 zG3e;DGm&-G276IMuH*~j${W3dy;N;fB_7#@fWB{)gkhp$azdv16{PNE_iXp~5Fcm% zH80>xYkhDamhb2Xo8VgK4&Q$H1M;sGlS|)hi&D_dQye=Ls(X%faeU&yVL>lb zJ($S7?oou8QgpV7UO_p*4Ouze;nBN|>YOa)!`OehBws352R6i0~yD`#O@a2<5Q4-sL29E4eJic$z)}o)uBYmEHAb8^? z9sc!!RxJmm&=!%2yX?DlE*}vZ0$~QOg}&%hyvY@&?B^a%MX}GWjA3`@ky3k-th`b2 zX!h|u^)ge95Q3_9XsnFuOR2T{B#lIK3Ut8hP*_5cG;a|`IemUuL0{YV_}i%?QJr{L zWx*3YVUMC;$|cMaDSn`?ynaWV0J`wp#~wdgt-~{yeKr3Ir1(Nvu2{}9Tn4BA%AK?f zPHh+0;o#W6>A+;5cmjq_uGapIW%~%ZuYxTB^?ZFz-;+5N5LlEQYh$S$pa#x3`*XN-?`VG4t zJaGwON^EU>D~-Fau<}FU9;&_$33vZ|kNFGb$Jb)lQVJyK2W%uoh0cE)Bsh5EyaDg# z%Q_@JA*xM@`@ZYf4&B8jJq<%~ZC8rZri6%87@m8mj<41ZDHv+d`sCS^SbIGzL!{PM zS9?2QtfJq!6oarPg83Mnaz%s4~`hd2TPqBI%9t*x=v}|r=GM@ZDL!*BXyIUvp%D0J@~_g*+|o^4^>lS zVPulD6{D@~N3gM;$bt1fM@q;`!1^D{4=T~GpNh>-%k`NFQd*Z`FQ)8wazAfimTvu% za~n>f6^o7aao5qC&Qr7RSy!+3dwW*b@~pp7hFws{nMmUpZ4PpjjOr$h{dg|n@{}F2 z4@YqsxYdRGM^W4ip>MJ;^Jlc|%g@u?IMRLjQlsQ1Pg>wz`rX@R+r^O)#ow>anlkcSnCLQ<1dnJvk>E^nd z6hH*c3xv)eMbzx6wu%jm6LDiaeCg&KG4WPmX$BaBTCeBP-R2!5*N?Hru*XRRtPA*- zzp^fEU+hDPtNKX0)oc1+o$IiVHwt@uOsk?)>^o&sr#1ydh30$%-zixF?@FjO>eBnKIQg#=}wU(ZxLKCx&i%` zclxd0LF`(GixO9@BWD;oYCQ>2UF)*yFYfJ9x-q27z5j}m$h8ghe7*x)lbk!$9I?KT zY?Ot0FKP!^CLFH13E1?KY{Bi53dU+Ihup%oUx(benSfDJU>w?8;;hHEo6AuNLnKv{ z`U#M5zT&Pn#mzSlZZ$BxZnG$-xXq}eZ(Py!>{yvW&pDOjDZEz4IsJ4;N;axDMAwnr z>N=+H5jP%2K7hFz!LUWzxe|MJsTOS_T`x58WbCA&+o0Q%wX2QIO;mFkDL~&WCq!1e zCwHMYMT+No#uCCwzbjc4Q&!UY(6qrzx8aPf5`ib+dg3<&0#GpFxTA=ygOR&vEDc75 zKhqx-{N0}AZwkBJmEGHMGX_Iop&EU=gy?97cQMgG*aJgdQb1x9udfii7hD0<4+SZl zA@m<=`f?!qKItAf?;e2!$OvF#L|+bPv=A4RK^jj?-!OV+8U{%ootpLQ!1s{s-!F$5 zls|DsNnsBtAjvP55F;yzNV+P(m>&H-i(C59bHR94cwUoniXl&+wJPNAWf}2rSBBLV zZWE^f1Z@)2B8=I_=riK#)?&Q5O)&<3h2m*m>Jtk z3cu*{BZqrR(KD!4^u)nLATF?OWt5U3M(7hf)2knvo2$#jGojEtFd|Ai|{hzIVcnP&+USK8aJF?=E4qdSvf zW!!48A>pOqXhFgZk`e$_LM23sSTgt&CFB`TsSfj)KvEYJX_5BB3tO=uHX*!(;IHt7 zCeC^+AvDPPbg4yY^kA{ZkK_1XkksRHUc2M+Jqj_uoN1 zVBi#ULZ(Q#>Fq^Bs>o4?Gl>Gza`LwXBnf*$M9Nf3f=sY-#-vQKeA99IB=aVG*qTQv z{@Iu_{Xn{zLR6AvxOO|n;NxDE)T#3}z&>*cb*4%+;MlfRJWGfb=@b4^eUZalCG9XD z*7zDnT%umQmf_i9Rts)xe3y#UHo^F9s{5tp10zKgH8@5m-dD|h^TS(#SZDnnsY=x+ zOJS&5{heZ*BQGyKP&KlaJsl(q{;Ts!>WGfnQ6m;diSso3)mJ5^^q3As&{pZ-A`R&h zGK8#-jt(K8vk*+|e>M5iMLToyaKYH;!OLgy)zY(#(OVB+c>h%HsBPmUV61RO)pZGRi-P!hf8dIHji zs+UW+Ti&Sc(HJ4o;h3a|2}%_fS{Il%6l86Ksnm^^<#bFERe>=6^KP|Cg*orPUlS)9 z6Wnr#J0d=Eu=S%$>X5D)q?a*$wi}9`-intnz$o3BKJ2R`u)Fc>>ucg86`xMu9y8c3 z@*X*?AVM`M;S(d7ir&ph2p#F?mK1rd*3p+-LDd~8pgmjgZL(mMbtsUj6|>?RJM$vx zgwq0A&|R2ujKoJIb(YD7d&w!<4w<$qP!E_PDe9`lO1m4iUj@GH+Wd@{FUEuaLUSlI zOe{mOQ*p%_5@px*xpamI-Wf64QWU3=N}`2n@Qni52S+jIDpfbB+F82UtRt0b>+YJJ zanqk)LLf_9hm#$2Itp1`FE+a)v1#%ey99>4oldOAJ(%-!bxW({r8$Sa4S4>k$3bpS zV9pkw&`q1r&tN}yxna{)b{P&<#%(_M?UkD1#dwCQ%3`J^{(0${{CM>|VZ5xiov*TF zrBCq*E{#r=tBge#$fT~li`BS+p&mDy3wyh9#Ket-z3w*X`gV+5&7K^Xw__`BvM?ff zWeUqmQ6{5rYG-8*VNWqx2zhbW!y+@w@>HHz&OR{$P9u%^m!cvs#cxwoIb5h`7qX{M z;Bwwm$%zeL?!-UeUslj|EMA5xEyCe48#Kl`n2pjZC zQRZnm@qd&bmUr%efUew14F<>Pm$9Fxr!vr^qF{z8@)0y)bbkrapb?2$Y(|s$NfhF+ zF`e}6-##pOC7<_bhp+4%GWX;MbbmMYWN$CFX2qW2ZS@C8XkHhvuPh}ypChfj&GXh~MAi?%U{&36+tqea zo3UyM;kl?5ToJuFL#W(P;0gTUb(X&aHT=}=OS{YSGlz1g#3#4MpRSnQxCYaQOu*Df zCtul;rR6b78Lq|gUUT1s4QL^6w1a*}p96sdP0A`k15x>tFdD!OL% zmDYzX@mojYWgY8_{E8|#Ik2PBsdBBhGqVizg{vx)oT6mi zJts@nl{PW)4TShJ?`>X+bNQ+GQr@lT0J&X}?m>kJbA_YV&%O!?*=zB5Yi$Ax=l)Gm z;Yau~QiX4=qi97>g)~|3tqv@9d%Vy1RpxU1D<{t69bk zK-n6FY^RWDZ38*Uq#942hkVaFuC8!=TpW<*8+U3C+(+NR6d3ALaLL*3V@;fQka+RN zLdxlOX-#LRQ1Use+YQ~#y3)$!ilId2xmKrN_l-MO+B%(s5(YQeR}NS^ts zR?ByL_up)O;>O(1)^%`lzXs6+VJ<*GcXuu!PD{E^?4~Tk&}A6<-vC3`A*|1@wCjwtt8fie|7Q3Te$tGz zW|XtbRg2{0vHrk}nbf1XJjVfz{3o!DOZR5pV@Gjw0Sbe@#HphpKndag6jIU+?6;;? zzLmaAa{4qJabdJVl?!w?doHZXh}YI+#)#cb2{p3Uh{aD2<#bzL7?*dihJt6|%zCtk z-86g)iWX%w+%l#_yb9g@r6!^4sEzt<8y9RRqR8Y7(Q1qrELMnW%bxWlMdO`zv7Bl; zusv!^zrprL*4|g&LPL>pg!gGID7yOG9{OoDZ*qn!&%j5`#cu?yWG?THBX%W&)Yx1N z?Z!5};DT>DD7r-FX`NJ*`M{@hd`|NZMY}PZrt+2$Pw40ILOI){D`gbgDp3GY88PkQ zY&b)~)othZm$vn-86K}#IXELOf2G~%8ES`n6A=+!(P3ODONx4;AW@$RHKxR{CoyuGu;G5V4xeW0Am&tGF8EuTIt(@X^nYz==_U<42XSSAs4oGKEI4v;URhJjYFuM z(7l%l7db#9!hg4m|Hq+IUYhj8x!rX2q;X^~0i}P<(%$@BL=ZLn^lBBAo+r@zJx`te z7v)>`ucU za>chOPWO)>P(Ymlh)}bFZ1}P5IAfj}3(Z*BYc6gF9i=nmpU{s7M1e={{(lLv^}k8# zu`GviekJw#pMv|ejrM$BxrhF!1T z%$OhY9pO9DQSqW;N5vvnUwET!5yxgyt9Opkk!YEVY46ilc%{U6eY%_MUEC0>6J*-# zpd8InTQzPZIy&1t^HEoIHgR*RuoTgm>1biHgjgR-vVarn7#mBL5EbK-zHG8=?2|r7gc}%lF#~27tX*{Wi;T5A|cOEFOaK?P+JI2Mq+ED&R55Fc6Nap zWZAYqXw6jImHv%ffeXCJ7I!Kdac!=<=qITT12?3$-|lsO;P6%luT98~nF!Jq*jo`g zTQx-rZ4L{g&Gw5skt$`AT0vmu7+i_Tjg0x8S5d!qn-R^-f(8keL<;{EBHfAjeIncJ z*C(12mYh$jS z3wnh~_lDK$Hb8AvX!@g5w21*K83-80>fR(Xx5(Z8nZ$=>kl2ZaPZ60lhDi8f4xt~1 zpJqG;`$}8MYMQS6zI5ioU|l7k`jMImU(Q-TqDo{s`T~(As;~>h+&L#hV`_1&3w0Af z$`jlULWKP}!TvgRAFR%hp%dOwuMukxwST$3#qjTvhzCFtu?RS_;IYZf?aWBj z%ysEUa2b#;%nP#m0_gzbe`L>^v!lvH$cpM!dK6oY}DF(WoArlnH>C0D?V`UQCOE0MX~*T`W2~Q_8E`;SEt_xdQ{8V^MEL8 zg)o#7L;tR^4l})EZdm8utW0HJ!n9bDu5-vMT?=OO%?m?9HQEo;9THVja82hAKH}62 zbJ0Ni<7l`e`ZK~KKOry+fG?8Dv`lqeU)KcmU)BTD3?W%Z)0JeC9)>Pm5-3onr!EU> zZW_H&Im_x?Ejg5i>~@k6nif~BvSvP~Z*vnw_wO-3NNV-9hKi?FhA+zBteQr7-)+to zeRJ_enfrpX9&S1_C~08!i62{AO>;s)yE_s_5*VoTvypc?gQMRt>>5Yh=EQ9zJ`2LnCSzAaHcXnKN2Cs+6?SDm zkUn^NnA_^v&1)x#68ZPgGu9ye!`wjK3O+>oW;-)1B{JPuQh>n|I2O&sb{IeIZf zo8lzE({eq^QfFc2y-jO#GGGtfh@mu!!diJO1C6KEKu=PQbUJqM2Au$rcKP;7<9qCt z%69DdY>gesi%U4coEaobq*xe0Y5A{++lq6o=D1Y z&gcg-KH{)EF=M_JW}SzXBNV<;Rq?kA3K3+!0e#-5RDHPOmMv4fr&h`l%-km)x5ldbqWtaHnDVgPX8d40podjJhJUYcie z&YfODggoq9Zi@VOG)2Iw;V>&#*!Usep|3!{xO;cHQH_vpwsXLG8sH%$|eoYp6}yzLh*=ZXYSMRmHlxX-C_;#w})iI-{V@)LF2 zqc33mcKl>}yJG2*5c}LpPqr}lru;p8Lf45~X1A^l31#JGLr1omUHB+ICGNuea9+ZA zT(a+d&wAV??1MhL$Fc#*N~uzoK$KFmO_3&S4_p~!A?fcmP}DS_(j??66^+4&Qq!Xj12-u0)IUKlaktGV`} zr0kZeKGUa~4y#ZQzz@2M+*yKZBnC;5(1qn%yqzqh} ztcU**6OJ*h^1A$%G~R39o*9bW*VBh<=p1Q(K(ctyvx(Ftqj_&u*0K1l z0MFlY&N5F%N8_o6W90B$8@pBMiT1hy%tOEfB4{3SbHk0`G}S^Fz-JL#=V#babu@qmuV>U*xz-ah$+j4GGV<$R^O5S->vPu8I3R?OWZx9$R;PfYarrYo1v znHpjl`{!c}k`s&RRbJMrXYtOI7@^5shYK~iy`r~(mwOaH^y!wO`<(rsdJez47yqu! zo=xz_o*DIgfLF~jiJ;e${71bW>*Ze0&sdFL+p)fI1Aho^Se7%2m$59klRjfv^x#FR z7nopW0!S=1JYkMVu>qbG^)fVt%@ZKFOn*+94ji+9j*8cXGopVuN$MCqLkkgYa5T{l zTju=^F*pizX|JE1=x7bLi`NxMy(Y65^!)C0E=r%Qb&^Gyl#aKvoet{bAJ#Z{kF> zC59esH4lr!^$<<|4TgE2)I-NTK`93VDYP0xbLIg#9$Fdw;)fmddvL&-1=pfyj0*Xn zw#&B6Bbu0c^vOEfsh41|`Kv_m0womk0wT` z17UJvVEVeA#}eZCOHnF*6m-S}zkw*?Mee$qYWtQDZnQ%)8pM*u(cciMvP{&q*}|%I z_o0cXu^xKRs3dcCQ5G7cT*guO!OIzQH@s;;T+Dz(Qc6?gwhhif|U`;u^Ze}ZfK^;odFAUiFY!?zcCjCo@oBC^iMQ0mJo)V zulN+n!F@-3$E{fK>j>0{&e#@#K)GIUS)@^t0eluH|-W@XjTwb z*&}(eJOuVnkr@X=Z1q_ls?L;L+;qcM!}3frNgq$?n^q2s6Rwh%Ha_{PS@+afEKsVqKPztdm2HCyZibBRrHT zJhY(VFTB|mj)A@&f2W>%h5H)i4EktUT?VJLKeedru*ux7pog+{kely1$Li;T~_5`sM# z2gqXg@{M})g|?G2)jpIapN$ucIxTN*YwevBoR+Abnc4JwK$~x{zo)V_R)lMZOe!&% zQ9DLFe(=lQHVl$7qBzSHP|b3Z{oD6W&-og*PrmHEaBsyMmEM4rW?_PCd$h&gr^JK@ z9GrE<6w9#kU2i$$iH+rWEzRpBrIEttS7&)#?c7<W22Oprk6$rQ19s1pC5;?yeCq zPyt@VIu6(4eT2em_5p~=&|!_K+4}t)4mQt4W|r)x@+4f=Lq~P-Kf^%1%%mFiXT*&T#25rK`~F1 z@nvbkWHA!_g7JnxE~S6f`}kM5t1m0)=R3lG^CRSP!1E9L-Jg8S|0%MjKY1CyvmXBL zP&WixM!J7nq`P-T>g~0?_8hk?N+jfTP8`0beJbAX z+jVIH$FwDm(vhUrjot(=f)3k5&qUpLp|!;-3O5}Bb9`d<`$x+_f$PaU3Rmu;RAebIjtjuo-W zAUrVBVQNw4#)=^q(m(odWY8NxMP$Ues68A=mUoJt^-oS?q$6D6*<>f4#KEvG*7!kVxF<>r+(_ojBMhUqN^ zE9b1`yrem0^Khv9ouu!6ny3%dcB(IYTd$T7AGkAVrAVVqHeOd7WwsPn3*-)HbVF(K0t1 zk$_b7vk6ng)c?ocm&Zfdx9v}C|Swj+I-)HRmjCG8e%kSIs+;ua?N#puj@L$$9bN| zc^nKzutkZyokcxFTSH9)PDHH!EM~_S6_73QhvZaKQm)>}no)e+RuvlHE9sC9XG?za zxV%qjZn(i`bkj2$TcvcBhEq0orH=Qh?Rob1PjjfCXX(LdI+`N6v8mOvB6_&0pwhzz zXy<-YG?D(6aBKnGmkFsjg(|~FJa5x+plQCZcHg)$yQxjOapEpzIdR;&BeDx4u6@vZ zSFKx7mhI-Klhrn5yS*y(9-By|=7~57*9oiGM;q|3AXF_AX2JrzV)GjVl7^IuZh{&~#)_f?~R z^=VZW`rplCRw3u!}+llavGKz-t3$>f(WZKkKZh3Y24 zP)DAmC_yD2-Wc^rQpqjk)`E`0VCCRfFy1^VgL}h4TX^ny?vcDfXT9W;*$Wx3G}7f> z*RG$mpzeVM3a1G8al%+kY~Le|Cu*`=_wxAg(eDMjL+#9FA`z7P@aS0n?;<8F7wqA+ zUaWP!x!NCUF##Uj%Xgn~pMa&mzKEPN%^9pLOk$BNdF+0ms`^0JS6Ot?li>rLA7OpE z9+R7*AbUHe>e}YxEj(8V}U?9$@Q z>}dXHhFfv&JK|L4Yl)1Z-1ph$a(|lX7W1U{T^kPCk;0+v?8O;%c6Wm;CvPU`=0=AY7q zzTaGRfG{ab*$M9(*CZ3Rv`aI13%eV5^baIqwp)h{oyKY4BJ-a|-$8>6-zcE zB1ak|Hq9G@BaoC7(E9+YcMx5;^YZ7vyPoxQk-~7v`98M$`BT z=Xr&5U@jStDc@!aDx3JK#$8E2S{EDyOl3^(FX*#`tGyvUrf>6TnEoxzC*OPV7kitxfiAPh43$ z1qqFU*md_u+HTVqJeRnucLlB;VOv3b2}!{>gvpsc$RH1|ApzWdcuyAX z-r0+08%5>L9E^VNm$2?#SWCuUV@G{2KE1ZutdcKhw;3$sN$yy?aSB>eOBOgD;;GC< zuxR7&=GMfkg4X%Cu4y1sj2T{{@b=v;8%kNF^O#h?Mnj2B*73M-_+6w~2PaZZ; z_dtoULu!>@c#d3ML70_z^{Q8uKh;n`+QR~JWk@#e{GGbm&v&bABBbeud(7WEig_0~ zZ5UF}nwwm&H$2OYST{6tuO_BSUVsLlq66GO}~9v;UPg!l?#a+;a@cPF3Hzq3G)2WWuY!+(XW&Gat} zfggxcRuD;e##03`z>cv6kD&U=+t9Q=VD*QTb*v!b9LGx^DEKlXpaFu*7I+BDRGR04 zQ|N3^%{fXdvk96v2$pe_SQKUu=8`-<8!ZpG{gF5IBP05?fVJ@+6dyOjLJsMSV4)C8 zGrC-c+MHncp+^-8w=@d7c{IH0Exj29%}e|Adsd>q&Syi5evr?+>fn1QsH)*#mBIcE z7_|CdekEQ(E34YV|6bpC6%VfB!Bsr?eZ%=1BKeQPgPReR)a?(GToRR=iqV;EWv{GG z=Dm)JZgFIJd*SNTYcI*dSUFoS$v0b*j;wQWJ6mQfwD+n^=3cU#oV^S8Em_?R4O5L> z6GSb^fW-uvyhG~36{-Bqr4BeLnN!C-#2Ghfis=kq>Q%aCbD`SFI!*|Y@E0tyMg|+lLny4H+xoQb_i1z0^Hs=NNPgCfJhwp%->;*6LEm0Obf*e6 z=tfd_v9*c_+rjAO%SUfl5Ql=pOf%~l*m?%Tny}{NuRHv=o0Mba?0DJvmb<0BRK{Rj zxwK*ad9?=ZTaN1kqML|=|-xlSKb-Z z`z9MKb_HCIibmb>7^|^u!lGxd04o;<+OQSXrfgWV`Ko_!71#Pg@vIw10@94Vca&we zCCH_73)D!^-)-byA0`LL)V<1>q`RzDJnXL5?PYYyhqHm|aagY5!MLR>fiDV6wHom2 z8C-Q{Dr`4q-QJ$sK*hUg+;LY$U574t)qdo$D6-5URqY zL^lN8R=I3c5x?$5Vz|W^^p-SQS-ajhRrLM^fu!5bLyb1dDmjDZ1APf|vbM5XXroum zsi1kY_uBqQt$6Hk$Sf0GFL-}lORF@yGyCG*@Om33o8!*x`6n8SIoMlw6ohT+iJ!ic3fDWVLbSB$@~I>&9s~+)mZHZGApgh&`FJQXTkDYe}l7#3(h4 z%&U%rniz@N(Ml%xR7StnZn+U{^w{fmX3mYHdramdt^~d-?N)iZ>A*S>z8CmK z%aO#nCm(6;rM+$KlWIh=99UivE=7yfbQPHz9Ye$LZ2ZQgR0HdVAlXLM+M&I&m(v_2 z9+|o-q~qpk%p=D}d*ch}{&2n=wdub`Mt>=Ix*p-Cx9so`2vgLTaYLV%uA`AO)x7PZ zj3s6wX6D9>-SR$w&6^HejSr%#AQDq&$p+JJHFw~Dy|4RU-;+P0>i+~-j9LLJyJ~sy zw_Wx}==AWj0-B>mq}wO@4!8FQ@(M_Nlen>akOC1 zOndOSnM%#YZImxh@7!FwrsJcrMhGxF-dzR@PzV88u^bh5`f z>8N|V8r98r>%?B$$h^h%2GIXJfie?PBDA`-oa?uZY7NolR0VzUA>jVG;L4o6>kS)m zvo9Pt`{GaR6F)pfTIc%Y^W(%wNy(%>&qoSn4`|bVZYRi}QJj=G=yK3U?HIA{Qu}*a zkr%oh8`IfIy3(2zMKaaGhsPRl=G)_x!B|E53c?U=!)2J;kJbn1sleZl$ z5!ZHIyg|=C!$bZ8c&Ltr!gn7YWV^(ce+Ue^tRU7iFeCz30UxqrX^Z~Yao{%hMMJNY ztgja*JkjMel&lvilG}CdMo}qR$hWL^n*kA9Tj#ujc-X+09?hdyfB8P%wnkN#_}a-# zVewqkEwSl2rc%p&(N57#rLYN@3eCqwPyWGhAweG#j`C2FVYL?j9vDw1kqI4~0wHk!K~ph`cx6fLc=rByHn zgB;tnA$%ej=T;~%*KXH)CTVZ?CrRAr4TDc4Xwq{@^ix-gw~%}7A}gH=?n_1tK6*VZ zafN^S@P4cDgf{hI4yP)!lO3!K=PC_32tRID%Pj?cAvGJLHJB#`A3y_=`D42 za^if^Jptn)jyaK=gCwlB-Qr9wMG5f7SMU zC${E+f(Y}SFPJ^EB-!`EyVFlE-QaqFpB-NaBG8(iNic(eD#IB2k>5(P8BmbVYDxBg z0gOYdddvS|EqdXM59Xe&Gw>bxbHdR;!?JG|p zYrgD%Ywxb0Iwo(!lwdxGX7uc}nVy$`2Iiz=`4^)iZ4k6^^GqAM0cKpW@A0p#r~msd z9BIsgZ)RmK_u=ZHNFbnj0*~oOP68v~@J{gYS5xHBIagI+iVUX8|E@SS_Wk<4ci2`H zr^Xrb3QvF1-1rZ1gZn+#o2$%Unq^qKgRjpWSJAu8JyA^oC97dhYw*n9EW^@@x7x%PB^w=zeUu>z~2#n&O0I&{J)Q z5qSy5!YQdW+1R_RdntKbQE#GPbIbns^Fc$;U7c}b?8J6VTUVWptCiz!MSDBPb>6}+ zVmLSrRn{GmM~=hWhKy~|f}cePWs^BcR2i$9NE4@n{U@DxyDT>;k0rCm*bq_(2AnGP zdcmoaOb9cmTku=#OUVba7+V=8EebnmZ?`#=pLI?$cQuUXYlzYeNF7ZGIXbmuNZ$W> zp^+z00~^{UJ>`LLA;r_!s`k(VB!=)0${m2rrk=XRA;I~eUJ<;?Ox&jT&m;JoW$P4f znzPHKU3?C#cE1_W&S57dyxr6jX#}OlV)Ym1DG7<=?6uCfS~sTdOUD(4_3ZY-Z;lv<82uvY)fj7n z5z>uK)g*@HsLZ?X{INH3M+eJsym0%4Qi&tk?K0VtA+76k`5NM4dsjlY4RlTTOLc23GbN2Qarmk?aIZ#PrjPKU9K|lZDD03U+67&Thjzywy1;ZK_7LsSfm+C zv^o-Uw~J_HTksC@p;=|~6po#2%f*JA)vpXZN9G38wH9$Y8q(H{(JiE`9p#O{4%rNg zp|Qpq4UBZnqr9+t@_N1E#dE1VPwM3(9*gBo?oSf6du?^DGdbVB;hNUf{Af;9RC(Z7 zZ;s2pe(9Yq2deaC(gniPy|H3)w(JYuqFcKMSqpb6gnmTjbA*H}ICl(>L}7 zA1BrCpqa8(atr$gi?Rgz92eq9yRQ|eHoZA_AXY@lq|yopI=%wBks5k;#>9+(G_BES z`k3$UU#8!GLh})B)-N5N0kJpZ1vCkyLw3MeV8j8Tr=A0ykj8YZdIVHRprWf$hm5G4 zV?Xjb4_}Xus96IbeyqHL=&FNKu{md98_Y65-_z$Qv)Cn&D%nAkxRJ+Ds*m@tNZb`! z)w{%p+67G*G(mG~Fg{EHXy7B&ju~K&qN@|Z7v96c{2n~5jB%{_o;7o-6X0+H%O5a{ z#{nYcc-1;D{WTrdwe%)TfnTJ(wl8}Iz0)3CK_v4nSP**o29iEZmiFrvVY`HkNMx->wglq ze}7Z{^FEt**ydUG>d5m38=Y6@P4Oc<2h^w8v(O5KF%t7m3?kLhwU#x{*N5ux$M%pNL`E zORwH(=-%m!Na}P5zxB4$c9{_@lC1r@>0&)O+Nc|-__Tlm6NA@-bp??iU5ce?0lN>- z69rPEzRo}kiMcTwd_GtP%_JhN^p>zaW$+F(3s7i!PSR!w-GN{!RmN!o&}tn~js|n7 z?E6*_k*v9?jQZ)zz@!8Pnn5F)6db&}5WRxnN`w#e`T)3nuon0Y1Z-PD)OdF2!YIU>6_PkkPe8N(`7)FeCIXbNumrSKL-^jMWAELZBQv|A4K2nCJg>+ zDr5yw2?V45aIP;FBGJ#76KE0(1${=x=A;C^KI{R4Aezpn;E3h;py#1r@(I8=}} zP(d^&3<(8%t4-9QCnR=*>Pm`x2AWjW;GrV#HhTFFXYvCEe?RD|Q_26DQ^`NVW3Dzt ze0)Lwm?I2Tq`%*R+6p%V>0q#V3*@|`J?k6a*27Eeo0ZeXnN;~ny+64Q+|k&h2N{co zpU|#7YCiko*i0E?bQ>mTG)JGPOa2EJ^P8QUwhZkNPCv`(lag1iv`PG?=0=9PxxuT- z2CX}!P0O6NM~TtRV+5y(UR4;=5(i0c|2y}b3h+nk{1WyT0p1_@{5({@RDtGINM$}N zIaNogGUM4t-A0stD7BottmyDrPCl*Hm)!hvf5PpxW-m~84nGLzG=hoWOfy7X6w7Tq zX=ypp+paswFA9SE%YK*o;dA$O<06h8g$=kGT4EFR>OLPiyAx@nd7sKKT3SF~R7fz^EE71Fd~tk82GZ8sCk2E zl9FpRnlP&=RT6)u+2Yr;-AFn5-6m!wTN|r4SzW50*r;FR%9YTY=!z~JPh2KH?*4SNE7y@*Lh9TG@t*n3jJJc~MiH+J zvxDwAJ8MYsE!XIG-J=wwl0;@JzeqR@G9j}S1P4QvG>IvV8N!je?|sok8#US{<-E^} zxEyw7De4Tmg4@@C{N(n?X<^<+(C~7AjFEg_Fkj_u?C$eX`%n*FjT+(va--7JnXk0O zk9kEG$!Hu4YKnw2*$xBl3y@(s+iWsQ1O5W-7cbdG3s{ymjw1Je=$J3JO7Gm@ ziq01In|ay#`sKXhOxMe;)u`(({cKFfIUTwxnX!|Mod|>-)5}~V$XUmm2j@-j-N7d! zP9;x~5A3&Y%+b(G){yYr|0r0Ho7!m4a3j<@F0B1B)dP2QEPS~!bxXW2c&KHFbKR4R zruBQHgcCy5x4uqd|3+IpIgj{^-xj z0DsK7{<$Fkk5TByI{uY@h+dV8{*O!97wt%_j4;R>wV*8rAkM4045(51Az|if!6~@h zRU3YBUf~oQJgdh5HT01@dgz0nM4^9rLw}Th{{V_sn^CLwCrnTa8F)?efvbBmK(|jk z9Mg-=!NJ!89+_)0-;qva)Kr>Mk@&2tM`|rDZ^s~J`&J>mZ`t`J$AE9VcPUy)etVQpQN(F zab*0Q-NJ3C`U6>FnFBCWo1XHVaAe5bBYa|>!q3l9ezG;J*|`5UT4Kzqx4_oJ7;Z)( zAkwxbOPU{Pi#9D;>V9@d%X2%PgOsUMhxS?08I(XAk^9x?R@|kI5H1N??_2|2QeoZa zahLQDO=!}-eLBV0d1BqWwhJc0{JYGjICe^Vm^yrY-L*UDWW;6U%Ub)iBoDm7a;;@= zZR?zZ7+>!wthxjfhRgB+;=zcyfAcmc^d+nPc>NBI)1h;YS`)~x*mlwWKEKxrdt6GV zH}P$h<4EgP`^89h6Df7TR>5w`!9ule`C-^su(#`ac%ju_5nb%oxy-x|&y|eTRJVe1 zqMO3Obl~j9&Tgjhslpu|Jl7uf?pBBVJa8j11Mv>`7x%j>g3UM*#!UkwJa;+|PgEgF z?bj7s#}_Q`b8gwbbx8+zUO|tUqQFu`J$*(&fHIhC)Bk>RL$TK{zHZ7X70*!WaZgte z8A@l3KUHxG9xL<4^rMqo8}P_})OIMtO-xywzg8H1HCn`FY-e#id*WT@fKwD#QjCT} zr11T`J)?A)t#{ruE;!Oqz=FvAGb1d9Gh;BShs-H9#T!bB-<0T*;(r5M3%pP~3?c$0 zpKO#vL6q^G-gvJJy=~tm2(2n_$C?65ya@ovh?-`nGPtbV<%uTNXHNQ8_PG<6g%}?2 zhoZN%EB&H_W*Zi6m9RC7r+jFX+JuecB0Kk)bR89B=XcfJwMm$F1~g943Ft<9%8+D; zXR*8ATQlEiFq4}uv1m<^hNZ! zrtoT^@OYO>RTkoYce)XMd>jC1a|n%a|1J0U{yhH*W&Uz@ctU@-!*kJts{$ZkhiCLp zeVpG^=%VIfY1Em2d*}L2Gg$qDKd!`@Rbk~-P`C;TmsS}{z-D7n>IRxFLMHfoU-o(eC^%mP$2|50{&xlxrkghk5qU63cjw+&1J0DlktTHM*E5td6(wDOc%f zd3?2mlW3=Ujq8bh`W9mBzTTZmF84Ac=uT74^K{uC8QJ=6+UJP#K_{0rnh*C8h)(;A zU*lhATI~AhWaG%`eM2CqFkSc?AeK3HGhM7c`?HFA>{oB0O#!%*QmOhH=To&_+Q$lT zB#f+~Y)-kMUpx$mZI&*1_ zy2z@6r3uq(59RnMbEi?F5c2F#J$zhl=riMaGy=hbogsZWO8F3D!Z4O>d+M6 zU~oWt?1)T-fAX}OeQ$l8>#+~xOMz3r6;{?zKT*}Wvrbj<+h(is->f}&^2ETMYrKz; zypOh7sbrY2T`}>B71{tKqVo>rPQ}>h6OA;jV+yw|qD@v1*U=#?WOLsV4yr)YX)vz( zN{ru&j}~$^U}p+$tNRq+dfib~-7{$jhg2BY;wpqi0Oo&1DYvN!dR)*{(^2{=waukN zbT?be>)kr#m1yf`iwNriy`|VP)_K=u4V$j$zGWzU31yVZKb$XLrRBMIegz>r$xO{8 zP|)}pxrVv#)hZu)aNNCxQEi>X+1`b0CYCY(o|fJ>encq9ZJNvK#0QHz%C#=RjZ&X_ z64z4KEIru0ra+?(zQjBM2Ws!zsv2~0&FRHAAqqC$k7)gD&LKFqh5kKJA|hk<8igT} z86xptOoIl`ZZowF$(aBuGBfCtzRc}N8ktaI*|q33%-4zYh+D5lY+|Z1BTLZ8y`vO2 zIjZ}T!r{hA->>ug!dPfN8u=HW@<(9fXWCioDn~8$QNDy_fdy3XMyr3KJ&Yt$ih{E^ z>W`dXL0rR2XP!c8x=BVgR$T5+iVDk6QE2Ur6|G|oU3qjTc`{YP9!4^rExh`oq6Oji z-vcsl4x-Gp_pLxuB2k6mcmFLR`8|&M>y|O48R3{MP}}@1PIT2iR}De)4I56z zlAsQBdw-ZwEn{!hP+sF%RE%Mybc_Dz`7+(26+}T=S~l_Q)I^EniLfXEW%r2aA}*o&V*SJ@n2TtSf!abqu|9RlYVbOs+X61vE=H0^n$fa? zTi?Hvim?u%CPCMnul^B`p3no;R_)zJORURk=m7$$cbNnHs_7XItjUo5-|OJ7L7uL3 zCouKb&~OyKesLVdvm&&IJ!-nNqp0E**-T8!t@d_Xh;FXdqoH1b=#V&P75f6v8LZE5 zz*kN5e#kr#EU9IXea-j2d5#~A@yJW;J^jkcBrZw%t#s4`kLYtNqMHLt6Hh|q1Kvj)x9&wQw(`|W1`Afd z=~8^s>MhC|h~vt&QS4#+W&ojc@6NrH@PLci7pq?}a$oj!0S|~&^MqfU8;Bb$ezb>u z@~mXl$4=-?F8*s5q^U#ud>?IzJ4u&&F0WP5wm)2{{E5By`p`1I?EH@|&q!6H+-NuT-Ph{O3V&1^0P@p@gOVit5|I>koq|l~pwZJ1a8nKUTMr4fq zkM(RJA4aj=Zg#y4giri4xI%j*O;h5U5)S2mvvau(=m+Eud)-*;kHlA*&z4V9w8;yh zU~W0o=0u5Ee9@IbV_LMzE?al>`I*{H3EO6+@+!ARO{tGGL`m%HhabQpJ+fELPg3M$ zJXvZ_A1RahHtx{!_&z2^dNElkTkDQcnw8dbkCwquA5E^0Y6MjMsl%ux2(5 z{&>NH&jf`=Hkj?b<#cItcKqfy@`}~uhy*UbN-Oo_Q!#}$8N_a9u3PfR>fR2hkCWz< zo*(EYdcEuey}L#5?IX!*Qk?FUs2pBYRLk}1w?ba%cQWOG;#q-Ak#2bAVi2dG&{v3L zntc|zKbOF9ldw~WA!Kuil=l7x`tI?8?k%Fa!}N1ST;o;k1|Bn=br6<;#((NAyh7~q z5CX#)Xi`++H&RJAjpv!8R+HUaz_PsFxuWBg-*~^s$pK&t>u{Frswtpu`p)8R}_0@%*Qe85>~NwuD$g4=>Ges zzvg(Y^yxWyg@|poA^rs0%PR=>`qJV3a?Ur9{UqjTR1KjX+i@JrP`Vgp#%`zSPCIP5 z#FL@}&q`1Tb*QL3_yktv`|^FyDF2-R|F0Ube=OeqMUJ^DgZ(4?{Yl2O3i^NgimVbB z{(Hm)zT3$8QDyv#d#G^KTl6D-N-yAS(bRb#N1vm<#jdAUd(X5Q5iK-N>3gk}u}EJx zSv{nOA!CTWWrf9}{lJ4*>cktj(@Ja+dD)=S#I@JgRY9#Gt+~6~SjxuOUEIi6%P0GO z+01wY&ehTk* z-#T)S>9X+6*CD~>1`G3p2J`ps-6SB#3CTud-oFprtkuw4h!F}c$K;}zZ1$K6w07^( zkz;`2;Jsxg?Vhpa@^m#e%n9bfp8Lrzfx7Mi244)_POl)YP8$MDAFYm`#iea{ty7S)I@y)1(s z=(Fcw27u&d>>P5REX*yB8Ssc}rPE!0EC}ig6RK4zvh2cIc+^5X^$(oXino!ACgZoZ zWcm`PE$zB#mWBmxOLamNlZ#nv`%qERohNkM$h|$#X`qEoA z^AJqzE>gI7;-FTu-IPvhtvSULFEn^vM#x;DC%Qa;_Ut^`aL<5e5$^*9a8`0S8m}t8ALxr(p@39vFHuYek!rACmN7w!!ATV zYAvVho^>PznV;b`H9I*KRyuTMeNaYavoYceF{?D{qS0(?qx6T1(N(sa;!Smwg6%x@ zQlrT+R>ZVM+emf(-sbUkl_aM@YpE+d_pB5?3yQm#WEvHlkJHDQ_h89!6Y*41x6%2p z^XbBKlSXF?qiWz9l2YyZ&tiFBpQv5#Hd8lTCz&G*sL&5$70Z_DHxFOP6l@Sx$>LS{fGahgWA30~s_TbV+^lREN^5QO4C!X#d3n_#n?A&=&L4atTM6Gz?D>!ZzqmR40UyN1h2B)_E5dXJDCIM+jU((Y05 zwFH;tvKUK$H(8hHd)IM@P?f>iNlQ!=xn}A>ICsMd-81qVKk@I-X0qN!?x*znT@>)_ zSJ;!8-ba0+Ym#a0XaZz{rnzzlUA~BoG{YS1ux^HM!*iUDF@Ai?(g<_u>W?QgGGn9= z<9jDg^-Rl65}|Satnp#|q>x!d{jDP%Ble&8(z0Vy%fvUb4Gg!l>r`-x3;8(GcUD}= ziWt<$6Hc$ZGnBzk&m7Ze49I;+R8R9t-XyHpo}q`{TO#Ym+3bOINi${L-9xj{bG3Do z$~eh=(PU~Vce(2P9y8N{!-)u)PLVX9HbS3tBl-jP^al^gC^H2&741NWc$Ph_6nq3@>b$6=UhU~--UwJd`hU_gFibh$FomDT@N!_a@+mgI_u3I+a8I$ z$6KDrw}hmrJkPwdoGeFu(oll9BU0uGa#6(|BrT_m=h$tneStF8FN+(!1wEZpAD8pT zX|S=!RF0M!Y9!05*e_lU2`j%m=>2N6c3>F@Sy0f)BlC2>2R{S5f5|*zji#-4zG?+F zA9M_mTcNh)E1+Yrkc=4~o&*)8aCA5b#6!_=!j%v`!1#8JreTJ6&(shq!$(S2na@8V z?EQ;-0}hB8~O5pgw9xH6_F@G4O~FUJZ`)X?uX$x4ZP z7h{6m3nT`dZ?M2tQ#lVqmV(;`mSaCM(OA?+G*wq&S3V3>ph{w8F@n=q0XbhJJ__7|nQE0fwRsP_RC#oehdX`$A>TGzj)v$Xh{p zb|PVwL|8<%6q-9{0itbM$hVQ+34c&@C`Xfuz6}MRSq6ppkCQ$Ctm&*E*!W0jy5LcW zW&ox(B3T!dF+(9_6r~#mQ^7CYGoZeOvK2%qV2%2RQn1Y3dkGAY#BT%A%LnnZs#GM+ z7F4-c5Df|~1j_x(D~L}`P=y*z8}l@b6?gX7PQafu=rzra!&npIm_rv z*4O4n`%*8vM;ccq#iknIb?Rew`sWygX|x);H@}x-=el5`@I{N>OTi4dWyIJqyO%nw}fssXhxJa8ys$fYzBRm}O{Ue4@)GGCMG2C_+0( zl(uJH=?{QQ{(j&7xqHr)c0L@tB-5(N0b95WETqS-j%GdB;~7Ro6wg;#GdoABp6S*U zLnJwa>tRusHG=bLS=?NH)R-8w!sfd^i8_Mhh~$~Ja)dpX-l_|K~8*`E-^DDrzGdyY3_yA z6VrpWTm8n69*SezWzn;PaaY@M$m<2MA*X;sXF#cbq)1bmYSf)E#k@m#11;k!c}7~p zEtNX4`Y)&AD>lzmeoxVUR;@lX4r>ArP(n8PW9sVU&n8vA zeQOe&47tm$sy%zkbA2bCSB;gvW^NxHO_u2lu%hA1Be-fDtn*7N?HBD`MvN=f)fKES zs5Bc1T%~S~*lo?GJL7oH1cyv7BS*n!;qAS{?vP#;1}{4t>&%D4$u7{L9IkK9U8LUn zVb#Xp!Ah0UR_>5t+IiMW!rRGlQ-)AO#uYbrLKT<3ZtA(|d{5;jaji`#;aPUR@!;62 zMijeA!K0xK6vFZnB8mDt;m}Byr(M%5tZ2qVv4N)80J;bY<9v4(Bn_ z@W5f>M{52L*me2X0+2EZ%ORtdh3@6UNrjq_8~g>kB>xP@^rfQd*r<3zJ9rXBm8el2 zONMfWzx?sh|LK~0T!yVOrDBG^8EWNYCO@bN(<|%AUR0n9P`tP9UI$7TGy=_Ljq>V}X zf?Z)-s%G5hKw+v?B0-|jin^UW#52~_PaEYE0tPN#byN3TMlEWD-ET?U+Y^`>XJX`8 zJesOcImXSww=icILN<5*Hq-68*1^HGUaR-v3PPSK$`t6-BDBu8QKQoOM}ED>YmyK$ zEHhP*iOtOV1nTglDrK7+lw8|yx2ZkdrBwyMEUmo9O4VeugKg|bCi}; z&7UfTHoG5%0~@HS0@1$Q9E#9uV|R29(Lbk(^a4LId0$R*&$No+Q0lso6O(7dn`*QC zz1~N_n>>(=lKz=z5$Tdo^ivN+dN;n_zWIskr{Rzps6zeY$AUWJ^pyC~%#f_PXzj~+ zzZJ6ev%?SUW%i;gn+Kj9M7Vx>CwXS` zqT^Vb8Lre4`*yuLG9ewlx!Fa3R1C+z==dORYk{IC+w~j&SM2}n5flg zc&%pa`#iMIh&JED>TxBOhqO@2^e;3lN%M4R39ru)Gl^%{puL90jZS+Nm&I`w!-)x% z1E01(4$Xf30d^weA9o$?XI)P&5cRj}3FhlbHWa=O>VW97avi9iwXiU-Z&=HU7)xGA zFiGKgnX2uyh}NTOlUW~1^?JjHnu>+VqV{a_0p8x^BbBW>S)smpO|P$BTmLeGB{0x? zYle>@0JQQwhiF#iCA+Bcpaq{26XbSk=&@RFduyT~Gwz;dhE$+HD??w$}aSM+Zudc-B9JDkxEVB zk*#24eTNMbBoo1cpJ-LQrS|A;WIq27{Z^Hr#owV5)vDDP_;Q-FR9$12)-Da58%wgK z`U4~}E+UX@F=B$mUbm2~G9w$w1o~&jb{h#aAw_^$bAJ$tNrj=adLd zWo5sH5jmg!X+~&>A9_J#47g0-%iu7;PN(I=6$K$o{!#Fa@AUwM2930Vo$B$M5p=!< za0|njeG1QpZ@>K zPo&)>(xFQ(&&vv@K`U^VnX_7Ri53PH$fx$WCAJQbOP)WPZ7yY8iJ*#Vt{^P094UyC zf(r1P3&rtW4bI2XIZo<68jD$OSYF=hCud*3n>AS9n9u%`%fR`Mo>zzY) zftqgDr#;b6M*2IyrMKj4Yw_tj4CmUY0l(W9*Yfz#W08$9EVH!>Q>|!6%50ghk%VATMz=7_J>Qm|%8x$s+s9+6!K`kr zOHRw`F2QZ)?4*;o6kIK+P8Hi`1ZF&vI*%L0pI9(P-p4DbJB|6Q|S zIRS(U1uV;aOd)>?IK$9^MAC^h z8MBZu;7Y+T{T-oEvh@lA3{_dNGNKu$Kz>6f(8Kw9s|h}6gC*3q)}_k(2fnWX$^kCL z&d9m6!WC-hxoI{i5z6s|wODsTZ6#PZ=>s(fODb!IRY$uaJnq5wAhU)s-O9+p-rlpu zqH%1fW~^i8i)%*aq>%qu%g`n+;mN5>_5e|f>@a9}aP5-A#yY1<_65cH`&5s7(2wMr z69U3QN?MYSH5_!k`US2ha|#^FOS3XP5BSvW*f*%0kSbFfYft)}f7e=U&aUW}awCds3hv09AzswSiPXCF?2;>6!x zdae%$EqPj{V|=b%3CORQH|px^LOSaEPzx7Zp*@WKmq9%0Og-0ggea!BwsOiSIRE@pqWle}PEkI7~f*%9Uk+;%~zDq=pk{m0@m5+=u+G(vxb@?XOvi zbA3*E7yBGKUw&5E&wC+Jp3l$c8dn2I<3yI;d2botu@C{vz@m)$7Mg&1zyFOSb&IIu zQL+|JVZv8JoxzA3TEVOpgaxs3XjDiG$sEJdLTUFgvzhKL z=UE$2*&&SR=`~Onpa8uxPsB9!D&d#ZqJF`QzGt zzd?Wh1StJ;#{sc50Bws|5`aM)I*Sf2Ux%qdsGsgQwt^6J2A!L%Itiv?)pt_s7oov8 zD2TP;@o&|M@4#%$_c;0E1^m7~|04|cpMR89=a@fvKz|ft{{pO5*59Qy-(*f{P@@ri z4#f_LA^AI35J_<|=*2XM4D=SEFYo3_zpyTWjw)jQJ*rQ;g0REN@MKRwpS!6}W8SGd zMQ`3ep73%$(FmN>o~;Y>{#eCn!mQ_|OVI`^WX1RSI&9xxZgKlUjWy6Gj9kwZF-WO) zGz`cwNl#iqm>c)s&K)2LCjp_%Pv8LF_OxEmza2MSK@eh`!8`%p6c#u($_9S+Rutd@ zKvA4Q=iY|uY3Law%@P#^nz~wfgykh1L-jfq_D^R3$d1210Sepzia_W- z(3nP(Vm-h&HJoe#t_4GJD~PHaD+s@HbY-YP3`E5blwL;h>gleo?&`zvS9{gwu%3jq zC1SMu?`=MGlwgys;`xYE)6>`Q^ReX>JGjjh_$#&Xd;mQ0@shRf`{e9N+MF(q zzBC^n`@I*pBll?}TW3eIa@{s6jtR2TsI8G0EF7-rznynv)=TJat0g^gK2BXIp-RiW z(2?=paS}86{F`@}qBVK3=FO|vB&YZqNS9q}Doe($j%mF~|^oxB$sSdcAC?__Cg-oqV+gMQA8~7&_|00VLj(wss%WlkGGk@>?uZ4>MIYg{Yb6E zY@WADNwr9Y$2pYcA!l>p+9>++N&jT_`AEUVkp#aCMR>|^W_0l;(3}hLF21t7ofiMa z*pedcf4uJc*EdJoY(@pcUsY}-3l!z6UwW1k@+@jiy<*67#!!E6dqc%j5n?&bF7|8H zS6gagP~e;9{(WdCgJiB;>!J*SL!{>TnpThZhsnv1AsR(D^QZD_5q1DkC07)Y8+y=ZkIOMxw3ldJfGM?!`W*Jl4?t6lq$Frj+JMeA68;%Y(Pr#qIs=vzdI$9F zSYWg6tiaMlV?uuSe|FHPcEcW^{PB;6K~WFKqrna;^c3pD(hkgb2{9Oo0BY>+muBv^ zV~6@O0jbMa4&+8JwT}Rb8Jco>ya1eyC%i|f(vB|Ja~S;aViN>Tcm)BzI38TjBdE`( z^)_~v%E~yT9oRdkS!@h`_zq;$epji$5yMXoaKYArze8q{(8FXT?L3x~F!J0b2Y9+V zg5+pdbw40>?Q->j!8JWXnUzBID!usVcay^5s4oFf8fs|}ceF#Mpf;YNdTDtnep)Pt}Hc} z+a+{!&}UmQQr=#Xi6N?`&g{m{d_QRP(6R2^q!_b%NWuDO1^|Le3yYH!Fe?M z<^67GO2iv$bFp~L{;@2!P~MWarV`uB4rcMe{G=r5y?YHgbaQXD*6bHu_9=y4*gapGh*(^|A(01xoOG@e`i>XQY6HeNsrUAN;t{X8~H<5T>LbAll1Ye}xWo`=khy&4h z2!Fs|hA(*UmEc1b2Em738mJ`(aon`8^n1Ye%bH&VD0 zbEcegKj%E}@44sr!4HralAXQRzSgy_buFU}79qFSx?xi0hdW2w}nFQyF&$RR`agXPY@=wDS(fLr?zDu&BPF%4d@bVz;H3W&O$-clieH zAoU(4J5@iI7Xx|?Dv$f2&YOLFOju14diSN(;9y0XV?xM1=SW>(F7SS$Tc$9EW)8@i zkal!tAFX!88Bev^s(uBYi>H_zoK(g4ojvcXqs@_ho9k%U20jiXefaL72{`2-(WvGm z7b)){k!t=-^~&35lBeUXDWKCJai;M$fi9$%Yam!441b|K{*L`VKC{?$^Zu7c1F#mh zr|BCLFFa&sHf*M4>%Qyag+d)wCs!wDPK{ke%>Y4syxM06`jp|ii6@^9tapeXB@tcfs(?s}Q-`KH@|Roi(ueV%)qF0#Xr`IZ z@juR7;(wgQH(=MGTiF%m=++sUkR+uB*NGcr@j|pXjlhGKfLzLAs;Qg$!0Q)G3LuX{ z?%z`g)5TdkMflak9;V8X8~~B)<3iN}Ab5(`L^sCX z_x2DT645F0=i0OPb)6AOh4MdfOSsh<$7SDWVfK=bYiCFL&CQ?) z2#~Q|!=;Lmp!}#OTx?f1>2R%$t{ZPjWINlktC1z|^lv7lHf9_1D|N)Z(_V9#cgl%- zmqBbKBF7PiIYMf=G1j~$D&@Lhg{2-`bw-VL)Z^}Wuk<&)QFB0H|B>&>2>tW!8D;^P z>5KC(NULUYs0tLv$dO&)r-&9=Zl4dUD0LHunrg$cdzuqc_zvH-)SFM|StH$LkImS5dvG4+@W+cP6Sn|QNY&Zb|ka%>hDO%@UB*nmWMl52( zh|a&sJ+Yt8EqCeb0)yMZpSkIFI9eTU>jidAeBBn=Y`+b1L(*nXo^;&?o!tg4EYb35 zhp}tth!ub5G{L9ITL49AMbM%J1H9k^VPLzYVX4pMMF7C)$h{3xmA;Ey0yaZcL}mXr zD9LrM;(>e+EgCtF^JJn=qG(svIO*vW7G%RUVlS-=xn>Y9=?dKtq~cc75KTr~<36vy zeyU$=)L(Lad%YkjFYVoYOtxGA@Cqfi(jt1`l!#daIug;2+7$l)qlMcdXAeoP0iXH> z(x2`OJ2OYUBOm%Z0r(%W?DwbqxfFom`&hm|U!d=#;-8U zxDCO>S9MnHzEmbE;Oim)P@mziB>+=KGJ%ZKCZ~6iQSzU}gWsmUNX_W~!=k`{M$)|l z&{b1*Y=J$=W^i&8zI6Vho(_ZYa=rh2ZjVE)wBLVxEKw%Rb#idZC-KA@ zu88j3&`+R2uH{OOZ|3FOsf-_NZexZgd*%@iF;t8kT5*)FEgIMn7@cuNbGY{i%oVJh zMO!era;QE(F`-@D?ikwijcsifRhrI-S6rLuN%i)4ZPB#B7Pn!qqfPhMzT1)ni1^Xz+@xvM5v9U5t-e;6fZftld74QvCYo#TG;yk1>TyFwG z(T;IIC_0JgAX`<%!$R^!pjGj^SG@uv-k73?!kT;>!=5OI-*S4v6Bd&i%eVQgQW&J) zOWR4ru#9s#S3Z|NgAcvtY<$37!Z1uF%R+2eOtXa7GHQm8qwghgWPQim<-JD zR+^cTKPm>Z%c`bdy|?FCbed^Fs4`@FW0z>Z_(Yi%|8XW{8Bz(EcF1H5ySw(tlW0Mk z=X)!XEd#?tl4iIR1xkJW9Su8-*7YXQf-*`5@@B^$$8w@$-+9BJF!2$4iVES|_oMJh zat!Af}Zkg zQ8bW_5!4S;({+wEwZ+_y2fEWUFqys#z9%tr;Q4aWw2xAUQTebyF+(lqiPy`H>CX(P z>S+}_vxz*e9(yZ>pu7)`hAuu;rVr2#(W8!#tbVw@uf{!+aHcrEKvlx0@*s<~i37b| zT`?#aCsyE2>h_1lH{C^4AUaUx(DL=wd0&@(b#pDAvRvW5UdwXfp6m3H(q8Nc&3=H+ z_+0+!udlQ3O6cELLQ8&ELVr7Pzr4!+tzUw_Pw@H&l=DYR`&}~dJCcEK=w9?2y2nxk zC=$qeKRk4c6%TOZQDY?Jk^u~0vcx~zZ!jeO(fWR4-2Sco{Owh~jIJz2LL0)Q=NTtP z7MjuUY7$e0^!!tYjy1?0z8D+V$%IU&`m}7JS*B0)no!k?c+9KcLwwEIQhv6ZukBO|HJ0 zRugs?#>E>_Bc2FTNpd%SzVCnfP||QlPmC^6E7%RnY6~vZZY$Ue9o4PYbys*#=(xFq zHb`O%0DufEu$Kttv;jPWlFe%Rj%7b-w6DlFn8&c7)pXmpVlqB-B*%>>yia$xRVt9u zjqQMUZaN}nJTF$8!o#TIDP-)`X!Z2Ddq;+^GVO^RR%S2CD`NT-yHBB#PjWDO8mM@r zF76qVjjNJB+(_;W%=Es-XQd=zbuaN%|He$SNTzOljC4ram`F!4c^1H9E@-(i43#+p zn|12n{POJY?pv+Jx0;?P{@b9`A;w9@g*cLl1TY)jlq%8w_To%3YodkcBbB~jbI&Ju z!Gka4_U8J3z5u55olw~$3Lo3(T&gK@h8}GvhMdZ3_N!0N9J6L!(BmR>3BqsZs4z`oJ|Afu69Wi=1WmEFV#- zTS?sUf=7+56(a?|$S5tJc_^N7IjgXT{nR5RY33f>ppd)mO~IPdeM)x|^-~4%#{}vv z#6tcqr0mT@iglXJjBDth`um~TaXbYOvC0WB)FVWF1ltj3J(Z>|b;)L#>kg{A;}>q! zmYMVG?F$+?wdW4+r*ML3CAR6@bSR9SG*a#eA=N&dyIW@8sLZAnpU%Y62>8#k=HGfB zouS{sl944?KB8K2G(z(@v#f-HE(|6?bC^VkkTI2a5Qe<@n!OVUEy6o=N$W|QZU8lpI$SQKnVM06@9R8`fBJHBq8ym*;jIIzpj6_-r z4>hj4BWr(Gx2faI-*&pASDA6!x5JZO$dFHD3||9e0Vf4JeT?vp5qy-IOL>Og;5RqShBAWJYm^Uqq|G{? zx)|>;3bGvIaggkvZv0__LvGgsp(bs2UpXI#L5xt7Q&I39D>n41H|Zk;QT4%dA_xM5BNjR8>jGQA|rI z>3L^fG3+$V7WPn&DoonZvnqKj?o%F-sR}=#xjm)6v{LSD2KV zM)0VPf$4!KM5`yb9?cVesDqO+*7KZ3l%#u!6MnC|X#;}l;EQhncHg!k8UsylBxM}| z+rYwMn_`PT)pTDkUl^sEdT$kwS+uQc$g!Dr`^aS3QYN4H$|q#m8tGVkhMS!;aago; zs2jU0ba}e&Q4o&p0?uqgn#*^6 zy3kUZ2XWEWcwYaEhP4nN14UuhCKlFcyIj8K8vjby<@a3U@1>t_aOyt+%D~?T!+p;+ zz9O*yxjgy*;asB_f>{RYlL!y`aOEUfZ(X-2&v! zjxbqaYM$Ju-{c11l^IGKBM_@|wQBN-w~I!=RQ3B4WSgo&HHu=_@{U6xgnjRhrmBp% zFg2ZV)M|))jEu1`!mct+%d9>Q+~ItjSQz4bvQ?mYmft@^*D!$jgs`GwVVVFTKXkw+ ze@=-x-%9sQ1iyHNw=wNz{6_5hEN&D5kC=5?Y~}Bj2Wsa?D7^@~4yVxbjke;Z%iR8W zkSORkZ~iy&VFy^NQFe?$HnHu&=F)lqxz&LfRdMJHIvmbnPinTo5BxA|Iv}7scD*Qc z>hQp=`<4$^cm;57D)rRb)ilxu8oK`Jt#>|^mhO{KQn>q%s532KQurb%IXvnTeMVg>r)Zg~F zsV;t0l_&W=mI!4Dt{QvCBKo``GTZbGTB^mxHF1jnzNSmeAewJUQ0~^^KpA#cSJmXl zAmK#I-|IqQ1bXF0s4=L+fC56M8@I;WVZSxC?gkX6=bK^k%1^$y0Oc>|AveBg#SywN z17R9U{_Zyb_$SN9umWBOdV-!!l_S>$0ht}7pTfm!=T#8@15RsH9hME>+AlPUVf^nyp zC4VcIF|Od?%PqcNb0Zu*)8hsrf?k7TmoPU-BkN_q*{Da`M$IC)x;EJCMK1KH5Q^Ft zx1P7vW&cRtRI+LrIRjpcm}VX;b5B-6>~}t9^Te1-mc)KQb20g3bjm)w-Nxe+DkZW7 z;}H^Uit%cAZe$G(8E4EaTHmS zAP03qJ>$-#53y#mg}%b_`(^N_MtQ`ty5@!{jZW+}NKBocAom8K@&JmqvK43zus}U? zt&}Lq9T-&LEg`BATX(gW0D#p|*BBi}x3<2aAPDlSk%Lv9(L7mpw1oIP(*?GRkHdwM;%eR_F+f_ei}s?u!1p4&^E}O7xhzl{ zXyH(C`q4?`v#imD&UErP<7+tuVbJAWhuP0Tmfk8STXShdSr+-7#vU*Pb7)H3>G3NO z(gWc5tJUc>4Sv=Sw?0JaY}zW87pa=0>Xu&lh6<#XSUllZNpXQ|S_jAh4KfSdhVSj| ziy#v>_8sIRs!4?#e?Id}rAX-0xf%GyYASan)uYn5zcK6$cfP}TpDL|fw*c-yd@p#OFHKRGOs;1-do6NTCxUIDeUKw1_$t?xGZp6b)PhM5}d9bx|#Ab|Yrze+Wi`S#^*TBN(v?M}(NbZ{HZI65E}$K$i_3Y|8nOgL4<;%BVbV)(^Zluz@>_)b z169(mQ>pJi{pzA*{61FiJJ|m#g8jp*FojD!)6^weX)#rh;yDHMhp*91=p7@QuW*a+ zwm~I%1NFFxDcVQCcnXrtLN`es{kjgn2qb5<#uja}jUnmVpghKfXkulnKplB^{Y5Ez zrtukbTVX5PYSUvv%>%Tz`*=Atm#feo{1%JY*4ym+^c+iBJWD$OG7_{EV zW_1QWT?mY=HIxnGEQY9jyVZY#<-dWL-+jt1IauqIhTW$XNHCJcM}nA8sXNzCtx9l6Qx5kdg4g$d<`#t94uFF zYiYyQXpi}PN%ctOVOasGGbV$1dB8V;S~?1-lp|2{_lYt7&T#&%30a1)7J4}SWz zDLdA2^A|DaM~?bvW#sUlk>zl;&;s;SwbNeH*Tn%anHp1r{2RWR$Jph$3&M~S zuR!uWsY_>hw;H)&n-QzS6mfMiOb`k4Pyp(rCcF8TGtoV3Z?U3~YmlBo_q1=|mWYoxAc|+u zhwt)yyH@Kb@$ua{(}!_dZ$N=J2Gl%xk@8U{_EVzo=MiR~mrd}k#%bHt=N24OKpiyf z-NCzbE4>Yt+e=(H7M+3krO}GsD2qOyW0|J7Vsub6^-b;Wg|UShA62`L*cF(e_9EyJ zhsv&VJOR7zD&zg=_W}19C%~7bZzOMe-8X3gT|#kM2{WTm>jl25zymQDO)W>?>6HcS zAY+`5F&X}nYD7#L36yxO@;0iexLv}TJ#tK`t0E>~^4wmLazUt`!kRDwzi3e5R>^`F>$#$XdOln$?shHA)8$FsL;d{!} zX{TiKO}h_9b)@A`QL+R4W4`lwBehkiJ36y-GF4dW3R+&)@u*~`m#_xV&nK}$ZS8a5 zHtZ}IlCCuAfZpgjU#i(7+<3gvR<`b}!<3eopp|oP==hBzZ4WN9&^K@OwdhruXhP)c z-W{&KL2|SH848=z)GPJKf%64uLkA&(Pz-T)FLq+u6U<(tvz&Id4V+>foB zXO@Ac`|Ov<1wi$Bn2rKm%aO3f1V6wpFbky`jV#8IVD%6IBNE5#HyDWsLX<;R3;z_81U2fy$`M(+2n>CMa~;^Ay;?& z6DzYFtIG4h%kge~f^Pt~&n(;P-nENuS#({S0Nf(*K;|3g$`{BFS`*tKH23;V+z9u? zm4CtuD@ig^*S<*g;ae^OK=XHr|`2G=q? z|AiBXKQa{a-3jCuDB)XF`W*!EXOAJb+X1?P@-HIL-@VrVOMdxX*7Ltg*7F}i-^W1Y zanCeWZjW%4A#xyMJg6oX0=Nx-(Tk?X#CPwZ zh_9NBLKf_Iy)z*z5K4)U+>Q*)AS4nO9=S9AcyE|n;9Pj>eyDQ+rM19M4G5^M?Wh@C zKvhTTr{N0(1L=2a>^}YXv|qg|%jzorutzzp>uH11{-;tcuJD;=q5Q1&Ft4L}j|pfF#bL3~NasjE z2b>%xBS9u-d(12Y)$7EyD?j6`KL_x?Jn1`*OO9b7t=MBbMj;%?JJ7vU?%dcORXsD3 zThA6iVIDMXVl{3G*-FSZbst*y^rkkLla?78!IM}BA}Wz-^q#pa-w)1B;tfwrxylmb z#3sphH`B250&CYB`}Nf~ob8q)9NBj3vG8^2eIa3{$q0xamPk18SiGS+GF z+_S3>Q{UwjT=B^gk?zVnqr)$$t;!R7&xQv@+^E7a7d2KV3uIT9*av%tYG`n?51+rI zXQq>)xuBQ+(fosYyleE7paxBX^H>gXJnapR)j4L9^oI>*f5DM>QX4;AAUnhJ+aM|M z9QTmps3%@gN>_L+SaO4P4{?n9Uhxj<21*x99F~i?2+Fgq3t}_HgXRgE^0VP)N=Ng$ z3&Ro#$-}eygE82ptHhNT(4pG5SYTUaBNgpwTC(SUnH=2;dK>;Gcy`QQSSU9~_He`C zky-G#7mGkm+I1mbo+Y<6&Ds99siu%9SUzBvV?~@ZmOo1@cmbemh(+<~(T51FgLa*f zljTo+2TV5Y4Qq}C1T73eUeNh`MG;|QM2wuz{qiF3yOUOW!;jfMKB#IgoOf8i+wC5E zQhms8^5v$3azBmFJZzn`ZUDzYs*xz)B{GVa%W4=GXrArnWo38iYu#msvgr*(G`$`N z(ZudXRgAF1!3$=_p@<{hj%kf{hT44il)*YOpJ(*NzUPT0H~0@cz){9lc_i0L#cy*kc@8rJv`YAaX z1S5506}y_?ApJ!OWL0gN^>e%HQ;;Nk>r2`l+NI6H^(SNwTatYx&Z%SU(LgOHOG%(& ziNmwE89yeV9rjRk<;Nmh9j9p517*4Ez{{u(pn&m!RO^q-ck!%D9AKnwiJfkWcLH9d{uLb@qIl`U?Wor%|NG|^I$ znu&xC=R{sM4#nAc+dhSzY32F(u z*q&iGJa2v<1{E!rZXDOb%7_LIECZ{Ua$v`hyJs7mVb?BM`^f z--t||JWVW$awR(%C0nr+N38(XyCVnc?`Xk<0z!D7S2el7FeQg6hUB=>JWLE8L(7Wd z_jpj%ay>_j{?Oi3uks=_Q#{2(Q2LOIi;r>QZZ0(!vh@(OiS`58VM~Q7PJWs*U^+X8 zUqF~=`p6NaEISt`o<}BIB;0FqSj>B5a5Ji@7RotpMY~0en}LsRgF>`1e&tp@CNEm* zN1a=a2{mZWb_q#IONR6{h@y46b&N99lIBaelo*c$_Vk+f;QaG9(JOMK%XRhquh+Po zqu?Dl*xVut*o+cKSlSF&T=uIjq$v1i0+q-^Puju}Sr;TSLBuLxyL$<-RZb=5d(=?;GJF_!H zCq=?z!E*U|^-8bDGCXT;x{SUJI%GaH1Ror<7zX}Xm}~G>cXxNSI=i~cwpf@f@<+#d z+~&y>PpPiPaIzo)O0Pu@C^Y*;|@dbhm(tS zf+rs3n(yzf`xQL>b*%Zz%|ZG*#j}I%tbv60pn3@q44{lI4T|_=ajxq<{%y&PGbZoH zxV73HruLh8NNyQvq@KjUpj*sf)X&mIRZ6Pbza`oB>qO?K@Ss`Kq73#mBzCLF zoJX9cI+00HQxb zFigqsaxPi68|+>#F^jo|JJw4Qz+?*!C|U%bw=}SAUhEDIwG+T*l@sv6n5O*5Az(|D z%ihGHG;@B=V))*F0D5E|Eq2_`c3ij}Oz58-HhM%DCa@h;7$cKG+f@bQ;Mb2 zuli<=WU=x5L6df@l5@rv9ITI#ln?;q-MXP;JU~94o(6dR_$jA$LjXJVz`xt6b6ONQ zACQ+O((=`_DwnHid3TBelmeqgoc}tIFfyz_!;vlGT}EW!&|~K))K;G_HnW_E zA!+bjkJ^o{v3#=0m<*vmti-~|Z_PWS%zNNYFI{o#13c=ynz&t6Y=g92nSY)?l!GNb{XT!_KR~K~pFi|H6?pKqr1Mu& z!UL(mBCQlT8BO!?pssrTNgmL;|IhM(mkgV2JN+KXWeoXaQ|9IOlh;V_PgexaL)m7J z&a~w)sPycgleUT7(|+IrCGdG0zotmCIN;jmMckMXAF~N~h6%vWFBe?rl`&%qL7G-B zqa9V0aOYoFW*Oh_=yodIv>UzK9aqXEKc`IdK{YDOpbjC1ytwdy;-+{lU}29eH0n`@j{C zc8P`2Lfif}+1VG;UL8rLMze9#fSqnn{?dR4-nzFD`HI|o%@-Xa5fJ0ykSxks?{I(e z)=JN*bJCCO*Qu%Nfn<^JI6j*P&3APtN+%S+hDoa%dgx%+kKJ<`!Xnnkm&@*3v!`L% zIDEE2&Aj9!W$Fb;`B{u{&$|Wug7P*fh+bB(4xQ;;b)R`oz!3wPfq~n=t=o7pW{j;5 zPCg2F&H?u8mE_agpvz%S+n{}4{?ilmHi)@z)g9>m3kMv4_d3Ij1XCcYvqQQ=|b+YAp+pr%9sdhLFDJtrJvjWThn)5W~N$DFlAM4%CjT zfCA({zAP+H6xB$6`8)3+mCm0xJqG|yE+hJ30&>$BdD{vWN4g9Y90G|7bU)4!I&(kZ zw92eZ)v{^tr3Xl2si%RK^5qunI}JTMj6_XKj< z3P`0YxU{xG!r<>K`F$mSyC;JH(_4nWZ#Drqq7S#n;iky0VS%p1vpxw2_bLlY)l@rZ z+Pk&tH9lNhnwo@bp;FiPgl1aWO|sX-YzmAH-lCYuDykjPdYh}RL6I>r6fKQhvtRX~ zLQ*FCmlpb7UG?{o7!@&H`}`nxF1|E5_F1S)%|SUR(@M>VgyfVI0mn-KB3b2m+htHnNv!et zIhstx1zlD+w6e#z+0pN9b;^ydZ2|MXz^cxGU*LD>D#f;b2-*1I1 zD9w5g)Q@a+4i4tYK<@EVk1u3P-a6&ADO!I?|a-Lsn6CS@xbIt{vrT&Ho5 z8qr4p5wI-K``qC3*Gu+y`IWzL7mqgM9A@4DgN`e=7xq`w%OXcqafY1RpKciJ*5Ow+ zQgUx#cl{It`=>p)LEOvRMp{sc9R8hlu-1vNn%*c?enF*Im3;kV)Y%ilh2A{bw-Ppe z+6zN%*WtieafdZ)G!%;nd$m;|@TUZi06b45YbnV!K8 z1H)m1HphYKzZi{%&f8tWU6ox1mMiAHHZ@#fhSPor0 zZa-ke(?5Gb?5S*>4Lte6C+?UVZ{K7ARszY~p59f%Y|aSrN95dy+-ghSTivuIJB{IF zppX9gs$K2y<2Pvq{wft|HhXCfob6Z*{eV{D@*2E1jvIc+?q|6TqdFeUI#;%b#QK5| zacEc~ISW0wF9l)B(MHYdH?P{eayn+8s&=yex+M1;85}MdRT(Z^PKwyq>*?co$WF(r zNE52sk#+p7mw}b1zMmb*Nx#EhIzPqYP?Dk>Fkc0z5)y%NQSg280@FBFA6^@R25;nX z^v;3n9S0H?-F$Tp-|y)NC*j1*8XuPK#haKb)Ao4skFg!{?EADTS;%ddWNqhJA|5kq z=`3jBHql+7KAb8iHIqM|hS}?rEDV;FFw#Ww4|&Kb;moBfzUsL+FM&qE*R43F_|fNC z0UO%uh;9f7h6k4vkwfrI5?cH1;>-$$F7(6VDk(`_MQqLjb!IYgKMFly6+HB5r>q@r z4ArSVT}@BD&EjJ;dJT3~Jv50_^ z?7_qt62d??!8zLb`R(|7wwY~t8g}v1O__kByhX$^?B%6BTD=q_nlyw}FGo z?_@9p7{(P3pgGv>IBE}W13*s0DflVa@)CU*NTRjc0X(Y*{%SS242i%kE6MW!s=q7% z7)K zOKYcq`di-eQCvmT>@VLc!-f-{ks65%fUIlD0?F0u=4@og#fP}%TSYSTs{2~>m)i0& zEc6vUsxGx}x`6WRubSfn;RraJ=%IxfnY~tPX+_J~qi@b^gXhVpCU~G`lgyVXce$0X}4?5GQBF908Xa{*MK; zU8k%tT9qL0wgj^sE42&zh)RuwrGNvm!n?6y=LM|eDmPl+OB8FZMcs+ZP4>wMk~h0Z z$=K`Te>MW}cz;Ea9hG$(-H?5ME)_NLE;>hI;#Q`}#N%ucn7iTdIEt9r|7ycrrS;?P z5Kq(C*c@+Ec)Y*=WQ_0qbh`_kCU%^@UF$~gq>%joS-70d`$VZ!R zjh^3n5zEcre5Vy0T+x*z)HQX|q%%WPQ~YsihfZYPihR0Q=qQ@a?Qg}00-=?fZ(Alc z8|2km-WtN$S^;GnuwUO}mUMQiF^|L=%=5zuBMp%i(3?wUL8<`CdTNlHw(0ak;o`}i zo_)z=j#~Yq1QD0)hi_xf++$r2|G}Taw`t%dQvBR2d@y&?57B@&ZcJba22dX47$H5k$v2(!>~S}i z)muNEi#_5!4+LgS-NUmm!!z+Q=+Z^hYcmaNUmMr`8cy!^t{efL{7=Z%w|G_LvI4~$ z#pdVEVxw<&oqke5$Y_YMGx!0cnJx6}Zjw~>keYaiv+b(2lJ4UVg!!IA7#H%6)}xEK zy|Nl*H8Kg<-E*!Q{DEs6JEG=u1y^|6meQv-+Avl>{A$)_@eQo3c-Rcnzj*a{s>Nag}; z5CX3N<;<2i0Hd|{u1xJk)|sXNq7np%1Gsl(zs>$4q^;?2kN>{PHU~T2n9O#nyS;B#>ql!{}>ZuoN&&3Aw&O??(g~QJ#DL1e5;f1TN$%q5-xH zFWC(Ych#b&c!eSLeD~uf_&n|DvoK0#Etgc>Rx}+>(S_X>rY+0s|GN14o1b>cf!o?7 zishlSA^_CJ0IyO@y9mr(ZPvv-z}>-7UxZ@lwfe9%Mp3zIskBKr9+{}s2;fx?aOyXp z`Fk59_FM zt&*??6c%V=w6G*DEx?IP^rPPU=)9gXT2O-d1<){Wr-R-`@nnn>JA+Pu1f%cJE6@oS zQ)4B@X0AKipsB!4_;gcIt4J1MWT)kayihJ^M2d#@=Rr9MCnt7n2y@`0Jkyray~{N# zXf@`w4KJDcZC> zJw7t>o^95h_7x3@vgE64FCTs7)}$a@WZ}zXKfJGAMe+~t**G0Bhni;!9DjVPy;#t7 z#`?xR4JLehjrktvX-;=% z2FH~JUP0e$qKB*S*19K>di57*#=fM`;MPXAY$>PVvzGXuD6f+Wqz$@CZ53*Z1z-n2MtEQX6@2rY4^XDO zIjBsJbOZk4a0XDSgaiV}O?s-=ALM&v3n2LF5kxjxGmPRtD?!s(71{r zdFEfzE`2vhU>4R&Zev_Mk=7RmyVm*RF$c5bOzIL0j7$!WIj2E~zq2)E(|yap z2VlA%t5J|a!-w+*@V>g^fS;jR_2V{onGd+JMV5#OGxb300MMo~)jxXeb{5W?DKg@@Bf&Non$>I>me(*rDsv z4-AC-uwu?nZ$dj+?k3zeK3@1F4OfMOufP;*T%=^zd6H|prbyfrE1KRkAi*u%Jjubk z^QU`&bHr_H6k&5Ra1*%}w#v{@CPWo(mlw2UHH@1%@is4IbzFIW=UVPEeWo3*E|(o%t`R9QUf2^078&5YU${Hjd= z%l9FadfEJ1E^^}uGH^|Kq8S;k-2P9Z3FzlT^8J}#LGeGX1oFFa_IJ4R^`QLE+2;Ru zjkA9r?wBGvmj1{~@8sy$b@*5D%D)+5{-SWl{p8*}&WJGC)0mvPGLq86+e)J!*i7GO zKPwssmv#kj!492cI!^+hZsg6Znv@hW$1aGTPqAMe$%whvy|17Hx(xF*qKuC37{~zx zJ6ZG>3z(1ZNPe@Iez%N}nqk|+`5xtsp1l{l(SI++K_Y1_rpaEUv^%?ked^Jjgwruy zPcN^%$0%n6;>G0Xhvc6ev=Q= z7~`3pF^-TXYpGHDkM*5>Kq1PoBl{Sw{sOIlmovCyJqNQU$#t_@&!=Bg8U{P>LxVR? z-T8Ubfy0%9RmQ{mrMTTArH;HfDRnHsNZA7D4cj|z7@gUjVkq&tzT&POF!Ehf#4as9 zPeV@6H2vY+2TJqSIjR@@38$E+dR4DIAinPLw~vn3W$=trd$7xlyd}}3z~qu-_T-XG zqB4ER7fg@rqMFsQ?RNHcr{`Md(*^I4K6{!9mAI?+re||qHOwu}X0g@K+k8{WIM2Tz zh;@PcEzrCmrh+_Eg>to;e(Y;N>*NBh;4O{_!!FB~ul{@x9?sA;#i@Z4imN%_uA2Ww zi4+8pyt;$LztG%T8L|IJyN?rSZHsbZ{TyCzTo45AgT%4j$u-G#a0|?C)nWERsp`| zMM^|Wt=?`(N%W*uc&fg&K2YQDZJvEMo1j-bKf~{k^k&uO?1k$^rV;g6oK_dD0^}&` z!YIj&(|h1a&7wcaPFXgDs9gTNgbj3uA&fNja1|1i%>G50E@LSXkaC1c=rQx>A)wTt z*!3pO-687AeWluKfKrBmNarSdS0)P*0aP7RRculp88~W!s z`1h$n)eLNuUS(+XVP{8rsTT2eaI&7orH%oVd<9)#$idy=g3S-T+WFAB z1IMz`Rh12|sOoA_H#|%6T1P!4YRJh-;gz3<$eA4a*!JSpb=d8*a$jZ-LcjB6QI#+n0 zQB)Nd(tFr61;Z|$DXYI?%x0ZDJT0G$^Q0B{NMB#N?!DxVkrO4S1wgf!%LI`t){PncWhaUYf>JJdX`H{`xZ}b7mL2q5%28_xacEkA zvIXni^2*~D_WGNKdrAfR>#+obfQ7SWi}R>GHfOJMMa_0l^=PoSsCQZ6GRw|n51>i9 z6|y7)R7yT=TCP6$dJXM%2Rm;^9C$(mr*}xq#lPV$()< zyzfneM$@5O2Hvo)ASU0~@z<&khVtj1?zCcpw~nZ^?#OjILH^;sqT+szvV&^r@ZiXv z7{dazPFCY=_ofwRy{k*X{(3e2ROUR!kPQFV!z6F)3?wvuI3$DYYS`r_XV>BjGw66b znOl~ZbV2oC)mekuR6+Jz)(yE&8&poAh~OZyw-o-`3QMJrLSQW0iANbi1{oTr+KDGh zIg!V1VL7TZs*CC`uIWVEutApW{(4+@Myw1Fnq?y!dN#}A98q(sYl7#@LV=?BRls>+y1Wsnf^J)d?B1?IpgKyh$1~Jy zYjg%|6&sF*qUyWR(&BEj!hm83u;%gg^?mX5>&N?fZ#^4%Z$pi}kDNDbgRK`JgOEU5 z`z8#vp;5jKnlWfiYk&>G$*A#&Xkl{b+BdZf>?-U=#AfC;XyXUK!TvU`tF;de$f~h4 zz!qTRwdeg->PMh44Y%I8dIi^2S@{#hlnoqPqCSgw0$UdXbkHDdB|Q?ie1lv_uP%z9 z@yq~!u~csxbQA`*r@Oae_k$?h>wqmmByAMB6^~qF>awRKOdD*21jQ@i8^8nIhLZRu z+#qHd08-DkMBz)%mQ(&2V%R}~Ewlpt$O9Fqby+}BG!sC!Xy<#Ek6Z(@ z{!9c7<7%0|6G6W|`2R;geRs$FInDa>1^h1Q`3p#TfEw3#A_$lOA|^*oAiJ)?pAmoNv#P`tWlW95YgABPmvcSZS+PMKB3>q|w= zn_S{}i|4SvdHJmP5v~+1{7Cx6cC=^qGkKZOS#_FaqhOhxbFda9;1TX!r%_N^L1kfc z6ZSHuSyP{O4ab?Yysk&)p-+u;Qo5)qH}rFJDg}iuv!BW+-*qyON`qC^S7+o z+aP+DQdZ;OHkL78o8ymxQlm6CydJK?=2u8%OL*{;lCUdgArZUga;sT!Hxeecj-~E9 zRB8zj9I@{X@fv^{*%~fehho>X{y+A>;L< zBne5Dn7dNQHic4>sgRH+At_AuP=;(-!q|5+)>-bk=Xdoyr+Uu$KHK?y&-Z-K_Zm>C^U`%S86u5=!w)+@g+=$7E8qQiAhN z%iT&gxG%g^{_M)3L%9T{Y*HkejX-vpoFRSVs#>Gk(TBOu^Rfd*t45pcW7-CrJGaa4 zEZWe$PdNxPr8`}Nk?;Z zdf5vs+4sk~T@y|T{i`p&bBP|AiQXyHs?s;yRM6ce`H(zWJAI}@$Y6bEZdH3hQHP@~ z)zH~wxg6deK6X*VCYLCzN*GZZV#9cdG8sc@*jMm-mWJr3#a?SBOLxY_2d{NuquHYM zEjg?=94lWwQQtG7T$tk%FANvwU7 zpV>Q}zJ4iLnm_L$C-_C*2iLkTL@na>t3kIgZbcY5O>V0t<>{`vL6wU+XO-R@bJEBc zX2a2~z6)V$Ey8JJh1V0$=-&Lc6*l)?(xh>t2%4lN?N*2);~I#_T=g``IgBVJAIWKY zIJhzgYRagTZq0n<>+P^6+`N3wMh7ByEqfJrQ~TnRt9;jYz1K#hItMrRFoTH)rF@9G zLL1xP1fZ@6>gZrgst1hn-Yw zX&%NanhBc{tQ(n~eqS3`{`Zgjw@vZYR`|aZ@%e8Jo`056Xf|B*^#IN0%KQUB`=1pd z|HcsGi;F~Yc$9y@b67Gd2-)5>VsTW^o9w9O9qG8F>tJods~_8|M>v)$VxgQy_5eru z#VezZe5tA&XL&7U?EZ=d=VSe{5+=3Gv!hd&#=Li-r zYb{X9D&9_Yaa&uUzM}NSe#I5!XnEsOWXFpLkpl(%{E|wyH(Xm{zxd8UQz2cmPa>ZZFfK6FZW&LiyfXblvNyfvP$1u>PLQw{4d8 z8VoMz%W}U`b0ulZ+MVz<9!Ae4-Va`&a*u0A`=mGDH6r_01^h-UJLuBv8$`eDend#VVqaHU%(ShZOi`*e2V%AyrYmPYOI~_JM7k%(cvTaB#Xoxyu z6dMXsOKjDq&^z50?8VcyY&0Jqo_~maROe|fxdvphJn@Z{eap3r}@$(hBMCTAnvf3nHhA5csDQgr{VGyC^H z`?qAnnUCuWMWgCpUuG9@Rt`L_qzR35oog-&^$4lgwEeg)`*}vKnP>OtoDTQ6ZHGhM zBGD!SHTGdo+kA^MJ&QXh@1#_I&U(1`@R=PJZ_eqOO3QIxe@MNU9sS{CZEwlBXE*gG z;AB8}b1v@a*RA^-;!UhCL|yJ7v$3c-=^C=%UVQpyYRg)uCu`=PXZaVoJFn-jj1d-) z-?7xJ{_y#(UKOf8V4GgLrqtC_=vagFobbm3b^f1ECyEp_y#oEBN)u(sK0=a!N-#{u z^uA0=)Z{6|j0v(xte85T9P-k^y?P;w^gf?bquefEkd_!-e9ph-Br#}ogIIj{;JXhz zj3&C9xrM|OFH?GDy~{ZDTC&e8c?&tCVpA;udkq40-9N@^HpK|JGo;1VGW%AKIHOU8U4}s{w*=i z+^hVtlW!eyRFum z?B6&47;`K$LJ;~%#47{K#SdlEAre@+y6qI3RO z`pzOVGfB|CV~ROQkYlD6vKy~VaZQRO)@NK4#~;XX0$|GCx@W~ccNb|!iR_N@RsI25 z=HejwLekvTQMI1V3Em!yD^ZldiC=y)P)F3xxOM>awW#wC9rz$;u)_G|Yd?G0%%K9a zgweZHi;9@<)11@f#b2g2RzwzZ1P#8X3IaF=(KF9?ZhKiL4~U%=Tk4FJ_wR5mO1b~+ zm4`Xbj}{$-=+O*CI+%^uPLJDddolB%W!j^^lKIM~+}Cul#4@c*O7uq7fxUXNb?bv3 z4;YeZXOl)eiwuKeaG)pEVn$yE}*x?+Ce}?0H10{VqzoF zQh;;o)Gdcj@GxU5co^&;u@Vr~iy1!peGBT(%c1||D`)9wX2H+DFZl8Cm0>f-Sp8F| zCHk@b8xc)wIi;tM`=x3D7bk>T#&vvMf#2{Kd{;mJm;m*^B@xYHwO}V#$PJy}BnWlP z6j{4oR;MoL_be9lzi&{YG`Cn<}-&mHo5bMLV8(;do|;oShs}Qp-h$&mp}=^ zDSU|~Iis?ta4kL?mzQ;YSO6-1Gh$^GrhBg1KaSq_I_l`{a)->5G_hMMmLkzCfdTu; zvxQ!@MbHpGwl-8HQXa4xms|Rs4L*CW!0gtW96dqR-6p9r+6T;@bhTe9&6Q_K({+Zx zUV@7V`&##CJe{y=E!vXzA@k!NHy3r6C3O9jcX|T666V~hzr5t>j-`7F-r;Dr(t<$+ zcCtqqQ9Zg@GP*0Tc6O;4DBO09IuRKfzp*ZUW2}{9>d2I`p;^lMr{AR$u1|5!2!&5` z$7vG3uZ6^eIFGS_DXlAw#m(wKW#U!`Gxa$8&n>gD9z7YNvP3}3(rku2sa;c6_P*~e zK`GoWCtXBSsW9%?HN&n%Ke-yi&xeD>s!Yf1jklh+-<&*A6e)5wK=%I2h$hxFD6A%B zyxx?NyanQ7Sz_McV3^Do|i542v90->+kY>pVoBilKs)Q*HrnqoOkZ6@xeO| zDl9*-Cdlsj;fh14T2FCrRP=3>8m4_p&7-VN>*%PEqoV8=(4z!@ppTU z+~r}U;_w`yOe{AI--!d+m#ASMx7HPAo4F6{AkJ8z(k#iI#i*pl4__Uw*c$97N%S;@ z`;Nl7OP<;}?FfGWOg!gd?x2K(I}fv!hZz~;W^$L2r?$}xe|95a*E2bw9O!tbAvgFn z5xpb_db1^vkvp0ou#|pQ5>OATxx!;0xK~UpYUW|?yAKs!(hTB8!QB)u0koII-8&_U zK4uHS6$j~!+&Xy5EL_E1VMYctMoA#Nx8QX9KKVMU-xMUiCLDJx^QG+?jWVMA+iCuy z=F5_k9rJTg>>nc(|HtnF z&+^lJmnL7&(XZO*vuVBmr_y?VcJ$0*opn5nBR>+2>;@DZYzjM4vb~%V2$`{Pf{m=w zi{FfF+&Q$6G_LN|h?X3HCdY)}csRo!HQ-+kf66DJ32zx0BwA5Dsy*--CMR9`>TQ{W zj5Q(|B-g*ZCV-SkgJMV9HT0@GDa#ePdQZC%;)@->V8dz{gooj#Y0kO7SFSxcADi)Ex8Q*fqH$Ze;5PI6w)=tNI>3&a5!I#aA5g_5wN{R%r=(CNw zaA#_ngg^sGv#B0EX~$qXS>(7gZ?az!DNYOK zAGWvj)-tKMrP&pi6n@`{o2;)ZYgDgzzaiDe)4ILS`I&E*IZD}KHSg9&Hd}np1Z^ zH8axmRZK&*w!`_Yo3I;~A1nwaM$dR@Hg(Yz5~u-U`@z*$Kjrh&9IKN;$<~5KddpP{ zZzY?B>DV4hGS?YTs)!PGc{V&)&=4T;I(c^!-bIUI+s{-$*F<*lbLXjJc^LmM6bQGb z&{J@UI&vOxM&$zHsKx0JQhX8L{K@tsCz=(AY?m&@vKM**_9=cJFl@PgK<(ts1RNrJ zytPpwJD3#&Wj)VJoKcv-GVKgQag)pF5V~~A5VZh)(^V6UTJ*{|5s?Kh)`?}X)3J+s zqxfDNy_mEH>R;ahBjST3gdC5`^mQe1=l9^4X4iR`-gRh0;dFE>a7q3xT@bYHD%Zqt(S^nPKwP;gtg=!y0 z@<_RhSXJ_%`6kVJ7B!@G{av@20M)+v-4ls=LU)hJRwt@`jCIslBHtV0siq;1KIAMbQEtJQ|I0vIWx1pL~=d4qDE%$fEZXjbKea# zvC2!n+ut#Fs@#{o^E6|T9@hN4y{d6kN1F5^1I6A>wyH@;R$iT#(x6tFV_kT#0eg&n ze~J)n_fOL}1W!U+p5`j9uO~7I;tuivWV#?V_?g3B62(SOd`lFgf-Xw(RM)RN;lCyt z{Lh3d{w;&3f;Iw%qA#=K2M!+Inv^oYzPQR>d+zkemh0NCZ*ljom|pJpvB)VBF?Ao^ zSUIk`>-Nr-d;J{~8egU^_S8?_*tjdKB(Yv2)|8ZyBr~ngEt_2SUi4=Y_9;KDZ4^0DT?4rxR#%`(@XX1xnP&nX(FWL@5g3&_$bKkk^NqNhXr zaE36rvYWxl<6$~UwYdKGx$;%`ATgmpH`;JvVv6yVnRCd&&)oaB>bBL}W=h;hQl5f6 zE($>jm5ls0&C=AFuEy7ZhzBT!#_oNSWuTBwy^)^1uXaVkH>+NlYz3UM2Ph}niD}7Y z4s>#<^1Up9Q$N}Y)}KA0nwJ@d0ZPoLS4(8#;UWF_+pMky3Q>=|NkfL!!~w;L*Q?am zbhYkhoVSa=)#j;kX}<1)$MeM>oXnz8ayi;Qe!1?9eK6Vy7O(gH7_| zdnX>~UdO?qFMT}3v26TM(iB&rF`i+b(lzk2o990<%o+HQeF$`R-NMb$(iCPopk*yM zu67n^U8YI-66rj64=tVKVWtGu_<)AC?Z#xJ1d7v8|;Gz0w zi|7A)zxPjd{>>rQ5)=Q%bXtcwkGarsXm0f{+#zdzR`Oa1$vL`{B>Wf!uS*iG(Yu98VtRETk@nWN5toU3cm?Y5WgE0DQyLPcEn~H3%NQ zKiJNW7*OI)aMa-L>64^sNr3bR30B^g6iy~wOp(E*+(M}&l|Qp^b69~K5BS|g2Zgh$ z7x0Y~*Q0yTk64xh(-SS3f~PiVQzGG|gw%1(wd9G=>AhdGGc|j-*Pw1M2ALat10HAt zA`KJ}JMbg1OE&=5f*-`yx!mOd6$Pdu#9E-S@P^-xAs?%q%yQ7|`01EvNOp|)pN56d zwN1%~aHVacxAotfdar(+X<>m`mPWlDmL2LK^0t>^MPDfw=ho%0sAIj$#a(X=EH2E* z*g4k6SV7)}D~vu?nHyA$xOXPR2RvyXrJ+Q9fm z77j-?w))}EtPJMR0`r_lj<%8?l?1U?>HbC^v-B+G$DWR^Ja1xMvF+{5Jii|M^--5h zugsCqgm0WcOd{-Fxd&~dmmE#c{1WC%Ep?5_FgzL4wnb@kxy=5@k!Fi3Vxu~SYnXYA zvX0SII$RRr_`*9oPOSdmOx&6VkspN3WxL+L$T`6x)H~-GAKkV_=*DKtRam<4fOyAN z_YQ5*N}1FTc$w0P`bvgAydr`yB^tIB|MVCuUNqTKN5#x2_{l?DYC-42;6#sv=;AjJ zQ#!bDqP*ezbmuiR3a^e{W@DwVlvSTu(`tRhx`IxfM}HL|@-SwHi1^_J=Ee(EBxCvu z?a~j+o5`PCobaR}UZv{(0i8nESkWklMOq(0#4`3ojAKH>6Y-dgdK{UO;aR5V5*))n zZIJCIrm0r#954Fx;Og~hmRr|{9Ld~Yc+NF@`vca-bschvFxU(Gu?d8grTMGT66&Cq znD*_xax29pr&6r6R4R0&aNf;IG$Bq(*lsN$8Il6Api>nwYG*RjcC-?PoW?}cB;+Sxb%Mj`#mg5YK;g1*V{-?y6nxp3lNY-|5( z`wmWFp8^^&!E@9D%&OOZ#hwbv*Aba#{TBomYt`)cJUfj_d2emjGETpvI8e9B+X{E<~hjvKFtC!R}beN?9rzTm0JmDs$5CWu;_7^BjDNK7 z$!qagJU=>6VQaxj%*+qlv{lP|&=|H@hb=T9S)`cevG`63(ZgPEPVkd^PWE1jy`Nvl zT86w_>dYdX;yy5XHzh*tY7d`6c^GSFp9>sn)W+`}(!UMjSLDrn%=&&WYBu>n{D(Px z@CsC{A6k_w4b$`q>m={Zuunb+B74RUJ})({HO@R<6D@UO?_0H<>vtdSn^0Jd$kK=+ zDv~P;j_4bJrn`CQqQU9+FRO8K)1$gdM0_&TdwEXd98tPw8NT_MB^p z#;(f8!GuS<3XjL5d^|Sf?t~6JH&8e53eJCa5-16hMz;e)N7P zdD3Ot6)u0>iCyl9iVx)-cQ<)zlKu_jK=;{zZP$JD%Nz3-{sg%MmU}Yp zI!HoIaYBs>T6&CLbswC8Eg+G`pZKFXdkLqBD{K58?6$9+;Mb_p&$$T`>k zVcC|lz^p|WvsY~b)Ofry|GQTz&8+vv$5?Yf6#1@ZWV`lz{_s|j@+;l(B>S3M2j_cl zjNdgCw>p!Ih)&PggM{1D<8``5W-`<_^FzFJZ~GLCxePdSt;t;h50lyklp9~$CORqI zewg&x(Eg&M+w$PfhQWq?f+e{%+3RmzHrCifrt9shm{=0IqxeKqmdLU6!&Z@r=C^+A zQuA0ZEmCMSu(@2Z(PM^t&|aj(VC%rJF18{vyYXd=bPJQKL{(hNSmDr9rom(rq0QAq#{eZQjQqj$}umf&r-i2q+4BF zo!6W(V`gRH{&2Ulh`SbAmAHTF8Cy}6lp{m}k%}(jfKswmr}B4JD^gZ4eWqL{!)%+* zra9P(EnuCv;#Dpsb-hSi@yxkr3)yhBGQS&PPg(1B|4vl6Sl)nNT;DWN@#*lSYn11; z13D_ESM&Pyb+>-7MDAOiOH?*+GCl7xnE=}R{s-nKfxw99f^6*)O13frPT~^y69;%% zpG+s>`x<|0T%`lv{nSQ6A0Ee=fcyaOXePdI<#+|9w5Y%Es}J%!g8Y4Z^xL_d{leF! zgJ#)&W+B5YWcYR8(f>AN*gVNMO8jJDsH5)OWn=n8!Jg)1ql%v)$qxFwu-S1*)9~K# zQKU%Sjd&009~Gxn!oCC_>DI|E>ICKI~zNGD(@0AaypecPu=>sI|Em^Yk#hOqUC7g z49dgFeU9I@ z>s(Z`oo4tB3%oif$>Ft#9Aj?3)kyIR=1P3de#+kWT|d+v@iub0T?4y6vU$|;;@I1h z*Uc#D=yga(-q2rNAAQQ$VrdwVp?2L|KYL2{nfl&dsLWJjAW1*&)M7lG}xWR0AKBv zzg-FJEZxnlvi~MMf72s1>mT^v@DIqOz2%B>{WBZ%6t1hinQPoRR3v&{-YlRYBVf9> zXJ6dX9~mb+)`-VY>M2d>Q3fLT)qNkCr|aJy>x!?b36M>gYHDw(t@Ubhay(4WxNCd9 zt9fk92_aUW2263-Wxpj7e$^zJCpj@v>OH)<-7W1rq(C`6cFCczu@PF4E9FQOJRPmu zwz1Z^b8*bl)bq3zsF2Zk^XCJf>?M~U{KS9aNiwRz-K7~i=072uui&~L%hWA3HN@?W zJzvmyT93)lJEOVb>_>CmFi)pf4ObmGtg}k`ezvKN!}&wmC{f1^oRs7P{_g(8Zp5cb3 z?>-!)H7Cr1tI1x*-DJM_F|NQ?y;#}6B~rH7z~WFO$Vp#RWxali@Jd7aM2S27)2yKMVP+9Whz%)&J z0FQtrUY1YdVa8!-^qxQHQkY88j)GrP7*IKGvjszB@LPC?5JV`zO_+gRiu-w()(>zp zDMJr+!gJyQzsVu`1c?Rmid^84r2cDga$vw;P?$Hitequ^TLBN7)xnc<@PS+@xVwPi z!VPi5v2|*}5Bemb8wnDP$PmeP)tpJ~VQ@Ps&RhJ9B1i`ke$qIlT%C#Px*$}GLZvh-Qo4*z zjG;-ZC}OLX5#nFl=^3+j`rqo|Up;ic|0Xm`)-tQQXI1yVrk!ro2sFl*WP20%Sj*}_ zKs-kNyx5OY+L&Id+fhD0F62b&Iy-Ot_HMkLD5Z(q)D&$f*fHvOr;e!5;C?zq*s5X` z{my|2=?2-e35pz}+gwd|Wyj7HHXA6r{1Wl)Nm0LHj{jC{aaGn@0da%E3H>_fV|le> zBZ--L8Lz@lT1pg!6;Wn46wh6s>`Z3gP%^7{wkEsTZx+{}ZURA9K zdF|%rti+XG-N}l~zRT?5>_t}OGWU1-O)qb4Whz)l==efUbnaSx+%xgvVY>m;^HlQA z=k(rEg z6|)|8?1Jrf$@wnfyICyI^jX7Vyu9`MZ08273z(NYPyB6%f8)CPy~bCg9t_vMc5MD| zrgq?{Pd~K*YJ$7k5qW1ZuieEn`_Prni)(!EmV$4;d+o^jO#&Em!XL@~DpE zT=-@CK5{||0}rA;MqY(4c$mX4Dy&SSvdU-=BFv&Kf)W$l7LQOrKfBkPyckxx(}9v9 zNBxTt2+qZzoopBhof(D;dvA1!&|h9_sYx0m9lX#t1r?J(rrpn-l##r_FI8bl@n4m{ zv;|}N|8?cB)lUdyeJNhp#?67dZ+p3*6$|bG3w<;h-#0TN0y1lMM^f0aA3+#Q%x%yf zH@1U^5kNW#7LCm6+46-yPAB|^4Zm;uzqZWyXSqveRsXE&|JPLgoB5`()B9emrtl%V zRSv})*Zb=D9N2F&B9Xo|R^*kz#tR+@)^vb}TU-cKjaNOT04%M#>@AuQ9SWa+X}S3{wX)34M| zh`!FwOjJr!t4pYhlKpr;agk-}XC2+8Mn1-=YDz-77vmY)oi0UT7ilgb3dojcWqA)Y z3HOq<*J?XHbv&oK`gPa?)wf3%zq?0`UU#Zo>KxqnrWCj6Rdhl4u41~w3T;vohw1yH zU6-CvU?ia`Z+BaT&bcbF46gFy%sq=+Zn#q4H1NX(KSK?2rGDXeyR^P~D$km|!7SRVYcCS~i)2JBwB}ptn9o1#KkC}P+~J56iKcQdFnwiV zm#dkidD!9Ix;sNC@u9c6RLWJVmL7_UI#GMQ_S#`UQ?|2)umjOC3~rOI%{6b(Pii9x zI;x#bb$+ENX`1e6K4`67b<$Ft~!Bg>K?9nnY&JoFW`@%IM~F9#)@ zc(fbOec#Bw%Tc8CX7MnuA#@6hRpnu#;prK)nGj^bzT@?me22fEq8j%jp#mDm=g#M0 z+zru(K4Gwl-(pJ7CNC#+5Mrm4iK#|&qEI~B7&9w)Ws(m$!f|Ic)?I z@3 z>RT+fbk5)#r&zd9wdueV!1+~oUi@6C-)@k>-;25AN#jg|EHbnUD%drR0kix!wf%gY-fj`zJpFGQ;GEl8XFZn`@A{dWnC)9&VUi?2$zOlZa5O5jfj0OIb zGe$0grw=KPakZ6nfhj~eMMdtmlF%|m$`rrYH})JA5}?*o#A*GOPxY@oth2AaIwgK( zCFabQyU*&#SsnSWsUy=JowxMVXT@R^8%>Iw7e`=rP}hZTJ3J{O88i5{W2jf9S(PAU zo90~_ySlvMoKf(b8k?-t7PnoGV&mSS{%LmM%>oxsjZ0ox)3N6oM&9V^p$l#|Fl+cE zih_YxFLakg+)6=10XvX!408vL6{zjCZ7g0q&|QHXlmx-71N+!8RV>@{NwXJt6Bdxv^!_E`g> z{_(+?&9466$gUnCthMA}PWkSq3tkoxZd0OX-4%16zJG3^+r(Tbnn^+ajwH(hr0cm>cimzoORQ z@x-X4CT{J@`NsDTSL_Uph=@7?LYQ$8`lo7Z(!@*e$2fE=k4V!uz}I3XoSK)c(Cg43 z^nCD_&ER2?X^2p63tc4@x9bA%C~MvZTz&5i{e2HI;{zO3T^k&X zUYqCUC)(;%9oBi1Jv?(;G`_wzPe#==p$|%Wd)$13L=_B5;LD?7rs#|&9*~#jX_C07 zQI(5MfJ^kxBgMlp%5e(IB%L zHFG!f^%I-4QN5WJYy~8OBFMuu{ZLDp604U(P;NXl?r;v~h|MNF{lOysj$QdT%l$W3 z^3T>K&nna3-y(BQmRT$@KS3piH?)Xd3n33 zT`#lF`r!E1u-gpv^vm-;>e7pcR-O8&Ypf&>RS}zoXzij57i{+tlOk_l#vQRMarPEC zX${WsVBe-Qgz8 z4H*FAIfz}Wii&CAy9xIj*=(&&FL-hjfuN6Xbp=730@oF77VX$Gu*^|$H z&D?z(>AJZI>CY`U-1Iibglj`erDBpd-1-EMt7r#r{)rQ$Hg&=9@Rae@+s~61NvXLL z^A?rvuu5o%8Z?NiKd@VoAcq`dnmk1$86MQYDWTh4`meB_w2>2KFU747?l^`!=bNgC z9~|9KW%m5-ne$tKWhQrJMV+K7?R!rgudEmGh3*frf!LzpcS_2}#R&6xm zJfPxhOXgv&;{rjhlUiTnHOxPJ$1a#R&yU9qGRl`%lR8? z=piLX;sLSH>cbV5zC#yvjUqyO43`I_U3=2e>=d^wP4-4;nn;E0(nH4X;cH@`!w+R; znz??W@I%i24k*ZWCG+MZZL*i1d7$wNhf~s_E&=0(J*p86XvI;f>t*xzJz0`yBOocK z%*8O>Is^Hb=|$C@whPL%MFO2m-7+7VhaI5!+&^t%bYC@1v^Ur!B($T);80Y)an|Bj z-sjB1NbnGwd+{`9V%g7{WzO29Bm?Hyh`tLKF5v6N1(P!Sf>0sm6+Z=2=6=}iv9R1@ zrwp%$!)K=Ui)9pFe{2xEXUQ&B0^^2mBfUKg(nDqF*4I-`cPBB5_h2wB-I}(j99Amni;9(w_wlj}_7CABlL4DIv!cI|%9NNF;w42j@?IeqGqfS}>mPSa5xK6GEu zgsBgQ9>q6ye!Kd|FIT?=Q^UJzl!YFr4kgy&Gn2p^A896`Vj!dS{d#Vg0*mz%9a@1D zxO(Dw7rNzEFFbq(sO?*=4XM*n_V+5U0w_T%1x>2_IimQ9iJu5rui)ry7x;-xbBp_M z|Cvi}?T<;?h-s4SK%o#!@4=$SvBQ=S%lqeBE6P*OQU!IEJMVj|HubbntN4OXJ_#1k z&UC&}seYk&MayOVtC%^05vD#(@W_S^cIJ@`1Cdgr%+8IXFB{<|=h9Qy(+F@OWPgtO* zP^f#4GPrw`+%!DjE5+7;rqpl?EB@SE$DOsfx0nDTs8~st6xZqFn#k0)Y9z&!LPO8L z2ySRFLg05-(a-s-G}IS$%5w&-9H!>xP&yNiEUyhnr!b+VxF#Vu2D&Z}*y;S&5M~NA zWz2^q-+Vm5-oH z-&FIMjLK@z+PN!PM3f)o5%pX_%Tkd_07%!vwAZ;;c*lvTm?84!Qoyu&gaw{M+;E+Fn2ze`12QTFgWLZ zcT+W2Fdj%e(pW-z_c_>UI^4^`@0@zQ?{gv#b3wRQVc1uDLi@{@+v%TiOW}S~MmCpK z4YPKoqO!^~L++xTE(kwA5A$IUml*WZgHiZ04EIo`MH$AkSacCrvl*IPZ-E2!@k-p` z=v~=S+g!k{8wXJBX-&zWM}CyhN)o{I6~W6iNlk6y&?T15axF0bFqV<6H6B# z4@1R{V}BaX7>oo_=&ck1sNb4-8Ay$>rMT+TcF~9_z5r&g>ccZ7zg#{CK4AwQhlB>vP!f=c~P11NLf+v#DtNX3NL zk+n#MKnO=#sO`&)HOihB_bG&WEeFf@g5`6}WKF z1qhkD+{oZD9O!*?W0ISdkKMVfA%ABO>^*5c7ur0AXTei1lQ?^jg|Rsf`;-D>pGSeT z%81TxaK@O@(*|;2c<7Jq^k!%QSKWr>xAYS{QpQ64A zY7mI$^eb0}5hDvsxf?-{39TrZ#>2>56V7qp6@X3nF_`hysS!M47+updFqP|oE;k3= z?qK^)jzZpI=nQma4Oo)=H{dEXo{dF;g2Edz<9L|(Hx}&aEpY2CcpRlDbMJ->Q|4%o zT%+%kM5RTWIoW|z8{%6#Rx9v!o9*J`i~b)o6?2E+6wXg=J> zDfR*;-Z~Mm8ONwBgb8>05r{tIr^hMuU>J#zE5RZakLTKP0a(w9omDALqSB=>G8iZ59V zIZ-cDCU*?tiGfN*Y=L8DJj}}SNjK1kufzpeR|=2c`1u)J=O1yWOXJ2B+fmK63@osr zNqy$|v3M;G{BAHCU3Q$p(w<_Gn(k10kV6!DBQ*Do*JBQY-WUY4L}S=)ersERh^`0s zEQ7}{{d~_5zUddtMPX&7;}eK%(HIc+ln@GQ0SPfEtGR~;gR>EU)c(}rai<7E3FhM*M&?PP z+IJ!PZ3aACMwK`9tu;X+&3k}v}-v~ zR)AA^e)t!66bG^kPsgHxAk*k7B@8>jjgbX6C4`bDP2k?$;PT$}@X0PXl0jsJHW`A}Zu>Cp^ykh8#FbCAZQ6VBT-XUL z*yW7_#9HD0X71S3qiA%+@y5QZ_^=|*ZV`kO%){*1!!bk_+O6{4^I~3HT7p-YcISta z802)2V#4oW6KcAS7uI?9d6GGPfVkbc8PVU1nglsb^XS9}qUSEOr{OpY(`)(HX0N&biKY zlmerNG8phXbn=Tf5TP4F<3{H-TkV*{XTB*d=%&>EiP~VgQ$5&_J$u>V3aHB94|nwbnB5+tWAsEEv|o&5_+|Uy@@tRbqmVjYRXi2EE8EAWD`4 zshCGtDgn{#y;LrOjswLVp2}_$Hi52~5qN*>Ml8hUedGvhLIl zRF)}_^N=Hp*GKF!v2`{~#MVs5#!8iV2|TNplA3v zel+F~{er_n$_iw|nn*O;D&xxbDlPk{6ds1;Yd)EO6P?@mB)KT$MP^&;r`JY!Rh<$A z(u25_LLk1=GqZ!qtfWULA@XB$95fKihvYNeW3hx={X9(ZMY_fS`szgX7&plu?3v0b zWH>)HxV@=z6z*0>jx%+_HKmz+EvPJ)kKI~ptIk{p*8Ip0Jfhk`VZkhc!A~UCi~=$r zsJ!?t;X*qg0|g-347yrV9G9Gv1fEYz0gf$y&Ja;M3bVZYtby>=?BQW9QO1Ex5b&od zbA_;h6sDDzE8$T$kQhNj9YpSGRcTb*6+&_~)2@RV0^#)*&ss>>hDh4L z^J1|1bKnjIBoG_YzKGKQ1kpfz9cd5Q+{YwD8=6PyPk#rtX||!TLu`86nZwj5wPunQ?r~}pa7N>ShPnW z)`MRz5kInJ>>6&;1QkamcF}8~OB{V1LkwAgpE%qNGcUY_Cw}m=`2Gas_cIJ>QI+{n zP}{tlJAUv&Y-rQAyH^XZq4qEv?I980b!qd>qlLQ1Lk2VCR~q)FDoO_w!y75|)nF@w z$4us~g7{#X^#gd^kc9%7v)sQB&z9XmL{yIg%h%P@k1NG%RMvl@WWXFPcT;pLpGpOG%V`Dc6DR>$)_7$no z%{IHOwp3)7;jTHdl`J2em)mj#-wHOz4S=KGUhVK|DLmr`(Kdi(Wzy52&W%hD9%kQ{ z99n_r6eQEOBDMA=_x;*V{>%J0dJI!f65ya zwpFOKD&Awsrg1B$q?^o#glBgC=tcaYoZqHq9P4NBj-WCD-GsV;L6au7<1;1GHKlzL zjRsabdTe+QmgC$hS2v=)`oS@G-@_lVnDVEGwGue1Z@{vp*=y*xCDiLD4vGUCN`F4A z)k{%e^1nz%$fGCl^gB#FG=L;XghxIoqH#x`xlB!f1Jv&h{uJf{j3|$L0ouI*F<|9i zOu)UKpO!Tu>S0puZ}Q@nTX?TBRQ2XLVp7O_H$mGew#D>y(Q;iK-+C>I!bJ4q5HHHB zbZ$mG4^wTekDKN?vHS-6(a+^j|BgU(DG&3e4;7pt zKFMcV+{%I(-A3skQ2LZ@DA0s(RUk@`}%Atg3Ntsh4K_Cm@iA#JxA7FUN;cxeBbbX`xGteU38vRl7(N%A#`BR0UYif6bKfPmr zQm-&|)_VEe#;3%4%JD+|l;H`&38&_zWLntq|Hs~U1~j>5>taO&MWh#r3P=Z)ULzn~ zKzb)4A_Sy^bfO@jv>!rL_+|apKqcO8w0i4ox*^=MbhLwoINtu)tZq3oyq-PlfH> zF}KQDtD`hAx&3h&|B;u*BJ_(O-5XvvhIMK7{M9Hl95*vPMeQ^7sWRq5my<@Vi$3`a z-Ni&kT6%i}jg|IPRGV2`+*0=F;278}G4u{g z{md(+h}JlQw}>?8d?kUaypl}I$=lH#Mb#TeUt48kJDxv%+K^*ik`-Rr+Guq-oX%BT zJ_i!>%$6FerJ`Z5gw@Z}&bHGi5+AT_GolSnwH9OdQPhoy&l`es8F~gG0+@*Tvb7E z&FCauqAtlMf!&boX0M`UuiY$MSRZip;+Ejjb9XIgC@?h7W719Jh|3=CzW=ekR1WW$ z435}?6xXMKjD);tQs>w%QZVD~klJNLM1T7fd}+8_Gl1}6_fH&`dp@rkS;!+r3({Dd z%q-Ijkd_>(oQ#h#ofSbMoLW3GY!}X0cimd+t9Oa0R0^w{;x2lLH9DUtL1`v*GgM9G zG1{a`__2ESKw|!cCQ{E6(XBO-@fyMc+`k~~^YN`FtFw&kS1?V!$&3BYUY1wP)9S5e z`GFgXp6_I(Gq39U;Gzh#)W>=e9-4`S zKIbW=XJ2jf54`F|f~au2Q4eT>B`?Q#OabEov4kr;+bi2Z;szOp-F5n9%-4NnNSJTK1iCLGhuwlg^B}V{ljsTTte0A9^@cDGhbLHdB z{4D$EkH>e9@)L+jyMcHJ7WT}lF0{1LFZt&rNS z9qgMxS)KaQI6|P#K*lNDRs*1Z>!~KTMd=XhAWnPLH{{&8!R+Fio}wJU;wY zCdVJA0#NEC2p9c%k{h!pe{K5f12+?g7m)R?)Z`kZ(fKY~ZLEK5^34dBgbW$8Xbsx= z+`{2KqO;eHZmJwz*0`Rkn7iH_Eeafqs!QZyC-+X$y!1Wd^NA2k&C)h@eH=F^x=HF< z@zS|Hm|PvSuVu2W6A3$6dkO2UURgf4ak;-%F|mz3cD6%PGQ+q(eY|F@RfG`jVb7wv zBQXDxNCKamPyYFo6eKg+Yxa7(A$NCJ zo@9fDdxBJNQJdK%lK@K#0o1wscnFDo)N46Nk6?o1o`XE|3(2Xu?_3L(BP9AsLck|xj=!L!TYLc&U6rs z{my4m1nR-<| z`xj^o|FrP`FZ+HTe?~r<@qRJApP+@kmVIlx6lmg4jBP{w#vyBuRW4o^kNwE}wC$40 zv3d9|cL#Xj{+%bJ%rI}^)eyEr^57F69~u|Wyn$yBMhqf-TmB`@`(Bwq<2xx><9@q=8x zn&gHxs*^$c?Cy_|dfJ=U^Ylnz2W>SbX~HRr~L);Q{)g;b9yOUVpoi@k7FcaO_A0Jl|;x+Lz7* zQdg;`ztqPx#(n)mAMLKk5Jc7@eb-WHC-*DCtsZr z;q5DRm}uQ{#FLU-zCvhm#!c+!28%^?E<%%7tg;-|;Z!T3u?n9OC#@4=sUIq;I=_8N zl6QljaswZ;^YZbJyzcS&{&IQhwZiOI6{4IeQL|H{k&ZdhqIGyL=nz~^!qcN{JM8Iy zk{$j`^ZnN#Sbu%2|3lv$k3r7|mG)d(dLqw(E14-SV4NIp%A#*tUO8g=~#u zY|<>G?4z!>TK5T@JM~Hs6YfLHBn`mH2nqS{D(hig5j)hM1ygGtY6jh|sZ`GChazfS z7^5G_3EcK|;NK54&cq`yusNWmEH7x@c%qFwEWnP6=5W;Dz+S6OAolC1jb=sn^DpCy z-q)#a?40H$_26iC)23b6uD@>>NWB?z8F^-B2B9aBS7;+#Q00DPFCAx5Iwd+_$H=gl zGqdx$@SKIj!8?GOU>-!?^FoGv+g9;oRH*9>x>jc5i1V>lleY2eZm~NVtI}Tc$aTJA z(?`jDB7LMa?~?N3b9wyW7eaPTM}T8BsfK&hFuCtA?QDAx5@ntu1xn*~(#OtalL+2AkStE;$ark8CnUWb7>AX<$yKau%(yYqgA z96UOS*;jZb=H=J_d`_OOV=F(4nN^S$NppDIEM3Ks>qWqdnTU zcm(MYERGNWTi+-xaS(6g_2N|F4ijO1)oqwQ1N)MZZP(*3JKYM23b-8lXiOl_e6j;@ z;WlRt4%IfKzTcl`d`Xd>?cLPXrk?B-=>l4&`!`(iRO}D(M-w?F5-NaJVjGEd2h)d=w3tCu++!i-;OSxQ zcQC`m0oa~ljHHL~0WSf)n;rDk_-H;L_dk?ce{(aDFM>7(0pdn?Fd*us4`5e1w8n?z zM{3}C^vu)|BkX1HgHtPXUpYmDMRlGXwGGx@BL>R;Jj6h~6og(#J$mx?1O{+R;n?NaKlkC<&aBnD zyP+{-4MT2Ul%Cw#p6V8`4_awW46XXojvvOqI1<}Fl!Y)Z#t8B>3L%(9*AEfVsk3QS z<`(he)!B3J^oE4!QGiAx>A#Z+LsjqGHY6f=^7H!<`lCmt&JwSP=_S1c;fhy# zhd~899vGnCu|xoNXB^yS^5V;1o`C<>vaElGN<9Fm^vy4!(xIQAQV>4mHG7UOX-61? z?&4nXfKW3$W*QyV3W&r8XG19+)(lA)d{j9%V9EH?2 zF>cgrYK?n$xMu~&Lr|;e&r@*HIb5R)SAFWkURfPm7rJD4Secadk-AYRi+|^}MUnE= zK~6!MmwX!8rxTfO$8LTngA&N`TCU_VIOvRfeL#&wW5Z3&M7NMM3qiG-`>vQg zUj3y7gqfgn(VvaT3UqPN*?0OSZnB#@yuDrWWa}=O74T;LVq~kNBGz*d1@PjjUeq zb#ssoXkrr;d8tFCz@N%C_e2}ahqd`!OpZ;uN@|*8laL9W;KM9LJKQXfp>^2TnrK7U#03 zu~k6mVW{=T3%j9%C5NMvNj+_=;w!T&%Js^y5E^#FHZIT9IM+n@#G(d7^;xT4GZfS2 z2}AC7lL7ky7!0=ovRgE&zx2N?i2u%8qrw{In`33Z%D^aMIytIQVm|(9;aZsTa(m-8 zMmxQT*m6f3(n9Y_I|5 zTiKb081cUOCiO=JW`qP=djwA|k|$>@J_Q~U-C;{PkeR<);|FqWy>1d`&DU+mX^$D_ z2$$C^o{xRw&MxcDNiKQ0T*U(RF8;$=IvXny!|OW{@eayU6N!1{#Ry6hZpm@4GSMYdA4V{tOB}0Oau9I(y>kYO`{==_}6z@a+tS= zYTq5zl&u7%n+XjKeJ4w_;=v@n2w=?b&cvZ^4~SkeI>Etej$;8ypV@p+a%34b-s<7J zt$elF!+DD?MP2ot2xJb#gi&R{!O{k1x01&saKa?-$AMPxV~$0%xQ3| z#OQE*4u^$I5KRv*!QOGl{X-??|Cx>4ylt$aVrXk!mXPHBk)2s%=cTHH_KmcGKIm3p zy?5QP*YGxD?4kORUcgB(^`0@dB2*yNN7%f&lUPVoqB@giHB-U#SeODTL%Z*;J>W4< zc-z2AFvZ&v0hZS+(~=&RF&MLTbmDs6(W#>O3v4xL)Ob**F--r>0!U~dkccbc) zgj$a5kIrV$J>CHr&hQ$~R%9gspu@o$&Upa`XAMT~#xIXf>R#$^93%%c@eBIs4Qfwh z9}4RPSVEpK;wn2kKpYeLPPVZ{EG1q*?On$-{3%7JzJfafsDC!II>d--IOz#Lkosn$ z2(RUVf#DC%0B1>E+7-5C0C0`qjicYm(md80p8US0(3uJ9z8~t1@QpU$MhY4;_V0R5 z0qTurBr;+aX#3$u_|c`iKg@$Z{X*clfbj4Sf#cs&Wc@jb{6paQA594&-wTXPOnkE( zoieesFol>}D0$oYLPZk~jFOD{D2g_nimzOmi<3CkI}n_G{JfvnJ-5p73Rcw{9gaHG z&l|hwp@v8){X7kX6mJ7UsyR>5HBRr@yMZjuWN7wh1lPl2cn>-lR8CjnZ3YXK7nG&s z8W^TA#}SpwqHEnduauUtVGnr%MfMOY(BhpK8vlt zt6wD5!rZ>^2?gkCp3u4XyLpNUJlB@A=6K!vgAL?AC@bH;QP9+4YowjS`kn09cQS5t z$7UdC`CP(APNt(H%-ZYJUi4n6#AD-fk5&&mZ%2;{>6442vkVX7q-YXidn4W-KAktl zdpv2q=W*v8TQ`3Zl{PEQnZYch_$<9NGsCNhv+=9Sm0hW`16h*oy76Yv(#={`#!H-K zmbcSHMz^G;r(mDP^7(~P-ZIeL3juRM>No?)uu z+1CvRFQPEXZfS5YbTP$CE+!M|X6x^GC1P=Cl{?ty0jQBv`xtrua1ku&O^i9b4O`~^ zwJ!>Ti@50qqyrY*6oNlKNLcc=18LSlV`YMF?b*L)jdqHlGnDoXWJ;hW=86HN^|;1N zpO5Bf8Xv9TX6iJVHvcT%``%_Urv({iO|w4BrdQrYkD*l|ITEx!=0+Lk-AY#%-&~7@ zi>4M>8Ke z@OZK1*~<$$Swja};05S9ezhE$GZ>XNR5%S3Rs0x8vlgEqcMFU0D5+yV&$CW@)KpL` zkY#amV64pzY1s-)S$Lf6;Jhg7R{Fkep%{D<3;7JA4sd9@5{F%+9;;^WGkK&erIE-_ zsfUoz*3UEo@6Ru&6gpVv8z42j><#w~Hu{xljn7H&>G)fMUcng9c3bQLT$PElq~5YA z%0*K_f^pWUr6y9Dd@MBWqho{tF?d#qQ!>KX{!PXMk*{CoVK3Mvh)evxIQ);^NawQ`Qdk+oW~F-w_v&;WCrfESXxE2qZo3 zBQcHiuCcYY7(k=O0v7i(D=DnWi3o9&P;f1COFoZ+-#}SCBcFnuWinM?V zr5J;`c(2QO$Kr#Jv1|LkTwv3UoAvPqH#X7-x9Fv@4CJ7g`;$wPR5A=X`vG}5X!cKT zo73w<1p|cpxFw9E0Zw5yPe5aQ(#J+0DkLjb_~k9}-S`FD54CY(XM~E{C@&xL4*tlO z>r_BGLFAM@-X%*hCuO^qUauA9o(_qKL(7R0SLq+bMKtMF>#K7~N)ARuyow4cI29Sg zmq6;92+GB>ki2Amai0F$OUE?>YUy#|aE{xHe{vUDcEHB%}c_H7Xk z7-e<;!Y{i4-Qw|wQP%H(fP()}E&s>yXe*t=ZUrreqp&gd4Gerg1iRFr9yio5zc$I} zLFM6NZc<(CPiY@|SXZ6YxE%(*R|WV6F<+E;F{*RZhx-BCWjFlklm_Uc?D?ibc5~w+ zoofO4zK{73#ww|SB?WpWyu7dv#u;)Cmdfvz2~8KmSr@1ya7kL} zJa=LQyzJ**9m(iDr!Cj?{W4ZVwm|Xa^AE3!h)I5&NUQw$rINPF3Q8V*_n_8`nZnjq z(xZ3NiUKc9?`g?+DD#4|GGAmpk6uA1>30Rfh(4`{yQp^AH(OGo;lZi2meh(}^Ne$gg%tP9?{=@I?9eh4oigG8n%F@Zb24uXF2$W>GWB${*q zSLedGZ`ol_;wvpzMfVAv%?=H_lVPU^Om~G8*ht;g9%?V4-iLpt&Q?8+TML2j%NalI z%I(*#XF{zHu64j$&NF2zW7+rXcA24SEL`{_M?I^gJuk~wPk3vxJDvuPmE~&PHMvnVam-%GPoXgA5 zEbsq18XnBPh(9t?-_TIm_POsLE1Gm)?Vy9JaEGEI>kNyH=wmD_0i@RbHl170u(JD8 zyVWqc23#%x(N$(~$4*N@BB?!S$R`7O-mj$5D8GQWKi;*ZNqX0Hpl+P~A;Q6@XtHkv zHrD1e;$2_9hPg5wl#zto`%VV1xNrVRn*B)2T%q*YugEli)35x__Lx8Ik`p`B`@k6% zyYv~h&)W%9KradiA;fIZHUzUpj09d;y#t3`Be4;U=K~}lOU&y(Bg=suM`vMQuyc?j z`pe(R`fCkI(WEZwLrFGeK)3}D!@oMw(gh2rHa-L)y-4-xKl6OachHdSuE0-s$pAZA z`2PKVb4?uU^cmK~%8vtk+%m@*SaPboa&N1hs-pQ4!IXY$S>!3e<(j%JkZ#J|+YxrX z-#fuf40~>Sr0hlW>J(b zkP$^T);4aj2Vyy9CZA%#4awo50SeT3xi33~F7FJE9wy}72KS2bt6WZqrQrLbeM$35 z`v>wJ7(;;2Oa_w=KmBJ5eagO~ExiI(|2twn-0N$-355uT4W9BWy^Cp5Rw8l!cZ+1@ zl=afrkgBUeytt!J)d!6kbgMyh*h>Wpfyl47^+T;4bH7!AZ3fmY>j@$}ob34uDvbqv zz@(=4i+VOU;ZQca$mw)m#5$JA)c=)wzE8wWmy!ECZ79El+r{bFxCCB=Ug{jED9em~ zCI96bPCj>zfBKx_Ul6;0cLM=HQ0FJC7P1dsLI25%j?@c36nUdcjIjq6@Ojz)jnxY9 z6YT!LYX9=d|D$l@FRnZBec%ro!9SiHyjwq)L5&MH#}t<#ap8^C2}7|(Q2ofrbBewD zT2v`i&e_-buwSBjHR&SsZa<7se(;cdY9@$h_B`b*l`tKqSSW}5`e&U{-6HkTy9udg zP$YAMn>*4db&k&q+U}XJVz9*1Zg*xd5~*t`pE0}(?+1s#0HQ*G>3$&PtcW_qKhxc^GKjs9 z`blwv7&+=o3R|Q=ERmG#m@JJXdK0CIB;!{-aZi5`neM@2Z?PS`s1=;~PR6(*VJ7^1 zw?<1}r#d<-j{h}BLY9CM#XawnS-~exTTR<>L>#6IOm8yHMJ1LiH2X-KjC0wS@fZXr zpq%cF)^a9mX^2A+_00`M8hT41CEy!DVnOZF2f;yuv%)7|5R_-05sA%jEk}#vqjxS` z88`DudV5bZDp%$O5XV6zU>Ylk2ZJ@5g;U4>X&Q@*)ElPr?$5JW#>v{eJL02wHX)m{ z88taT2lu9f&rAp32pMM^#sp+_Wc8v}r-WlyDfIr&4&HAH{6C)5ui^0h#(#Ud zJiOSEoo!H1ERFnbRo|W_vys1fhdPye^VLg@12n6JXXKjw17w=sY*8$UN`{2rFud;b zRWyEO5XsG~jD#g`ruDDZe4RNPyO3M_25*j0-OuYKJP-v0rwEEi@(CzHtl64-`%CC( zS}P@)kn6qIsf6ZMF1L)daQw=yH{SjMTOgIhNgrfeaHk+se{)st-6kzl#qHj{f=P?9 z$`SOWRscy@FpYm8tI+>P%TecJ+=6w46z3affrYc!DGrWYbt%=?t)YDG)TyWY({FX@ z9CXS)uG>m)n8mkc(EhzjGH==#2L@lqF~0M#0u@M&NQ0!x@6&ZGC?F&NrW$NzRCPz-u+8V?V`&z zc@5Z9ZgLhKdKUNQ-e_s@1T#?v24rSP>tLh0e$XyB$=|oAY`d<;bn8%eZt9p%(oDq* zmsbdKnTM4c^C&$~ft3RNt%js;x11+@LCCk-8|QcN>C*G@;K))-+fN zHn~Drtn0#rLJ`a=+gZ=ExVdy_!*T}ggpyU+!T#}#WB@}4hI_^ zg3b*ew)kK93-cW0PHM>5mh~AXHLHPlx9DM6u#ymxHaQLKTYUy8`xCY#W2SMCq`EQz zLm{91<=Gq~Yz-4iGF<7%vU2f6n1-V(*o#EHvXK)Yfb+1qdIk~lS)N(sGR3xzcr0B7 z%`4{188=b{s5$NG9Z9jW=h|22NmQ;mmS(cA5b+BH?8iPAN)3IhjeZAr@3^3ygR&~@ zw>*#g4&W_KVDw*SYd3;;=hJVmYfn0dMIM|006sV{IJPb?jDtAkH#*K;Ixn?^S$zBX ze#}GuYD)&ZB2$WQ*Z7S^?$No0NjRZ*Hw7=ZP%q=^h(%P98%d4J# zb#J^HJ3#a8rC(I=Iasjikk%W_zT@*RCHb@svzP=fjWWK7{!ld;4MAVAtdS!95@UR2 zyGTU5WxvQ;2h*QU#bfv@M<-#O#(-%DAg|*gM#>551SCLUo4{MEQ)=vjEld&X0R6;_ zcl4wj_P<;-4+g>JqXGblpAW_W<+SHi5NJ06z&gpB(ffi;uoZ4?YT{}JE*9qx*(nb6 zx%;~>)??I4yQ1fu;gNuv7x<9N3Rv_X0PGc@7kC(;0Lhd6C|Fx+J9yng^*h;Jx23#z zSZxPEc<05THDpd|*ZMDu_P;4Q{Knw&|K#PM?}LA2vHy4$3l!Ci{c^l-YvhY?!bb); z)J*lrGCQ|s1_MrLl3$ZmQ%RJ@QT|d%#Jo!&-@Zc1>%P;TG3~3jvL5i#4J{Y49+@u_ zcYEe^C{|{0Hj#6(2jR;N2`mPwnQTU<;~gw(#q21DpFuTr!)R&5PPHi}u&FS_b0U{! zw^x0_mNdsx)~bUFkoi`5dqhQyaZCU|mRdQoNuGMhddM@Uva&SJq;D1=KWubymDQEh zsx5`xI$NoprT3-n>ZYoS%f4_UAmCNW-z>xq=ozrfGZAQ0SE`h?X+zb-I<<@VK7=}m zsORMlJhNEoqPOnUc8#xhFJp=4USCyBKpc=VJPwy1)9<$^0NW3=wEgzX!fl@kPGwIo zzU5VBQxofi`1uR&?$nT$9M@1aX|Icb_Yt%pCw-+jZ_iResI@hN6>KX^jT;dLTG^*m zsl#iHHg`H4QZDzF83$=}oFP-3pFfOjl{-%cZkSSoWI`x$FLY>{NFWvFhu+r~_@F9G z7E1m*%A9%XHT7P$TMpS-OlV_@H`)nV=9BPQU=XEyhr5}b4oe>5<(Nj@RBhRnkY`q9 z96WQzl#Y$!ZG1r7mGv)b8v_@>t&=yWi+QHmHc{uX&S!lD3>o^L4aFG)16nBNRxK`O z`EEnot^NY-Qs2CqtW;FI=&y>szdbPh-sr8v)@||;&!Ue)9~Pu*krvI=$q{ zE@R`w=(a9-D-oekoyog>fySypKHw6*m&On2D-!FboHAdnmfPpjbrE%9VPH*`c?hX& zy!<-ltVe=vmpRdGCg@6=PX=dZlLhESn9Q-~8oGCrp;n#w&il_2_Hw2;dYhZY7j|}@ zt+X|~D}liF4$|l@wn}eBhd8+xpI1^@y>1v=WyI#Ha!YnD0$(kbHykb};V+(kppy?B zX+}7)=v0sP78UN>aRcLGWWczjPMAxr;A{bnH*&%$kz#g)Ark?)jc_Wnyta{0oASIu zd}!1#zSh0)Wo>Ll11@PM?sK{=R_%52c2M_r z+Gj|OQof!%qtwzYvtssldFpF8u@OM3%TLaD!5DyNE1IS#%+?_Ny70d2w>0- z35MDh`L?|!D(04Mw@bFYXa1(vx|DDVUe4C$mNA~)kQ0Nn1g5-hu zK_=WQpb9C@0yMVI62L3`fNCpsBy8RM`FFBD?NrdwAQ;Pm*+(x~;4yEh-+^sGhr*M` zfz%cWWPx=m=)C$+$o4B)AbbFxHHJ1ZY#qdQm@ttSR6GM?|@`NO$(1+N6%^53YU8+VLvT`0G+~f!vpQ$Nm~-* zVH2Skg?)+t^V|8S@`m4IVf@|!;verl|Mnn9{fF_)KmFKwC!UV`P|iw3$+N88T??Wa z67oDW^5Lp`fAcF|rG)P6a(VJlYH^+$`-rXKM0nH(pLD6?l|_@euX!lcp*S&vjbFBJsP$N=LnU`~nQO}{)gC6m zM!;D~eRI!hojiSvrNZ|i7hB-6%Q6DuLU$MUKYr5IPo1AT(^88WbTUel(j4iaKTE&X zy5!~?ct?1PwpM0yI3Jhs4?}VgU($_0X2yrUGuL5FG!VOypCY_Kl71Lm~|L z=4;H*2EaKOA_f~g`ES5Lf9Eve6qzu1DZ?*&;?o`~6vk9*ib0xI)c0;qE~($7$+b7W zSYtBL$U!YB@PH6=i`?~+tIk(C_!xl!yA@qa*PZuJir&urROb0|wyj?L zp*&Awi>F=oTPl~p9VaoA>HyISXNz@g05OO*57bKKw%QgqacC*2Rr6n3f0Wz((b4iS zQQ){z*2UK|@7}y!v^36DnvNQk zBz4ZZmc8#^NUq-W8(?I&ITYayoAJF~<9r^Brkp!p)tX73kAjUY->=IihT5*LHb2@= zI>_V~vJGVKi>wg6KZ@qJj6C;cX><28+r4xpyRj*3an2SRk&G@zFH%@82b{om>??%r zYy|P)RPAjBoQ0#ivZlk++xs=`ogZ_4^f6t!fcOxjwIi3>_qtt1uNeFe^sVh|R$Ka79XJJfO`1-u9C#Z^3{@}~mmAZLTA5mY7A%`c8!XLXjdU!42G_cC z1jodjIDY0bfXwh=aa7L6(M5@i1`gI(NoZGsnV~hG7q0=58=#jfI^29VBRVIk!KWLy z02S@__VP3;zWy%@cE5XC@YAwCb{Kx-1^coyjXt7V1X}t{gCqq~AN8Ry&YRS{2iv=* zihczQCqF zz>VR6%yNJA50{XiPyQ2!Q>a5b`DvaL*%+sY)pTJ@&g<1nVxWumkEseTgqe40^FP+( zJmq_ia0_mBL{07f%}zEjc7dKz)Xg;Pe5Ce}L-Dm#7VmyTp=$4*t`di+QIheY$ick( zNcV7si%thY1hU(=cL+1{eH37Vzx7|_M!$0Ji#_4!pNcw~6O|Zz5556&n0lf$Gd6~L zA%Fh;$jsUyVTul#nETj1IjtkuOY(~D_UO1w5cbw_Ug{Gz`N%`^-C_JqdHU8NtpAhu zLmfBV7r(mh4!t(p106M8O?>Uw?=s4TU@2blmwus-XRBaAf~*aXrfd%%?~zK^FhG$7 zbgRHihp<%Ck8#zVy>CAqi=cl~_d`I=Hj`ukV4zW8XPrZ?pi2k1rvNqSI~m<#0pt^; z6hgYwq7Nvi7J^A{7rFm80t$i5UHJn6{X04SuUyAJ?FP^v%Beq}M*l=XbBejUDkr`A z=>g<3X!PiAPdi{6G%=J_v;tlh#~Vl7tB6=K>NC7!K)Q z6@v?~Y>280(g@o4!W9)GRX70EdccGKBhm9vHm&o#=yX$E>R zH->^U=jLwa4iFa^C;TQngVHiIomPqfsl44l8;#+@dxNEN`Z`0U1>F+s)rkqdF1gSH z`P)S<;1QJIrMbX3j+Qa=1zTHLpqFOiZgIddd#5FdboMb2)2TxxPSmMQBCHj(iD<@{TMf)34c5A zRO-_6*{##ik@6*d!TRJ0bbiH09S8UL3U$fca$+sJU74)v6bUv=0ClH~@z+T&`U5n? z!Yx5tpJKsc8y~x{+ma9Bcme7`_b{RMt2@EjFsdNwp@CgEN<~`)sV5}zEOAxW4>jzC zfIABU0p0sffd$9EER_FCi`g1+g50d2V?%Xadez%i+Lh{rs5}A23o^Xz;u~>1Gw}y{ z)F|@P^^u{KsOOTxvH&P47LiWN7#^3(eR?Qwvd>&YXcURG>~rAL4>j2~Fjife`s^|a z2LSX79A$vrC83D2Sx;Ko$+Pe!)Gw+=o2w$_a-%pbyl1bWMB?b&E)4$w(;7jSi^+H% zQ044(ElV3Qz+;?Lx20%o)s7%_Rm+Xr@OCU=-*~QAdDbnH`0kstFv$G=cQUB~j7}7&q&rw%uvu`g3H(i7z%X_OE1%%> zC`sa#V8)1`HI_=|R$cJ*mrtvj+H)`RW+}E$eVvpEWRR7{Hd4#B(relJ&g#XBP>Ej7 zH%$7tZ;%xeE_`Ik#G_ZXa269P*${Izo=Z{(O%YQNdSG0~-knniE(o%l2PT$n%N zAO+E^Ei9Vf+N3I-q)FEpS*%b_E)wy>eom|Rq_?BK%-G&qS!{{=1UNR#!k=0APw)E^ zLt0I^!_rjszZpd?HO^XVTNLN8=YNd%ic00oG)uM98Ph}Z@}RRapJ%-Ymt(dXE6&@8 z4I{daqeaC3MT_>tGJHgpqmO^0|f9~&OOsIJf?jX)?ft8iB?c3zPIfqu;Fhj*E9ThC6~C z{`uc(lF%oP%qd5IxT1-yz(>NTb>Q-}xU42KR3xpMxt6{nT&JAWIS@3$J*r2NN!3U< zb7D2vdDZPahd)?;y6c1meLgSHxf3L20co_413vjv-D=vre3jaPp@9XxjtlS zm&{cW-P)`#-&hSSf=n&N|M@fKup@V|8W`Mq3yR~|zlr+_TSy8G(;+|d!X7@SUIU1k zz$tn)78;p5fVerAy6Iqv*QyVKTO zG*kL!sOV=FI*_lCS@)b5B=B zNhzq&vZXU1c(3+DeMMs5cHb;lB4rA+zayC<%T7>RbgNa<9$HyV1M4V}8bb<8H7 zFPxYv%}HC)@Edwp6_3zMbB5%7nq%NgLttBBU@$;l2$T0lE$CMQNa;VN1R;iNe&m+EF0LQrnB04vYB zeH%aLV8F|(B7{(H>5YhAVB)X;7_Yb5iRq7)z0EE0Ao&pi0o>m`+{Pym^_J8Y@xx4g zdexEL3}m(ue~HKmTTaRYjsX*Dk|U|wK2Uf+j1u5Ok&}QmLsf^JBE29kkkX=oH3)@gj{+&!L@<^2&n9vCUg!D#XcJL=t47H5d@CJ4+tVGDBXB{;_?syim zQL*h{cBw&OA7~bum_KBd$>WsCNfbT2J0~Rx_~RYhHOGtLCO)ytJg_itKBU5!KBir( zA1Z0Ox0t!%POxMN+7P(CD(_Hsh5GUN@!`knWo=j8J@wY773e5b%p%3IPw}zWgBs z{5$Y5{}2NHjoAsEzZYQDs&BkZUQD=#>z<>2W;R&280qik|F*0t&XJwAx;D+z(o7~H zfyge8?=xq2ruY<(a_thGRRmrm4CS9yt+1=Pc+7>>> z7)$q91V`Ru1qtZo=5EO$9h`8lkVbehN! zAcq|jh;ly*Bu!nsy3rO^mPVv8YF}yh1nHRT$cRkl+%mJ!LmR#Dp(|kpW}-UDP16O| z2q=H^WGC?Ext-yflbz{36r5hI@|8>JIxf}(TuF_L+)!@*L|uNzlVq@Ix{Wun%7^}R&tHkedTPg&Q!Qq@m`$XE91n13d)#*@W;5UZv-SR z5>wIwq7qX=`ebOuF=g8wokyH4O%5bD%u`%Rr8@EXkXp{|EoD znoFD3<=gDlluR>;$F9jn_3#P50trUW;y`u~ttu_MgU;X&zOTfRJepK%L%2js=r?uWc93?e33zsg`r=z|0UcSk?XPo0a=a{2TqK=3 z1{6Zjss{_W-;Lp2vwFs!m^P8GWM014Z!V?rZrjcG0rytV0mNkuAgii-25L9zy{{Pb zr2JME$gptoWeW6m{&SVBUGbpLYV!ItpIOzI^hqW2rY)u^w(&dlj7>_Hk_Ih$ToVQ> z>bOTc!FK%oV(qiZQMJ6vE!tv?{1Vp5yn|-krgZb-(gnf>UYD=iF1LTXyth~%5$b0W z;ecS~Ro#T{Y02rEi}pd6zm7sjUI+1E;9;_k*;;@~no|GXgwXxYeaE8*ug5yMYDS13 zTY@+%D%BQZZK^6BB+4~qEE=Y+^-Ko>qrr%JVqc(HYm>urq;1t)P%NcHWP+F^T8 zOa`gY;VeF{ydCxaJcdn6zA~PUY$jcBXKJyFb)}0iX8xf5`yhzed~Q#PPImr!N0|f?p@s!Ti;vUCp;2xW;wx zm&~<-zmUYlU;zItzhPCsm!w7sS%I6KdxltfrY--8kM`zEDlIbC z`Mx6OWN%Fiv0_D@eMn_)HlPo<^jU--4_Ego+;T zOk0@D@cy9y=3yIX>vn1)Z6USKd}7pcOzm2Eby?-Jg6!DJ^D%%k!Ucum+Ve83i8l6& zrcTfqER)~M0*xZx_B3FYyiJ>)w@Xli{t8c?qML(moR(j>QSSqGpsCyp2<8^;n^Euj zq3I%#F>z+ac<|b$ZTl)Ga3GwNeca6f*V6$Nk$K1is-)qy&}<9)bk4qwJvsRWQx>WI zYvTN+T2xu5EPXg)m`-%@FZgr^OG28cVk)TJoG&ufiaqF$s)%$L?@26&juc1e>J7Dz z>KEw6SFO~D>@d!`tQVntXaP{2K`20J##MI3X-Fc)pl=}EJO9LD6GcN>^M?wVn8!fQ zJ*DQ7>3tFB!dI*Li7yBG3ehR;AY>9HjW$4S@aC+ijTgB@@3<3B)82@NObi{^;zZ`J zp5eScs=%i(^YoGei0DG9z+}JD%HtR7cUi;8DU@CQw>+{$kDhM8DCDJZ-7{j_#YvBc z@h7*h42ZHeE{?W8J*Y8DYzxJ2ho`Hb+{28!A^rK$VsznkbV-W8^fcG$R6J?uQ6pl zX1^uaLiz5)x02kGQ8e2xi%3H_ojK~`xbApaAQ8E7mMK>SY|_{XjEyIo86rDOP=x1YwbQhdEvpwc$~<%rUkyfaQ`CuN0om$gH( z7~y;a`94*pW=Gh;<&L0y!h@r)mv zCUr?p=2mr(+2ILTXSBZhps37Rw6JNv%WkxGZ>-QRQLk&Zfi#$YgO95@Z8ZD2DYM0C z-2}jbg`4oI%l*z$xCcuW_<}Y6J-2ukL!SG*2AO6lZh9KP46aBBU>X(w4#mt}a_1mR zvWxF76glWnuR~(%LJ3&^B1ZfU?OPz`{;O1Ye@^cH*<9_{U#HZ+P4e`6ItOBvzhDmx z0u#7}UU^eVD8u_p-6ax{LA!H^HwsL+uH*7 zV?Z|>Jq$kGPacYPG;MLZHpyNV@2CVR(YxWr+wX{?j7YqQlpYfH0+?i|SyEl5!zA;N z*rgMeA5ydjcT@5NP=!ktviiysIw~qmKJ$u(gCTi*pE3@QW}R8OC1t?P81%)Nh0xY} z4*!e2w~mXlZP$hIF%XcDZj|nB2^DFnp>r%+KuQJ#Mo~giTFHkPhap8e2T1_|X+e7E zoFRvS8TZYz_Fns4-}kO>zu~ji`u6fKe?LV*xUc&<&f`2nC)vf=%rohk58&%a;-jAj zQO9WM7$#ZcYV~EV!5MtM1fV1TQEc}&e#6qH|5Yzx#|9J?zJx*76^hK!3|oNtEm15P z`t7VszChvM(01n7&R?|MPoeFGemTbe>51+loBt`4y>bj?-W>iFNvsMSdX_+LC+UuX z*NsdZ-}604XQp6<-)-y`9(w1y%-kKCj2GN-da8Zi<6UEpbQZAA0%6Ja%P_<(YsTHF zKtSecPz9mC$jEpRVK#L5_OZdoB4bLI_?2W)tGCW4I=hFYO{Njy^kU~UX{WY&m6ey4 z28$*L!A+`W!R&!|(a`p3b~eY>fJV8xnwnIjAy^9Sn2)ymm57wI&a`Md61g_PJa5(p zrg?v^$t}VN1iY|w4)UoDy+Cof>vsL(D}a;Z7q+7|RTzRt{SFfVWy>hWM`x~?9eQPw zCOgabo%RJP+ep_57P$iXUBkl$DVc7FRT-R71R!X=L=1>G9mfLe$g?BWbI>eGYqHUj^f<2}p5 zV=FU+(4(4+cVskK;ysrny)K{AL}w2%tXj5Zap@pwA;E~HckNAUAZ<;`h=7{@L?bvK zyg4PUI?X8L;K{qB)k!-1(kvh<1Al?iK)`nB;1b^)Y}8#K2tY`iiJc`$s8Psm@7kN^ zY$U6_+4`3`kzF%lb2B`1ch}1(J?$voQ!;2*&daz?d{3jEG;+HkJ?rR@rV^5AVR}~Z z1?N0r`)BRT{@m%)_QfC;+>vz+?Ox(h0`WGk8y4G|X4JkAjgZXS*)8Q_ zRiqiYx-EWF<{*%*v(cbMVOsRsywW4|+d?-Pe%Yi1zr_sqq+A^0F@DLJhld76TE3W8 z4^WyL;1S9YWaU73Fn0(2%Y{AH`5pRBPHVrJuahpI7v=DZY=Y%yecQnZLo&*GNuZg@ zWAJD#sC7nU2;P?QWNpcR(Zp2FkJNNJ$!5)9)5NW=M|5z>OV5jYY42Ncvybw6@IH8+ z;5h2+DbgTpZm*MBmSC6hNhQ3!hs~C%$WfX;3cYacX*+Hc9;q zJeP=d5{$&P)56;ig!&v+0N?2X4^x-c}L%3@B|?% z3;FK&7RG#THkv@=%0M5m7244sIW9n-yijDVjr?$6lA(22PWFcBTD<^mn6&`M7u)Ez zKx=L&Mzu4J2;jy)u^yI!+uvFxd+j*og?&K`WshJ4UA*9QpnlqN3hCwb`;Mxv6x!MX zgNr4q`E3sn5AOGob67M1{k5vthuuXOn&1D%%ldC2YW_f4e~zU0=RnNA@n5-ov+MkB z*TMUSbGrsx%4j+j?$xf2EXGQ8OlSCGzgx_q9#_{4m-@4|iTk{%rrd1ssJC%fLRDWk zzva*Bi?XMlF>d6?Mi8V&uSR|uTtQ)%4c>xPfV*s+#J(xZ}bfL`Q&BdD{Uw0-XSM!!l)fpd{(20OCJz+5qID zA`gf6ZzD(A+WuZl{QnKrM1DcFKhx1c{3})UpQhmcKzgDrtAP>=NNR|b`Ab_MxvId) zs-#Y@CLpNoru}Qc_5aaM{fi&{HYO)!(g%{?LJ=*`Vf^XTt)rP^B?+4tvPXLCE}JS+ zXp|l&{ZQWI(DY%kkWtdwr;!{h8U9ftBZR)^*LTB!zR=XBw}MbP>syX{fxdBA^p$We zD6uU|-j$LZGP8ba>D)SbfW+lF&mTB%;2UfKnY@B@`t zsGcY@M*Zu&>Q&n-c@Z1qhtjlMX7a**mCHWKOnoD`Cf}Zddilt)5;K$L5|FPD-Y#z7 zq2y$eGLC(pVW#O>nRLUM zFi%r59pbV^&Mm86ZKlCWEd-|>vhxf-bRGn6*25ZI(qb~AYc07l&AA;fTU82nR*}f-lj}%E0j#f|o=l;0{;(Ci zz#~s(348s-(r&egGE9Y6mlu)GxKLF-ZXq;n(fRvrPNc>ZchtR&%{rky2|!t`5<9E+ z<>_q%8X7f}B6R`9>{A|Rm6UK~;wl1=Ot5|Jf<5WF6)_Hm6aiJ&@ZsJ%oY?zy`tcOm zwD)n|?1((O@34-_lwIMj&-debiZVP5DJ80z0w`|k002y~cz`&r*U6EI&OW9WLuD$^ zlaYk)tn``2ua;!WId@F+_#m=qhodT!vk&mTvcP85h@F=ruEF*ExbvrrS{JCi@;yTo zmYXI`oyTF!m0)T63cG4QGc)HR=>w3NvoeWu$rqp~!CMXE@VWr&*KXxjU6Q*#wme2? zU9#>?k^D3`_%%FMYue1h;q|Nd`r`PH^$2zjS~QRdfuvlhHIEX112hbD(5p6w?~5t-^?zOke1 zIt<4vD~o(Zk&ja|*wkAmY9CU>Dl|&RQ#{$y`81>ikvk9VUvIOV2IJI&NNV$*ms^yI zqMLyXwn*bB983mOs!H6y)W_ifv^+c+nH7w2>M=!;5D1Ae%x@8Rc1lUOrJ!%N@X2O& zcH%B_`Xlz+5elg?GrI^qK!#YILF|M5cEmqDvHqn@`_Iw8{%PmruTEG*8W5$p03zq} zrI$376^;~g@fI8zvsKwGJk=+T9UjCM@fSY&&qxnFUO0A}MpM2Pxfho!(b7yRuZsL$ z)9*l~msyLnBd;&+->i^_ zJIAVa*`6MUq#GFPjk#b&AHuhlwQiJ3cDTXjFkrDACjeWOlaV!iaPGg!Yl8i+yr#bq zgaY5ju~k=}!22g&&~8B>#5RcykdPYnSs|Yzosg&R9v2b1AW;0RF9v^+WM7Z$l=_uq z{}T?0>aSSPpM$xG_|FC4ol6cb1bL!W4+}ArZOD&UzfOlAL>=tJRps=GK4u#ZlSkpN zF3wCyZ=}Cum~`u(C#k8=@HZ+3J3@wNt{|ttKnYGbE!KP!P&yM^E6h(yn=)Sw$;Z^v z8DtK}kR~RJdED9)6b<97*>>)X8+8G})dBxv?U(8-uA?M3rh`qqUrnK1%^L3IJeUFE z+F5U+xDqPUq-r;`vOB(I#vmk7lcj61L!Nxl9N7CTDb^xQ*Ljd9?#WD~KII9+r$*0K zmFth*Ea;Pc;dii5%qW^PL@;@2x6&ymJ2@-Tp-7XKK%o}5t3)A%Npt%u0VRS-ux?Z7 zap-!V|FM(L=z+$qe{mR{7W@Xo$7I3mz3)Ta$fq+$^vL^4+FiM&?#%jgYoOTinns^X zmAb@Qma)ur?v6S#=!4vOMQ?zXw_Ae`5BMBd!5q?~>Ibv4W?6m^J-Wqg2g#m)t}lT1@OZMt*jg@WsYTgD<>aRj`=+udJ9;LxDuG=-}uIAx-6hNHfxXU3zNsHjovft{eB%8`VGGmnwl-#cCY&D_FYUS2=pkpJb^ zd6CU|`{4X+^A*BZc0sHzBxSv?7-Hx5gXr#{yKVMoke9pkn6Rv3g2nZF`HO)KZ?o5C ziJB9qoae$6?q8cx*ps=&HySI=f5YN;1tku5CgEL?pEAvIoR~6~$Re7NZu@Xw`kaed=ON zsJ3)wfMk6JPeo|JQ&4o?M+EJ3ops}*T!MQOX(dmuQQ!#Tf4folC{}*sd(Aj|@^EvU zA<`QKjRk)Ui$QfYU{s$sf0`d)w3(`!$#Ybw4Zk(Mn3-gw$9el>P|D(5@3I<2w@po2 z!=hx|Tv@)y*E}u^XGyRGy3eR*Z*o+*lrvdV{vIaw>0yEkDSULJK`~>0ZFdYIpg4G# zZ0i(ItJH4-^|RXz>plRl;5gnY9=GZH0yNpWT;cft+&cbsD$8vA*Vgf;?E8s- zr8rWDei%>ElG}*gkeMTe+q^b_W84y5F0K zyyg-eAnW2c=g5aVTo_=(svdm5ntq>5EK8Nxwc>AF{z2r@cphg72||Ted|m=-xlYwoAO9e#DC`?K zOj@(D8zzf9G(jwg>DvvN_&YD$-`wc^)=R0rz_<6dvALRV78U1%lo`gDRo6HUl6q-! z<)E51RJ%7eX3yiYA0?Ur6Eap=-p6G%;g4)H*K;qA9iyYO8ov?`%rdDu4v?H$RJfM$ zim-G@6tUPoo%rn-UY1;NjrfaR7hj0K()Nw<0!4v>Sv5OJE-s%4Pyhi&~rs+80yc)D>$-gGE$j z7x3q<8}v*ablwEqk@TmgXzRE@_I^9dPDTX~oEIOZq2GM9iMJC?^0<(D2ase0nz743v)~QxmZ+XCUYT-q=zbLP405luo|5?x$IgK#{u4L zr-xVnJR!60Yz z&N*>q;-#BpfA6h&-g-NGZ|1DoJN1vYn7lS9(Cm%GYaCb+F5q*KyN`C^KZuMQaUoYznyvecR!NjSKQ&x1TdsT70&*C_x;?09{K{M(Dt6>qnS(jyVkZ= zw$`6{YU--%BF<=i)A&Dnzi)Qg5wk@1dRBT1*H@yk(Ml*#vXytTVp5@TLZ)KU3O$UZ z)meu+3KX5-{63M`q0YW=GiV$QKI~{5qV8>j(qHU{!#-WiOb(CQoO@Q{G95ab$kX*2 zuC2hT~@L!kW;jnJebykU>gy-M{nIYOb6G7D|Q0{0Cb}z>`@{kq-(2YP+#5r zp{XWpz5lS^X}d&ptMmcQEpKOEGD&UPNVPn~wU?i+MFNA*2l>-W&LMg4mVDP1o_ve= zLl)1RR5+Xl3f1ww29(GF?dG2Dl2Q7ieX}i|or}uJ`m*K+J|&-x z?@T+lpPo;{sD>)3qLg`2cQCx~JP>Q+f~#Nq8>%awB($O~GO~MTF`oZs-s>560o}#% z$hX!n677R|YlVO_m86?3N3D*<_<$l))q?~^5juhBr;f`UojVg?TlDpMAeCEvJR(q( zI8&y+gF7uNk>w$$I?+3>3-i`D%HtaS+<+z;l{Y_#)F`;o3%<44&*|Nq#3ZwOhV0;iB_ufE(j`+8PH106pI&c04%9&-8Gn_G;4LiyS_x-ze4GC*b+ zH;y~OO9=klNAG7VU;dp}KVNlSZDnoP`j~%7soSFEy{s9{NGE%>qyzz~kYWI`lt%N4 z(Y@#}?wl*siE+`>p?Q^I&m>|To#&9YI#!^^TYOM=^n<8nVDd9%IaY04vNk_USe#5F%%&#q|7p}_1QCGOKdAgvk#Ww2fGuITFdGweA--Z*os4S z-pF%5oO;NSMR|S_W|kS?QpF_DX)V3o;DiShbfO=J?-n7}0^BR$(`x}$F{QBgvuZq5 zOEFvZHj`7I1dlyfziM5+r+MM3jY6|gGUk%CHSd7GZ!hLt78v?K4-5O496yeEH0OTZBio!Ul}-S*H~pgGN>S zAiHsvzHY+qibC6L>#}DuV!@5cCx~Ooqj%U5ecWZBPe87$9h)DI;Z@5v)W+A&fZ)_* z{Lca1e?~_4e-iBZ`#vo*kPVHTR_!^%JMb-FkmAm6LmnFLmjQUk1@N8c&H{J`dBT1l zd2n7*`EDM589WcoFzD$4Krdz9-%AbsSD>N)op;E8+JEI&s@_k&bIxg61-#hICJB~@ z%+6oLD$>n0qPO~U$bp~**RL_}*B{+>%ZKm@?oR4I%$V<>s|fzGr>5w}gKLO| z`OB?3Ph!+{kjj?>XM{-1$$4SL0lGjuJl}NiGvs}yfomu1H`k`9J;4pq!xaj|AR;a$ zcFiUtMq(x3jh%zvFE7T*#h`r5Vn*`v1e1N-nHbqGb6tx(lT_9J@BR^Zeb6oEa!gm} z?1Z^Grp*=-_35|qTNvrLDXu)xYWeH2Eh)}K@~?q>e6lWufu5IGwKeknvs%8ZaO=oM zC6nDRGj{uH`~~ynqTAcr(ult4XWgPDdsb9-r8JV6Lbq$0I664q1@oeto96IV8)_K# zciymdvr;oHcdkbiIpr(2yTa8bQdghcq9feIgkSQMJLl*N@rx4o6^GP|z)6bu9LA-2 zDi?6lr&)-+Re{k^i@jk#13+x=-$oor|E9P3F#je)Ig%qfmJww-vvG?7^Ki+YZ8&hXPAX`GpStWKUC= zjonRXs#e02O7kA2*5b$Q9Xb4U!x>({BWwnTfJCqXK$!LdmpnJX3~}wk-C^fOI}|zl zvh^I-Jms{fNOS`$vh)V-gc)jG0=>k;l!;HoTG+3lQ(xE7sR+&q+Y6!#>k5Zfv=LTH zgGJ=_I=S`waGo7^NKd!sijWcFwmzL1+}q77m1Sf*_K|8+O_E!RaP)5`)TCKYdUUcp z`)}fDD1qYgM&D2^)c|U39p(B4xEpVJbn%wRP?znX)ANAm!AKh*=)h{-dX+n{9LS(@ zL<2i)HO~cQYKul?RC48R$INt^eiOG&$b<|QrQyk$d#vq8y+hjeMLR`)6dDtpz;Q>E z0h2kpFvE&qt%w-qqXUh#J>+2rJCv74+w;gU)*H4P;}`Ftw(i@}uH9D2qU2DLJTyMa z`CbDbfDv4MkF6JGi6QANXV|9CuNJSwEx`z=xJ0#!~pA~&$xLLZ3wh3I}R*z(yoHTdp z*CC)JM;XQkK(vwoIAFG)KTbQQx%qDb5&pgl{9o@a9M_={VBW{iiJhcD56DJbap|k3 zKZpdR+Q3J^AB>vG&woKq!FPe`i1-r;@J{@0&fiOo{Fw*fpZZ$HPhtogP{B(znw{A- ze5VOeiXTK1YD35ameC?C%V89>I}rdZoBm-y2H};@|GVQ?Na*z5ua4iJfXdGQ`1;H1 z_75K1e{V=W&v$xq0JH$J37L)h+&Bp`Rv)P%=Q6I6WU(gTly#It;D*TwyMy1lz=pGEIhe zY_K*hy|(Fs$btJoy^lG;a`=0=8~7WNQ-moYRE!H#u+))Zn}gx2oUnFE$kt5@>!ZuyUeA;;OS`vJT5duAhy1W2>H@4j1AKzwL2EoVCR=VbtyhskWy*_IMJx zJYZUVC(7vKyF{1Ii74T7ZEhmbp=qYAGWoq0^>3Y9e|h1!NJgesFha)(Mc;yUAaTP7`GhWAQf#owuzgwD%nz= z!tTNQkS@Izwp`?->TfEEtdyv%OR-=x_~BEijtZVcTR^tN-!QqvETjGtoMZWwb)-I+*!T9CsdQ)OHbcDS|ybv z4i**D%ya-=n%%>z1al(a&ZI%XXjPh@@znl;Dt|x~a193lRmcITa_bMM;)e_?3x0X2K*qxt>u(<4bEPGIt|dB^4AaVM@?+Y z421W3dLPbGe0Wh>c`(AQE$VTq>?g}EU^LPr!ffOwr1U*S!T#hn)I5k?CAracF69PU zyaOHIJ9K>pzVza*!k{UPNxzyje@oE&Rbs)1p|n=BWSyx&m|e{n0==+{v&dgg6+T2l z`G(!A>rWZh;u}ch2ubnrb}G z&mn8$j=%pM|2w*^qxm-sG<*$1d>)k(VUx53rdYZd)vA-1W`gmdT@jR8hMP&a?-vPr zA#TI0^ksPVOwk8V_7)3E+!|Ch$M?)DcK1o~-0i?sexYTb)k}U&^DUjnrf`*zZWPSvQzB{TpGrtzVAFku+6Gh zHrDHyb{ZHEive_WeDQQEkt&Qik8Zk)1&VIwm0D^0q_mJ;B9UOf^hCBWVRmVJh;LQ$ z!+trd{oJmbSa7K=Z=z0LzeNzW4zAE`g3rE}u};;S_+5#>0#Givr$W?NS^uFft^aVw zRky*PSzxP_%jtdCo%sZ`6rE>4ELrc2qPA}Mc-5Hg0n)h-x9 zoR+u0FCl3Xkse_&P~W}6AbIBuZ?H<%Z_dO4xNqdzuIaxe`lfI5KRp(h&A-nu>2|+qq2cZ8!R{r(0uy zILVEr5|;eJ(pc`8B+2?WpHUS>6u=L9f1rZ&P02;Gj3g9~&#<2KVTG0Q4WH9{A%xCe zP9YWGl=d7kq{xCEplSGkY@(^}HkMO1R<@sH zVAJ_S->0pn?x1a~r_`>;y86$*xumIa(US4f1u9MWae7qigC~rIwXD7hMr9gaa8RZe z&|y0VY=v93btIz`qw~07^l8pAWmV5MZ`ri`lUZVaPU7Z&xuM~&-8QcH&a>51@>#67 z!C9=hYB+Q=XAO*tSsy(mKT$_6b}vB>mD-@|mmd9P>SQuOoa0yOQI?rsXbMj!#k|p8AXLWGpKAy+3Qj+=~-~KZn)8&K!V~HdX>g!xSl}7 zb~SYqOW7_eiSCNH-=Rbnmt`CKiqBy3)@{#Z#mHf}S+d0l%#6IC?{&~xdA^>i`-;4` z+~4Rnr}t9oS!2tbdX?}s-*|xQqu2=zP`=}ec8pB^a4ORIHB3Ew451Ulzo&1Z#Q44C zU;u?BK)ZpqmpCmt@XKP}DXUdK6-JQ5>^-?lS(M4f5c}>y&JijZC7=WEr8RMxZvdJf-bg5*i zzv08%^|y|0sPCt7J;(?s*(n&^YeM*f(Rj!6@$*nnlwq`V3IirDdDcx;?du|kTJ(!V zf+!X?x5I{2%~Z;c{@|6LKl`zac1fesByIWCX6i(MM_$z!eDG`gP`hSYI&v8p{CfwW z17!w)>dy)(@<(@2GyW(0g#X$b^5;LD=k#ow=YrIMknj`?Uz>uPkFuBc!OkWB$QY*+ zcEE=f6Nl?larWnz%I1lcXKZ0HIMf_Ot@(8{Q;p@dO=$`&TM|p^XQcUl*l-<8AMLQ) zPERgQ6-n~vS$VLFgT&3KO(z_K$8mTMzy>Hr@5C-MIFAtvsp1#g6zyMgJBp~_KN%dj z++CA>VI`WTt<)cUC>0|AJxuA==bVnyb5UVJN!2K_{;>};J5eNC`@X2n|CN9^CCfpLxvxd>=L2~ zyA3mwGTXi8^kWm2+m_dF_7g7=WQH0pA6EbZmf3lOC@KTQM1+zI%KlNZ(k(ZqFp;~Q z!eEE-63%aq7q5a8+K%Jzd)A9T^jO@<=NpuJsq{EcX+`Ww59Eav*d~-4^?J!DuA=j{ zC`HXSit%<~YE}M-)k;XR0*+CswUlmGcP89@CX-vbUAq7#@(wNeQ@4jgCH{0$YupwLG~g~q=_Zhlh5nLFwK+mt%25Uf!=Lc zJArv6d!?u6nJ5BGovVkKqir*jOEWEp zQk7T?aKNH~;|oX3?8hH*aijz`PnWfQLN!oT1q}IGZvWIS{XdD#F*a|09>CCse4QnT zwJ%XChPWT+k#SaSXA0eL#=TMti(S5m;-)S{#cz`kA8?R1+M!g5v5{HOR!I0opY^Ya z?lDdW(eqVX`3_C1TgBD}S2WnOI9U_HD)9AzR&XrPZI864s;So_>p+uNtWj%)EKw-9 znx%3=$etMSC5F!e-YJgd!ik4%qTB>jti$4E1B3X8;~DIcSPpzTv<85g@q?M2YgZi4v$8h^p}GlGtJZn@Kpof$K5!AGj0Hl`DdUnVOcjFaX?3dW0W2kvKP&*9Av2;gKW zAOUqKqYD&F(X+nG0K~mpH%8a@RTZ%`u}KbT5=V%*9r@qWd~x`Jo{ua4u_-0CY(E6Y z>sc03``Y2y1(Cie<>lF7{2u?FT_{aQzW#~Vq7E5;-YR-AbCefFybrnCngXcPHcynZ z;m01w*)kkxnbY*IJO50O)PHN*Wb6kKpdR{TfdoBUAbC(gCvFcx510X|3&&xU7UZP37YE< z*z3R0iMKNiN^3XFFuf})n5lp$^j*`{XyzA8S~9q*PbXL?gQ!pNVEz_h5wUA(emV}{ zANQR-csMt(bro3PN2P>}W>H>&d~!@7M#BzaSB7UeJC7u0oaFRBuzWs!JF$ExLYqt| z+(4{11$>y@LJ8CS4YKsoLZJ51@p?v%aCH?#-xn0Uu2D*wpQFBzvzen{8ZFRS%GT-v zQ?=i89XoEc`>mR8W@>O7X^Iye8|Ao(;EraOI9Tgc4p>2`+IfOU$gRc!wZS@$)k?-k zTkF=%OXoHS(zY*HBN8+Q>xJ@4yEJV-ug(ey39Mfppp$wfcx$LIDFQ31zH~ReCs`2ownnGGqp_Pq}j#~@KCEc^Dim?B(46R^4A&m`wR>R zcebjO1$P(P3@efysvD}~bJLip7U_JqzP_X zUn)A)p?NuT*h+f%L4vVaT7A&K;7$m*DmZ&(nz%|S)0Rf$*2)ewu^ z+I8T*v?mdE|BYz`z>HWB;(*OkQRM2-Ts9+|S$<7^7#8EK5$cv?YHdAXTEDzf0ZOP3 z?Bkh3Nsg0^v~92#l^x*0HQzn^4?o8a1(oSLjcIvssrY7ahA4yZ#_H5MAvB^hZy-DdGp&$;1vp_ zTJ%`GI=Dfd2_1rOmm=>_Bxv&W ztghd>aP`v#=EzFt2Le5z5JanMicx~Tq=80Sfb>`_{=D@X0sI=x2P_RHeq*}@!-%!& zx>}Rvx;u_4aY*l6Odfu7bO}t+C%gD!;zN|f=9}l5DPLb*o?QkHajo*{N+8Um8mV9g zltw2y$-8xWof)%DAgq9P?NDOkVl1$$G2xZhyv523y+G1pPjhLc)TJ4(4B^>Y&nd|LA@E-`1^kwVoq~U~<-J z@q@@+?3laxtkfbrAG$6Blv=260dazfT)b8PdkR*RqDjQEu= z^Jj}T$>>x4o-#ZgACr@#_Zu^(nx2@kGT><%Eg5YLaebEV8lltZ#r6F2YD>tu3(V`E z9+ua+%f{NLnq_fZ4bKn+3t3_ z@;F8O4FUUFVY!QtS-}L@Z91+i8%N27wXG69&JPPfFi@3qC0q~XTsMjkyMGW#omdv5 z^w`1l>H7V8d}eB9R`%0gYafGsmCoI9!>UT;hlUVbD{h9Rh~>K_s;;WBvMz3*rlGnn zT!7o_IweaUl}R+UxB>3ni>`a#Hgv25!X;NUSgz{JIGe)cfo)jk9k@PBp`*4xV__sm zzD~mk$jgem*wInjt5I3ARt8?%wkgdyTa$+8s7%)rXt%#NZwA9t0(G(Ka|4k99^X#2 z;c-!#=E8e941}B-;w@4Lrq9(d55u% z%$iq*Bp)`YMq^E}qqp{*aPHU$<2T0;-ArrSIBvF6i1pI+b}pUh zs;5ftDV*tix0_yVO@wJpIQN5yuF)GOiP;Hx+xaVJ zthm<~FWC{}#H6Bm^8oj)eiG-;2a#C!JVr4&rqtFNq=7bpcx@moP&Wr#q!)D-0Vv|@ zCb@YDay|9&19RKWtJZJrjo++m47J2kSU^80UGK=a(OIXi!Z*iiujEesvEh|nMxX73 z>b!~7`?N(bkp58Pct7BTPV`yV=wowhYWEXL5+=>izVf{IE7JC3o);yYk?W1tCsns_ zU|IO7CnZ)iGZ!8^Y*u|ccbzCe=nW=h#`wAe`RN2|WiatYqfhUQ5FOB~ z++#8&j>Wl}lx8?DT!V2GgFwxRM{5EG6Z#a!%bP%O(tX>!V%5~sxbA`19Au~W(Pfsl zEFEc2YFuCW`)uh9W}&o3e~)+}UGKZW>CSH%_;~_!XR2XPO+a-4963YjIsVe})QdvC z^@Pb(-$cmR9rE}Nm%V^tQ7^l@#!v=#9*A|&JH+^9>d-ufwp@HAA8&5Dfjro&$sAwm z{PYiv>i-v%lKsO6=^UlnzVt6R^AljnKdKWX{*^ZKXVN);oD7ESz5O*kmzKlXm)Yz} z57F-1%gWcRH`v)mO5IjQ1LY7(3q#ruk}n)hnzmmdhoM^DQz~)CfB5DQ=D|yq{grSY z&yC${x$Enqz`=V%u%n-*eNKYHB|XXhqT)B_j#=`kWX5`x2@RNn=0rU&!_rdBFN9tPMTpN(fo^RkK9d%#kj=KT9hDoOt5 z42*`0eRYL0RgEe`4m)?HE{>js6dSjnO4ipbQyOY$Uc1`y&c61_$~kkPKBum$;bjcW zNy|xj@=e1A2AcH&B|4xaE6=A+r(0Om30G5+A3+%Fc(MT*-;&Fs4`h17`fhpH{@C0F zefJTO)Qji&HOG$oQ8@B}^9D4X%~wlZjBF0PR^079VlDi+Qw6Jz*%F#F3PsO`U+2jz zE(6s)0#a?xwa9sMC2ee-#I4kAszaKl6Qf5P1*oNw*W%Q`%Vtuih} z2Jcy<%1G{k9{Z$=?zbfJc#3MGC8iy7(qrtCm?aQskOJwX>8~wp+B@ zCByg(lDX-G%Y3o7%rC%C%;(DD8p%=3HszLVV1y&>gQikuw`Ud;c6M7sCF%{6eG@Jj zj`XO#w;4Q-rH)dipJfe_f4>wvvlLkLzS}ncSniy>h#S{K6&M90rO{sYDk)>#GxgAp zGDmAzGijRL{`VR^n_!;h5(O{m65q2xmCX5cc6w7NVWEILtAsuokKI%;D*aMkibiqsdc_Vx7u=3`OWxnMdBO?VL+ z$v8J3`{@DaNRCV~SbI3Fa?K=aj833ODr!_p#}}yD$dq%4>-9KOw+}?9u<-0eobn&p z@uJlmU1i!nQ-i6TJPUaJMjuqhJWRRJBH~BNZ)ot2)}`V5sVnYI&=oAaRnZh%_HsoD z6^`H$OmtSVqZUx{e{lb@22CCdFNPLHM>|^y)0a3*zzMV4U(wuNFSR644qL+)cNS-Q z7Z(&!820Ry4RD9-(4n%>J`3m{xJLflk}?0QG=AW40q~vc&d;9k2WS%Vtg4{r34fH+=&#tAKOMIK+o_jvnlzjT{7i`CJ^`6SU2+kac~o^ja&%oOEIRp7#~nQ5_is;JmhcpXS{8 zB*P}hED0s(>)hkB-fFVKUzfoJ5~ZUXUL62@X)igdDPl@H?B{1Ay>88KxyTEb+{ZWx zOS=UeY&UX;BD)8M+T ze&d6hy}|Ug3~{b&adr}?ICGqMhE@G3+gO9^3XWY>7ut3T)N|@pCcHSKR$e`M<~NYD zmgWlGzyaL_DxTQh6Ge#eLS|*n;h^=Zo#+)iWp~D&uBTsaO037ln;zWrQE#w}c5(%#oHkhGFI&bUx9{UgTTZp|e&;yvdD$1eke^^I{Ex;m=XJHl zWb}xQsz!aDZN#V=R>a#^*H^y*1XOU<+weT0-!9LEkOv&oKg)PB=Blaq*^{Pqp+Gl{ zPM|l%$QA7j6dZ!AWFFXG9*K$`9ahy*AqgH0Fpmm_ziQ@=3X-yP8k!lfVt@41Dznv- zGNYwD?CHG8^Glml{x?*|cJc;>4Y*MSU+1#v>%`x^N^&0R;2veYGv2zm6JVP4TISJ` zL%LUB^Cc;fWU(2;qg3}I0`Wo4p^ta><3jtSsR?%*>g=f}~4?*`6ptdGZc9>S`CdbHV;g?gx6SZ?x@eAy)~!IlultDolL zSD|?SGD{i$JwBfUJ!`w!<(n&{CIB<~@B+mx6zih*P*NUKN2?3QuRZ|2%lsAVhk|b= z-(Pq-(rAqyllLj3)^1+kYdy)S9;`yVaCld_a*w`$9jtNReN!?p!FJ$yc>fKKYN^oz zW8*Q;$`#tj7m(~la8e_Gol@$R!`I8I!udW1p>2rZPB*a^7zu8slu8x?l@&wN3lFxD z9JZ8wK@T-Xl~fU6r^%UYU)Xwm;PT92>mftP@}xb}1h*H_lUrAcNv`CJb5j`{3uG-? zc3oLM*rmAEX7ULo{_#nXMzM|HZPz@5JUtzeXCI?LQO@u2*d7_K0lgWqLSRokAlexJ zckhvYVU}N*bTE^)W9|mXV4>_{ zP;S3Mu?IX0WjQE(t?Gm-y!9xR`eLN~$eq@ndtW*DxizNq{fka0TPWZ2u|sZw!YMIV zRx;5?$~=N}aK*R9Y>dW-Hg&$1BS!Gc8q#U4rl7Ve&51!=agk{$R>Crf^|n`BgYtG| z!`?917OHlj_DMwv4R2Rw7lF1i_cfQlSNix}o7+{nsiS0s_$Z~OTFxa=?HY>>$wS-E zy`0;c8XkwtwrakNi(RTxjXk`Ub;`ZRD5FoEFjzq$&%n4IT`1SAbEPi2lST?cq*>Ci3@1Y2`%+RP84@q^!4p)pG2^Vt#1BAqW5 zN+gwaTIa#{A7G{?doan?QQE^-65gv7dC3VT$d^T6&bMVzVN}D{pL8+b86MR{;#g4_i0;Qs%ZUOK&_&4w?w2#2shzVd9_z0QAc4!X_87M1in5eIOc+U9dBOg@jUbt_0Y<;4XMCTanQA!LiT||lX2+S6IOQ&V=M$3e zaogE>Wy4|-r?Lp$FbK|6N-3r1YDWGNn6z|rIItfk{k_q4$zc<=z3cz>OTA2LWu8;- zb{)=se(RYwh9cRu45DjPIqH-C`PBw_W4UD6+sm9&?)Z`CVPEb(m6eoT@dU^Xi1lkP zcERp7PmT)@)=(ehBM=3{N#Ks4>(0UJe){{1Biv)Ll(SG(5DlDgbWtV^Wg z6sngoHLFK0Dhi9eB<(&k931M<3RD4qu{v)}bh^HbQ_onYK#}E4`m3RgfDhRETFW}) zF|o6ZxwDu#g5IeaRt6>ZDgaS|Ay3VO+1TjP$1Mn{X6uf|(`x48dWX?;y+?{z5ZwH zuGGJc-DMc~6}$Tr9PU4UrNI21=ky;Z=+^_;P+(f@d8@c~6{<0JA3ZC1zA8EK%RI5J z)}ifV<4;qaxoJAJ%&+#b@cU{KDRNO(%&Ah<4z#gfi=##Js=WVKJ9MUE)XbS3+9c2F z-`b&RVtt*B0Xwv%x7bFR!Nv+M2Ph@TMaQ)lY)Pw9>AKAuumoidDY%I_hJ9Bb_a)g5 z?pVaytuKRixMLsKm#5QENXW>+XB(Mw}Jr==i3;x61^5F?d_WG6c1|-oX ziji@mU^8EvOzrOBq$kNxXh3s+bh%56PSHxarM(MzPnvfO!Py_^sIA{|`ycFmc|eS7 z`~MKq!blQY9z;FwP^w2z}gY8Daj#a_i zWS$*HmEQL#L@F^{E}e{2H=R{cA=&eebke_WJKka1h;>68;%!IF3l|saMRN5IWlVAI z-$mQ^vuuG?DtFD!6|~>`BuHm zP9oFj=da8%Jt$1{NF#@>_HUkRwSL`|lUvs%n+IuAMVWAo6O?S{`;%iid3FAm%(49J?I}G5*B<-+b*5?62 zC5j(Jbj^0awULQqk0FEFLkN z42APr&IG(XFtrPZSU}i9_2NaiLyjSxEvFTYmD5DHnVeSjj4IB=P$8p9o9gdY&X;wE zZ-ll&gqjym4Y^kfbDmx(r)7~}*7lz^?FNA=!{oF(5m~kp7Q0!>C8Ck@H%erS z;lbrQ^>C*3lve96rPNMXmf*P8biAAvqZ{)?_AIwRR!3(4_Vd9wInE-FDK)tp zL#ftaWXuwdco=6^qjYGUoF=G8&ARMVKXiR{Lj#6diAA1aK^Fvp54#R#R6EvMy~#ZW{2Yr6u0`>gxG_o@E@KX2Xo@*a6(h3Q&XPMYwoq|bj|3KhY4;|(b&Nte>Kn%S_nCe z9*MNfCFlE;%2egFs~E2q5syG;^}t_PVU#+HjQRM5bBO4y&3Hs|fda94fQWNHx9XX0 z0;6VvP#&gV6#`xcVmr}(<@TBusbH4 z6}VhZJLggEO`7;ID2v5d&iw!otLb~aKs#BoDQEPG5UZ{TZlzWtyIB4D^hGsE?+`k( z=lWF?ZRT1~KikBm)P7AqydF3PAwz;Zf1OS9aLyj2Z>K!cjK0s~U4U869we^sEAwnd z=X9?KpFo2%wQo)H(A*XEa1Ff71F(h~!FA(|W^GD`p39LAg^y8DxR@cQ83BZHG{tGu zzWQc4t%s$YMtRUL3-Rg-Of-~Q0SpZ*rSQn{<)Cs;+oZtBSk58fYK#Ofu2+vJ1Jy>~ zhY>S~Xz5!;dy9v>KWWb{OOt6c_OYNn8zz*5<@6A9seL4H@$&-a|9(P9uD8^f)e9Fz zh(Vnhed!;U8a3lz<}*3yI||VWCyoxFA`moA79$5EBW=mad^nX^!+r`zpacf2?9_VN z0+>v7!5=dTCtL@E=2M{gxPRCj6*`a$0TQdlfc)+}Eihl^h*%}i6Fx&hqO#|qa@rBq zmz(&_O(+3WA1$Ya&KA3XamztSBP*~<`Op8sa!qnsisfRW1Ak$;x|Gfz$m0O)H7IwH zX-W9hfa}2Cv){8Tj5I~V`7LuY24OXGO**3N}Z**M1qiN*l{c?o`6mA^ujV- zN-G!;Te05zft)q~t2wBk{fnsI(;kQ!==_d;(4e-vzs%?g^^9 zFJ2S#1XlePPFGY8`r3Ym9#@5UnVaLE(Mvqz{DHAaerMNTn?#>~Bj-YTx_8*JfvZcL zQZysR#F^Yk+Le=u&oEt^;Cdi7nhh#Mm3+h(g$Kk_q4zO(_EQSO>Rm-!83(W$$4G#$ zD=_==)$OdOdpR1!iJ5NMcWYFVCeA#oTIHS;sFz#2v~p3doVNKU@NtoE3FX%gGuz{x z!)l?u#6>R9<-?EIi=HFk^5A6*xD7fPYr4L}9($b8f`}rC2r6ViDW%Q@Mf)E47}ghi z#?^(NzrZDncFe-Yc09`#HUxT1G`>2f?!pIp)vgmbA-gWvFN_lbWt*&`B1V95pW{ zsnf>|8!_S}a}aMZcZANCdw0!INv~@5mKm~B7n{(O#?2GiH+Xram6!4B&6u<+fOl6l zh^U-)ys;Pc-Q{?2WY4X=B+g`%5M4X0j|` ziIYzCJiBeVXYU%l;=l7QNM?9AN?7k6>gjbvO3V$P2Cbw7dUjk!-6vWS#Qx53T9>N> zgOX_rJY!A9dPSaBG`> z-d@i;D=x@sL3>W!A8>v9Y+FCI*Jg8Vx13@g`=vT(Mbb+5LUZoS;H%^a;=Yg(ad(x! zREPh{%GRsb#AEDVf?uq6QO{aLEMC5Kg0m)b>G|aVnGII`#U;C}d<)7?YZ>x-Vu5nd9WNxW3UTOMAKvQ1Z`g8AH7R4_K`hT9#L z&C;Y!VSf7B9cMJ15*Hkp6A>fpYeQ1{rU9E=kF%O#|+xAIp}`VxI@qBwM%2X z7xJMdTg8MWqV|_X8aGadNa%{owH6nXd{)FZ<~a8~Z+6&9GiR5xl)l#86^{#fz2smHg+O&7DuB+AE4 z{{`t%M0xAtzb6IySt2^wE}*cZ<6j}3WDnUDKTPcuQ!~)+7}(!n!;>u|mYLqUdncnb z9rfC@_=4`v>_TkA;Umj6_6rZ>8#QNTZp3SBi+Hv7V@2pSh%@!Ab~AsJ=vwsH!@=+R z?P1Ilg>RCEKhK*VYO6Nv&gB{E^DZT}?#rWxhQ%Bx9R5P~33(s0kuAP~mF!59x(IQt zn%I#=sjv$Q@O4Ogk?Q6AZpkaY+oa=_?4e7SP0(33!Dv&_T+MK<-rlGgE)MasO~PfS z!Veb0dWO!X3!`E!7!NZc=y@vehD%C>!-6BGQ|;#`&-012Uub0NXQY1Uh%V15aiZgv zkH#k{JTav<=*gS*e1&0s&~mnmWga?DMtq#;IN9;o`_qkk9lH=&`Lxw5RUaAe6O7gu zZPO1O*xS}T(CnftVfaM9NeMgNo-aRlxa0iM=c}SOXsEN+hpisfXXCJ6I&bgYrx7-6 zJAIy2rG=jiUpS)DYr|!3ZR7ZRqt?i2ld&f?i68S!n)+CbDs03mrL@)fUA(;n6OX_D zsjyK_3sIRp`*inbY}S~t>L#RFYW(p^9~TNX_isYhus(qckGaWdk&L@ zx|^@*k&bND*ea)OuUbuBSEKOqvB%6AThD-3?E@RyN5`M>$HMM4=n5_x zv@+$iZiV2cM-}1~!w!G>TCV^na?P!>sUSy)B-{Y^dSLOVNkNVMcRk-;_6a^q?P?Ra z9pYtM__Ev>c(iGV#xi%=j5Mlip=<^Eh`o-DHGMn@9;o7{lgC*+sBm*dI)aXGZQO?@ zj0$|%=9GkYL@+3a0e-rVafK&`l07+!opaS=y!r`B>cMxD@_3(ERo!O8%34!kCtSy_ z^|IrQn@zy(NVEZ$%-X@l-{FtFPhG@#2^>mwiDIkB@H~m}F0qJ-7&sZR`JAI3ZdXiFxKk74CcF-|;0g{@$-(jJeIxH+`lxv?Z8i%rpC zI^&{)Pf|#gnO%~gtT+#C>%}Rh?kDY7@|FrL2D+3$v~$zl=8hOiNVH@A`*kZ-16fal z&v;!rI;Z5f7L^E>)D4ix3duYPP^!5BT-}}U%O-Qf5?BalJTBUzn94F6Y%^?T`4~6~ zFEPRZg?+eVX3c2FGJCXxGrDm@ebql!x~%g z;xWuqIwY9p2k`Sjvf#-ZZ(DZj7s4CD9A!&QyO6LS&qPS@k(;g=cR4tAwC}EQ_w)fk zQD&az;&tGey%c;@DB3R8lc;{Yy6}EI3Nb%ujlj`UNB`Fpt-nnqQJ+`yd}$jNqIUnb z;S$b*%JSxsvL5=p6p}WU|1P?DZ>R~$6#)%U)b?fs$wRdV_H?W6hsf5)(*-s`SVSDX)*|vi;MoygHLsq&WHt?0VH%fv99O zJryOTJn+!pV0LTUUZB;_GYG6{IpehKN}|a#b8hP~wzp$K{~(xkWr*PxPIIjfZ^>bdER?1t%y2aF^>4 zc#L$-b=e-|d`VIyGsWw~szK*i67gL*Eu$VM8qmut(V78?e4xQGn&O+L&)-0gsX;4o znacByTCC}T?$a15@DtW}k4a6ked&okvu0)hg-6nwfv%iMoQ_X0hp3{~ll&ICV~efOP10q_e6;tBcEnuzt#7g<&F0O}x*%lR z)$MLi;G#_P>qc*7t5GjUDrj~iP;5x&{;TuI9Z!f;-jwCAZ~U^FMFtB&p1S94?1a@p zfeyaTB7JCk9v}!S6=+mVD9=B#ql!+!zL7y)Zx~9lfMUK{9391_#n(!#( zz#h$RMbXbTY|M_GX)7V4D_zn2t`O!QBc}y|DE404j$XW`tpx)PB4*~byc)Jq1Fjxf z-Y*l4|YdJUS9<=y`0D z7gl-)&3Z6(&^3b*2ZKiF2V2=vn0?->O3-?hm;lg_AJwFFIM<1Dz1kp#dGwU1v(@uQ zQEoc^GXgJK5X0WBE4xbuQBwG}~nO${VGe4BN4ssVD6OwGlk`c;0S)`UKf9pMtgku+Z-mKX+-BaChKk zCP(Aji=z$Sm}evTQydEpL+Zf9=&!4g@p(OvL z&#~UMH6>Y2DNOv+-aFCv$Yu1SNMm^3+85b}^W1QjToHv_;>at?gh;0)l z@QkuU^hbx^CR9ZF=cO8@97|Fh8OR!#BBvjeEJc8@>8ic5C*#udT;0_XKw3 zXHM*i)-%)4v$vAuv;`#KA+8&+zlUPor+;fo-%Zl8RWof6qmIPPTes1rFLB{C0jtNV zJF5>a2Wx$OHTp7i_Zw%*za4MZgMgt&cB53wY61kIynuT~PS{1mepr$9PKd%TMX33P zFL4MEgTrz>K9fUl&fA%8nfRQTSjZ-i#Z!YISxBeW)vHniY5)M1JJ#1Vd~STAzsMx} zhMjjO!YMk_B-44wp~h481%$V7$(hRR#lBdZS)82qxU8)eJSo-=ZK@X|%z*4FT(Tno z;`Ip$3}WvG9jwSa1`}QdksTM7c9nXU(Pk8&D!eCa4ExZ>hbrCGQijo99!U_gL$zN9 zjyZ-eNrnd&+ZThO3pIUFN~U9kwwyMGBjRTC2Td|b1Y7(&Msq<!d#ovpKe|;68P`h z?6Sn*aA4DEby6g3z3f7gum&L6+h?Sj0MlGrfC=$#gkyGT@!R|7)9xZ+sgcH?W-lJH zc-5`LWm)~+$FG^bGW42#(!ng9JoV|#^%2L;zl%BL-RvzTC)x?Z10Kyf8!~Np2x_GW> zc8aDJYbmS8Y#sXQg{*}eBW)thFRiODncM$d>PR2Wf-Z5NwA%aJT}WdeOpEa!AnGS5 zoAF@f=!JG&?_7MI>u~v6Yoqg6eB|__rORBChI(9xwlCgp-nZ1Ub$oZ8>Y>J^Wdj~8 z7fqPZP++||sH5b5`}4ll`KUg&+Do;Jex{7kB_V9D)zVY@KY10u_iABj*%^)Jwbck| zQ??fCpStP%{#fti!(2p!7mur9W!sp{=iBR-c(QQ~&moEeTId8r(H@=uf|7 z<9+$z*y{9nSMBIX`{1kev1fD&T$}g4--tii7L-@nzTaw08NK`Z(uv6amK`$fg*QEV zzy8?EN;CgP>eR9qyn^wg9(`CM9ye34apsCASX7`(Cdqv8`o%_rGP3K2dwNrDnl2Ow z&4%0Si?AMWNOyJp$CAwn3r4N?J+guStR!jP(>6gU3cG(#GUor z66n$4gB1)FHOh`yLz-s66;~Y8=Grv!gg%$bJMInFar3^rj{63l+pN5f`=5vp{xOoM zypH>yFMIoE>g&2nOWbzW!es>`0`i2$X}_chXWv%;;3hleSZvEo8o#je3@i52%4j9MQtHZd!DYeOaXkKCT!u#(vq8mx*d6 zG1mRwy|Qf* zg!g9#%zX*>gx9?{uKK0(ry^r_wS$9a?O~kKT)D_!PP<8U4l=gX&O2=|$Hpl!rixfQ zbJ2mW+h#nkJmUVuO~dlu%F#thCbQoRD@j%xcK7+(OV77ao;o{|L$`Op4VcS5oZ5D8 z(vGuMAwiZC>UNf>CUP;>cIv>*%O`JfN?X&e9;@)$f=pKE$6D{?-u&vt88q}2b$DjUzPn||vt5#0d*jal2Pfy77Q%P*f{j(O6 zu6@mQeQgClA^to~G2_DnP_La0Rp*qSEk7$2Q`-$g8{uHXN7Yk-w!~rqKYaVRb<2hq zm&LBywW`ECV#{7@ciV0DRY7B0kKCv{+P`f1%Am6oN7qSQB5X7_=^nVFQDFI2bwKnU z zlcmQW`*FB=qsE*|ZLwx0^{b-_7TwM9@(Wm)(4n9Ee!Az7gjHR(&3k;q6x$zbCF^h5 z{NQrl8~6ATg~k`76HhFNA69(7T$icc)s(X^nKvX({pipENs0F7D+e4*Y&pKfYlPF5 z_BCk{GL-9>h4Z{w%@jm1Ebd2T4*!m2f0olx9%EA;WBW6F{Lg%+yaD>(YZAdF8TBA# zoxlGIf3F~SKcCe*R{QjLf2Q($liO0)rEar~;VHt~d%~h-z3OJ`pnkCXzynd~JtyfM zD_>i+*lV3o+;7{%E2gOl!<-TwP9G6Ee7-Nx-gN#E*CVT<=hZH{aV&Bv636L+Z5Tf4 z(-yuuxp?%DQh(hsZiH+=an_5Wu`_mPFng`rS~R5BLk+wBc<7;C+m>w^G`-zOHd{I= zXi}ZKPrx#DamIYdD-P#ZoaTDWJn4I6>41QG>xuId?A7j+1iYEoN2oKcu1_Clqkidg z);PWRxB?d;P5#){jQdDU*P{)w@t&So;s-bWs}FCpx2@jcVddiNe=KHi#0Z`AEFOQG4HTc%ky+lt<(Cls%6h zR`Ad+$@s;IGV`JF&YoM^S*6~!ZdcDu{mUDUKe~bcc8>qSx_>j#@V06{o9ABVI>*K( zzWR7fcl*W!4=*5I=E$3sk(KE#zr;9~46<8(JMFm67QZ25uMON?QW`O~aBO{br_Zlb z=C1hdbxNni@#Vwj^@jVGqv0wVxIfa#W0>8%QBVBfSaoSLx7?$CyEJejJ3Lr>Nc%5a zui5(e2p0Ev@#)-tb>wEDu&!>uq1&-HeW$GcrFM~ISO1|~!`0!L%HV_eWa$ZKMCYMR z_oULenOditwoQ0(^@w)cjZEj$)s9+bo1d)kc;d=@V`{$s`BD9G?VCFe(6yLyM0^_j|qoR=|M{pZ_x`>pfy9u!{orc5nl3p|fv(65_j_ksGQ zlDNoB_p$F*Z@4!meqz6HZ3PHs&QF?men`2c)1D)%;n~WEihGpjj*pFaeZN&vFR{(} zFJ8;^pZpM1Se$e~w=~;_Seh<*m<7?U*VBs8uWu}11&1EVY4zJ67W7>6aqC!i`w1+R z@gflbS*Pu>=^Z8SU(weQ?&9V$i!8sBrPVbnIKwj;x{ZVsP+>d40l9Rm)AdRUk~OBz;kWP(pWD;pEn zaNgY_JezgzlXV287Xjtp5N)S#+K$w7dVB1S9hZzTvrF6kI%cGM;beFdN*d=u47>na zdbbH%wE{oMf{jDPT!V3K)kJ8I>|d@6k#77q$4@vInETx~7RL ztpC^$N!xp;D8pdg>KzSYoi}NM;DTHybe%;n20L_gV~N2+>{Ciw8FUIEarURdXqx}N zCVLVs>RfTO#}(G$K2RKMGrvnmxk!h{cw?6vnP#Y)H8co)lRRluX`3rPi7}aa7}kv6 zfdF+6ei=(@B1Nd45KAXsFi9tN1X^=E3llE%rkv3F_1IIY0ujBZApU(0gWQ4ua$9oR z0T^1xsEjW)9?c;ZLXzmHYTJ19j4b~!<83ype{+UGZSU>dw#zCeV|-^WS+^ZcjoCl= z%0A(e@tk&_o9>cloK}LNpdK%;T^q~Z<_RY@TKMGfjhICJmYfzIUCn8+dj{l_a>)EI z4S5hD@BW9TKxmt7kPbLQCmE2udAw5=iAW@nS%jQD9r(9gPJ3cef{7CD0A$A4l1`jS zu#S0QlrH3~s(=h^5)N`fx`EAnTj;Q$nH+y(Py*WIo4o?R_96 zZCe6~hSRuY4|+Ww^d{QGB8EhCY+p*V2xRPJZ_XI&r&`nVnCBv@INg3(d$&xyW57*0 zEnPr8WPLa5mzZ?uAxyZIPbOUgdLM%{^#L|zzydsn$|B`7CIEiDisJWg093qFbW1_;tLtQBkhYWs~ea8w-Wdvj(?L7D-{J6eOC273e}r2b0sfLN4sk zZ&Vr#{MaFp zFegmWXV|w{sy5WWr*t%T1XQq}*HpnIp-s`(u{C}p&eALJFk1$r0?t$~7aTB|+U)B%ZX zI5gtGm!89$$AHA2r5?&@iTJOm{C+?XkZ>rld!Ykl0nT>)hKMlI0+%_k$qYHI;|a)J z8g-FNKE{Y)=x;Peg^&vVcm%729zU!HQNUYgU{&rQxl?{4tpb2RQYQd91E-o{p*oOf zzhnk^KZH`J*LzEW@V^Q>CQU#jL6E6D30lEhwtyu$4osboxnm`Kh3W`vYs-hmj=vHs z^lvXwE3B<0#t6~q6%@GfNuIZ4kDO*7(=po=4ck>p8JvQtuKfM5i?NzxwjiVNSkr|> z0I16sfP&aQefB%TwZ`CCIeBT3v(OX8^rj2=x~0p~`{AN`Fc@#X;{$`VNCs|?K!_z+ z`$^#a4OYtxE;$LOw*HR1-!+2uuEJ{~0L2(t|NGgwfC_O&MYn;D{S?$25hK&`goCd@ z)H9$gf0gsk2)2;*P5^}~#3f8%*sr64Q5f+%mYt&Z18{|#_+(NkSmga?c6RZ(gb3lB>85@M#yn3%j>Hb=0lG(nOzapCWX#DwMd{($cni~y zOByc1#Nlr#Q@bp@Wj44YqN|{LhSTd^@uwM8s%?#;cA55j0J-Whgp67XD(dhiuBOm!94R#Q}VG4(yF@=S15OHPQV+WId))vS)b+qh!pf1>rmliO0 z@O=hS`N?Dvi4{;YPa|n)_KomTLv8Wp&05VvhQ~eKd9T4g{lKBtDTII9Ca5X(&bs61 zkXy0s@{soVizaYMwX(C|k2o>gK&i6JDKMrYI0UT;D|Uu6pPg$E@{AjnJ2MbeY@Ce` z7=UXm@(rUt(6-rD3}eIWstHhHQ`4ewk#05^vNLmoi#XlA!4VED2VK7$hqLY1#mEyR zj|`)~jTVMEi+F5YgFzn+tUpLC^lUpoSbL^(z{AO0#Gp3PzxF+dFM5Y-;5Dw|c8V4N zGIGpGF%f;mL^k7}dIGUd?e0Ld6l;eQ_IHAHv1d2S8%dg|9nJ$QjND8spk>@lKbZ9< zlzfN0BG`&GP616(D7r_5auH0R8iq?&gHmzN#HD>aScHiR_=1@-_C9JYg@Q}_H>U8n zd!5RD8zbDfO;pN$&^iDMvm>J}4ClTXBPd;xI1!fC523ujEsfbwDB|h|_?Gp=>h*+~ zQaNoMrxJ7)Ly#v^_vMmWJE2fA1amJ#Pol8$is?uiBa1KXOAW*Xcs7T;-UrK_p{PfW zo2OXaXp|J`w#_t=#J5A48H?D+3gxtn3ZlN}y#!Kp?t?nir6o@SV-~0?r&&$wFw!QAI>193 z3~cAXDwlm;B?ngdhD}^d+l9B7f)_91bOI9g`mTE*p-C23sv8XAAV|*M-AcU}5By07 ze<#k_A1b>DIu#K18-G+A3PfBh!oRGibt>K?T4FFCoDEp42`n~UvDj;fVzFOI-bid@ zh9WJDz-hov=3ZRl1wa`oh>@u`^GM?dSaUvlV+{HP0XL~>5G=Gn1V=0RaE#eh^h!4K zl0vinzVD`QeV24>CQTcJ_&$@m3&bYu(;RSsi&+hQMJ&G~aFq5lLwA2`&QWlm$Lbk4 zh2?XB*ucsE=uT2e!9`eCY31V5&pZ|2+?+p>23L^g;6yAZB|C(}CFi2NCI$I5uwSW9 z|1Z?XHP9M2B1FO6^apKz^m9!vP1i+wK5>qA#F-8TF8=V;Jui# z?7+{BS>gp)H#kjt1``}m^sVxZvta)jSv+aq3=pJyAV@b70h78ScPey3Db@V}1P~tl zo|+)1R4Bd=Ax4RWGBg!V|3lyC-|Snpt&(s79`bV#Zku-)8q;_C;raFSWOl0BB9E&S8*!1AIY)9{Cek(qRA)5gr_9|005*zD5eTa3w2FngGml zz7b}KM_UEqmT#yBCLmCIh49lEyvju8i4qs_-4?*5D{z6DLMJJ##^1V#F!0JzzDcyvc4#?5QQT9MoLh> zCRyptLjFF>wqo)ZW`*CBg`ag9lM3Aix@b0mSAIY51&pgzJtNpgC8 zRUfPtGaf2BrO+N=v%gosCRuE$ZUzXiATgUo-U8J8seE%jD>7oZs1y|C#n19RRG~YX zzvzzdw+!p42>l+b2CEuUJMM^a%522H~9xmKXEHvs#saBe2& zH?nJ2HnPz^Hr>dD;Yah2X<_~tB^_ybq;4UPMrUaiwGy7gm8m7Tpl*;3j6;n*dw@*f7c9y<1p zD{^7&?>KlqI^TL?^{(yJ2K!rh)vOlxYQ=hNq<74~EAlI`HmhvxW4ATiPuO-m#eToxOfm^z}Z^^Y^gooupJQO&4z1=6hVuFn=EA7$04F< z5QeltT=GQlU~a=}p=RWVZsJiN2)y_>HJx5P4^7>@lhL@T%9>&N%3sQT%zCW@8{qx0 zyJqczDCR8j$X|5s#$DFcWfix&0QtaaG3Pe zeKYB3MNj|{bx`~3vmt@F0+$S=@{-9y*zFD~hvnH~U5RAt2b@`Xf;o6`D!ai@R8<;7 zud;(9;}BD7A55>`&VP&}O|U2BLQJ7$XaE<^dymJc35snaA}Xi$^pByshQfi!b5tIe z6hj1aL5ClqXGJh_L;ME^>2nI7AWu)q;vplqWcSz(5l-I3So>vcmLwh+aR^ptn@!Ql zLDZObNvij=pv03Ul<5Rvn5=sWlY|qAmBO!W4p|S=OsoIGTpCCF-0s+wAgoa!~ouyt()*Hs|U_7YQQ1NEPukxQ(l}UAaG+ zEvIdTp!9XvQ;)}xrXdhzs8Qrar^ZBLjUPf~qdo%v;$xw@;2GN-jCY2htcuC0|~D{klsr(}hw1> zx4Vv29|$>>*19gGRATTx)^EFTfX4S3fyofOhWds0WxcRxpF{kXh$6)QH4IY;F`<5# z?%u!v)tj8Kcbty81${~z`ipj4U2qzvb6P`Adta$o>DSmc_G|nu9Rsi!8UoOdy8M^m z{co{BSSu^^e!6)%Xpt{zfk=qovAcl|SOCGn)1VA2zC|5W8}dZlft;S?)CV8Cof6)d z&`y;gk08i~b-mKKdhr{qjUIxr+jf1MzJfr^P_$(fITfbzI#!Rmn*c$uzaJ?Mk=>>g zl5t-QLw8*^0Oo)80=_nF*Zs|3!rZ@${CT%VZRR!T&NDEinaFRc-O+Hw*g0k`5EFgSDDCtV6UC@O zeoH7j6yB&v7#y-7f8k|P=(BV{%dVvsO_mwdO`LQxdyu?zg)c@9$;O)3Q@D1Ak!~fi zQ+CUS-gZNrWQ9G}oo4`AEaoFiJfGWHs%SbCny$rf_^{rSb^=*yM$b8mW@U`JNz)uJ zHAv~!n)mwZYV;(#ZYd;jRRts0nLEd>n=vXS%V8bJdHNjwhZu>;(Un(98b=^+8G;P7 zyfG*7)!MNha@sT&tg*B%rlh%sIu`vHxzK~Uxnbzhm5QPfLIo41DAK09y#0`wnOZI3 z&ClRu(BSrMcZe#&qGISBX1cF~;7hfbwgSR*n`fieaH>`$dy0umS5TOBd`r6&wey{* zFoByLWX7dsjQVue>h?Ik=rsaX1_sYTiGg=6f&bGkg^UD`#3-%8&)$PGqp*<$NkVAI zL*Cd7T+;5j`5u*ZDraju+Z+VKn*y~4VnHDe&6F0=;2Egp~M5J84$ zjYWT0-$FbeO&P;^l+j|1ZlLdMIQ*uKYe?^uJ}J2g6=+?PnJo&M6dYCJa%@F7`>LuI zWZ551r5-=T7-joWAvdEN_AL7nCY*Ey66H7%%Z=yCG~rU{X+X-#6|xftC{}JK88mAX z_f3M)s!}oL5DeudAYyBlEmPWmB_;|3@qU!uF|I#}hl7pn;U^kaPMJp9@~Po?uVB`Q z&S^llN~z^rYUYd5nCD|+An7~kOV_|PXl_RB-9b&naxKaz(;QpX44=i@1!HhBF05_MI0@Fr z4m$Q08Bu%daMCz%Hsm;uVqz=S%tomJn8hqjf5nXkUz!e*U7@;Yv5DDN^GdR9h9Vv7 zY)PhW(LE|`Atu^w02NZgFO%)fXjmuG(vh%caYwYtIe|0{Xtp-zE}kK{=FOGeWJR*FW}XqK2Wu%an?t zYNWC1bHY9MxZ(W~!&q~9@orlw{>j$DW~2mM$$?+E6~=y(CoqxG8Fi4H#1Ej%78RM9 zOUoFFA1Px$7{Z=1b#6JN! zu722>ax7ZRmV~38^pLRV@*CFd(Dc=^@c=Nmx`186s(DO}8sbUSN)q8}1C&Q~lAVgA zm=Dn^25F8xK&1%U^iq6QhNVQZ$a&nNXe!*B%v*z#ImqkV{0h>YQaJT4B&hi~5hff{ z&ZL}+;6{(+mQ_&V0G`H}*If8R^|v`=%Etfqs1eq7mFPAT6?)FUV!Xy6_)c+ z)*q{Ze?Q*p`MGlSS7g|rZxRJYy>;^n@|IEt;&A%a+VN!Y)YPyNw5G^WP`0}H!5SYq zt@zq(DXAJ$zC9c8cNkFONP<5WC*?yu!}h-{)cx!WMRb&HeC{>zo3D z+e|0cdx!8_(6?-?9_fpebXYC;+<5)>jWG{*G+NB6nuV@Mb`G6_^cp#O#~Iz6-TMB{ zDXJv3-L^KeWy9gSW!o{rj?LPg#i$dbDdNxl>rw-D3z)J#dJMAn=Uc{ygZ3|z>EShr z37`RQ!)|w#Zd))zf_&Ib4ZsABn`{Oxs*u)v7&3*po?3QNx-v*FHe!D*CLCpreL9PI zfRB7gRue-F@DzB+`WQ?y#9vF>FvgAW5(;cW4c}X0I!sZqFbVdzp{@F?CiH=0$J{^- zSWjpoCP-VVs5oMYE#Pt^r~v-l_80iiiQ_@Gx+GgP!i@s4>aual7;db3TSi{vZp{E9TF6eq7a~NH*oA(+y@>KY zX0v|=46k6H*wJqrO4g%&hgOSCFO6Z9mr2G=eId}-G#krDCSgSr4vzi-Z@E!t- z)h!Pd-AFeM+MCcFeGG<{B_3zkX(W+03f#DSmSUE7;N1a!g;n5I!t&Kyv00Y-gr~AK z&w~%VrWT%)+7W6a-E;4FrHea6isg9+dgNV4M#Nx$;)#^$2{H zs~os63vv2&K4GxJMq)Bnl|T#uBX%faF3LTTSLw<<&kAW|wc@7m{Uel&X_X)gfBi`) zl$`QK7!}Uig^A)zFrw0;B|?Ny;bbxl4bVHDab>?^GO$x;Njj5W#AJT6qgAglr8Yy0 zL(s{qp>+I_uq8}1*psraPppp0=05e5lFa0l7+mBAtC|Mr^&Rq(P7sV1p$TTErzMkG0{vsWBup??-)3y!5EDd{ zYNbLDX;T(RPSh;33DAEZ({B}@*qlk-bgB9>7e!<6ZZslS^LtcDf2AA(iBh4O_bomw|A6<79k2Hy%5 zdItiI&T&L_Kp_mO9p)lpeG)=>=kkCN*|$KXPpW);#ZlY`*cG16!LmLO!0H*Ou$1ac z!OpPaBw99s7z!Un*+P-KBC|hU^*chsH10!9Q#h8*B}bAV;053W`-)~$5&Bd3S7zt_8M0dmVChps}z3o}9!0z7l60pgxVTYh8rl;>B@$1W_X zj9$OrB+|dv*ja7dGJ0a-e#*cH+apclkKOd}>EBaw{BdRP4QfeTjy#6rq`tz|0Rn`M3HPFuoYvqc57@exD2U3FdhXJdS zRX-uCl=VPLSpS5uRx=@ZtWwqkDPjE+!dl7H zABL&QdLSiOe;BYTS@jdLN?8x2g!NAdYb8^E7^W)gfz)X%E?y#%4O4k%rPnc*Dn_*~ z4}4)`NbxXzE5C&IuG28DqyP&c%C=&zunx`k+9lO^(mN~QgRYU2c!tRqXN4ImUMU6+$Lp3d5y0-vGR>Tt!)3f5&czS zY}GqG$FldMf@^pSoBqaKPV2r7B^qdmsEkRBV91QJ+m6d=Sp%5Vq{*tJWi+LsA*am} z$u6Lxz0q=7)U!@6sK=lo7UWBz3cl*@rmg z7%@Xc?Tml*K!fME&EM*mHqqIk6@Ci}`qN}n<+QNgSmug}Sav$PF3Q?Ula$jcSf!$~ zDwMt*sMJt7?R-FR2|`SR|K+Wv{n{R@HAGVLs9wdG_zeve=5R!dSW?@thB0#5p79)V zYY3h2F(LcvQip^cb1-2K6FJS`XYG>tyM?2Zs9yqm68AeS180jFnRpba`@qGb_%hD! z${P_m8C}I!qkYa1VYXhgnGwz_27?cwzsuQZ;C zzaIOhadOEgqeI54S7;~qs_1pcD1;tRf(nkwX(GqD>nE%(!Ubp0v^7`y^fWp-cU;>H z*;z(>_{^nQ6PA@`Kh@b+(RoVV$4>PpO~_vDl)AXgz_f7jH86dC!7pyL#fVd{O}iR6 ztdtyg>D=YW{8tCIbyIHNwi z`CxI(B{M6@viuJeVylbn%tGvhxUiRRA&ot)qwailLLV!B024W)G)3L z8xT;Jb60nF`XJ#IV-?>^L|Cq&$&0~7R%l%9u?Ot2WuHC;cnRl3KK(f9nd(O^n5DwN zptS~aTIxev*(|QYj&%ADw;4*%|2ILebUgm6$2EmVPWrOj?8~IAC+njTnBV}IW@qoN zb3M9U2OS}hr80)#;fv#JddtpG#st;xrLNZ94@(H`xzFQoZJuRjkR3m|(kWyvZQCzt z(Ec?@YdGQ2V4K6a#%M?qikA=el+zmU*Y4%V4vu*F|JZxesHU=QZ8(S+6fn{V3PMm2 zR5}|P1tciai3m3ILhO)UN(BiL5=1&dM6m#clAwTqbkc(Yp+`le?*)>e(#wuu*pg)P ztvcTwchtDAo;r1^>V5Ayb$*BgMzWK=*IaWx^O?__%h@_KuX^~AV*DoU^=h|1opSl1 z2K{zK4_0AD(XBC!n2=?Z2qG0$8FLg1B@;v&WiYD{hjRhzBlI?xlyT<|=I&@*j9gK3 zT2x=gmGmR($N%YWzUhPh{r-sls~*t|uxB=LD=f--O9T!VKpRj3YvJz(!wluDPm~!~~73 zCVRt*vy&%nlkhZyB|U zN^AFq?IsXjU0FN*yX@KAY~it^3)? z1U*Tur@)*0I3l8S-`w1UFf#2SxiBw7=%?)~EtnGYi#M6Uv$1=Ho|p7KV!HWOj9<<% zGt<5O;n|0%pc>uoaudgEaol|#3p||s|A7zw{r2W>yCd==3`Y_N z9H*n-tkI)0pD-C{r|>`REzv0>}3??vn%hyG|L|B(A~?ZT1%W~e!(zj6*me|pVr%0>9x0Z zY#Aj<2-FiL?yV^8b_^bItKJW1sm7#_Hm}Kwr~3>g-IO~R(){4;S{|+-sSOJ`OU%ZN zZJRB@4xVFW!lIDsTRWOxhu)?MJr#J8ao@8vtl~E81o&3P^!iIbJr?jfJcVPZUV0Wp z8YKeW^@ci`cAtV#Ea|S^MLYyJSXZ+NTD0my;JK#iO|Ta&=|LT`#86SAe~R!I>qvK%3@fzGZm8hGBZH>cHHt<~FCAGw^qLE>a zM^)dT<=Z#JCo-0&j4}(JZV4jYCE~cr2oFcGw73YigK&VTsW(&ND>Dk3GI~2QJI5k$wlKJxX z3p;5k+2Gy$a_K1IJX4KzfO}7G&%h}+>ISm&+`tf{I_c`mK5MK<^=gg=vnQ^_SXbg$ zNn=K&dP~~n!XXEf7eCS;-wfaN$P)8*GztfKTX2LE##DYPO;+SyCkeqmFO=_&5Ebq< zVOM^d{b2IiSpGOsu1DQ!^R@fw#y8tkqxG?I%P1)-?M)l3fQuOnmE_yq0(WW6v)O6Jz z&gZO?L)Of)l~gmxOi|WWIF}J4=$0Uo3OCczOCK=%>bvZW?{SNEa1_gX@^ie_L^nAe zf8Cu@6L;+PHr~%`kUI9^s>&`69i=Lr<6U`q${GR!Z2z1&Pw$+M9s$`pEjpogTjIB` zk2QI>%6jF_mFrO3+yBY!{)d{w|GkT$mpehOyM+3uNwn{5n-%|y%Z)rp0I9MaT?)K# z5*eVDm}YUR5Y=^WuDA7hyu=QLf~}5Xyi57qSRAbv1Iy!QY7(8Y0V_JPnftWzO2)?X zWHF>OS%JKMqO(;;$#fm;NYC`4tZ7*xT)MJbJ*vXWbXfhCnI(FG7h=;B>d7I=lFuM~ zU4HloKkQ87?v}q{4pHjo*0|~kLcJ70)52{zK5A4HM{a)}e`9vtWfKiM z&8ClHT%APP)p2VGc;hZz9XdsDV#y_(vyaXde?4@hV?jm0upq{~Q$0}8yy#@)$YAft zURp5!kMfruKMq2r2qpfV-OsC!{ZIiG^Dw|J8!4>K+?1`{VyeqBYR&%qnN+qJA@37@ zz3Y-nqG}9;8_UaPGr`R_>Z)zUJY?F>ixX@P6`J zSDxSRazO6Xzl-B-oBYYfQwFpcQ}(kU>ryc zpZUs`C+_Uh^+*qbh$omMNm`8-ky$=RC-aqc1DN9GhuKHnHmq{>d7yH71J7A26bjR= z{dX4PpD{$NdbsX_OJ*D@K6&U58A z&mhmV^US>Wq$5~#7vge7ZC~V94ED$=Zaw0X6MVc}gnEuhRGLZHT#5C;T zRiOmy`tieKP}auPz<*Otly`9~&gb!4S3K|2*m7TR^F;#`Ssq_H&XEcSsfq(PyvvG- zjv%|RV`9jH>Y0*LV*8Q7riZDJ-E!KAoG{s^T zGPZBK#S1+pIF}|m1VlKOl?U5xPF_7mcoOFNQ?v5%Mum-bw(DNk%LebA)EEB4Jnc95 z>Hl5$1-sn!7sD5r_LzPPWVIXn3UHYwy4K0k+#jn~5|Wy(tuK8f9e$Iv!($gn&bC8- zwYTl`IzdGgYk%^YsDaeY+esJtBt!1LuXWo6Jlo@?e?736nOJ z#^I)~PwtMGIu1XGKo|T;M*API@M}<+_Z_eR%0L%`I_u4xrsHP@Ag%fsMt&r91JU6O z9m7AVGa_}&_j0~(ru$OWpi+{Tj(Jh!*s<cfaPdR*(OuaO6zvL~7m$4SFCZDf1POQPM zZ2z4O5j{XHu;GYKv#P-M3nL!Jhf)h|YVl&k6TP!e9USw#!NyfFG#rmx-#vfRYo08+zJr;oM^@*8Fy&8rgX5hoCPz>p%QguEwiV}?QD@0s zk|<&KTthO3OX0|6+TYJtDCo{-1gKnTWUI<4eRO(~bVgUO{nVuB3dbPaGkiE+h$KUl zg)Ur&#_1NtBPecK-=Q29i5)ch0yEqGaeehP|6QsI`o;x|ptyWt{s~74_#u3Csc>4wP4Zj6`Zi zS=T6q%}DeorESlcRdK!J9`pq5;U7eJ;6Ecxe*PM71(P{#wBEl|K{ZTpTM!p-G3bjfdLV}j}f%Lo<-m$>FFZ9WU&Pd$U^+; z$dbhKa4N$FJCH048QoP)50JiJ5&3r9zBhmEkmH^srT%pG#q2B#zK2iC18y=b5`%E< z7-Y2#@Hs@{cG%gQo_`Auoyfh`Jf_%Sl2+WSOumpGzBIgmMOW_9mGJO)@nA|diD!9{ z83A!8Z#lZL6#C2rFO=Q=;&HaH9L-9F=IMR)AW|d?K;!k~P!bwSsK#fMzi=L5@4O=g+BKOpO>XQ8_30tATBRt4K$xorV zgmd=2x>^iJtQE8v!P)YYO=U%aR(dnCuvG2T?TyNF!C3*)}F1dGVkT4<@<==7s zw&8hS!8DZM-=jBv)f-`E+7SK@u!kr_J8q^|r4rkaTzjZ3N{@%Qvf5m+K~Dr0J&iSh z$C>BM-9o1*8A4s_k$ph0YFBrpCBq~F5wYUK) z-R6egTJTHp#YE#9m7-~lPx=KO1hJj6pPqWyrycglr^eJjYf7}$?~20tWViEMgV1Gv z76?zILUtBk4~=rfZXj>+S<#@(;gDsChVOq5K4IO-wVLD)Td2>pxXBqa12tO#&u0(SIV5Q~g$#)Tb3f>V# zxG@WX*UT_dq;@B{t_!hv;fJc{#0%*7;A)wlw)K@*-8C;>^pd~kd&57BLH~5tOdtsJ zWi$6WnU+K?RO5)+(AQ_wVFzsJBnsWkEjNcs*MM(iM+LetD2*yk?Yp#>Q4M%0UDN$D zO~&yiUR!KEy?BeQHRGRo-PG6n^yhA~|41GFZiK^*b3O*)vh!h$afJWnXj#H620QD| zWY-m*LSEsD6eAaghSd)Dh?p#lMpHKx-xIH(&!PE$(Cc|TAwT3kve}s1Q?C*KaEk?B z>y!}>k`ejn{@+k6?Mb0`rB18cdw6(Ud0V|JN>VwB{PI3!W zy=Htt4Cd&)Fq8J9x6O3OvzU(dxj9F+mi1OTd$KuP?<<40uOb$T-8Lg#6)5C+_}4Kp zRPmpX=l<>SWB*mcCBQc;!m`pQ6GxKBsnhnpxARk{;{CKqS?TWaSC_()GA9@BBn7lc zv8B>opM=>C#G0fX&q>Q#DS8J(H$Viuz(fCu;MZ6>qRh=C)8e&PJ*4B-*Cpmha01zz z(z2ZCIMM4;c*iMhQEPX9lu7kFDH)M+=VuR2DI{nnpANTp0dUX$xpi;=9*U>t5o{W- zDdf{*#9$ZO?#lfP3n4=Ik*W@`v>1;ijp@nfZ^LsZmLiQWx%s@nF5xJ&^{w>T#C>v=J?#?3J8gBf%4D&V-ZUy7 z!GPWz!r4btLlN)?Ae&%aY-&Ftpe|Vis*ZDh1pCz|u1&aD`uytM#}iBL#(X7}!cWtF zXxw1MdwqhZ>w75oZ&+!C%TUaRFUu%}puHF=qBZOYz3m<)n!xvH*zn#n)&_oKIZTtGW0d~ZJe7|Q{2vYeHSQ8Q>4)R)$-Tdbb(53D;^1&(?KY4-DZ%HCc9-loV{u`tD_-fV*e6Lc z(0C(G!d=AcNByHoX#|S{)h{Z)-aj|hO!!Axb8ZZ@(7NS=emz`hFmxQuGOzy%FP z_hB+8{4?k`+XAqCuo$^xp8TV24j9ZsciLLCjZ*dE9mN?9LA%pciLdPNq+(LITZI3AJ&L77m5tZR#qS=n}zxGx~PHD{9Aj~7{Z zD0M>S@UT8{uHZh!@0Hy8(CwgZ@%e0B zqNR3z4D|wDdQ{@!m;4ohc^cgui5w>pgJ~y;^71J2Zd0$WhF-r9Tg=`^qi~Y{oNPtI zM+IZg)GJGcJ?gLNWi!--3E@^<3;TKNmt-h@*&f$EmSj7ohsC8%ELjDc?r-gR^VwLa zs&ZFxtjX($oHsncM_a|fS~dv3A{|IF+v;YOSU+;pDm5NYQ_GASv6%WRm!#iVx?e(C zNo)Ix`$xRPWH!C9i61WcYV}apJjX6JJ<6^0@+%jyGA$am+wt$FHLiF1|trPHL~f!VMvEK&FT7GxZx zA5Z95T_aQ;q`%kVKf&ogQTbm5j$k5iv~!#@^-C!AO=i_zq0L!W}xDH0*c3qx{}u3 zYotKo1klCmKs06?*wx`PI)e>+ezYKqqBAp3g6;0#^PZEwT!G?b{s(LRdw}fyMF)YW zzCoqkB5l#WHVKT>5*K`R{A93B6n^$SGe5L<+lNy2an;9&LZ@eMM;+ARc>gEdr+)NT zSKbLc4~P;DI9dWlRL!$^l#g5sE_N;uZ{2<0IK%Y{sRJjV9(Z->MCDh_YvL789UNO^ zgCWL88c!F7w2kNQp$vmAFyF%-gYW5NJnYNVq4!b)>E4pCVxuJS`NbFXSwHi8Ezb+9O78{p56SN_Y7^_&Z6;88 z^mdJm|Gzeu@vAik^m5-{g6!Z!kmENr!g%2*g$kZ*mzzpqE?*k>@6*Yg6=j;?s&Wfm4ZT{PNVFUS+^`|0Xqr|Br%eexg~!I*HLS8sXc|Af>+Mv+z_t@bOKry9je7z4ifF*xDbG?x{f zpX;r=!$RqB#Hoa^@p`Rumo^Z$zvF?WnbU}X>E9H)=`Op8_S`D6>vW4X+mdnsQoQ}C zS_n3pah#dwTMR~oRIjI<^6Fpdh>&-mN4v8&hy>xVAE5g6j#{gQ>Ava&?b?=V7Qp11k1bJ9k8;xAEb}# z3qXWE0HCDjMuE(*0}4VX7)A+hm)K6E_R>V$Et2k3Ua7UTeMr7Yi)CIL6wLMwcaS>PzeqvDEgVmH z7gLLM6)s_7Xf93O{O;Afa5_}fc&+S)t~Y)E1yLc^y*$@Ng%Mwyjkg2*PpULNd?(oc zu2$Y=ovHlprWu|O-Ty>M1HCv7c>LSsT1n6R*cN0>F53|H%3GUB~szT&AT59gU1&@r`2{=9{zda=(-(yA4D z?G+A6Ds-y%%Uj!Rrp6su$@^7>`;2gPQi`>(Vhr_PQI^RrLEAuBQ7uhF})kVvw z?d!SMsWV@2&M{o4YwJnLg zGnK1B=Rdk`Ze1;-EsUhijH#!tcQ^z&5E5#t6?-{y3<1LVyIW&7XizY1-j^dUJpb-h`q-&)d99(^}|U_O#O4i+Pj?X%&*1N7|DxS7aSB#D|-=z z+Dwl-Lp-6;o*DrzH&G@|>OA`B1w31xTs!Ne`piq|Ah&{gjw{!o*|KT!Dd#XZq0&Gz z^yw$FV9!Cd5Ugn4C$Btt*TXrV)o!U+zDyqrJ$_)5KDJMB<^b5b_rRvT1mIPF!MGSzh#I|1BW#qpLO6+(6$f@ZZ=wA9G#c5tXVzPGITqmwJev*h^eCxHW&JWE8q zY6_(UrShHUjFLnlcq(lS3vX8V*;6RNW|?4IXHj2xPVwAODh;Ga(&{E&GD6yzL5Z5@ zf+wbU>DJYsXKt`B%6F-p!ltmI%hwTW-KA>Za=i14xi(Y8BOuEEE+cW{F()l+Ys;D-o}_yM0^NRv@lh5A0J|Fhc0jti-C2+RVaRM;9ZbTB&3fA49N-(QUUfb>yBS5gI8;Hb#+)IcB0#s%dkx-WBJj2Htl2=F8XAdTh51as5F|VxVQ4u z=EPk$AKf$WdSYZct2TG$OgS%Xzy%%p>EzO76h}<1@iLssD&Rh<^k-^z>I~$o<9MVE zK;u}bp73r9`Sy%Zy$vwI!L{_VQLMiL({u2%ZSSrtS9E0EQLy#*ZuOUb5!BOvk4*Ki z^LAc#1~6n(Ko1mf4l#{D^9kP(ZYpP6CoET!Se>3hdg-*j_w=f^Mv`9VMduEVA+!2p z&yZ&+3Hwl0eVEVrR<`@ufS%7Ey1dTc;pK{<^S$S9=nMP?7l&=k=STz2Y~>mJpc>t4 zjw2Ssx(tPs;bll`2p2}WSm99Vb?@hM3`(ua3N2TnzYN8*q8x-Zfjy4qF`8qBHb#EP zo1%~)a$>@VW+09^V)>ncaNmX4D}waCg79;eR8LX>|KCF1QBc(L4_BDG9PK2@5h_$k z@`Mb-lKD@S!cbDZcpC#7fharNOerr7Cit}Yr*z4D-m@|>q&I%HBID3$467M-fyQdF zXwa)5MaR`!2u&uhh5ezvx@+j`mBMsK@j;#aiuEJnP<4WM^rH!uc*iYyONaVR9!*Dg zo65iEE&p~ph-ydAg9cH)M_|DssTW3p3^FVWOaKy3G@cb^yExiAlqA%0akjBTbvVJz z-Ety>(_mr=$Lz^m!0ydwE_u+WaqxY|7_~QEdV@aIDD1~$?5hBL9432G;cbfzZpao= z4^U1X81o5cIpnV)>U4clRmXwJMw`jI&K5z~`j&m>p?6KvU7f!6I!uQ&59WQ^;4O>g zf6qpI{>*t;zrFngu_x0Y{8xisVB0RcAcJL;{%=B6|F!*+)0Y!9S*T$CRutKS44LC#?Nrt>iW`{$jn*%t1_i8= zSe;yo&tlQD@($9rX|2`F{JQ7z&S%ci@^2L?FD7`cA$APFi7G~#Rt|)%1=jOoyj;{>m$ zGN0ws>ys4Ks%d9nyng6qUL?=bVI5zt|5E#kZwn_@kSWdTJcu6_@^j$`&s)I$d5WC% zTC6~*bAVvhwDc*CP9B`Dzh+gr(`^ZT+xNx&V)9C&TkxQmWa#{4D5pXtPek%d4e7{gGz4l={t(C{(}(8ru3Oma98O~bEo zyD;pF5hj#1UMsf8hA()Ss*dbE_DMA(aN&un25@lT=XswCoNAsEVx=&3)dV{?y zY@6(YaE24#HXn5!2+tAI0aJq5s=$@))>|<}1+zNcD3Ypi?yvmM&@kcGHWqj5Z6%&! zdWw&_mVs6nFw@vv*~tI?k8Jaqx%F|edlOj<{+DNYkdt`GItEsCnGFR_7SD_u4v*0o zqJ-j#j^4`>h*?2;gYfry+tglt(mBzVK?rAT{mhSVSYkwndV4<>XQg3h*k%^H&uv39 z8o!r>{C=T07Ocb!@Mo*869u_B)N|BEHRYHx16yZ%1vaIi@VCJVpl0SC> z^$@V_O(c zmsT=k3iXB0hBjPtWTiW2zEa-R(*8#zPbM8v~QC)XKh&*oGp>F@>J=%_LT*k_Ed0 zp9rl{dZpFy!pYk3b*4P;g3DiP#eSFBV*X~f%cy-LyXJmSV#KLFH*hk=b^X{{vgK%;e9doNVhFAB9SgPq5n8Iam?|S#*+H~FF1Ac;q z?^o!frxxo#Yw8#xaDb`F_nawFYXN)@@9g{g@@mX0V=+O3pY@&OwW2eKDwIa8W3uz| zXYbW+Dvvy3bcO#8@7!qED0H4W+n7BN6%&OkDVr&vNoB+uIbc`xb8O(`v!$8^X5Ord zFS$~6FN?mms-G|vj@LM8Y??m5#)xO3ItI8A4MLbNd3Fxo9v1zi{IT zhHi`o=|-0!5H$__`*-wwb>N~9-UUDYLd>q@zgtPJ1SD7;&ZRne8J zI!X_p&1UZAF08rAxxSPz~(TgAN4r@I;IvV}9~Q3hvE9`VJjCn`C5 zn3oAvZm658Cx(Xy9Or50h;BKptcZ$eo5 zyF`l^s+)Jj)zUsGT^;D7kny6ENN4_?r-pfOvWatNap0RH2DA3zak|<><})-~mbe%8 zqs4?cp&*;Owy@-low^dxBE2tkL5+`~SZB}oleMH;xULHQq2_)$gO1`B6O=8ztTb}( z-9U+-l1tBD3UlHIM5}p3eR3F>=JxcyNPkKcR69MCuf^op*&fLg& z63#w_3=bj+-g+6n*%Oq)@p@?^4>!KI4_uxM*b$Gy)vPX)Mmsg9xw>8GLxN$0=ieDT z55mkPew)1ljYPNvGQ1Cq2s{9Hrbd@}$krvjmHZeF*0Q--P3WDwwqBM>NnSp-#yd>w zN^6=q(+F@KY%B%k(X@3$B@w~Vq$4%NqgGX1s@fik@c4kD;=9!&!) zD7p`CUPh&OfH@>5etYGY2Z&)-RjYICxXGlgu*Dd*W-fxH0^2YRY6gF@r-l+Q(H>ry zGw550dWqPH>%1_+?_NEo!Z(~&ODhirBcozBN^juhYLaQ!N1@B%H*mB^C%_~L3+^+i z`cXlg=lhZX{3`fKt5>miIIcKEQ~dSJbD_%H5~j?H9m)yXA~(FR79>UQtKT!pO2PI5 zw7owN*Y-ht>#l+42O$xT;rA2nJH_ zgPkS45&Fq0(9u*uFax)KRMKD#C-ZrAxZjz~4F?ny;vRYGdZN(Hzx^?PXFq+HjT8HS z_J+U0j7=cb2ZN5;TR<${hdZt0eH&kD+<+enZW>Mv2dOAq0ok(I!)Mv@W&_-9eSzILAFmVj=Y|$7&Py?|p+8tK$!pHxjB6--} zzWI{>KHJlG+R^aj-;`&xNBClIj@S#dF=QdnT>=Six`eV1JA;qS-E>cE61JdA5%EG71dW z8344M)x{)zZ2HM>S~bttpS3Y>!Dt(9r)6jPc?@sCdXDhLe)Y z0-yTg!7{419=@TErFMVw!@#UEnD_7A|ElTTH6ZcFjyf+|ST8oGvQ~GU`-{qFfQxnwH>MuM-MUygREwndiL~9)hrd4jf z*sJCL7=`3e&w~tjj}&VSZ2$aX3EP@@4sK?jBt%M?Cx(X=+tK0mP@YHD;a75C;uhgJ zxJ*UFjZM3o8JX+5i*%(T`D<{g)B!NbDT|FlUUoPDhqjN@yo^GT#NDq>dTd}?M?vHI zRm_>`^XXoEpnu!^no`x(iRSoi7k$y2AM{JVZ3qgi{kCEMiV}!tKD?RQzsNeq%~F5m zLm9-eF2Y!5(_52}UDQTQnBH1u9QO{t+-*lSn?4nV7e#RrEB58O+&teLBp0z`8~Sq(8w^UQO#$TKF_aj8#1ox-I+Ftz5a;~C?Zdz9K-Ge zoM5HjRnPSP7j|1ZiKdySOk>3bgEsJfCBX?$K7(8vBQZtBkalwRGRN!4Sl6&!IWU%C zbAw|*!*6g~b0EBa5R(6 z$fyU8fWlAQ{**Av@N2}1yN;}Zj6>Bu{K~1lCETk_&zY1Xm&6V`w3e6O^R%ohG*?P9 znMuLUQ~=5`0KE4MXCL@^0%;l%#dKPGf8`bmII1C#Qn9u$FY&!dGUl=7EH(Q4u zYvV{}fXhAaqq<{-+{utd%-|4X|6C-_HD+aF9^nNI-JrJ7J;4v0f6@ri?ajR#_4s-< zqD|8Rh6C!AyGHDb%s;-HXf4??ly=Ho7R{o(4W#CSVwPUeH*0X{SGY2*8D|Jg)Z%5U zaO}B1lJ?q~;)1KU!Pgx4HM03ndy#7ppr@+veE%!h4_s*xq z&zp+|k)sl3Ti6>pXShLRceO|`*)_jeNF}Gz1!~5}zU{IL>t&1{Vz!0khYx`h*?8+Z z+bCM*Ju^;9_Mpg+hDS@r zo2i5K4COxT5Gb7mJ|#ppn?BCXY^w0b+MtgPB9 zRAXytDGGV$`t39Yl)C~bPylwu!y^q&0&u;Gqme%35VPK4YP1O5m;{fwroHH0@Ccwr zP*;6Q9MstPO1)IBv2;;Oa9T0W&1K!ZFglipv;OyU`M(lKe(^~EZ@==dF$@;h*MbP$ z#GpCwiypudD$Rq%-C1p)cRx|rpEB-Oa^<8`?b+}Y_Ozb?z`QtPAa`V z(_LqnykpwAdDd6^@=t#*t{8YbtT6KOsey7IIK8LM`OhmQ0i)tU21X$x>d)h(M9Dvo zle+x#IH@P?=t-+@e=g}bx%LY6!#XT<49C`JyNO%!0vA#K2ZBC1zmFq121ZGtD!)9SMOt4_7;d7->KnEk@F!bzLb>Ib?y|UQ)#ir5g zxQ*lsqY)#rumN+P9v)r2j&lSm=`MZV7t9Hy&l=Y!4nMoiFgsrzJi=XfYT#{$jy>K1 zC@T0@-Z7JRJ{9Dz@f~yct||P@GoqKw$hA)>kkK(nj-CtB&3e{tC}bDW1#YGfC3z1- z)hE6YQdz9W2ajwnw$0bu`pWroq>4tq#KBJEvs02LKAOj4GkZSGw)3*jemgQ_)^i_I zUDZOdg&Cl)4m|2&8!$p=2psKhQHSMsPu|0fr(S;IcBX6LWtribblZ$MH4TA*K_wN< zIzG3&y!5aUF*j$!(->wLh;2_mXfs51kZUbLd$UCZxdh!1Q#`SxnKDVY2*Ir#oN?^u zsKADg9N!?KkXKsf3ez8s%uh+?c4kUQhPHOU&XQ^tJB^hjZi1_rQZ!7XML3Z8AnV|K zD+t#x8j2`&RJze(*1Hy!R3R}8p*2iz<&5Kcc3Un#QHt`ndYmjya5la<45+L7K)y(U zPEf+hkQKSVtrl0T7M5B7EjCn1r$TD8ywJ#qS0@n%m!;V?Exi#x_VTmIpi#C|bN0lE zq6*C`X?ym*D0_Z;C#HMqTNVsXBxsf3Y)S_k#1l+!3yvd!WEgs`CkT7bl1nXST2rR4 zHP^rNc^TAQt;X4BIY^DH7Ic(9EtS;l<7H64>fjNZ5wDy8(c|T@LAX~>30q6!!NB(S z!Hm^18&eWBBVCtWDF0#eYF8qS@@zw?i_$_rt*m>?#}PnNJ~Df@}w-&6$_cP z!P^>H54MMW9Ag>5ZI4apX{RO;1t_#RT*I|UbUXJ$?z@*pCj07@w(ro#HENYv%ugd^ z?mtj>{C%i{hn2w*bwQ3Rv3bb@Q=}H`Qsu(WhYVak{H+B9Dk$EsZh$DNC4a!!Q@B4h z*J8N5wkep;55N5czB3B&ztmJ~Q0#1EbvD}r?m|{iv%QeEWmJjFjk%I-xq8xsaWjjE zYYG(gscLoDH1F+mV9Be72OE`5aYH^gFOpsseKN@x>7Cdf%l{I5VQ-ueA;yRq9>W)ec_G`1FHP{2ecL{;px}Oa zvBsntN3Pw7^B6MmVk9`hrl|yjvGH)1cua9%Fg{dfcVVBd-KBLKoc(n*eP3A8!$!3f zmCARbVW9THA=TRCFPZrfKi8xR&@*JP414y@0kWK19{p@$iv z#Y0k3wt6P?FsVbgx3zm&q0~^PdAl>enV0Z>-h&8~uJ%z+l!dkDGa zPc}si3viaf8lZ5a0HlgQl*)1@7bPH1hc$z6hrf4&_Yi@H000JocY+8ykViCKMs>NS zV3^+ty1#TMyj&8M)`$Iiz<36zkNm*D+;HZqARPJIABF;1I{MpA3b3G)iX2e~Wc3O5 z&q?M9l3Fv=%<_Vc;RPr9tTPyXO>N>^SxI0~jS_x%VpYL3j_{)2Zw=JmE-tA`5$o$y zNwAF{)QO5f;kpx$)qv<-xtSY@qs2LG>e6WAfiI{HQ`h|JO7sbv0_Lu$TBlxD-{LepPz|-{x)Tx5Y z&x82*cEWbJdhB2}{b&;7X3nhYW5}XmR>-Ru8$}gV%R6u>3r06zGE`}NCNh)grAM>K za4g(bSyZks9)xQd|Jy{YRp4y6ZX9IFy*$qlQ}JWAvGjYsl+Wm@G3WI=(n`vq9|zOh z!wsRbCU1;`dc7g_?y!ni313_>d-AaI?#4^o3c+bBV894a09yU_k*nBFV~&&`c*5R* z*b7cXx~G{IW2=HLaHFXg9$qo&BkfRqdb`EZsoz2ZnIUuIVYBvl$WM~-~iLylT zZm8jl{8l*RMZlo2k3e0BwgzT%;XF4F+3QHEA%g+Q?-N+=+@EwEcY<^8_KO}1GglmQ zX%5_^k^WBj-Q?%z!>)S5S1A@nHd}O}Ps}D5Dnt&Mms?`S7@!u+1@u`kV6khhh%Vqh zQ@BOCI!xErr_)$ewOk&Bh8c9I5!V@)k%I%b(&JkiwT~v7+?pmymdBN->^Let=JA4O zivf9`EOy2pnrBD^>nX#cwWDHsl5naKlUhF-;k2YISBth6HwJgn{hG1sPE}Z7oB3jMEGI^G-M?{pCg?!!1GAUoR;7`$7{)Z9 zCxH-Pe~`$xOX2?w7q@g zFqe3ghHLZ=VLgx9YcccCqS=YR3i@<6C;OSRkc;0Xq0Uc6r<$~sR8IzxL)#*C_vFKB z4b__=zsQB8VRWOONS}tcAe>;({dlLlwkkDc^i{ftq&WxuWAY$l`{uZS-FF9rcDnxJ z_P%+~zw?j(PiyNxCv0o9!9q{#V`u0f(*|8Xrz_xbuy|fz@yPJU&y1z1adUVJbW3sH=39o;bdevg zH#8-q+^sTTckJ%$!V7ax^QkIXrN$kzKEO&d5 z<^QHki-f6o(7Rvfdbj%fTCCmPoIvzezG33+`6!5&yB6IC%`E_q z!(kbf^KDU=e|S^=4V)0!l^jA{hoFgPp(jaW|;|MWPg%zG~H>%BZ*&)4(ye90jD*bLRZi=m?F+s?1}#Ln2R-g*!F z)@#1-M+EcLip%cN&lf}MsHN^v24lU&6WBU>VbX+Tp108zmYI0O&4-9ex9+q;-=$rz z4{N#x9Le4zgt@bT50$N2Z!*XEf(Q7o^PgEN`ZuQM*9-X{`vJcH0B3M4a0Wa||7t?$ z1@o8f~1!U_re&IvizWl`%^!tF% zANv4q0uW9DK(BKHoKQ8u7ijz?s8!SD9V++1kZ;b9@Q`o!dAgF0* zg;%vs1hI#>2VwIAyRO%3Y+D{uhxoDQnxl zx8${S;HM#{Y2I5|2yHmmE^sowNEaacfVk zUKI`Nz1w-BszW3(=yuNLuUmwsI_7%^KR7I;YQ%c=(uze33Fd&1tSJnP%tbCLZ9n;-VN^^W=Y z7QK5}ZC!Z@1I5HogaMn-POZtL-y`e@z@6cXOzyxki&?7|`o}OE6(DLzVK1sXCiYiU=f`u}gt4BG4X@v&#XybLR z&tnc;RPTH=Q3^`98%`z8$QOs;p08^HT+m$_@A7c`WJC)^Im-X z{{e2Pe*(bYKnwpc05}T(b4@OeZ7quH4MK(I?vc)NdLpHoxKX^K@nU?8sdfeRVnhKm z{?Y+QgVvgOj0Q#ejAK?uo8XE2>*$I8=|BdJ6w-1dtp(0~$weA__ zjHO0D*yJ|kPAc0eA6}t59&k2m_#0*!u$R!iR2W#dvpg8Tmn~E$I`<6|>$16Hdb>hg zQZDSy)R(WPIl(nYa}*BkJy@aX_FRaGZ)}B4yt&fs>t0-F0saJzc3{x_zN>iqNOI&A z-%38Vcdf&R(XG|bw`QfyMETokAHOcFH6t%&wNx;?_|8HMGRX-*aN?W?QxkZ0C{$xg zUK&U|bs9iwVPX?i2JU(8dd5Z1pBKxdV!z%qyQE@V`9!z&Y6yP~;tV*0LNI$!BR-sB!@Y$UeG~T%qifl7j)jkpefqq6sl5y^I?N(LzK% zHwLt3%ikh|nCF7C7la%Sgyt|1Gl%!!8|Ld+8~B-D4xSGN=j#r%L9s`{ogUry?kzPA zI)Upgsy~PI&2c9(b+M$gZ0C1X^km`W(HT2VWA%0O17|J+70fY(^O*aX864maYS_49 z@sw8B8afgmR;{dRd4!d`iscsyomu;F@!Y&;BxRX_##7v4t6%o&Nzt;6c?L4Ki`t|V2Q9IXd>Q|eNA8%w_ zdx}{r^vQPo1(;<4@ZBokFkWN1cyl;-&YiT89S~!%xmkOCPYq>x1h~&!o^+M0k`h%^ z+x--#5YtD>V_&sDzh)+T6Y@KL_0sM#0e-G&Be!*bQAwx%ShM$>Xs=p%QFY;y=+W1o zX?{~GPfMR(DzsF{5{tXCRX}2=KA+cbBn^SRCA`{VT$Eu0uOf_FaVXq_oP-gsd~0;^ zDg%1B(z>b=?<6>g6H|As3RDD5F0EB{OL`GK#gp3~y-YGse=erN`DM}f{J@fU1z5L% zAda}RzF{KCo7yHxdV@s1Knj6PxI)^L^`I#$5Na@$?Q&~F%pQ12rmla)O*&;Iki9fy zbc;q7=ELu&m(gY_qZ1FSttNNk<#MTv2T-wXxWgDX7#3t@X;%xv2MFz!?Wittrj53# zr=idy)8fOsVN>kQ#Djrb;|3k=?oaLP(4BY)o_co*+(aFMDO0oa`(R2>i*Kv5d|ch- z@7Qmmx4T2@lfSsdzD}K^g^xSNweEZ0ZtfpzPPAi1Xm&kJv|qY9Q>z}c6O5}yg*P^% zqLP_FvIuz#lVsTWPUT?18}n|LIG?Sht!~!REHiTM$ceGurr?;HuxR%lKC}&Aq&tBF?(Y?_IJ<&- z4VGcuCN3qdfh(BS_=XgjZe?1MWZk*+&AcTJMa?GZY3c9OXUQ3^I_jn65gzJ%gQ_7W z-Se#~@SCT6!(>SFAvmzk`$4Q}1Mk*f1Jo6(p>aCX&bA(v?s)VCh;EnEI;;o_&Zs9# zcTBK4j5l#ui?p8@%Iiy7q{FeL1qn^#Ne>_NI=|2q`uZN1F$Lgk);)pw6Vmgb7k=I; zh|!OO*BqiC(n(z1_G%@>hNB)p0gfLPX~V#pm`c3Rwl$lyl}*~GfQzIo^HH&1`r1Hr z=ixf18>l~S@KI&Untp(ZwODpi3pC3@A51 zCQk4GFKVc3x*r!NMed}Fob>f|>0ud;*U1)eWO@43(nXK*NEYnE%{wa(99R{ z1!&Rlg05*BS|eV7{1RVeF@E;GSBE1UHrUkQW5()%L==KLx%m4EW`=@--e}cp9pBVqYFREWHqJIp= z=i>rN%o0bgJ+72GF*&yqZ4Ou)^+gvzNcl>|`*5G&yLXpz*gd-eQn{4Gy40O?Zs{!@ zHTFbY){27ItG5mb-BKbofGg_Mn8F&94X-=)bWGeUeEhgH^kw7A?iHc5_Sfwyf+e|? zk|9~kJ?={$Jfev?y+!s%xC9JDAzr|sN5JmS0j#B;&A2yU6=$iylZt-vTq(6=W&;1P zxSX8Ij9&;x57aRLfC_fle6zH?1ir@E)}oW>ICu18g-5bt$=iYB);__f(<-!X@S$W% z8HC8Gp&}p+DkzcU(oQxazcHGLL}kWejeXd1jNT`pPp2<^cX`Q%GI^c*_dGhP(5jO) z4~cNLzsdf6AJ<21KPtHeua-G~!2N~71-El3fFs{q(=bvTs&C=PTjneShqkcYuCS}< zHLbGr=E2rbT0`Nx;JlA72bpD2v>?$9PrfENPN-;2h88XQG^nA@Klc_4V-MBz6AH^F z+#n`B3^=cFV;-PQ0MFAORn_E{we7cB;&2ir3$rfp8T9UJ>vfO5TB!+ zp5<~aK3;(1+pPfth8sF|ob@>?eYXE?jjFW(dv~ZE=-d0k)_lK6VZx#zeZC)Pnxuh2 z16rofhg##<=M2ZQj?RI!MU5i@dY3L9Nw5n%z|Dhw7!;anuj5E+_r|#GD3EELHt`peSglHW5;AFqTGeS|o z#njF(M1k!igV;Fw60HeFVm~vWR#EknCSt}~(6gm}Hn#tT?5zh5h19#jr-iQx76+_v z{?_4)bu7L#N1VX_4w_{e;CGXWLVq6(97bf^Y|V(tWKGa%Vp!)l5g ze9C)7vU=#;%BAv?G`n-#gV!(xt6Fkar{8Biv1`e#AK&e;&M_dW=P=!u4?UeRb_c!t zmxL5QGK=2ejGvIF@YdkjQ9{B$pV!zdXj zm&(fDNiqzwu8v`~B+?FN&2@-ZjoJ~N!aNgdhj?%f$V=6%7<&r6At(c317;e9pi;gSi<3=JNImR7^d%!92ftOXi9PPp{@E3@$ zM@1?Un(9sEs861T)lnr8!R#0Ea1rPyI9U2MrDu6>$7tT@|m1ynbG+;^x_P&Mz6bWJn5PG(nL` zg}@PY5%3DC)?+tRlF$3-v24bt?!%+rxsyvv5A_)9vho){QgZm((8?tZ9M#0cc*|kF zPJu*Lm=amvB4D*AfxVX)K9i`{{8`5U8&bWdL+AagU2an+eGtoYQIT(RVj|z(TSv1z zgvkIEjHV>ubdbt}EkL+t(?~!nzT0?`U=HXEJjagzh7qG&p?(bjl*N{>9AE-E!^EM> zKoQ^PPv1>Irv{sNxXD^PvYreSnIGN(!h?UP30Qs7FG28e1n4rF%fCB)w#MVWKvV!r z*)QPcZktKmw=ygugH;m3OxMMEkOpT#8C;|k}bsB@ZsPlJH zJueBW5Z624t$wRk>$92on@h5q*KdkE!{NQ@>=VGM8M26d-%-7iTu)x@7V`4(FkX(V z({H7IksIS?ssRV5-#cS^GD$dWW~u>0%HU|A;`_i!(F-r!;s^Dm)&Ql^$*{NTwD>wG zo*+1|QNUKRhPbMO6p-s-`&l%8ZSgTtL(R;IwK4q4I0{u3V#HF}hX|qS)OuZ5gFa#F zed22KY10BGF$~p$AvyFmeb;lL7U}yKyfMj!Nmv4} zaqcwM9n@XyJs`>1a9wMk*MK5tLwu%((WfLW?nr>xg}7_k7Bcl>oSg~TFP5%YW_Mg- z=R)D7tzQxB*zYZn`?m&a(tNxNP)#+C?TZDDxIQgV4IQ*(EMV}wYoLO}J6xp|a4vVXx!A+kEzs+tH3Yb?RkTfx?OqS!r?)w%YoJq|Nk^ z+r_3wkPIJ=(6HdI?|X8S_e9YY=IOi4-cdYjXyjlrk3iY!~1?D*8H@i1JIGkXq7^;5y-rG;(ff%s_Ye3XYBFh1Q7Se?~2IZoW;5iAXW-}5T z#MHf*qEz06EaRktQu>Jd+DTDl{jm)J+8N$tY1oFPZM22!f(=h_42nJRrk>WP&Gl4k z-&UOoIt+f9?D_JWS!Zj0c>h*uzLFok zS%?Axbetk%69UxCa`khLLP5@y;+c@?cL!dTO4gW8@&`uNo?EF}d)U)%ca#vX`MA! zm!?{6zvi2-9Toz`1~d?&2@%W=j{hkf6pYW)r1&M zjt>JlY7~KX-%=)rCrM z_ek5z3);9h8!Fit*0T!0yLlwnOz^-#9_R66m*tvC3 zjb-%$4{EX8c+Fs@u@$pF$y%=gmFldF1%W~Bq@23rBxxCENu8+Li=#nGqID~W$|^6V zb={beEws?X1plBU|L@Yx;eQp44Fc0eS_?P0S( zs#M*|yR6MYm+kG`&)le){KWT_j{|h9;vj4Z-9UaLGBrjoz}L|u%4^>GOjAPSs#Rf$ z?zG0G8Iff9cfB5GW-t1u?dBVaJHrR7&FzM25^xKrulZ-A&vB*h^}2jck=0KkYpBJz zFnwH<(V=GRxn_k$#zclLY@u^fC92v?ownh=n#z7lH!TLP1)4YtBFYy~4dENcZK>|` zEvma6-XmYgv$W9pU0@y8fx`zED5+??j>ciEywl;A+p-N4ZN!&yDb$-s?%~1))Y(0k zD;4v+Qsg-2F6IV0*3%UGd%aB+IZ8>;1Ml)3UkGC-OH4El!~x?ETQ$G4=8raCY!6vG z{OhIt*AVL8DTNANGpCIA^DF>>9Y?>eiQc%DLgT^GLxVGc&HXnEF`_byP z9G7%1%{=nhZeJ>~n>gW9CLZBj57g$7z=4AR>z6SoBvl+1XKlEaW$l?u(&ABk4?F8MqU+B;no^f5pIXTNbT#dP>+lO|@VS4yfco#*Nb`zK|M`E(_pKcyMrY@*J^0@rCj7j^{>0ILb&daZ-tQlj zSK)a=HMgM8i8hyUz?DMo8)n)waB&E}^GD9>TX4;~h++WPr>i+~pnZq12c3_+Brw96 ze=uMoD-*uWeCyD7J6JohjEN65RqRlKVo;U#p;FtxJR-GNpSt*2!-LbZ_XCWjqH-}9 z+=m?)4hSX#M#?Lmv=|M6Iie=9z0y` zox5v@VP5uV$HTyknrk-3v1L)SCZ;}0(=nq)r5V9`?jPbbdoOKy zJy9ZQ-?tT30+#nAZ~oeoCRqWFOw_~ zc<~E13aDxKx#y3hxW1-YT=sj#yIpT8#@Ufp(b!|Or|8(!2<_$*fA>)L-VtR>34UHG zo{>T*Bh*ZSPJ83G*hO6673DW_FL2U2NLM=W@MmkRKc?uMx~ z(sK)(X;oEU6ms*9p2N(YnD3?$Ffn;@5Bt6dU7QFojGo3s&75iToF{@|=e$F^=tYK? z@`pI4VfQ>EChg%5>@_>rkIuG6B)0d@^cc}2>YTmGrD_wb&x*Q?G)qs}98(6l8E=nh?*KZSP5`>$0M6#R(wT(; zY)XB+u<-r(eFeeoi{7b6eCoQJURYl8q-Ehpoqr z(hE$gHC$r!sQZFVc(o4f`_`-c__~P%&FgmU{+=t=t0c~@h)UjqWmRzdz@(of_k-ub z$~G|xw01WX>%U6HJ|>Fe+hIsIG^X|{o;KD%_{$~Xh~JH zRDzR3-IL?|%#_rPP5QrCnfGtuR2O(CCx7(tnP4TbSm#}VCsXW)WY098YYoynMfKJx z3-YVq5#wTbWZpF@BNS}@*8Xin)=yxw7r`Q;3YROpx=#{K zRYW)QjZK*gTGD;SY&#-89VSOibykbHV_D`Tpx7?fvO z1_TVU+LPA&@@cz&YgF=$Q*f$n$r6z9rMcx@+^|ph*dxwW?068D#7oAzgT$K!vpb2M z1iZgZK;X^Due+-;lQPNAoJ z`7ZwGk|dL%8Y?Oz6}N!O_z7RwdIj%k6FZx1vm*aClO9gTp8O<*mqHxiyTL~dK>kde zIeht8%8>}Gn7MAHLwD~SaC&h!I*t$3rBh8abM`M=%+$;bx~1N2*`Bht29T^O{*1E09`bOB@~AD^IUcTa~mMO#rdSZ#EPz}hu z4>v+nCm>fU8mJEvi8=(fGc>!7g7i^(CxIeU9AqIylMp!qdIjvW4ene}lrj_sGML*w zfNk3YvYkM$-g1ivRLx%*Kc^S}9QDr`_;UvSxidihzGGWQzFYFGrk=FvrErsZ7s@~bQ)ak<`#eB1e=D#p;Hz;hU$**=pw+I|BD1MP{=%bE}`i( zFba@KytFq}2AJR`0*Jn@G=>qt*!#Wa4~DRNK*50n_<^HIi(EtmXI#OC~txN)0m7D+YttFa3rIlFGCunzd_O zqXbGe&h4Xkf=bJlJQ4=P)Ukq=Ahr}5U&UFLP7C;5po3hKj3!V!%p4#+Zka# z3xXGh#{d8qDkN6K@Qz}x4B`^Ci;SY&e>Gb@{&#;Ckvh1%&c{1^UV#T z{7;KloZs)c{^31b-7i=C&npx61}N$8g5UmfC$OWUmt1iXfiv=)vAvVrt6jx-F_K4d zdHIusIwmDb?c8H_QX_BmtLw{sGW2VE>Xr5{eA3=`_fm&lpK1t{w~~@+=f#zE$h#Tr zo;7meZd&R6^ihZ4PQz09*zoZ+r`>~H&!22r{zbv4jGM}ht6P=j=I7q{ky3~; z0t#33P_Uber75M6&u}6w#JN>yhtir(9pgg}J-Dm&i5Gzv%obW+F0(*-sVvPfYS+8b z@}(lGp?=i`O1t;7}I8@jQ5F;d#2fU2PYz6C+ zz28T)9$!qTTccmv?nkBJYERoy;>mKwQ;AEDbGSDHv~%NQa&uWU^(raaX#b%PDL#jp zwO6rUFuWVolk~nIE)za_loQE41a~l~R|fd29Q~hGvm3q?cq=JYJ-OtxGImf`*ei>e zFaN$zpiqufuJ_lzz@sT7#(&XDCuXUKX~hcW{0={-k4?iuG=L>)=#7_~`8RI$8)m7T8ZTAtiK1hL!9|Yj8%0)Xf?NEx z4P=gm(wc0-9k%eH5TPV;+-J@jt)s}4*o0C`ENbX-2oAoO{?SgWa)L7C7}Dua z8Tm3Jv8GA6BNJ7&6SPAg>Q!d8e=1UIy!Trn#~+PO0Xq*a2YWocvN{87EeBq#FXX1E z#+r_-QOKc0D+$7vNvAX6)q_pk5xKDaUpCw@#`|P@ia9XeU9sjqlUVgtL7xw~1uFf; z2R44gJWL1#M8Bk~-8am~+3(W1iYDBb1RD{ErR9x7=zG7b+zbN;(F9AT4YDKF6^Ow` zfpTu)UF5UZg$y^*teK9YjxRh$r*}gHGI=og@lo; zyk)UTdC(Ix+gbx!Ek}+YIL@Kq&|XT3M%18G!8K#WRAePvuUO&gLyy^k>n!Cwz)Rdz zPRQ_}nT3$Uk>wyz+5<7J;6);04KG)VlJ>%5S;K|BZ_~3lF@phdcB8fz$URoE?q;x~ zZg_Fb&BPs8lbjLiiR(Yn_=VR=VEd?jKxHci?1kNSznC*bUlNJ{CFVUMb|UyJbFwyH zC@`qLCw9Tf@DiMKA5DDM`+Lt1oc4Qvd@$h*-}K$zJW5D}xj-_VPy`zDU1!K$#(NNH zXPaaJ#3;SXjWw*TRxI|=0mZVL8wq_=Uh?iCWTn91_p$jURwbd^`20c#41#Vyy;mA% z4~*2pAVv;>eK0;A&pt$_c^`x)ITtM0^sr)09mIfb#TdAiJwSkMn7ELE@jd1KA-9hQ zIF#Poza}=we$IG5Vb8UYt29S@T0e$-{Quvif;8WMfQ|Ye&vE^c3vC{-{;QAkp@(qn zzyKtT_j5-@VbD>rnQ#yL;leNYbA111bP-?-G;f1rfcPDtqpu0GIes(@Z}t566#LQ! z)(Jo!@S`Fijyds^1OBvr5Q1-r{>9LLolpO-+~4#pI1Zek4}!SIt-QCbXeq8<9>YWD z|0YoA@N6C(5&tYml7@X(V{1_)pz)WeE&vYgY()%|^`qV?e5fOyeLe$8yUv{9-6GUt zc@8edAY)Maw;q{)#s(D!>5m%>at)npfVhDEl75_U3{>zVkiv%;K%f}?GZFWgz)OU# z;AX={4)URAZ~P!P^%o3cQ2cn$oDpKj%Ox9=w0XCy7Y2IS$RdYfDyu?w5sWvtUF%r# zuGXXr&B-3{IqH+)*?kK1M7}wg`R4WqPuO=ZZx06X%YB1|(NS^yi&TJ~SR;d)gqfwy2z3h!!A-am5x< zM{rB3C1GDHYM)hodDyep_ndZF9OvgZv0~#WNu; ztcOBL)RFn~CPEjhzdpn14}br%{-l8G7I6Ynlwb%m;gE_C#$ z#=;LcW@3ARkx^hBlYD97^jg?CPSoqqC`B4k1^`#_%z+S7TV6hNEWa;?I|*ClkfeCorjoELi+a&O^HXw-!EIL}OLx&| znZ(ysEc^mE?|D<^aQTfRI?XMnEW_2tPYvr^6@a%b9g)3_is$m~gNeR160++3h-;g^rfaCrriB{p^Hs!~BY z6qOcbPApq$<=| zvdqevZY6JdcFlOWDhB!C^)k;5^~0j@`5Az}MKJw~z5)B6U~-=MM@=_>1&aQzrTyQd z{E4al0<1{?@4!@0W&k2z1vncIVs8aQA1cF62r(`FCTZ8X*V%W>YE0!hixYkP^Ft{M z{Ouof7FTUxwU4cL7$a*)I(jAlbmY3DJuj|td&KErg%HU3;Cw6SyH9h9CjZBL8n{&R zV3RqC@3p@&z!OL}ZWbZqU?Tf=ovjf!^RU(Ftg-Staiylqff!f&lcMb4p>6|VpW=xP z{imihZt=@lkKeK#RZJ zQje*jEH*aOijp0(Uf)+*iSG9CKQkj&n3?i%{}xltAHix9Q6}i@ahc9%zf7~kxn*W_`Ko7>sWlN3*fQ}?O1!?aL;xZn3(cC_(b9)YVG^y*=tY6bO`kb*= z8RIrT1pdoy@K-CsuDGJ&ad~NV!P6&2RYfTZIh&j9-4ui3x~Dn}byIdq*=? zt_uq*+gF28P%i1Hv?MNAMN_3oW3VGVTqiI{KUv1B&2!db;_lpXxWC3BWJBUiwGrCnT#vF+uF&IRuX_&Vz4>mFqnNRsP3H(0O~&G+9~T@(T=ahCB?s0^kqtOf+aL?J{K zUxurN7L2yPHFMXEtTwjJM)A~$S1PzuKzh2kWF#?Q4FU$|g{tglOFYx+bEwA7IZwo^<(PiMW z+5rJ}ciA9b0lvPC9dOB5FWq^}MU7(N2?NeK287kYnIp*#ysr zhxWZN4Z+phATs^v;wd$t@qLEP}*n!uHa!vzcB2uRb+PuWKL@B~&*ajoWzaVD!c}TZ}*b!t1hOKJI0lJQBKg0xNAJ$ygaZ^4a{ zv$JIm3Le^Yi5uh@EB1Bsm~||C8ip~ zwr#*iRBN;wd$VU)AmySf4{J2g1mD`ybfZ9%7v~Ko1hekq<|mf+Do@H1H|@JPe7=T( zIafUIqw=BWfl1E{BKg2((bx=?4OA|D&9!x?Bte=!GE100543+_EO^1-#wgvNJ!oOU!zkU>@Sa!!Cwt*TZ z&*T`^OhywVtEB_4T~evs%I;|dZHIMAn@@hUIz`J5&&%~)ry%uWuJuNOx$t}bX?`XS zk()zBHP{w=irW!6@Mihpk!%OMuc7^@Y;J3KwXxHsgyDgu@2T)$dSKjn8N&P69)vvG(LNTLan*q3D6#zy9}NCy)5mL`rhQ3MTQO@nGL$r_5Xo9kVcUzV#8 z|FHe;p0*4DOmIFUA2u1{ravxvYy)b3zhRzXWe815Kb7BY4Xsu|wzCEAzJ0TOVZn;< zxbo^13Gdv4TTT`_y*{#EhQ^3}#rH`bSi|aOHTZt_Sruq1s48E-P6}Mmr8^pL`Kl@l z*c3S-+7&*}KCP(yWNx}IZi`7f9yaCYl0LUJ&k2&;>_tL{DN8FQRn@K~9lScS+K&I( zimhrw{Ezvmtc_UEY4D2fARbi&V?&?wKsGmyp9@&RR8bf}w@9c4hva~4t6B&z1wUMi z>H&ijI@{tKX7$eoCk(uM{OnpPaDIS+S_w#hzB{D+Iq2U#rwbY~5GCH6p-EREfn1AM z$bFQ;lBs-B0_QWVnlrT>w&<)ZElR8vNZhfz(7dB@X-Ha|)HD8WV-Q>XYyyneIo=da zb&Ky;FpqhCXv!F$L0ZDmPuY5)+H7$ei>K@>95axb0W7w%lvORp4VVgFOt{5o9#F^lU;meeIQ2Wr^a006{}fl zCJ970f2SKqFs8LQ9`l*j%N(#!^-9qcGh#x6sq*%rR~G&&(M(<+Wz5-K(Mfd$m4PY&e3erTRi3 zD-}Cmv4;>)HCa=>?#v$-2Nqjdb6U{oR9hxL@5Mh8Lh>8hb^B4Uo(^#p`J?5&kqBjTqF4%6Ld%pTmxe&M88^ z2%eYx9#|gr!-GeM`p)AQ8$8#iUnI+SUKgcEAjd!F9qr`$4m^fWbnLazY-T)E2yL)hP-R zRrc`5OE}Y=>+KT0r#8{eO1_y2-7_a|ciOYrkQ zAF&I)rL*_g^y;#P5b4|8r?BIkCZZhD?`6848hoH}U|mXgP^A1`4Cao6zfR?CoI5kPL8$b=#Y!p6Fu%y&}V z8}B<#xJ;dO@kX^c;;&YR4jt~+{qVGN+NyMKhqH}VBqju78a*!wiU;nK2xth+|P-t~hx!izEwUADPekjGuOP+*#-_HlU^CpLplM zdPmQZAiv#9tTU{Yg}#RG{Y}sc>@2CkSacL(wA;X1wIpNCjz~nRl^e^kkEu~I>Y6sO z;cnyXDf7#6XA*H}6@vxJJQ;mpHA^0Phm{Wk1ypggFBzA4kJ|o~&w^X^agwQv{3L8aDc(+LY z7d|_c7BM6*#AL#Hc2%b%ffUu`mC?Dh>*aRIK%jBblLWfM%X|E867MgsRIy~t`akqZ z;Umf3=kGr$OLh#pydus+Bf>n|<3vQ)~B~UYOx3u&#I}xul$QkAH3f9L&K!Bgh&rW42zVi&pwD@t5h?NOdEmz*Ak2TO>j)YrF`|S>CNXr z*Df==cJ=-DinWCI@*yYDw*>ZK3ap7kq^BcUx4QuXd zjL%SgAFn0pv-A6k`?DsFi@x`gzhh|oZ1tydpL=aWL%*Yl<6nt|{)uSoPrva$qVf5A zU&Nm<>wgo>;=BKF#37|3+p~OSR62c3{&f-LndT{Pe$(@GrdM)aR;H77TEy-2UWYZ! z##1jxq}SG8`7|!GRUMcERRBVfh(jc&Y{2(Uu#WK1jRp-;+?#Zxam9otth5u{#42m}hLQ7f(c`34oLZ)q zw2$f*N8i(-E$mj(F|M7UAilnFy=qBugn~y%ocdlq)>Hl8-WUs0VsgDr6+rAQ85nT5 z^9xqO!{}~A$$jr~ZC>}yM;_&`ll+osCvqQ-p3}PS=+ti`4?NAwD}A$vYeJOyf#E4* zpcP*8yzpxZ$J5SgxlxB#@__#C8*BXh zb_^0iRqH+kr%dD(o1b2JEvq6ErN2V2$QKeExT8`2N;Y|7o%NE2ata zU3h^2x;Yf&8ftcfY6K#@f?5j+A|AY~6P$4+Va`y!6)}8Pnu4udnWEd-8cj4d$b|C* zDm!SO+M0ZQ^m7VN4D_DLiT?>K?$bc}Tmv!U3161*((&x3CSdNK3{n?s39woZPXOqc zvq zT+!s`Mg54#`+J=cu%AFJ1!&P0x<0?&MjGV2*$1h#&9&r!izMM3uM5z4Bs17ciw0+~ z7GCpTd5wjQ?UCJcuEqV{?j$4M@S#`OwuL*%3T%MjlPtEIeEP|=!SVS9d zV#JFFMmD;v?eWVOg{^8YtjuEBEP<1pt4rIz*kEDEtRUFRuDc|77bkH|khf#t`{eU! zM>3;d)niWap{5YqWfERfb{Uk1(n~(w&xg(ScR#|r;li-lfqGmbI~cJwB61^c%!pL0 zz(x7P9n8V5N-M2(yVdT>S~)Iyl)FsUBf}Ge7X!53-$Dp;t*CJB9E1QlXy^&FhboTu zYQ1F5OFhkuiPFbib`g3~`J~_ts!3Yilc%C&00-w<6KD31Zd83*@Wk($kH2WpX8P*G zrFOmi-@CPfq$&EDfh*kN-FYs-^2hu+);ds~aCjQg1O^)#K@ zA^n@SZNJ#Xe`_KC^|Jou{{=4teagD%%c;i*@pLTSli1hMXluF%27Ss7`IznKOk8(9 zZ8XtvuL1DNgqC2HMS>C%ul6U(zlOr3T!k8hr+HQ zpxMh&pzKENN?U-qr&EYsIhCu)IL-jsv@T7vlVXr%o^{!y($IJ7!S=CFnF^Nv^KUPT zK3u1;u?jn(12hJzh2OX+v%Zazi7EsWeKNj5uW$rFv+YvfCi>vKz(gM#pd?_81rvQP z%YunM8ZBU=Py9C`3l2>5S)>9c`k1GE>mk(vLIO+hV4}}j12EAi#0(t!rk&sTDg-dm z=P*$km=``7NMql)0i*%g`?=t#rpwFt6-_RW!1p|F9P_zK#>jeXS_I;B& z_~RMByN(hgUojX0o9_>JXsT3IvKK#G6j!bW53m{06xYlT^aa$Vh%Oh3G26?vpG^)q zkT_~cE_hK}dgUfbee@iKrvCPA!OH@hlkpEt@2@q2EMi2mE-QUo2L?K zw)c(?RTCa!-09zX&qwuBc+Ts)nHhen2d5r+-z_OEiC@J|-mv;ZQ`3)CSLGgv&$pn< z3GC4pL`zE;j06RP!@EwDBGdwE6N}LK`2SFo*xz#>zr(})1HU5qhm^G#zk_dJ z2sP>ftQ!{n1dUFa5x!y`7&c5wpvT3)x4(8hVqWj}{|bH#_{`bB_1>@pZpQ?u0yF_i zlY!`9sXvB#^Ev7D)8`4bOK5xA zsFK3XDcgDDV+OoAOxXxpEklme30Eq%=e5qJ9pag8wMe?1z0R9=ntC)@O{N_@7w|$Q zwRB0!QWzVCE)=eZAJf?MD7w6G-EG9A+F}7Ly5VvrhSM^S9BfasZ7Vvzz6qneF+I%4 zN`3cwo1=D^Ghz=N(675dKup6?+4cMlfYQ5@)mu_OWLoxYa?Fwcj29ebtZZ}ZQS!=S z(mb@KmFQeDi*rcNzJ71^EDz=6z>AIgcTQRoT3-DIF^DxD-^OaOyNQ4mfz}c649+FU z+g`=SOQ1__M^`_JtuF0;6rlQnF}u>-_pY9nZO`3`)K=#Y)a3^sw@=+y3v}pN_xcdM zu&VU(1O~b{becA?Qry&%xtK_9zijr|v6HthahXG_mOd#-GfI|WC(S#()o9tp^JD0Tv9fY z#m=agu=TUL40Hq{PE{TcO=B+8@(EVhgZ*st_YT~D4v7A04QA1J=m81Q8E~&t>5uOg z!u!#=i@Bs;HJ9!o`zZWlmD=W@z*GfNiM>Czffhbv&Z3zaX7?>TP4YcyW{!=eSpHR5YEv}Hfv z3B39HTYIpAc=2WlD6+q>7i!1?GkUHQ7_uP?Y3PG0fVwxvz7rn*)k;J3&zJXLXQZL( z3LKHvA^NC^6P~{KWnPT_CT$h#b35PJ8C4)$&JNgI<2fi-*b^Zk;r?mwgz& zZX&KbX|WT#BPSF)-2$=cCZEZDHPJ6yX8J_uaar-x$W3FpIa!zM%`55@ z!g8-%%)T>AdZU)QCE+;Y=awAy`JH_94UlPZ}Ema?3u2Ix{7HLZB8yb*I3L) zJ~wU7yM)THUee31^lN%{Qr7NnpYaLVqjP=NlQc`*P}A?bcfZ+#|5Dv6{$&H75Y6|RhGj@O;^ihyF zoeg5_u*1TQ;O_&}hW>d7mcz&g_))2Ns+iX}hUKkR^GfBA`KIOIx9|nxD6pK?=J-xmie(1R8kh?BUpZm#HZO(ClSJEOZGC{7b?_Ysj z;Ifp+pBtE;GP$(He;BU&c`x%IFA1b>RBH?=n(Pt2zV!yCw%tx@Ee6&f<#Hnk_z|;ld zz^3-`dMmc_%Ff?Wu*?2np!b-T$4Sj_c*F?yA?Yj6#|@7I$J6yQm^<%+aS<1UTG)ty z*pi1qI1Ao`>m_k}Ml*R0H8#S9JnPzar|KN1t&$@smUsO6`H~tum~x==Y-Msn9QEp< zKJ1)LXN=27bjmxnv+>+_g45InXc6Nujv32kjNa!p?({QIrL6CHS6I}2`0F? zq^LeKEuLgedkb1(*EQ;*`$XP$+e?0j|C2E9R>{HdM9JcJ_djFb3GM`TSn|zJH^Jcm zb-l2&Z+$)&mrtN=hZc3DS?zdJobbpaIK$Tgxw;_6KPy4=Txjn;>Dd^l2>%t6Jwp2o zJaV`AQR7MyLJ-PSe`b7+%@b76MT3Fh8?Apbo&OQN;C-R$4bVGHM!tzCEo<}IOmr(_jV+@6Yw_T|Q9$#?3g(mXn$kHs~!k$_>m;lp@-NnOQ$}2KA;AbQ2}wFbH&$uxc<=t$qsjE+hna8`q3(vrnUWlvxGFQB`#IGk@!de@*-x5E% z1!ON;$y;4vYvFtWAJBpyjXQRlGB04Puqq z-_*YdX6lp~+23>1HAYLw8KT>$PwxiTobt7y)DyoV<3CT_{k1;?Ka-dwjwTBBQ4Wa! z3)7o%5tP$G!!l#OEvF@hE?s`mhij)W5Ey0OHOejg+~_w9}T2->HJB=9Jml2QGDehY8NyoPubkC!G**Rj-%ksYV1fHdfi*qm25!3 zYni`nZvPp2U|pY^(!sLAcqV`T;ioat*f)XVFg{3JCFLQTTv6!`Fz$fPKTL{a%{E&h zH9nRpF!pwd%w#L&+ZPVg7g?xmH5}2?kRmg{PU^zp0_;<4EV51W)tR{7?Xks)S2;b!Sf5uvxvq0Q{k*$g8o@PZ}@nagleC zAh!ED(m>swx0MxnPQ=9K8Hmlcyf;soIKSM6qduqR{Y(5hMH^1+P0Y`@HP^ZdB!rs(3&0dm zjs1bZ{s~xFgUfh$mjUxL=l$3gIPGlSXi;{w(3t1N`TVB1@+QxfBmW2@t+-q+bAQR$ z73Y1@pC-#190VG-q%wOFSJ$*{#0t!*FIh!VH3a53+oWckY0Y-Lq*s0f&f|_ptTA)u zxF40fAFxDa=TSymaklx(?&O4&!0SFUsh2Zs{DbDT?vQNwQQ}kQ|J}tuzirrL4C+H* z4}unXtpj*?-*|6?#d?Yg%zR7jg$j2^Jsf7q-|Fbzn1>orHh(E5Eu<{2Agp{+7QFQy z`FJMjE%EH*dmeT>xjLT-% zvs8a46#dslNxy(4!XHA2V$96HSWal&011nPaPi@0?rfZj4{x|%iPsVtG>%15cBY7S zXK_F>!)M=z(pPi?fuXDY8voHw2~-1L(lUCQr^ufp%Cp#nHsKXU#|OGC6v;q%tVqw7 z-!b|K>8o;#hkPgm9{RJ&;D(v*YsRTF($6QD+b{56C`G5^HGX5h{?b5eObHCoR|G|% z*UMA{W0^Z(KkM^grcMK-_EAFyZod26rMunzhpqZqxx?+-7T@leEL-_0$b2Vz*0SO- zAA4y4Fd8ARyf7+Gl#EIWf$CZ%{s)+hXGhhH&!Vh?AM8Btmt-zd!`dk|{!}|NB^Y2B6uGXA~S&PB!k-8+# zbb{d-L$-j&BSruAJy`+1C; z(+-dFR*L4k^q3_-z3)QXe%xDsh5n-@;mSW`oF!*KS}gnwft{b6C{%dQFAlFIC>D=& zi)aMz>@gr`p;mN%C@lmRH8ZiP7^XFdH0y&;&~47z_RrStN%_PT8nXsHYJ z){$(dUf*Q*sk>QseI3p)BUA2uoSxtB4(f0?0^{uCj}@kYMQ={w3^Q-6ON{DcOjtzb_;23X7VH|<|5is$B?L`@&Nc9M>R#YJ$62g+FHft<^ z4MuA-d^b#UClC2*NJWm+qZVO1Zl3A0@!;moA7Q#()nzpM9y^t%Wwi+VIrBG{@E;u* z0}9?Q5E~|b+1d>%w7jy44ROf0ky?&SMXViR`AsknDHohY{7tx zKXh!IXBy`w>NQjhwz6U$6c{r>BtQceRq$n@}>TFYx2Hg0u+YV zH%XuwnD?z$%Fmri7cUlLB!T(@0@oSBH`TBff>IvD&V<0ZJY0mR@Cxf5^XWyw5#@v| zvooxDl^e;I#$FN?l66n*)6;4QmX{j%;(gUGqU2fgLszT$Sl;1)3t-RyKYjM!wS#|Y z75_EA`a_EF*L(PXw_OAL?k%TP(;m@!P} zJ0lBkx%pXaMl!jsc6xbei*_j?+-#j}EKOOsg?HeYEzOb z&IVJiLkYnLXKmD+$nozs^@VYkAP_MUbv(UB&!k$@NTI0fkE$kwtY-` z5fGH{NtNB34^O@Oc_P)8-BKePnX-dtT_4{%|48Sj4Cg@lyz%4pElg4(xnmd5)w)4s zbbOwH=pQX*{>It%k^a zenj+OAWDO5<>A@(4aB*x$$cDBJxzK949RPWn8L4H+7&2wrtRUUcHN)mz90o?Z0k-u z^XaQEqN(& z2flvr>G3w>;1B~a2keOEi%h?S2akRl_aBVk7?Hs*Xp;C6^$#u6ujb*mOZdH?VME|r z4yle5N1SiA3eMu-!_4e?c&`1GAn!neG+C>fY5u8#Yd?Ru@YYR#qh|eT4d)FFc}970 z*Xn3%EdDp$ z&x3*kj6OYTxF@UrKm~(2b>pv*|39qcf_s;R;-mlt)!Y~2pT^H7Ov)V%B{ZG+eZ}`* zaFi0@ZN);>IY73FBRE09#`3jAx5{1l3}zlb0M2|!o!c`d+G(3g8c3|FP~I3=Vd$DR zGnMT67OD-o)iITD#wXX`E>o~L(`0-RcB%?@5=_pfqa|N3vf|DgHbr;@v4%$m-q*C? zh04fNj=y08YaY4++3yTH#tC(^>xW+Z9n$D|$ZXtGbbXOG%a0W4{Vaaf!eena+nhl2 z1!zE1#-TD056s%M792%t`R92c_pFV$=d7tT7SSzM`6@qZ#MhA`&czKhX4#w%@LJ+f z>)L+Ip?yl=efzRb=OIdfUh3PFXA)nC;vVt$pZ9-_-~2;SPaIDBX8})%?odKkI4U<+ zlz9R*f~uDXLKjt3_S*nXd zJqe|P2A?!Qydk`Ye~ZMv)%BEuz9}*gya?={A$SW7SxFNr6@0~P>-kp6D@BN~$-epS zZ>ViV)zK-s6lE=ar_XIgH)Q1tfBjorpIxU6ElYJW`0qM1rX6ba)A1ic_G>mE9fwJx z4G_H3Zgoq8#p3+y1GF)l^LHAv5IX-V5`M?IHv8jH2vOIe17tOAUzR?G@1~d4+Gs2yob z+Kq_|nXDfQeTj3ZN^Tpw%U8kiOq&5`y<*wWR$r>_HrbRm(lPXK`#`cK?qk0=XNYOV z0tR16%UA_`8gPGs^xS??Hq~TU4j)me@nK99!+BwI%MRkIT$wWRgXRb~!$tex>6x7V z*9(6-dxj@-^8NA0CCS^s(x~A3O!;NVbHN#EJ8L1smq=6nJVKuj`tRZg9Mbwtj@&`^ z*LDxDp{!!t&vUtImTF+tF;LPmx<7GUopw^jlXz!0$#=h(*Yv0KkLh>>U5UC>sDNB! zlVVviLf4069G6&2RPydOf=w<5s$C#ulTX+6<16f}ZMxRH?P|Mx>UP4V1LQ&XwkDd| z`l~qpesE+BL9e7x5}+YBQ5;2w_reBkZl$gx@9HBff>u(0uT?BIJ*IoZg}aQ7u|b_z zvAH#O9NIkJu&uP1W^qAYr|wD2w1{`2_{((Ijfm)ALAsm$knpIo0&Z{b+uly_Zi#sQ zko=hsreEK+y6vN3u1f72m-J(1__kZ6Bez!`>6qHBzU-#IF@Cwv2~98=XA^1;uee;P zLhh?)$f25KKPQ+(qhPt_UNsu)VS4nMcVNb}9FkJYyqN4ze0w-yKc=n4KS1f)j;M{R z1Fkh=P96Abp!Hwqxh2{V+iD<}IUN&7nnPBsK{a`Ku|n$*jTFxKT(a%9&T?z6d+^>H ziVGuYw?6QigH9WU?yR4;s}T#%;1sZnvjK_?2RR6KI-~gfr;=M!Rz z%+#nu19{PbJ>tdkvD4`-y$B_}pgFB-y zT$H?Gv`tU{5<_=899K`0$$UIX;cu#|!RdgCy!PtD8k}s{OBO&t<2a zxkjmxmV;lgz?=s9@e`aDC=C7pvOF4{!l{bE2cQ~PsyNyQjfnV zh2aeWE=ECcBNRa|f~r7C%nh;T=2e3${;_gBOf8YPc}wN7&h7H{AV2RkLa|Tj&5T6V zli5c8sR@D7-CFkfh zd9mk=e_l_{u69YIo>pEC=EU1R2_hDKoZe8f$l=;GOTsp9acmbk43z-(*N?!yfzATk z@QB(7Iyb#AHE(lF?SeCR^FleehBAYcb-#=)d zUS7hAr}>YW66i*}qJC2*TRwC~;{_3ixW+6kkJ+BVt6wFM7nw_;}+WLbJt>TK8YVSPpS#HC_7=OJ`&^Cp z%J2U4!)%%5j4YOi?_~^_P}`h1h6RkmAIPgeg~V;;9}tzA?d1p#Tp1WmTOdkD7x^4A zNISdRrSpw&J90Yc*2C2sZ8jh-`%by}Hzri!;~9Vk&3=UY?7Y2rs0fMTfI$7&>%PqB-JG49GvedomSP*2AaGWb1?$$(s-N6% z|C%Z|c=3&Utv{45JS+NXD#Z55*fDASHa#;>ugFaYvg`CY5w8_C%yGm>V$o%wQWnOB z+9DB9D3YHZ1-aek^PtUN4s^3hqL)qTAtadLWsmxO0DveO&~TpMCy!L>g7*Ci!Di?) z5eNrt1-McQ7}Yy+JR+(--U|xb$umF|Ko!is#)=k^fa!(Dsjuk5o>qyknECBc^^b%P zwG>B0cu17GZU(8^Gbb_ytlo2|`joLVv%iP0(9ON3=@aWqTmq7nBGp-!nl9>fKN|4X z@_I3Ed)|}FSEU`6N{+C=6IG|tVaM@B(hx^8ihl?>IOPAvEQAO8B==67%766PzD6Wl zm(-X%d`5X_MdKW;rPyK611AD+wxuDldxkc)Ul3HpJlgnp9QcC5ub7(p;>xtGtJfv7 zxYl*9y&f?R>oleNYT}jGAcI1P=mY;)lwZ2ozc}{Q9w?q)B=#W5Hzfmeb$h>VK+nOV85Snwf+A>9ffrKfq(Lq?Oy-lr)#(`tA)}(cW=;+m(%7 z`!){jiBZ)Y+qTUOgQ>#EOJOX;f!hCr4hir3Sl>pnZO;4Fv^&>0i#)=$uE8VA8*BZ~ z=)N^zPc_pUaZH^_!Oa9IB#|pO*A^FXNV3fD>;>n^)VX|tGovI~dAsW4)j`!4o@#A3 z&EW0NocR*G86WB~!C^>v5WB10F4Uov|sO$xx*rCp42u)dqSzj?5@b9O9lE**s`T{+X z5PslS%uppz3i-besP?A^G@b++AVwfiz|E{>Ed=e?{i0ODm%M+;XT~++Vj-DIZHhFK zG(ZoB7MZE#2rWS2uA{CrD_nS*-7>!~)}L70zpO1+E@vvPy}@PE98RTLv^LrEaz^WO z?cMQfv8iG(v693L;Bw3c75=acWaL}c(+ja5I3DX;`o7v;RTRsa9>c>!?fgMkASV=g z>%R7NB;2k;Qd*~{eADu={acf*<~g}JeRDj?Mw#877=FW~VimU5*ELf^^D ztEI;rQfj3Huu00^*F{-7c`6a*c6S@IKP1;*evN&R)_>r~hYwz)=H9x1k$ft)xHFxO zW3y0s&?*#WXmBaeeJeUQTd;|IJf)7hD8D@r>XWTb!?z!QLO%r8{}j6&W;KrZxo_TZ z^^MO0>A)7Hoz;l~b3KE;=T~Y_AJFW+^&4a5^S8dEo#K!e$h^2w>!3#}jVxrPb>mw_ zzON`b&<+rw)JbbRQ2KD*5m`R2lRQ)r+UXK1j zhd1B7C0Ic=cvTYVqf*CEHM0ry*S>c1P&*qT)T-vMw~lwKuaL(2J*rxY4-l>dE^{g9 zT}VB6;u!Ky5F}KFb#IiM-`Byhu4!2!baEQ(XZSqM?=&NxWjEKJWM6-NB5SFlyv$q) z0bXDMRi`juTafMEFhg(1Qvs7nin@laPL#M9#0Y~t;Kr&BTbZ{3MyjNmSaTD%*k-eZ zB!$W@&W;mLG3X_RC+W2x-&7PA7-K~2_2c^@Ch%sp6sHqY>h@I{Qf)>w#gPO)PkS9s z0eR7Mo_OHI_N423E0SKv)yE#Yip&06K^W=^u?K00{>da@atDxg=LPFQ#dTHwQ_&L( zW5oY5X;!FLAr0`yze(w0E zTvFbrTmw>`{_?W&^0H*xl%kCs)Phom`}S)YaX}t>Qat|7c#z~wJJGvu`T+0%P`5=JG^qvVmVn4Xpa6q;d+BZdK`BG4%| z1O|uEkTvbXwY&(rP+^>ZjYn|+%yJ7&zhDzq0sfNlbIKF56+A=Nr}2V1yc8+OHiq)# zA;rd2jV0f3@Qm+RfOn2jX--i92D%NS#P@{v-_=Ms?pLZ2%RR5?1hZm{QcZ|Jcp$K1 z>(-tkJC)1z8v1 z;SQobiFEkF$IUrCqcq;9_b$Aw(P^#?)Aa*o{XW=^n-`YceMkS8395FMBUn;cXxyJ< z{_~JeuEXRH638;~$$ET;flze|DEiV7P;WivF))s%?*p09=i(yK|FSeY3cUgiFn|#d zHUU*bl_2!5)}u&dQ`8WXd*om)X{>iZZvn(ng0-$aAVcij#D^}*1xO;#bET{I@wkNZ zYPDmUEtGnlq@#~s#se~7R6Px35EZDKggbb74Yb09`T%busq)K zaZo1sSC`;puWY|ODf{hka^h&ffF6~nDrd0o34wWTd~`a1%wAs~w1PQBnwRwUk*WK~ z?4=4H6COyQ;RN&{ME~#|VdkuRK77^U2>#6nubIB*JhQw`XAN&YdCOn!UF5AJA+oNA zocft=kyo6K$0cv_2wL5=^XeAOP{OkoR32FLFwTp3A8 z;t%=G|5b2H3KLQ?+9KFV@e*C7Zlaiikw{+SszkCZx1~lDV9b&wM?Tkj5^Y$;cS*aJ z?b9Q1tGr`M*2c4-b4`}RrI547Vj8*uSLGEr|9~7-?OQnqs&=iEZanX?oCXi@GvGN% z9yq`;37r_$8ru+jBF$d7f+deKht6BqI!?@L+acS0WSKyRqc>@Uq zQ2syN4T*GI%QynU^lu|WeZgZMrnab#H?}af>e{p`?S@j}9?qF-XOB{sQI7C3ek2vb z^su6osCiX}TEnWBWBGp4GHLHR&w8`*gK_|HYJ*p}g0`~Y130E@9aIF3NoH?Vny{o0 zU>2u4R(qhDA!DK8dg#U3JBU_Ie)H#z+mqf`GJZNcU1Fd;adXbbI~Zh+xT*zi#1@_R z0&rLhD8LOB>LLuF4`f>v2zrq^zw^PdUw}of9->bq*hjH|h^aJT%7;7=N{IivJR00v z02S*x_RYx3n_)Fz`AJpF_hiqvdOH9%y z>A#Qcm#+Sj+8OYzVsn&)Eb|g7_|?$iow(QUx5Dx{{&&F66*20Mbx?u?w#a9@x2G3Q z$M*MLxdww)vCb6|eO7RDdGitP1FPS>NvYWSA|`L%x=?D@f5Lg=hmL?xY!6k>XK=2g zTD}mxXT0}%O^m<5NO**|ZLKf>c8n?-jIu}6cp-I(7xaWSetw>_T*-d*!KaRzZGYtK z`^rD~f$xU-E3v#hB={?4DGmjD>L&4PjPNHOlfg%q-21nDlTVi18g`u7P<)5jJ6yUw zFY}O_5A*KL2R=QrXOG? z|07<(*wz+ej_}FFZ4@sTm4y-QC@H zCh)tvTbCt-#Ky)%AJ}H;+#5YtLVF+`%L4^=sV5*`m}@2jpXz8J7J#S)U#DtP$GEJp zC4}=8vl#7q9CIXBsE_!pd9If_mQ~%dE~A9Ep+Z8f!-<^HmJ_oT$Ipn|R{X85Nn4D( zsHZP5bLA2>kk56@_m|842lP%K;F-jlIdlDYe|F*FV$8PY95P39a{OE>1UAOaT`xaP zG#eEd8m`lhCu4@Z#V(BGm_4$*MVR5pp92h0rzn9ib@syn33Twr_w3LF$U9*Hd#6dG zm>}3h3&YRMd1Q%yXZlyB;V%t^n6KvKM&pFXAgkp`ZuvMA`vkeLFazO+ybWYAmdob z)`1H9T?L{XRFe4t#ffg287n&XzTh5(jTw)I4QlT}qoVg525kN*w$`sHK%8?oiV!hb`Apa^3Gq0-Hw z*eqwL`T*pEms!|VoZ`N9r|1f08CTD!aYND_H<=>3qVW9Fdy2(gNIb2YnVxVn5=V~O zMKXfsBYt(t23fvL_er@|y_(lTgO;`Be`p$hh#@ql{K9jYeT4@?5d{x^L1+2gtG?$z zF{zK|;P2IdFSM`AzL^}5P2R0mgm%K%=<&yEJnA&m&a#U;=db%bHdx4*Wn4bLK>~Ry z#;@MV!JG$Pcs55=QK=7A8v`ks2v#wvMrQD7C464sqp=xwOH5QgN2td|FbCVL2B0YF z(usn*Y9F+0a&+Q1No5L^gP+CeJp=(q8@?cMY%3hhZxN-T%7Ef5VYZjVZHb^O3k|W+ z<>vRcd${(=MkI|Iv*!8nqhI+4*=eF`sfGuNJoThH73ysI5->BW;#ZpxPeV2nQE9*- zkYfl4C1ZC41oEDKY8SoGAi`jX=Ma(pu{kJ*LPBsE4@+O=ah~MpS*Hf)l1~)Bx_?Xi z$la$A<3*EK&JN`}SrT4Aw08(dWq$f?%Lt%G{Euh!s18uIHN*xDJNY1f0kprpHXY7+H(a@_EWhAXre>PW<2p8e z2w*`eAAl6&Ab$d~s<(%_$h)TyZN?V_jTZLNE)a1+BS}*P^^cVv(rN=Hb<($sOM4F9 zcsKde(Z}s2v%LzE$@>6JJuO*WD({zr{y(jZ|L2zKd@nes{i4cB1E|^vw64-4p<-h7 z$L6==5hq{84UEsR|B=tMt{aay*VT!KQ-r47ZzW{aN7pv$Wma1Y^hP(7T#C~v1>>e$ zp@QHs8*olLL}?b_oYEyIwzx~wMHCmfJ}~7`o{ys|v&9|GiXPJ&yx}s;G@o?b z?0MhQ{j$j(Ta6*63}$BecMfVI5!pmXr7Ki~O0J-(w1`$kn#;5ocv73wmQ+4%8=DkZ z*|`ot)gvoXoOsw8*BJkS^=`gr6l?+>AdWBBNWFOb`kwump&17MPCEdFaxgmLino8; z@rd#K|6Bh9_762#F}w9o#cAJQ4!#|f9}7|WAAvM7JC%V1ICZj{T+sfUuQHz(CO+Ct0PS_SV-rsGyGMM!AX{51_k|ZJtKGhw7 z$=(o)t;hCyafug+`L7~!(MFb3&<-FtJvqN z&v>7qo5?oEuf-ZFytL82I@jnssPx|k&h71Ch7!mGb|@X*K>{6L)dwNJvGqL-=eB1b z7wAyr%GxQ%1yPS>8BL-UcRXLz5Jk2#v`NxSp$j56C=s<^m(Z}sfG z_YPPc+=H1Fy5XMK)W`4sCW1rfg1Xjh9-;*<#75B#;b|bfd!pm^1>@ryOA)_;5xaV zq6EZ~I8V3}anG~KBu23GX z2lrs7Qb5TJ@P4X~zZXR@yrUrx0^i?m zM5o~S_Ps0kH+XuT{hIAYrUz-(AdKtT|Jhj2qvm!~ud=Bfu5Rb`;Zpe;kI10Pd>N9G zvg}6egx7CBh=gc8_y`&Ji6k&j7th~rVF9$Vr1Ss-Bvt$qiG9r9`%RhWUoji2LC?%4 zFiH?tPwQi#^0R?mAcAKgaJwf7(X1k68qbkCbhQNVUu&+2=>3gcdYw)u`=YJrCRZ(I zHOGj6YyRA@>(H{U(TtUXfh(5M7dK&}cETiDiH6MWzMUxSa)(W6n z)h9g1F)UY4)jGak<|L|RKU*2q@L^93ts|d#Wm@JOtbMNNgXCK51dVI30Ewz+e#+z0 z7LJs*#0c&3hE+B{uHX_D_y(80D$5xzDM%VO)+0K0W+ZTjxL(U`Jd?KvAGKX)BoX`D z_5G94`RB0gelheapHX5Q208%fb|>tG%GXi#X~7n#R+rjJ)%U}{{>+c(ZBIR6oa#FY z+M8AkEWfR?qs?OJmyY}^EmqyQ-XfOb&OxhLI=OeWtkqLnk|oglO-)tnuYy1Mn;iD7 zpL=yTNJ-<~J>_fZQTjO*u+O=c))y$$R<$Nv8-KE5~SH%=1<~ z>~6@OZnO&;EQjW2DDPD6`GH3XvuR73?Z9NvPafL~ipOk8g!PS#_irub_J(mXOc}8btYU*2JAJ>65a$rJBQ~2X*xxaEvGQ}?L!klBL9L9W z;s~c^Sx;6tfIYN)<}rR)J6zCixnz%4CtjL-bTCnXCs+{rDW!TFmqeS z&Bw6Ef+vCZuRhv!MIL*^! z@T|HXp4xPDfyd=S!}$+n=SyQh-x3g^>Nj}U7T?=$PTYnVW(&889+`z4(W$>#0c5`A zTR8M6pL?}3*XG>k*>V+LZ_!EF@EYw)|2UKXyy?!pH5fuXzAq7#Zv(z2ncB5>L_-l# zzeo1qFGqatd%it^$~`}wDvEB=dt~tJ6im^LQob;hMJr$ z@KEnfA*juDWezl9P+RZ6So~jnz+b|9LLg`kW7>b&jU817bBc)d<03EnMPPEDLhE;m z7Wi66AzlKgr|<-zgQeFCss4(&XK~XK|9;b7I6dqW&>LQuS5IU13^dlTddt{ekr_Nv zJt-E|Er4m^lgCd>mASvsdP^KG%^B)eW}Zp)l54ng|Ip@}B}HLAtasbfalgpq9{~=- zv~oW9doAM_sK0}(PffHQY24@@gQJ7w@Mb&dUCHBlar++~pKYPf;}>M6cG?bZJ)Y%i zmdCfV=x^^djSkMoelQnjO#YD|9|$d?MG`{bcdG00u$4MMrm+GugL;GT8J!5)SRstH z?98Pq4+s2oLXF!K*1o+fC@;)nMa5E?c`ZY45nnEQtguskfF<|%);WOYY&SPtdw@U= zT}J95ww;0Ir?+*I0?+$db+5?{eCul6Rn(VI*cn){e|PzT;F|UPv zGXC0!Qo^On(=Sh+EN;O;cYM;8o>jHc*NT6!$44ib3%iCw&Q)JAQ}@L8Du0>g-xvKv zm8bU73&-mXdJ_l6BXWTFPac;Qd6HtnHQ!)$k?3SQr1vS7OTrzpU&1)nRTTRnr`UY? z)SlvsH}hYn-@t@Iu`S<5JvU5H~Xx<;ch zW-i!T;zw5e=J)>mF8w110{lN9@Vmmbr^tTZ$Kd4mF~+I7wKMv3MSJL>-?_NEH#1*X zWhIe}W#3;}*(1vO(forXZ<-GPPXJzq$t7muB{4hTh+CVQZ5o+{s5WQkMS^i=Xt=ND2 z%#K`cJE}wNq%JFNA3c%l=ow)C9U0s>5e{J#MS)YDzzG+FgN$RslWfMU> zWhp8j=$a&6!55pv1u~*k%M*XF4`)a-!6bFrSE<1Y^~+7c)O;FMU7QB8Eu9P zWj|LIl=lS#j#Y1^vzs6lw& z9V(s@u>u)^w{oJ!R>8qs>ndWL#==LulA4xS4aKhVg#*yMVTo>a+ihL!iQ{YCwrxw8 zUTg6p#q`NyFr4JdCQe|2(P0U69Se3KAj|XW4G^8Ev4{aiZ%Je9eLw2FYbhWPBuQQ2 z#9j8 zJQfYM>lBwiyteespwwb*AhCG!E2hqc0bAwGtYN7TfOw(KSImUL7xXYzXq5vjt@BsR z?PlKTub6q*x)yXB=s4Bv2ABx+r?z<_Pr(QxNiJyn(A)z&{Q^IzS{eFKNO1#|773o= zSIo6v`c{h>>{!C2GpK@bLXo7em}fUYbF~W#wxa*?o^Taq*H_HoPSBvLM?y4PM9QRC zI&#=zIW(aIv>cR6gbTdTSd;n|;aa>R;fn(3ZjA+#e*l!RO5wgo7vW;)!Wf|B@Zb2j zc*GREV{zYrN&QQJWiUtrVZVeLGb*W z?G`#nWMdszymD7|tS&Uy8K8DTL{@3|-X6}CRmWA5GjB^Th+XJ;;)o5%Nxc2fB9GsH zp%)ur_RP5(n=@je)&wW74&~e>x;{n>j6^%@2a*@>&EK~p z)k$NEy0_$1BA*Hlz$8F7hkyoJ^odamHS{e@b;@12$y?Q2k~?(g4&`jSRK`B6|N4G> zoWr@vgoID)BTW|{St&Eot2QqkUx=T>U{3-uy_WFH4)nzqQq&iaYT)biZ6?VlBL-Lz zm3mK}IIsWw*0*$b$vq$P@un{U-h0{X$l*njlZBvr2QLd3h>pb$mn9P2xB(Wl70<*M_v{_GPHd}~05Vo#C~omY(WURg*UyJsw$ zz3$XB!Mi)JQcbwDM`)U5EqBN5hn`bNNaOY4U7pvrNiS{Bs4X?TK?`Y+n6$VK!1F1n zda<$Yx}dW>!vX4rb@BgSdsiO~WxmH}Qn88=N~agoiyCd1Rw2Tp7pk4ctdx?Xl7p<@0=hD{Mk@<$6_nJ!hqt*VOwtb%_Grd_^%;J@Ray;x#O|i?gkU zuh%8gTtF;x6=_ejbaQR>V68(Ncy*f59Onj%h`wA>o3VM`MzR~tW4FT`PTKB6g!Yt3 z2irdPwS3<%O5H+Xn3k_NC`wQeQ?Xhr>+coehvVeMVz-#|xU3?-A1)SgE8DDp4Mr!46ruwoPoF;XN-`oZHCm;)cwrHK|vX_uC@SXGD;J+)BZT{(51=XX+~ z^7AHLG|o`h;b}cO!?3V4IZW)XR&X$qr>dC6nARi!{S}|O`zhk-C9*vkZ>tK)6Zi|5 z2kQ-b2^?4NFyC{`?WJ_G!mcXS+?xC><*K?=)-eWbfyrR!?-2Nv<=kg-x|%^|E98)N zde?Gu1rKDIs_3kJzE;DaBGi{S1ELL{>Gh?bnoem>-uUUQd)JKbNyjeu78Z6KJHRJ#h zqg^k-*;wsu52ghML>`QcxaB3M3tD8AA~N`-oU6->DHmyluO|-%RE$xXytS6^TfXhX&tnRHr=KYgR^-Fwsz9Ii2>J6<@ir|9enf7 zoC4mC+67zAyGI)FwyMboH0Jj&D5<@nU>dnSt}L=D9%;|!UdjU(&30~R{D%A;TQfu! zy8$zA%%_iYQJ9_c4FnfW@n^|H|I&2Q@K|23bwGzx<7B7ootLe<*Y%m292scB25`)a zz%eiEC4}}86jabUO=WZ{BMnAk*fSXQz0g_B|BTW&U>adxNQR_xP-_u3j@&bzUB=++ zU^U?8tD_~sQe@I|wq}G8YlVWEC{{3pTz%AVoay`uPcVGtby1+1?eg)o-F1s}qgTCJ zM_5IOCtx=HlQvSdK2dc5+yuofOjS+*GiDOR9CE~M?iH_!x~X+NJ6Nj%8D%z3+)SmO zqA)(6`yKf+(;(^5{8qgRn;m){b?2DzlD7G+PLIbLZaE)X5Xn1m{!5~2E%;#|7`-I;L&%^=FpqH4_fA5Jyq_$Q>Vq*-SmNC4gu5a ziP$3ePg7sSJrJ)7yb466WR%jZ)K4u%^0Pa>xbXb@DX8mdiU>Wn9GbA-H`%ZhWvD^0(3Ad7-V?54(j`DRE+4U1#{qgq~H$o;;u)>T(Z zieop`M_Q`_FzuHhiLmr%>?#1aMRiMD796R0M=j+{a4@;)vn=bnVq(?z6<3YUMQzr~ z4mCF?cE-f9+uwk)xLcYqyw$7dA^owHzX*%D`Q9ijhdMhc?-jrSTf?Y_B-S)~}6RYvMOe%kryty_^S}cm= z8dxmus7S0ivDQC4JIZwJGRxVkEzaaPu$;1vR1v%PDl#$c{ouFOD}9Qtj_U#S*7EE- zl)~hP%&>NWkQyDKQBY`p{pMRs%qVS5iGq;gzhdo!8=5-$njN#R6VX;hOjoKZQ{d+j z8x&!W-6l_sSls6EP*u<-ld4yGO;^&24tCb~**A{I776$vsovN2#kUkBTWvk+U^w%` zI$Q_l_e^HdfGQXH^ zb9YgpZnbZ~!s^jqZn-z+tJV5MEFIKx^OO?;IB1E;;0rL)Lrt<_S{Rs7cp6j{V)0)j z{#cdi+XwG!+K<^UB3U%7;204FY*6p(Xom~>?jLbMP)dmG#q%Dj(hADye%SqDg zN)l+=B0>bDdt*vm&oZf!)N*99A7&-@ip_ueIIhDYaLUh&F4~jy9X<4h9 z(E5O8k^f9yJ8OaZyS+*cP$&a+nf4Q&CaASsQ~{{DTwx`)stR;14__zJ<4*@45h(+u zbc6CXi&dF=Yf4n>;e z@d0pGD=?b$wL{1WBKhV#Fey)|18&<9_Dm08Yrn8JQ##fQ{7F&4&oCR+UrWNyvPPjP z-iBOh@YYGagi_#;bWAA;H;BuOpQ^PodgEN0-SXO_FOsbY7(t;zHVr)U^tT;Vy#A}V z`IDcf-@k75dw!ok2S#A-wO9SF_o)B7sC9!n@L2jl+28Q#J^vAbg>U^~LCA-6B<#y~Izf``m-J!BAQ3mRk!2juK<a^^44(is4r<(B z8VCJAP#>T^Kz$fYABKK+jD%ZeAiw^9{2F=&!zbYT1c81a$Y6Xk7~+R-{GcBQG8o?s zhWOzdKj;U748}KuA%6JA5Bh;1gYnH^h#$W3gMJ{$V0<$e;)ieipdSb_7~c$r_~9Er z=m&xf#y5i@e)z@@`hg&W@y%d}AHMN}ejvzTd@~s0hj09#9|$rS-wcNM;Tu1s_x?Ws DpYLVk literal 0 HcmV?d00001 diff --git a/image/wechat.png b/image/wechat.png new file mode 100644 index 0000000000000000000000000000000000000000..962404c3f41f6794401877eb31ef146ec047e359 GIT binary patch literal 179218 zcmeFYWmHsQ8#W4x2udp{(%s!9%}~@Mu2P|t)SkO%!0 z>Z0yU!j0}UPOfl`T}XZ2U(x!zyY7y;n6r@t2Wc{;EC%*>Shy?GGi3uq^}Kaut(fl_ z*yOPT52ZdQxC?D3&oYsYLB$aM1GKk?(Nw@SF(6az$fx)JEW_0?9ss>#4h9y`jf!F7 z0A0T410tZ4Tf?FQx`bs#Y99e!VhPPZeh|I4e2BRvNfdT} z9(eHqdcN74wTo|kb(NuYVP6wGPMA(V8vf0!$$euaS8|mQIP5x$n=E#|Jy^8KeIaNf z<2cgp(I|9zf1%ZcC&L3JaDSpoFCU-Ojsx$YAc*Nsc1I5FbIyp4k5xOzBa#O(vcM6C zQCt~=^R5287OYMoFTzZhelN88yIld?D?)Ua@5LY7`D!hUAf^9YFC?4`xL!vnS`*w$ z7=>~A#C@T~ZjKx{O9;HekTdctI0@qvAJqXL8ym~+z`MUbb3#Hwow3;3b~cXO(oe7Z zFz-io?Za(=uK^{m;Q_7(wE)r1z)!|2GGX6%EB~qnr@3YhBnlJ{FkU z3a$;orcv4W^r0TWl&SJ4UyN`vvr8 z#Avr`oO2V&cHAl8XU7f%g3Xd0$(LWh5qrzSf=&VQ|I8`UKWGQ-KU zWg|VFFv)(1&~*_=n3zIXQFt}a+Ilwr32GSlkFNk`jtj^ELYQ0JuU&@SMQufQ>ze1+ zkG)cc{pPwkpxFfP5cbc-LvC8+`-e$iEj8ZFhwPS{4~O&%H^_9h6i)~F%8DAfgo4dR zS5O4y)loMW>g=XCdK*8|cMn?=&OX>6dR-N%om|fC2COxtH}v`{k1W4h$0gTh-98(z zTFEAljmt~1@+l6Bi>?;HVl zPEVltk~8zDI@RDwc^JVJ;w^rl8a(jm#o0$VI9xi~M2uImd6#bpYGM<)0|a-9w(D3@M7m>W0q{9xM~qoFU;CSvNZwYhLysm`HOt6 z>Y>ti(tttMJFp^K{v$vI;9uKMW$w9W3=VHPa}Lj~@+oZApd6Zw%`5XvxX7|#>}z87 zwZ?6$aJ#&_lT~)`**4)uFBE#WcmfD&mMI#6{PgXIRYujmF@_d1c95u>8blKUigf>v zM>Lz;HV8ZiB(vq#t&S`D>o2Y%*}$ub-FzVZk zU5CR-h{Tj_I95pMNw{M4l>7-AOqiPo_5aa$6Y|ZdtzbN@TwKm*Dbzh-Y(-+u8z(4x zP?LvO)C1)Ocy!nyR({l^Y5lA#@LD=8x-OT*f+ALjWw13TGHqqlb`IM;ZvP^nFN&kn z1gZzbEATkjjT~yg?RQpFqqm+GrM#bBFBOj1InlmyCkV+qI4hqUFvWj0w|8EN2LaAs zosBX1Wx+kelL8$0hd+FNxL|`YGPAWxeF^Nr`yz^d&HV?J$il^f(fcv_Wv2d`7u|6l zR%ODiRsoz^b%4iSdXq#@d3kysP3NG_EZPS^ImyXYW=uuumCS3Y8hY-od;g#!)4sEN zezWc(rFxnf5T&J}gBFdoFAIJt$&e0oBdgz@(SwZ|XXKucrXCLu&*G^aFs|!-$_EQy zhV8S*wxLE|F=D(`z0H(S1TV^4syc&q^<8v}fqdT}&!*NgK5X{uJh6PjC-mRn6|t*h z?uVmR>T_0)XU&`;+h)y`#K3;}0)wnu*mJ!LdQ{!=sC?R%x-Yd#z5k3FYKmYR^F_N^ zPNZ&liF-VK(YM=sw5?co#c#1KwD)jy_|;_ppfM8Y#^&0 zzQmQor|fGAHXJH%p?v16kR07Ki4xjCOkigfyzYxV21Mh!H_r z@>EdnhI87u$<;}5nQ)e}=d*hwB#NN2PV4qRnSw6K?C)wHgDR0Q_Rv|{eoI7*pbFIX z@80q5Y<*5%-um^ml-h8vr2lb-?Cl`+%a=tv7kz(<=eK0fnlI3dMi-wGU9>BOj{$gd zpFxhMIgA7Y?Hi^MQ3@I3MV!nQvL0v%jDJE1wsT(o5hQ@8FajSDh*!|k);?Z7d3?27 zj}AQD^6Yi`(|XXJ(ur=pIGG7Q5fyLo-EkHfZ7?z?_MyPB0I$^IlZXN_sN)4@Op_M- zMk4k{ajXpvR7GMsGCj2SD{Puw*aB5$OI>W9lZ%Lj%M7y8hNnJ;hUc+r=dBO&cK#{o z3;$zZ-(M-|4|s#uL2mFBGAOrBR`?$E+zR8$2^CmY#v!XZczClu`53@(bT>bBH-7cxCt)sf`QV$#D0GAOpY)5N<>LfK0?Z5mk+(peDSvJAOIrp41xg~Ynu8h|;sLW%j zj20M)B`s!*^8DWc-!nutc*C1w(Hu0Zk}q&jQx(|$F)@@-5?KdutYaNuj z+`YI8zgxK*iS9(vCDO`hY=4>aPZ;M$r$WqKMa3nXhwPh%kw`Vat}o+>MhACi=Vt94 z$3(<{I3FP!kI`*bn(Ec?z(M9$IZR8qi_aYXpp4l9L>8JMp;bDOInEqvbyFuuAejC9H}heLM2bNXbadv8pT`xZmg~fl&8A#@OlS&W~IugsHmu^!6b&QN|}si zojPwSo+FxpeLFV z@bA}|rIxhJ;Fhq?2gtYjAy-ib^1yl?Ze!6cf*MB3Lz6=z)cl31gX(AR!G{KO*Vvb{ zF^*UzsJl+G0L*r%KBZV(Qg*miHb}JddV}|5FdThy9o>1e6LNf{66l8#f@?uHPFEQ+ zy1sn%n{^ctmy{el+M8?kMohox2nH^-DDr^FBe!b+(q&j3NIEKY=Kz7ZHqGTgq444` z&Hk~Z$r}mV=oooN>1vhzLT#Ct}#w4$f67LuYX+yIdShY`I#CSo*-!#N-%(|OD}W0;1M^y&61@*{{k?Xkb=;SvM>gzJ zrj9&mq=Hh9Pr3Z}=LcI4+Op5abYM;(_F3qq?+*yOFtY_R*(dxo6ZOKM=;$$EK1TF_ zw`V=n2y`d%aqbr)Fj_LJG-S5}RWg&4&r!eRtvBb<+u>nl)kcuY#VXNQmw)|JgVf zNQ6p_%w9P)wz}m>1t!n=yCcFw?_?3hvR9s<(%F3x6E5`7M+BX((?fnyHsmK*0s zo{R0?yiFbpBj;Wz8$r1_P{6FYj0Uf-5wdXfK_~QLF_@RVpO3(Vmev8%TTZEXB9$+G zt}$_9bzdp7d<+D(kzxALXpBjJHBd&!!FUFb??1v%c_2`!nyh`9O$$%6FIo(N7{r^C z=__AryC|S7(h$yaUJO2HH#g65$mvZDOX@+f$zCC~y+U@xY+6!1a}d`)C&5cnpKnnC zsL)&X^LO{Y7xv01d7;J~yc% zV-M|j|7Rzj0_MZqy1=two+F@X1JGn4 zP4twkni0x~`T;&LxoTy=KE3}2OfU;Krj7aG@}5jshdl0CHD^hy5eKK@_$CfZD}3 z#Oj%L%YmmkA#*#)!^}K*#-OLEHNtb;itZeg4L_7UUWx7;aTi=)x>;Q+<}Al+wG9^E zuom1v)(_@@L(&232|t?&Pbq}Ei`wTnZz^>3PMVIV@l2%Rb%~Tu>44M#I>YDc6%rEC zp0Bb7*wV7({X$b~-J0@b04N#2YT&EZu(9Iu&|Wt$lx+$x%_6)r3yA7OtfK_o?6<_&D%&`!E~8BAg!TK2kp{#d=nBp z?Ed15`UifU8?AKjwEq7IM=+%G-xDi3pW%>mloi3K zd(&VK(uqbU0J3y@zIJ^+L3q-NrroIL4a?xA2?Ml=+$yt;^F4pSUvkaei$jij)sm$MHS0Oy(_5R3r%OB0k&LyEQYHvDOq;MKPXm8)eV+V$H=ZLgqPCOt|)e6{J= zRQI%vZ9~KIxxlw~vDYWnVJYEP8QRB}{Z2f*O|$XZbuaJ4X+u$J9U<3DcSo`^L%d;E zvDdq-vcr-G9RN{n?u=SLJTLMo_5Hl{pU}mGag_dprce-$F)-zv?Nm@yvh#oM1%QnE zpE(}t(f8sIgWA{h!Y*JmnM z2X{Bnx#GzB{`cy1Tr)We>mWya~NcygLRTH-)2{t_C}= zV}Z!1N6rjTbSjs0U>9SjLY^4I$g3=#1CFw+&4@Af<@h&(eCh=hfirn=BZYRqv6Np;LuE!#Jr8^qg=dl z1C&gu-Oy`M%D&_dE-%lzZrA)Qy*cVD1)yVYI?i>*ks_-$$x3+2U04D4oMm+`f5auM z`r!5UiCKFxyYPufb#j+!K05%RQ*TjD!s6o6!v%ebC#UO7U8xnIj6%2=^Mt-$v0X2F z;Q1*7xdU{#uXhGu%0-<04COu+YTUxeO;Y7S1L*Lk0D`^MDqr zU!0m;zEeSYanS&7(BGEvvRJCVyf>zDk_boetV-%lamRr!s9$Jmw^0F9ifoawK-N)4 zNrhcA!SizM>lcc&LSal6plyA@_(ep5<35W&K#Ygro9 z!G^Q$(dL$}=hfjcy3FE3*QI*Suz3vwVCIJwK1WJHtR@GxxXVSaJ46E&dh<}Kaa)`$4r zy#juYIP?xMl>U&n?`$?R#fR)_%}DNOJ<3T9{BbH~n*&SLt=nojyBRTW zy5yi3siNIzqt1XE-o?y4E8t>nRG2YQ^HQ5PE(u!I%*+d6%g59fFhC@PH~=(jhcGOq zSX*0w;AFbeids<1S9Y>leSm&?2o8@=le1U?h^Sz~+I41V_bk4RGdKF}q+PACef_Mx zcXR*u&LJKPXJobEvneBY$(iPUKYrkP9SM|jLWP@GbjJcStWIg;4pfNhV)nq}btc-7 z!<{&rVIk(Zy&BmwPIlx(+Kn8-3Bm{u>wsG+zgAP5;Xa2$n_@L>hHJ8QL7hvtQ^h+d zb93|c>s5v|F<9!&PAU(3^sM|@@3er`J4v%u7 zxrh@u5Ks?f2gNi>8w$15mUvTq(0e8)Z#JD;g1USx!p}?_MpgjvwAXWhFOD*bU2j;1 zhI_MFz6idtAp!)kS+{6Bg{sEzv})?3+qz0{_&N>Xgv-pXm<7a3&lLuqf@vGWHQAYg zsX6W6-G%`m64GOx*0D3et)4zO6-J{Sx^F{6B;5zSVyZF!2<;kxdS(h}`gU&Y20hYY z*8>zM9$wyhpa8K;pwdsv^4J$$u+wS(Vs7fjloDX{Kg)?D9RHj(mIeyJW0PzXWQu&$ z_Rb@7Q>;Wc0rv9+eQ|K0y|WF3LS*ZG5S7`FLr-)O^z~+xr~KS>Gxuhq>htVKE}xRq zYpK#+;vM&m^W)d~rW$NN%DK+6jRTvIR1^YqJ7qWvH;4nbc6U4KD-AvbrZ)>2_q-WkC z`5rq$U_fEv^yd2CgqR)G&QuMu>ePhC(1d#i*}?fSKcHdsD*4MnaLm@C1&{+`=&QUJEJBnNCSymCN%J-=ls&-di=0;1IUhgx^DU=}JJ z41vquaRDH{|LDlG*E7SD>O2t&E=~p3d1p$gL96sEh(?=TSa>3yO|Kd{$85pQZlSj1 z55N#{SxNZpY+x@x`+}9KPrrU2wc**M1g&s5g;=Puul~T9wBh|7Y7riE7-Lc8mEf7; zkeBkK!)cPyeHwcAIyH44fCeI^0uJdA2n0}{F)H_QULZ{4@t#Mv6dtI9MDGF#q$j9D z&!H}xY=8=RuEXYgC!q!!=G*lO3`~lRjotNI^ER6ys6}qx3@N~%oqZ5g^4=R@%7+H_ z7-JuHR)%I+9egdIRfF*dtbk-#cINza#$9xCLR)sYE1FOc&zlIc{T+HGSU9~Aq7f_# zCunItX2~Kz*K9V(G{B}kX z`Dky>!GHllrkt}>5@}%vs8n6Sgx1s0&hRi%!o-o`N+Y}5FeBY!1^pIFOd(n|=#}Rz zVeZJ6Do&GXBVhdlk1q+4=&Kw&5&HZl)_+l3LgRJmhdz?E33oRo=12#Z|5~bMh1BnjNZY)A@E|sA0U?F z7ivubVj}H|;!-I?V}jZqrKHyMSJt5{E6PG7dvG^Opc;d! z5sC7JvCr;BX#i#p2)Ym}imJxI_8;+kZ?|WuVZdmXfkKqf*Xcmg*X+XC!z_RvGCUr5 zba}2o@o&!$uz|q4n~Rf!u&Ws$vZka=%(q^Q7^rffe+?C-nrkm+$z}}A)mIr}ylnhh zWLwruwJ246Xu{I4-;{~UwS@f*-$prM4sp#|s z1o>BV04$GFSbMc=XA6HoGt&k~FB8Coyf($x-M9s zrmPT=XG(nex>+Ke*4E5KN^OKO@3`u1O-8o191T35vwwb&rkaN+@k5C7pSQQ>ykFfJ zfI|MevBJnArcHWdA4u#s)a~`g;z(ExdQuwwENizPC*yp^(2JGK%Bm-|IK9?GzUTIW z(U{+{qLC8}!>*tS3D+dav`5BEJFmbl^inM2Jf2xcuPq$Ld_$F|A^1oUIegnpL9{PTN7^i&%3oX@i#?n=$ zRAzqI2X+^Wja){y(p=TJ{BP3Xlt^4^_Q5hUUj{h}@@g5fXr47GuaYobMc?-!h3q6x zzrF}2pvTfk`(C698YT%skSHi>h@Q7OkBvvvS2-`eUw+tY`ksRxyQ02C-|bBq1#wG~ ziQ4jOYdj|n#_p=z1Y#m3g`E6wN9~#tCK9A_y^&XxAKYDoL>;l@wxr9amZ_VXPaLPz zQ&=0Pu5e8iW#KvI%phR}Yd_gquqX{9V@7JG8IiCo zSdkV6#gm79_m}Qftj!}#ii<4;AFbGiNap1dInfFSz7$W>W5_@XGIR;9(@MQm>Ay@Rf9e2 z8aZL=<0tx&tEYQr#ZjqCi?!c%-E)OghV}P&vC&;_MBS3(qZaMcoNq;J#2 z`BhmrRF#7EX3QlA>iw-s!mmjDz~;AhnRS0o(N^3j z63<45O6x5PlT|;?hj7@ z*jR9xoG}$*h*d~xl5KTM8GFC67!U+9=xC5~lhs)B`S2j|O{-hdw8lxN zTjI6GTX`LKan(L!@+&g74^WkMbF3xjh>X(|RZsmSf3)WpU5B@B^2`CBUQtib0xU3lF6>&)1v6|W4>=B=_S3C z=w@_?J&#JafPjE}!ID(`Ru@vg3RF(?+OCpR8vA%Wd|9WO!hM`k5;Vvj8AYVWG ztuq{hmZ8_~5{I$iB^i&UJWdi3BR@>l&*h_(w@|itrD3dLxe2a3$o-XdLihWmqS%1L zMadsWC>ILzQylriS;v}yw@xH#=21UbQi|PG`#8)rk?Tlw*?x*vj7Xzu!}?N%X!`Q!{5F-6rDf zXEK<~bPpZ;@tHN>Ir6D7gM0}`n8wsCa|u1J3e~Fz$_x`QO?yPHrJ4y_@4Ao~eg!`k zZS(c<;MD}QifK@wCb_JoqG(0ncxdcx!jNf&a63o!K6|(R;)mMC&n==Q%GL(MOi$85 zmu0AVdAx3lpDvE!4e0ztrj6^>sH+lSVRJ1Y$6^;GXAyGkr+d$n|cTf1)I&caJIKOT?J`~0x zm5>pb=b>=+XA4nOlTq_~6lqHo$Mj37OoDtOEt&(f#0Yb8FTCvEZ?1P552E<0o~ZVA zQ$8zyk{`jJq*Y72jKj}h&8LhbQf{&ARVgh~M6XfHl&A8DUAG30$L z)n$?$;{SSpBKbSRd*j#!lQOLFvdVIjcX4q_gDM8{haZS~IX-G^T-hXFllFa*=4YYj zYnQJsE1uwClTez1!JaFap0PfDk#d{7+-tqSX9nE1(w-kux0%OZE}eczuEH0Y3imbN&>sJk+>`*) zu+%8zD5$#7)&9X~=w|Xc3C|#wBG$Q(OS*&2Q6$^eM5}&`4lZa~nIq+GJ2Lji+cYpz zIS=Tm;<4kU`s^rm(#Eucxr8}BuE5x(hOvEM(}jF-_?uhAA={0{ z3T>2XoROkA*q_tYB2%%a0J8Gw#|eCR?8sL@tcT_G7>49q|M3RtA>?Eq8Y59xk@$^G z&BB`fXz*Vgj?WYApUVn#Sfgwd<_1Y5M|RyAPZXcW8f{_Wh{XxA<8YS!X zn%4g)IV#*nIj7t@J+&r3oi=&=vDEa8W&T&*9ibdv-ZCb2R((l&i85d(xH4ZP`{crD z1MRbfN|ox9k{l@Gxmf)P9A1uy*A0yi3D!wfhwBrxo28Fl#ekt<-HZTr5}Ht4;kzGh)Gia+?=wd%*}6U&R1ncB4k z$?>J7yXWw8qz!VsXzjPin}Jkz8?_cV{l7ht84LfH{_~!XA5*{mSudU2usLn!vGFaI zXEk^UO+#)OK;ODsH&`!S1c5l@-uoI+tH(-IxdOotb;@ft%5b;Wb-h%v>)_M^uAT z$NaLl9;#;;W0T7c{Entsy+t25yhZ9t6u9~QTiFnrad}m&P#cnY{yIC#ZPX{-i;1tU zLLnVt3wsrp@GZ`m+HQqiJ&_}TmT!86spl_xAe?QMmVMgv!4oPRnb!f@_H|_(pqe!Q zgwG!_lvMYOsLDY(#@2;qq-nR`L>R;{Bdvw>Yj;M~e}A6(Y&qYS@QK6vb%Vh!N=(5i z$@$Wq(v_@zy5O0l zuhdazIQ-x$;Cfg@7(k0-`H7xHA?+MiTB#MM_TY=M^;RCZX_eK0r-wf`HlO(5p-iFY z1e}JGh8NQT)W-XYA>}cPPO8$BY5KpF9B&nQjCrSGxmwY@VMnn;*`kRi_4fR3|Ag$}c{gpP+W{xpOGC1^M~REaZcPil z>-{Ov9!gEyqt%`q-)WQU(_&e>mERvRc5C}+T$?x2;m1SMIlSRlgZJOIZ%k-O#?#(x zso4Bt78brMoP03a^5ML$B#8Y@`;+GnR{; z8{NzNiZFt8jj(nEQ3X{-6o7RxinJ7$7;!dpk7|!LdZAMk8mFCdXbUL!t{IuRSJwBKEWrW|N;L z*j%j!uFo23%+@^Sf{Hn~5V`0IzZNVWVko*OwJ4AKP{t6GuQm!&0x6DJ+EFej@~L@i z=gCfek07G|0Wz+8Hy#A?cx!#gLQ>HqsgBdZs+HcMsPrd?F>2k-wsP;2o3vXc>_2TY zuxELhjy314yhq@_?@qf-OJOcT)65y>zuN*&f(C&i@ZAK%?e)HYr-H$(OLcqRaV>OE zBTL}p3y-#1)?1AGQZb^Yiy&Az0i4cUZ$G>mdJs%IQLG&f>Yew7OS9t0d2bG_Ty)+^ z12v9pX!4PZb6)lRzA9=&wOKHN`JnWx z{8NJ4jMFq>Zuym_-g|`TvgqUt(F&`nAkZD*}eHW=za7M zA4--B_CCpeB+bEOeJ}cs2%q0HgpK|4dq;fwLLAgkGt-unVvVuQp@m)P|J~M)WnRbq ztloh3bTJ)zoB}m3;Jr{!=UO?P&_3WM#KSxMN*KDuo`ne5Z#|Gnkl1D;<1zo!d57-w z23%9d&br(Cr3-g7wHJ;NIMml;;iBV0`u0TjE(NG4e5btwwsvC>d@HbYcYAR)VAS#Z z#o}R91C2K#*MD>2AWSsFGRVnefzl!87ftZ)tAa7(o&US<5+6&;B6OWoV$>>M8^r3w z)z0-q;uyQsh=k567dl^2#)^ynea@)ets2yz68+9bWLHOV|rPxJ#$oOS&r8 z*pL-O6ZB55TTGp(=#+-`fpv0tVvIGhK0ZB1?utl{>Wgl=Cj`GfW5z$vC200_wC5~X zi4msJGnf^VFn*$u_I{Hish4ey<1N-xF}%icmBNg3;YB=&p05hbnQ?-D38pmpq$jpO zVpOcBbeIZ?k$t9>Sgdr4Tp;(4^G~vLxQNS%Rl3!f>c2RUxwuyUWt+Xm(NX78`#BIt zH$v1-)Rr#;W5)<&?6Vi?=}9@?dm<}?Ew5T>Mb9Bq_~#;D9{J&$kTE}!$_j_Y04Ghx zn(wdS!vpDft}e)=?q63Ce(Pk}FG%I-8SXcCLvaZSUUP0uXX)|$ZNN=I;bab2U0q%C zVd%YfI~VRbkItKVu>uGr&3i&q)N{(fh(G)`gpAj6=;u!iGq(m($-VE-s)MG=r~gI+ z_lx*mBDKQ@2*VB|ad-AR(C%8&XOmO^!Z(MqPS4IT-!29sj+P?{&lg`%QLUl@V{nca zT=ai@&t>d^F@z5J*_n6&_H@qihNoqVp{nf|)B%jC9+4MZHC>x`u-lBt0}$_KyJ5ql zr(|!hHZo4qo7y&{wTw|qB07DZ*n-8A<=a2d8O=vahCQAYvUE% zl)cbM!<}vrPAT4(qU;~jXS>L4Zl~;2>!N(M-7$_Qi#{J4_baoPUpyZJVF^%icGA-^ zff-n_t$%+0Vy14dK@x^z9RHW1hJX1RJ*kFC4RRKTzah-Tf%7hJl4A=(+3|s{6o1G> zJ)PZzXFZG2pEA=2n;-kLz!l%s7u}U7&}rX+2t`7yZfranMgueB5{_>SC5b{gdqsKT z4!AN|=IUpB)0JT1r(|5qfYcrQpR_R_axx5Gja#O@yX^ZQ?$$VK|18YLRPV&q8?Xeg z)laNI`TNE>ZO*l6!EXp699!A6h@#9f8J2z8H1G9Pg*NomW;z@_w`94;PawWdghHWy zp!ZwF(YwRgxPAZkUVwVqliMtZyk=$Z;9(cbdRSPuaEZHLl&KFW^`!j|HPnl))lux@p|GVcL07SQu{PjOl3cjT6H+R_8m%h1xh4 z51gOHpd-~zRcEl}arS?y zU4vg8Z4~76`FmFdqU*^VK1M~}Miq6Ytx-{ajMn{tK4_25M&X%xbkK9}coN8S9e$1f z8!`Mr{vk!rMx#XaUrA9)(S!YPC9L?*ywW=Ks&bzn*x>iVMA%9{WIu1nllu5gN1#CL zLD?I9qRP02;541j^aiVioGylO$J=Bk{F{~q?RI*l1`?Zh^PE1Ev@e%#s?+q_nI!B- zT`QDu#Zo?G&6}FrY3wO_@)_bXVY~eLG4R6#%gy-txF8WHrt0(J2v6fu-L4$9Rqed- z9wK^YI;^j!uY>5zdZPb0N{+IB7?XF(ju9lL6>@X<>f?m|O%eA)zle-m=V96VdYfTq z1{@eRUWuJM{k1QyTwP6`H^g_#%IoDIo^Kvd;)*F6n5|fBRTMvcP(6 z7ddZ#<7)RCYj>X_C$2{PJGzv}ExHl1cY#q+Q9;0svHQ*Up>E~-gw~oJZ}9AdOa|*< zQHJG!h5y1?YMgM#Z_;GeaUej2`TcV$ZlFOI@rEKnH~!~~!5)oEVZjM9D9795jLy?( zu>XQ@+ez-h`g)4CJz%{1SEh?WJC#FF+3<$M)|IFiTE-nGBa(q@FP8oQ5Efmy**Rrs zcC<>~d8xLir*~ow6ac2_d*h&eFA$(=fnJK}7e3YuH&1hGYn-1Ducl(_Tdlvy#|(R zB(jQP$=)tW`%l=WlDoIaG}ou@S%r+js-2S6e1-4knTfych(|&OzWg|rF2nlJVHkUR zduwT?YXW;N#7%0GLYs_KijO4LV-BViu<4KAYZ%5QCgzAmGEIu%O*;y_OCKm{CdaI{ znF04LMOXD{YpiVr`r1L$dawh$fR9D#KPlM%-FN{M3HYeWnrhLE5E1i3k6d2vE;n=K;F@2 z13>3Y_zk%pOd}B6-rxk@U1v9#DeleU4`ibcXMNP(cq7f0+1eRviL$r+2BH>_VHVlz z6~bnKR90|;jUjeAssMfc_c{Z(gUO%+k+b%Y_7Vf&pHHEv$zqlm z;*-tcwR<=NMb9Jt0+GW={J-0F^PE4nWV`7QY&o#%lF;( z3p00LZwks0k~7Blo{YFW!Cc@Uo3a4JBvwnP8-t$-PCLoWubPb6*pTkDqg{YULrbbe zK|^C7VEe(ZIKFHg>683LuP4EOI5U(O*G?d%(VDL}ACg?E!Tf;o-4Mv+xeNIU!y zCP9x^{8Y76y1EegQ)a^t%H{7GC`p*J1RmE|JK9p3*!O#X&7wS>A9V%94*GAGUsmU1 zzbO=NEk_F2dC~5DvkmCx6T<5%7-a{2mS+bQ;ed*nt~9?A(FdrTJXtIx5H@RQiT@P% zAGy1HFfr_uasS6!0`3lh%;lT&SqEB?#Q>XhUhoJYQ2gp42M=|i_t9L#b@moT6Ueti z{Cyfkzymq%l3}L>v}fZjmH_fS_P9IuXuo)jjeWWNSoX*GkvjlqaKXo-{~Q^^Zg#-+ zN4+lq01mKd`=!teyV*KBjO&|&a0HO2OkE~cfm9jwl;=l8b@$ivH0ALZBG0i2pWi4w>=>Ex8CQP3QEf0d*1q6V12#W#q&0s@ z!xvGcVvpLVxd4UFE6kb8lold?*)aUYQRd1TrZ^*xFEs!?(*F*EJt1Ov zCC$hG_F?ZyU~pe z6D6;t!`t3Bil21GdnVi&3?X3sy_=0n0Y6S_K*nsDlH#Di&9@PMvw$tiiEW@sBd0Si z@-H{>ji$hnR@CDA2A1KTocrqJ>$9xldLWaH8IlU+R*tss z>%Z8s`)Uf)kvIu)$J?iLyusyv6IY~XoTFF4l(MZ;@cmH^rde`aOS`V>zh0u0nrY7U z?S623apO#nu^!OsK@zzH0QFh?;0wJyExzhzl^HWz3|Nbw^_z1GoSR!b*U+xEe_6cX zegeR02=r#xq1_kY$PUvLCeOb81D`-<-rj?xS$G(k>Xg*IureEn$YqUd0)$T)Kughf zypaIxffaa~7RYXcm*-8L!;fVzUjg=#Vc|!4AKt{b@*}5FIY6)j;FOT$L7O1-dYzTl zMt;Fj=yC^qHxRsdk|#`c*H5Jm1=8iSYs9qO;&FOY&9KPa+rL)P{Qmpq8250e4!G%C zc2Kaq)I{_@rN=#kLygl+f;1`j6;RO4U<)?ozJZyfm^gurVYlM~LD z`R42-OkF#q2g!sDuoTdIdat;Z`Q-zz8n4as3AdYV3B`%Z{<8Ln`+|aD^wkOC1fVyk z>A>SY*Qp`6Ri%Lp%F4=@>#VZk^iHKy&YmOUh>A(u>xhR0%e^l=)>mVBH}8RqjcyBF zV4FssRCCjHZDMF#Q^#2)r}V9R>t63n$GNb~aZ=*jZeo?1r7*PgBM0GLfP(*mhu?bN zvwWV;YcCNXHUTM!SD$npj(;Ee7SW3rGIJ9Fc$|xKXEWgNhKmLJY_}hWzn_Y4X4(X? zTP>{rZff6i+XaN4^7)zI&hZ^j07umVrc-fFZJ2Y%lI>de4-HVTj8dRljr+;QT9sL- zBrY_oarzyFbsV5e(!9ry!Q#u4|60M&krp9JB=p6pQ13Y&H+baFA3mxF@%GQkK`fws z>d?T4E=;(b&0TgPJ?SomI~F!dG1H^Oe@IbOp{Lmd1+gDTUO!&)Br^L@hoQ?A5c5Qt zZ{$638!tb0+(Qjjc5`cf$cULD1&w8I9dj)I^lE6DV;W*T&=QwkKAoArB*`qtm!kGk z3pbZw4ED$Cm1q0+D)+hcWcqc*9976ueN0RFNn6e5HYjleedaFcBwPA;U&*(%R^pRy z7J@sV>!ee!>B4~*&xJj6R7rH6>3m8HvQBYGA}(|W3#M(^#P@(mf)(*N$fhRc$=}m| zk@^}HtwKR<-U~pKegTC;H*A!*$ZWtvkwwRcrkX|#Z*jLDf+YO;OF7sLRO~*Of5O*v z3;Olt{R)GfhI|GxuL#queMk(mg!6s;plWc7toVKAg1Z>H-UI=;1^)Br&w8P3NH_~n z%aHuKR}PrW?G@r=+o&_BXzqp0B&&BeD>OQqm#rN7Y&H|!n^Xa%cAiSFW01BGo zhd&?E`V$<4AS7qK2DxVe6Pn()HT(t3b|3KXL|bS{?g{z*wR$YF0BG* zH;K~c#>jKg`L_>mZf;ycUII#>2cQH>D3*PHw~p)*dyxie<73AiFjXj@V!-}71yP*d zQqd?NiFhCb%k*{m$}Eqqiq{YC6Kfm%bAng=c}CS9%O!t>y7`$v_IQb_s8y`(?he*u zQ6tSOI&W1$MsInYPgJVieI)#&yDmD;xIA6932aVbBHll^6yi=%QyRV<9b0^DA?%EU z8Dp-Ef6SHbUn)QBkREc>d1KCu(TwNddQkN06Cens$CN*qvc?z#Rp@+)DG)GZu~y>i z&u2BTFmMBpy*9u!sC0&OL+K1AB_O7qqx_gSLGn7!>0{nhd?R_SiGJN+^-6UoUOQQ> zNYQt0^}*-8&${&7vAxcDPOjOwNq-Liu&I;ruH8N+dwcirsqy6>hPol$=ZqDyZUVfw z;btevnvG{O4Tm;KU+)=h}AOP`x4Ad4*q`>qnJg z?u83HS|S5*;1@>bo{w_19MaI0owq1+&mK~3v0Gs0e;f~R8oBVS;g140gtZ=Ypq=-c zB!sdx6J&2Mad%+a&=DP#mkp+9C_u8Fr4veQ(E+8M_9>a$HC!5(-xVhHKLFFq5_Tao zb8p^o-tz`3LA7uhbPYiDy#Wv-4P!n)(!ozy?b@!x?=Heakm_Ja&3<8umTBqbT05fc zWMmC^W1a2RxB}FR$|i_iZrT5nqL@iA>Az(%QTZ?y>(@?OE5_&?b!cD@bWbMOsD()~ zl&F>JPpgtBJ-2@SuORn1r3Y7*ubA$8W0wDirn3Nwdj0x1A|Nd-CEdL=2uOE#C`t(u zN-8KVEuGQ=3rIIA0wNtsDXkzS0wN^>Qt$cQcm6YX=gwU3t}OdJC%)g$IlDe2Ge+KK zY&ZMm3EQ{oVm2J^h_BPe(O8lxJRabq-Af0dGWcWS;cIGs+426QucUF`DQ^{NOga@v z+ER3z>nl)X2{=A0GorC>A;e5|&{)*UX%MBQ3An6t>mG^=g%$dcbu#H4M;sq3b)PyP z*C6IZluz^cXP&VV>8mbEtM6G&G&AFy7LrT*|j?=UFU$=Te@Ese`F| z5aEIE%W^4-+ecg06jS|DDtGiJ`?OJX7|Bkt8jDkuubmJxM(8wQwv@lIbo<@)JyIwZ zPL5Bm@B4iZxnj}igc{1YYFGJz+Wg^}2H(SLeAeQf&2z3F2wqR85|!$)HxMgEi`^F4 zs;qB}$J(w7R>o!=6_%$lU}TQ4=FJr=`!{HzQl*2*&O3YaaRziKNZ)P(mbIRFbw!YK zo`(2WcO_k^RSM#pr#sd@A02`5>qCdgrnDaQ2nK40PM5zq`#T5h$Wy=1+4FzU@pTY) zkUF(`vKg_6IH?st`WYji{hFr%xsF~}W>*;lNk+mhlYG_(Klqq?r%^zUo|(J=8cvE<$2IdGQl z{K#H7?z*%i)3_a062Q;nIGoP=i{jM4h0-n50jvx}f^{P2qG^cbCljz!j8+tvp#K^OBg;=;kvO<#1 zL)=!UgGTQM=CDe4nEfOE0WB`tQ3|aHV+LuGF={+iPyN&F-FEb!_J1>nf$_|S6nRNW z#0dhl_UE#PUYiTsEA7|>7ShI}co828LJKe%Zd>jg9upkx`|Q8@5%GTa@W!^?*b?(> zuKIfTE2?g0aW&U?6b()`*=*rUg+wPI^BfOzQYIStxd--bVr&(~DAKC*r~g$z`INl; z^?Xm$?-}loeDtakbM|q1J?Fn+l)t3J_>ZKIG00!g!1eYj;x+8&U-&;f{&0nH<%^MC zF;`6Aw{4^vf;^Q~y8!y%o5`tP&}yT=k7g8c99}7iWUyTJpyD&{P=j&^&kM1xB+rkQ z&1R|}!Ohs~yw+a(+JD9&!(&{7v+7GmdC9%oh1K9T>lYtJ-jZn2Y>CnZ}Z zvy8@y-IU!%-l+FUIm7saWgRwEy(nfH(xM}h(LyE(#ndme_P3p7pR-^wDfdukTRttH zb{Z6VAF+{5sdT4Kskf6`ir%_`q;iHYqhggm!@V>VKA;Zg}ot*Zp!O8lP}&iF6FVphU{n>9ZIQg z)U$g$N*2NcwY5@v_JjwXoot)oQD7T>;`G=gjZoVH4TF^xGDs1Px4ND?H8FGNMc>&C%d0M6C+Y>6%w7k zH?T+AitktS;Y(S{`a?I(&&4sqF2c&U4nDF=xDbVWmmSMc-X8mzB$?iv?lU(U`#tvM z@ZaPwhK9Fma$+or%gZdhj1Opboe{Fpf>Pw(N8iCunP$B)EIk|UP@8yt5}Se{wC%u* zikkK;K*@8rUN4-lzgTLBILs3nmV8zdcu+w;xm`abkmEHhwc>aF=7iCERKXScWO_ge zWWbhgJ8<#(^x*y3Uu%}Hi13H_2Lqh>AFY7_K=;zn)31ywU0uz4`g^@my;<$ei=8ZQeiuxSjO zc5*1s2ek@gUUW=gO{;5*j@{j4DetCv7p9n(OZv;nNj?+uX)-C83NM)wvx8CilK!Hv zLhEn_No_?*Fz{Xxkhtz);hQ*^vGdW;R16DG+LnK*Egnku*bX@KSo?OU@2k`ia_tMP z=w?xe$%I8ZQxf)7L7TnPlj95>@p~2g;$5`NUg;klWCc0mF2%89mx6b1bVMhe&!W@B?Dm9J;DDC3a6$e#d{T=U z?@!qq97pcWxB7WYU%&V9#fR#5zXxy3?-E<@#+X%l{(_`{7z;75DD- z@Iv{)Isd0bqRMBWvD_0r;#~+0N$<}OYQ9204iFYAa&kVtKs^JUCus`)05Qt$Y>vAI z!(4I(;%VOUnJ__24H*y!we#h2GunPR`~vCc7o00sus2)+a&}{5V|PG;&ICQsy^Baf z2z)f0D_8tJ5LkCWu`~^*%(shfzC+7;FS+#38{VZhM|dM(V?{zNN{5;!0ow3LjT8Ps}M!=YJWcj^CfJRTvsxL+o+_{`vQu>Pvn+*4_bFskvlcN2XFo4O^`rU zER=2zo%GFLJ+%-eJ?+7X^5Hu^Ef76!9^^GZM{5_m7%u3)7)1luU&BZ-#+q_*NY8zNGWBZ(6^62}s{J+b$g3b=~G%+>;=li^d_+RgUH;@9o z?*6a$7w02ojWuEpu$v5&;5-7sR`iN1d@(Mo}@M(!EyWVrM zCk}O7d`9qQF>CY9e)Ib}|6SX`MT2xfKZ>G(uToy=_e4r3T*vzx*jgmjc(n$sOtEaq z)U@*!X^E5iL$Cb)u^p7HHDfVcLZ_pID^tH?z@d}(aZ;q&X5Q^F>i$_=rpy_(lU&E? z6-I}yPt89lZ)^_^J)R9`qB0>Evt_ln^!z+wR(n+N^ggxTgH#zU)lm&~;T1B}Ms!_A zWGFQzN*R}woJAla&&E@fJ539%IaS|W&Xsj^wew{lA5ObvX;yiPu}&I^_+JApj*y)XTYH!}0-1mlax2d*XJgnsEsH`G{1 zoPPMFdETp#^o{hfa31lLdXrO-l)U+{HO-~^eWule{DdibF&byuR|B}DEyJ^>m;pr3 zqKjJZc5%1f>)Z0ZXy3l;{JuyqQdPh_<44WBQ7jc9Kcmh1ccHE!HJa<(POr739bQHy z=^FbRM-YtY8whV)PL`P7blbi=t4deYOKSXN$-)d{)_I6O@BHc*t@~!x>w56B|EBBR zJoBChPGW4``vo*P$T@JZsP?>oLNwSeLH1GSO>g5gMC?WMJwjsQYR?5IMpKM*r5xf4 zHEP=aT7nVjjbKz^bH7_QtLZ(uNs$&UV7xDc%{~|SC^)zaew}VN5rGFJyE<^DJRn}| z)?ECnX*&c$viFn3LaSxpFK_A7qrRIz=oXG}003Dae{d21xb>M{(7(^Tar_J;F^5kZ zK8Y`HRMZ514FK(6D5L!kJ939*7iF5iJ+g`i*~dhcp@z_86--ddJ7*hah+>fEegb}4 z8WJXucvuk-Te|OOv743W(MxmM7j$|hF;M9gDb5$pem_pQ(FIZIwHREetUTLZZH%Z|1@ZbZl?VLnP2wmONq91ot&WaW4#m9 zRq%7>56o|0eF>B2Ag}R5lat!l`WiYJFA!?w*cS+_;g_PfgVD0pJeNm$^`**lDvzVZ z$oI{`#Zo z5U6sNJyPC+o(neUO4vvTf%Oiiz?<)w@Nvv>dZ=Qp`vB!KT~B{Z+}E!-!8#=8;u=@hzcRl&mThNg~BH8bEF9_Tzg`-C*T%yt6~N7J$v`5@mK}p;EV#a;Vx6 znOO6bVM*^x4sWlCOHaIFn%5@7BKntl66$?-i^`m%t>4JdR!}ujr--uaH{WO!=9+U$ z=d)CP>Pu~zl6=iLldz82Pt79QK6BJrDvGEwL`H7j)>gN0g(!Ak;#$f?UyCPu7d|-kgD6-v@#ia`QjkBR5$E|M^ zgynWZuI@ZzJiXi0b}gpxJED|9eE8w)c2&r6m>z7+E`)E{cvWOdJNJ&K`u+Kn|73rz zPk}N>WBgxAJRP~H{EISO0bX@2iZ%Z3aE??(+Xw^`0_2Y++INZOk9sAlck9eRK|aRI zKkDW*eR^`^_nT+Wh^3>th;PK|1#B4UnQJuI50HUo;~V%7<{AE?kiUIy;rvniZrsD2 zYDlskuLDZ*uLT!tWnk{STR4l)6mir&BHArCf8)*!#wk5wIUq41-#B^5Yu)eI5$$sK ztc&qHu_!_ANz>oA=l?cm9xnt)9e(vPquFY$U(NydKA|ksR|Wxq5YHV6bYMpu$}IlX zJ6yXy?$>Zj<_fjC?=77zT$(kVJ z8H#cGIsZHr3X<@KVd+i388UIOFo@qSHqdok5W*TSyhNM-~vWIn{-_}x#c zM*jh{fzhYiaP?>;7W_q!C=I`&<>h!*mS3Q0wZcVR>1Rz?fdIVO_W0~zY99%mMtZVo zAc%fTn0X9wWW@i62>E$#6BIJ;Ah_z01jQ8I_4R#GP{541hohq;S{VXcZAej@%&rB$ zw#BgBxh-)7Y&{;(3pC_>Hc{#IpYn;z+|&A8gF%RyP2?V_KknCnL;rd{)`u%AaXI%Y z!t1DFKlI73%QE*`hik=#Jm@eHNIzFiAyiQ8R)+Z##VB*^=ig)5YpgRh@`RYvj|$n2 zf|Wm*D+oXH2~{qx8ADIFk_OOyA;UD!VC_ZC5;M`TNZrM=X2#`qzg1foLsUbUY}BTS zo^XFrQbeldiJ4Aj8>7of;pC+Biz!u7XIR=eP%+BZ1l4VfO<|Ng`-Y>+yRHfNmw=FQ zvCq5ba^|*8dZM>k(Z5^y6=bcihp-Lyyh{5{fm$M|v!7pXEwwT7Nt}wpyB~if|C1us z)?)xKic66qg}AtkU8k@FUB_i&X(_z>0QS{9NWhF?8@Z<#tBh&OR~Unep=9@#p8m*; z(vKw;krq{cG0sG-kFU)CO0A7EHBX<4L^gtq_mT@{CbnRbp%w*mSBeH(%8H+TihabR znyM_1mXUom3|i>=KrRS~JxXz)8I}U(hp}MK&~Xe0$*aXH3ltRHw>6XJtmE^&fEPYMsUk(@Yt11OCo@g(IHh z_an2>vZ_y(3LVq|O>1C9rLA+5JE?v{3w)^(r^)b{I!$^ATYnAZXFh&|P9Jzr!-xF&oww`Chsv2jX;H zC2x4SP!9&g-mn{xdqm@~@r=E?l(EA3AWEyyHWvR@Pxp7sDfXCu-EXkO3VYeJ?iQ*p zW0Q-~D#hNYCnjw0er%GvdZ>2L_=iuZi^Fp*ATRKxuGvIcwbaDzwW`CD_A@6Zr|%PH zcdDo#hbxANjL4(_Ha5{4^YG_p8&OF1BXVY5 zNPZ8#{6hArIH=wlJ(8I>gXD@-is~hZ_}0s91K&S9dkfn53uzx7I1o?YwiOqL zdo1_v{+)X$V$+Eg%Jdzg*a{+BB3<3Z@kkKQ-v)#sx_PmGv(mgS)L|&S4vrb|!kp{} z=uHKYG60O3<^#`mNc=k3ZCIn=d+_J2F{de~a$Jzfvq{{9fpcL)P0zRVWv(&*+5>1B zCrrOgP9))DhB9%n`?K$^O2^|V2YY|VQ{dR~m<`q->#U9U_afl;P!>qJ_dMkCPH}fe zvV2;r&1rDxP)fpHw?U3_U8#t@mxWe8%&!;tgvE=M8I^JnY!hi zOoZtn+gGVRn2@-k{R22iErdrzl5j25lpM+>UaQ%a>z_*-+!s6d{w2W9#)E?cAnzTT z=Bb}k@h^Q88Tlnoh}`NN&p~d)EGc}r(C$=qAFc(c6w8QMy4%NiC7E;C?MLD_6+1iD z<=(hWp-e_Oe@Vpg0c=*UN9QZ_te+rCbpVrTS%(0Y1KU9tPm#Kl;ov=wrIl5tq=&7_ zTq3iQ$6@r3yU>U~ktly7TiT+qD#!1Y-(qBd78~=-wg*BuAJ3Be-I}9+bH211YF5_m6h zUq)Gj$378=xPc@J_@wtiC(z z-q4~}!DSXIK9hF2@9b*7*hhd%Q1!*w%ph%ONkf4LH&GG(ML|bvR(W?|iJii7|Ihkd z) zUNc9lYnPDm5N^(p8F^yLqTC#|mV| z-Wb);C=k_no8d_uF3;x3rom6HQIK`p!8IsP{iXPx}NLcfW z8ix?JXl^`m?fF!?=0`{`nW%JCHWhgwYACcUwf@JR=kUKidV4V}BwhtskQ=I@hbl{IMn>dQ)=%Af!dgmw7sIdd*zW@8fQ%32K6(9RYRan< zkJBBbH<7^8qx#3{K0acMa0yGvJE1dqq>mBHAe&WD-BSLnMwtW)d8-9Vkx&~&3O)EL zDR+Kkhw<-Vvk#_6q)FuIp0-xT03g*^I0A}f!Dq6*OP6sFvQNnW$X%fQXbGP&3IdcD zwDONFrAH%>utKqQVZ9X>)-dyHt#`!TB&c+ExexhCi1SO~R}Za7GTf#FlwGgT!}->| zr$C)b?yrq(kMp8+!kc*@RCJe>=FP5$RW~=Mlk3SOF)KwF`(F{f^(HZrG4KUE&wKhF z5x+I{a|O0xdoCZx&q4;hPu!P~J#D zVrkt|iUq-y;G2NNY2|-GaIK=e* z2;Ck=f;Qwv%8S3#c30swPh(OMGd=-yT#vOiU=B;Skw&1Yi0xF=1e2q@<%m^!8T#Ede!-6y}tH9q5768C4Y&Yp>D> z+ZQFiX#3gID=&Ddkwzg^#@)5dyUf+}7oQ^T#^&bz2M>BBwJPnhva`7(mDsM8>HNEYH@ARxE>w^VZ2Em^5YMmz235wDDuk+g_)3tkbm%@4id6RLu5CL$gUn zo*6B#8J@9cz#U$PV<)B_PD18#y-qwViG!cgMp=!4nL-3fagnJ;`3o^*h%Gr%f)qrcXgt?P?JBy{6(qk zT$4H0`0jRauv=RX%*`}J*MB6(H@3PL5;}G}1mz0SbjAEl@figVZ;t z(9$m5dN65q3jBSg6TIBM_C2;+iS%aN#t zx;`zlY0w1UK)p5j)6USys0&e3XlZGAVjqJz=!}R7x<={ZO}pa+NKQW5O$ch8^?|dR zDedEYWuX{etX&J!2_aIF)EE}CT&fpWGw%EV*y*Qf*#z8QNzg#v0s=hb1_hi`Db{F2 z>-@V;7+&}ia=PC0uM2$YL1+g)B4rm+h(YZDQ}0CY|4r1itd31FY%xYbK=Y0(S6Gvl zJExWL6+&z*-r2ol8TjnBc5-p>Jj9@yiXgs@&&HDprRDx0Z_N%=aM?&9pq6h3gSWRH z%{YjPiFrT-_4@sGP1to>D2#-VQ7#7=xqx71_Ug-zPuYElf&~%$3;)3wB!OPwbzTVB zU_$5+g66~Oxb5G+C!o#9BRVnM^DabBfe0}aeCaZ^&|Qpxe}4*^o13+N8=IMB-1)|@ z0t*R6L1FPaX8s1P&juoYP-x^AY)zCm?G;d{dw@r^{mmFVRc zwyD~cosuk*SzP#9zs2KAxRz29p1+VReCQ@-^{2}5Zh-7tZw`&!JxT?Rm(6#s&$tG6 zChY}yu4CL>WtG9xWJ>MO!#18U`7H5UOQ5o_`i|_|ES^(sd>Y@dV;14ug7Swq|NYt0 zN<$eRmT?;0ex`q^lK@wEg2_MBR$Clu4nhU#v{Yq{kS5wl)j?c673y!X@mWG8*e;g( z!peT6imJGRs{Pc(qIqjOl)ib&PZ$*an2a?p*=d!GmMPO1@mye9?2VF|^OA-3B{yc^ zy)@j&(2)ts@u1FSyK|o&g~^XqvNE%*iP|@fGNidx7#LltliY(DPE4fg6sC_`P|LyL ziBcg?8pr2uOO_tsz)GiBo%W0D!VmF&nWL=1h-Ni2QOTDJ=ywniZQd9k)hPU`vVJF4 z*LK}b%ST#8AVmcOq^F{wd_FXrDsAN$8^>sL$m<4TJ2K2z@dHvk?lQ6@T88i3`k~#d zcyeMz4@5pw`Pc0L?`7Q=-W0c{YjKq`qUAvAb#ie z8eIT;T*Wm6{SK0H=Wr4yCMLAAM9WV9klg(ciY)`FWc5#;$u5ws#(di_A|o4vagI!g zVqd$}k;-k<(x&sY^Zw5F53-uVv?BKikt5QRGK3TcRh)m^RD0ciKCnM{sqoAflr=ND z`{&3DCE7tffPiS<{&G!2BJzbv!FRFK-=>#~d7&jLp5tg<_s&AtYs&5p$g2HFS5da- zWJD);F;$1C_E1$ka`}eX7fGyYrbm@+|28Tr*nJ|1=z0-F2((xlAYs2hV4R8~W#z$r z2?MOrt3w%&^>tkPH`_#z4u+21G$3FRZt9{M>^{fYcDy}ta|`ADQ~&q-i@-ckgkbQW zuuC@}XA;MyA^-gB{ zFe|VVkl)hMLS&kL01|UB(gpnZaSyz5X-M>M6#lKN2+9%sA(PK0&GEtICey=VmNV(c zXlK-4c%3J0MWA@s)liYLmwA>gCFl-M<0PL&Ygk0Fmfv>UMXk#FCwu&7cm(`^wmwJt z8_VB_Tf-|q`SSgsOjRmTfcvn{yiig(ou~Y?aU_`kis97jtFT|oJI0$sC6OqRoAKJy zKQp#W2YG4%7w6{8xb-HItK|2euQ`%fV{~>^7_Mltn+HnAJ)%PM7|`P|i?%zWn&qAL z(5_Kywpg@_gp(Fn>Dtn9*RZ%o3S~l!@e5?a{-lfxWW3NQVq?M8)Vn+TF@>EhljP=X zf!*^lDwl7iKYFGi^@)0G3%~}q2ZT^99L#C zvgK&y8Fnq@=doifEL13L7kw)9fxanw@m*6UE$tW)96CC5in^#~uP=`dM|QDZ0OP+e zQ=f0Ss!uji*@@vtHsL5Ng`?v*a)(*4L`C`5&JJ%fvA^!A8L}zHW+v^t>>@3t`kHb1pj`5AIdw3Thhw{R$SdrnMsa%%Vk;)ID?tTLi={W~Uc)>>T4i|+qn_=Ab z8_j(jeS3;8!tamz7<5ZBa;AK3^IiP+JZGlw;tGn_?QmR?jW8VJLc3Hn3{sKSS^Q*6je!tJFd-PcL9Y zg^jjw=?qj9Nc7;Dgs(KCJii0{1t%okfF5ZWTfNC+?2k-NKpa8v32*6dE*RgQe%DwC{ z^KI$JYgG@>tUL67#hCUm8}n=KmciHdHUx4^hOC^NnH3 zF22JPmxk|ulUezcLf5xjnj_X$+ezO###>rVI=7caUZWA^%f8eh74Cy?A2WTdEWnhk z(Vk8z7f)?xH+sns$H@l8(YZnGM$~H=8|OiccH>n_ek0FGH7c-sN~*|gV4jjlZ%`S{ z8rSsuv%OG8_xo%z1&Roi&8uizCG_=z^lsC_-P+-$(6TA{VbuhP`}@b~ z>@QB=R!#H)_QD~Jw6|0En!cBpSLA^>aw#V7wFXWuyW1VD$w0+O*P0M`_Rr^}!P`e> z(^H^jFKQN<#{m5LEtINn01n_8EHUjt%60&Rh8FgrSml8N_cQH~;IiZor~oUel^~6i z-T613A$@ycUk0$(TVT9xH__I7>j9CwYdeTBD17L6o74@48gRj*2g!QHl#%bb=^oD4h`X=JA67{x_9WOJ7VWp7B z4&zmNC>z-N6Jl~x^iJzh3Yor)VEp_u zo+DEus!PJt;7HyR|uB7J7{er#@T4q0Q!syL=JLN8WotGF=l@A-4I;QE8+mgk6q zX@#YK!Zi)0NrU(LI78q2W=?D-1Cy$Op&}7~fG5_T-#KM@*nWKP?G~c7zI;Z!)J?9p z0%ySy8o1w+MpGtr&X{Cu$}@mfPAo5P;)9;z9bU~L=%V=a*{EVc2RPu=GQRF_-QXJB z|6F2%{#!+aYGJTIaOJJt(@<_hX+k%m`zk_ zGB`HSVIlbsaMAUhfcZ>`nTQr>n`V!a{L`1+k9HQn4pAw=*M%`*XecWf2ONIe{POs* zJ3@XO3fKOX1RT_Cb8HG{jUE!AEjFGvGTsI|BZy3!|A4p>30Amk(`pN-6p(Q|VDcbs z4jIvfF6CxBbXSkxegoi?7xo2&?|DL@B?Yqgl&?0d!U>W{XeWagvH=I&qNt@wBD3OJ z+fS+ggi=GzN*&wxbKNi8(1}erRd%U{=8hhdOSO_~B*LiY=`7Kh8H$N}0j?a+Cwjs@ zj?t2DvXyDzhk1qBs57y3Qdgp=-MB1MUa1#y8C=7@G-#wzmY}ZrG>indcqdKmcEHV? zZC0y&iYhd>b72#;%`Ih$>@dPww=`Fk)IE3Ne+OI7$;cL62lD%_P$Y#E^YSwa;Ay-g zabUr`X%iEx6;*01V!n<`6lVGzt=!-3Khr}+V_|O?qRdCwYi`b7Hds~Y>Ot(Sy>$3Y zUNef-NcvHtaP;d0*67LajmlXR!%wJrqvEt=Uf3#CUavz@UmCt#!FQlTt4!sRGQy4S zkb2gWcvUQpADd-CiB1F6j4phUn$K0|Kf=00x!%FKV;9DUy4P9cBpaI|T4}D)u8*ax zh{$Mk-sJ>R)}$R#z2yLdLQr(iI9R_K})D_ln?(uN!1lG^{1zO_&Z=SyUdlsnM8pGhgru^bR-&>H=SD?%b4v zqp;_Bf>kdOGej4q`MB4txJR*I>aH zA{Kkli+~!S8c%PL)cS_URLAufUt1lVX%)3U*4){ovw_By4d2Ne;#2Fr88!pKGH&%#>&jwtE*6=qHiR6pwY?-i zo_A{|;rq)husR5l7q1Em{~O6nomj>nx_bM-=!bcP;}e1(;3{~ zk-4hZ!n@3kxTYO_ARYHqhHlZMWI8%FI-Cj6G(%C(I$VAGjY z^>eE&_E0(>`o7BL=VqdxQxlHt2RzdyC~^H0x*rYNRH;O>mFX~%=rW~sY$Qq&qfE&y zxGcF~)uba{Q&0LJ03-Qoi&sbY=@S1+Jjl zhj${HovQ-%#%_OJ9M)Wzy!OP_a)tql<9nd|!$!QO(;pJ=*LMQ7vK!5ne1A`cR@aiM9{m&4vKc9J*HHi0<;DMDn*rbq-Ad@4z;v8g@rV{hlfc0)2!@0;Zl%+v7sW^5;@Q7 z&=4lC8J&v1rp=u+K%$1SbRPGJ(dLNTKE}n<98jjV#nxmy7o&bILUsRoyM8}YxFBUO z0oRI>H$@ULW+WenvYU0@( z{ifvU#y2NLZI)QNJ?h(Wg*?edgEgV&!i<=VrIUQ(M|K(tO)#C_O@@nY2g37AUXC z601z7LrohysV*AAl^T`P+%2dS#*t3T>dMW+kfTpcW20iM$>B2B$=02Ma;l0h?!#EiE+E3}t%R+>%~i?p*FE<(bL1 z?u}te;xdYZZ=kK<5;M6drDDudpNJpEo!d2HhcjIg{x>G9$;7L%;Eb&>V8-I)e9~cQ z_M6>=WsMJ`i{j&<9m`=ZR!JN2R4Rjyp^uJ$n1Q{Qb4I)w>XL4@b2y+*H*E!{`nB_pUWGRLksM zG=3Z^Xj0`iF^q^J(;zqaxJFK?hf2(%h&@Dn0&FGQzy*xyl}| znXPv$MX0v8l8zg{@XpR>?i_td3*{p@KtqhBwGsze)Mf63X=|q8{Mo*Wn^PBdj5#qw zXdX$0>hz_yOPpjcR-R-bQO|sVPr#>%Cv&I3Kb$?$u?h$T&h%t1t?4PE>C?Za!PaV7koH9PANZ?i{otURusmW_RM)I_2KN+^&Ic zE-6pcUCr)@ZfPy$Fe)mv@{{O+CUX3u4qPkYAe2MLUHHtcBgU1>n&#(7L zD)yla%-Q%T#te)#X|xQ74A(_1Z{YVNPWqF&V2|&UYO*`HO&an&sdTa&=p(o`o;AXH zU9t*ARWBBx;o!1kC>3koo6P=vtk_U~joy&Te2gMVS0ZKTURiZ{VTrVwk4cvCcbhO) zDk^?@Q%U;hv*hM#Wpr7XD{sLeXTa829+SD*fzs24iN7ZggZ5hl^2on26G-2PK&r?# z9q~zyf>IGg6hR(25atjsqZRBEncZ%3${+P^N5EG0BE!_+w+_I3epAfhY|tdTR(fT| z@}~0z-IG@D2p<{|5Bhz;MT{Gp%(VO&dbI>68RY=c(jCZ0(8Hp5=@=jhidl4}36-e$ zExQqobF=3CFo4G23L1TM%V^aBpVf3NaXzwSo z=X9w%+JUhHsmx!HK6E0a5e|b_$*Z&LOP>n_E&`EBJ&~Po02^2XKC$-{zSbQW$eIRW zZy%vvga)l{y=7+hdUn6P{Kb#a_oo4}^;1j=Zd7-MV65QQp5&P9ECJ#_mEDM;UrcTx~%v!e5J5>zbf$6hReg zmQ8iS(yp<~f{2|`=eR2+@lK~yPJ?;rm~QnABetcAvaO$7m!s)H0}QHKuE9C@u0tsss59SrY2Hsx zrOBWxzI(rZmJHpgtc51(Wy8u8qm9)VE+?TVN-QQC;#MUMS!2HLtB4pGnS`3$EUcrQ zoSqE59BlDTRF3pLZ-NISk3RW%>QotQTn+eV%o(V_7DnG`I~OyTqw+Z&ycT z_6ZSE4clXEc6Jci*$C>nl}+N@{PH~=1`gSt|7w9M`obHevs+`CD6gjr@gtvokNIlF zH4L)r562J$Md2Lr>46URRwr;);z_dw&&UoQ%Lr)$to|nI(w7LIw)vYQFF*$`wSzpb zTW)Th?Rx}P;eX3STm8Z1_ukpHG<(#&M$>FOuO2e2e6O-V4$DhHu?~(EFGt`>v}6wD5og@{=bqujijs zD2hG)VxU&lyy7u&MMajoU|U7-T(BquV>XsaIh+1-+823^vP46&5v{`cA}*GA*2ZW& zFR5}L%5~#l@4JP#bbT+yI;9K)k~}}Seblvb<>iaOhTB-^eLYl2JA&vJS}9{%^U4W zZOpX?KZbRhNAYW9t8+1J-r_zZ6rv>$?+(XQU5oCN>D6d8wtd83j9G!QQKt@fvLmHZ zc8yllsT9*>w~yb}Kx+=J5K>tU%6Ul=60;Jj3$aAsdpv0yt{^2?7yBr^UsaidV-m-W zpEi<|tboE0A4|i~{1?S6hv{c3l}WwjE8MN##9DjLINaGiE6~K_$+bT*p5x2NWV}j_ zMKv}yj*ri3niF~q#9y)ysNm_vpfV5+Fx9kitFQ7RrabDlY_N48=gqdNoei#2XV&nO zLmWCEFOnY80^L-@fqaEf?eX_4ZUcQPdp(Ii6dBwA^7QdBNoHN^c(d}AnGBPVuaM&L z9fiFE+p1ca^f2Lj5947jvTp-Hb&!@(dC1uJb^gyWWMK16t^-8bu-a_e-Uq8$0s_qD zFTXkz^0Yd5?6z$OAUg6c5O^)8^D&gXzA$di!{Rze=Z)-bS zREkTI+iAy&lN#Joez}WgEoOV$!Mb^ZMk}Y~yleNrwrEqBH^Y;k5=Z2 zCXe1IAqj7!!f*URAiB2SxI#eD88uB?NkW}OVx#NpRDo;96_Gw2ykmHzfHR`EjJ`6+ z+Hc=)uO&fLe4bc-OMtRBBHdrz&W@L8$ZWi}FnX0Y%VKQ0bFf5)pP#vBv@nV3vi>Ds z?j0jsLHSo0sgXV;8RPF_QIgrd1mOnCa)bldlp1}^%iI;o?svufpi1Yn`Y;|rS?M}# zr!Hr;f<<~aOcCc2Pa5V3r)PY6ZT~oB9G}Icyi0+;=FR80MYvV0#sig>$y}|sYA~JJ z?!s)=_UPYnEy1dZ_SuCOu%g2|3U>hl9d9}v9o;bmyLQvFGpK@pPCm(1VYz_pZc!n3 zlNVNb2ssR<5|e*MXaLF~1kJwT`mS6#=r&Rbn>k&)ooXHd&Pd4pdq+(3C!5wvy++RV zd5|X0uaJ%Q9z}$ZF~FZx6J^r?((qN5^0+tIODuuw2)K* zZR3>lYrL*nMU~4!&R<@Mkx^f}?QPI=xCAnKH=tne(wtwnGlhMJy&RTX(vCSEy{b~_ zpuQEJ1}z+#X-xtdVCwtGt>3#98G0?v`~QGEx4I5Txg^FK(@8|}xUbMSCY4%2i#}TB z9Wc|Byr#9TGs6EHqNwwi(&05re(0v+qQisxs;s=Qr98%)CtQidSxF03XyYKlqMlS< z&eCsB8W>DCZtv_oFmb?KuOB5AF3C|(Tq>iEjjnKcwLX{7a(=e!Wns+mY+uH*5O?#b z?>dF5qESIwVZ45bnu%Nd#y@9Z5W{QJ9EPqn*srssiwP8!HD%h;v!i3TnL{xaub8jf zg+;0?Icc?N;?jn7;XP-@vB8y5FqthC&*flcH|%@yDmb3iwv>DJ#iS$)qbfz=3ohNL zEORAli^3P(Otb1SNq{Od8%dJBtYV452B)%7{*~y=t?L2*2wy%tlXvzEc%?C=zq;1% zd*3rKYu1%g>@83IIw7|pQ;M$9Z4;ep>5QX)xRJcat8C6}S}TOu*T=}H2CgaMA4e(5 zvG7rQ6G~oX#$@U+4@bckETQI~WTdN!PMcB7-4qi>F&&3Pc%F&WhAwNG&euacFmLbS z#pEj8P-BjXs?_@UCY-4`Jv4qmjalV7=?wxFRJQ;{j8m1FUypfi>K~0HU%o{?Of4zT zs%tKBfP}Oo;|h)6zJ&S^XnIF70t%e5a^&H5i&J=cm*^h2>h%9p0FWFXC zSJ#R0*k_t#>SXN_D9`pFEqn{C(iCdHI6sk{ZQg2fovDwpgD-~Ac>UV#!2|29wYo`9 zkO@s1uLC^kA}R|5g75rR)652k5DozfV;515>Z+<$Fow#UvSH*4puZ#M*H-!FE#BOK z4y-#*!SdRMW7v%>{?nrsp1o=;tZr;pL160?B(pVeeh_^J_KoH|*(gJ%IKG*39u{89 z=6RykAuiGQ=~9^dOR}Xdb~zC$5<}*k9&;_ZJ4_!P2Cyy#k}xsvVhWR*x_IZjkWn*G zr|65uRl$>C{~*zOxvnml`Q|0~5W5l9O&*2liKle!V^(^F_2AM+^Ki$LNTzZ|$8t2&GBZ=60co)p zxgX}9FNY$>4CAG-G>VCR+~0kTjyvTKpIeF{>JpRjTVrA@MUFM>bi6kDrsjXSLX~sUK$V@ho0#4v=K7Q|H5U`#|%o8OP4;ZS07p z)kcj>qDBl@g#yT)Mcz}w^M6zC_n>IVf_AHE?gUvse1pqR7f5a5O+02vC|7x6`M@L} z0Hl5d?D=;SW)v#dwMD1G0O_n-`f&oy+X{#RE3mBX=U+Kwnfi0V_I}_#re?JF5FUTl zrDhd2Q`o|o+x5Zt7R8f+`;I9SAa7*eJX_aWT?L&h9*LOGuBe)EQQ#wMk`0Pfeg0tu z+Qny7BTyU6lRrk-TJNXX?|!#bbj2ORzNLxvmh4$5#!>eDi7~6ay{dU&mp|ZkXq>zS z_9orN=p@aStW~1yIgc?=wAEiv&LvKrl*Hqab0Csk)xlpV)aRrbyN-63-?jaQ zd6!9rvIZU)47N)8jvZ}q`hQHlWmr^UyZ$YTh)5$a(v5`5(A_DZprkZIhtl0Sv~+h! zcL)NKLpPEl4FW1HCGg(s+3)`E=L3fxei(J z*9NxU7qcBg{{Cr`p3lW5WYi;{CSt{wqVhgbt!&;(!wtFqeFy~n2Y@*52#i(-AT;|B zcwW??nO<4)3qlU)^g6hXoh=xuXV;A#pLxIVKx@Ow8Ntds*>=Lru#SL{DinM^DXy%@ zul!(eIek`G+R|u~wDmwV)rJhg!zJj$P5gS?Z z)ZBFc;EY)kUloH(G)*$Sj!s1K08PN-cdYdZ{WLi_Jg-<=ov@f0b7wa4Q@mjDG29m_ zQ*kjCwo#Z;$`tRqB$X(haF$RQ+)V!X)wGZH>+7wiz=~fL&=Or$gEs}Y7?Qwbwp&KUDYx0ApS|5yG2 z)!opUkf-6uJj`vH%?AZ;+IiXf-#2~;3gMmFJL~V z#ecba5mu{6Ud@=zlzTVhfa4XRjzl>lYVggaRvx-T(i^h0X9PUFJKghd2-7hwzhJOru$Lk;dF!A4$4QL@Vs zD*^hZ1r-{<-`Lgsnf$>jSo?S;N&7Y)wSkqwWY_QBkpJeH_GMl)#L8$y3>sTXm*yqg$_e zFPe-e9CVnRn|-XU*+3LT04jyw|AJU56N=>Jt|28DvB}(8Zv;?$4Pvm|p90o~sn=Y_ zJ(#E%>Sh0bPr%-R{wwiOVyyd2ud{{$zo;L$Z(3f3MP=q*pIw7O&GKf;Xfu{?74U`| z1;&|{VPFIOR@-?il&zN2yp$cc?THZ=lK6!_dN%K~ooiD!_3C8V3g7$kl<}>)dYYR~ zee}iI*KMw|#ANr)suxG7y{e6m`-v<O|CR;dhw?h?Ez4h|NW8N_V*v>)xP5$q4!_tpF5zR7ZkpxLGQA%V`AYu z^5XpFtTubfeBj`Ajy_7yHdSC6B5oQ>r(8hGOGtaqA=PTv`tl>8C|xZ+d4b}|Q+o=c z5Sa`vdB(V6m(^-mYX`mjRY4uvd?|N>v-%0OR-d1L!}LrM!DD2eY z@wBv}5#yue=LNa;0u{k`aV4JXC^N+?KbEtPtRrJoB$4J9ejUIiPm%S}?ImrZ1(8_8 zAs zyQs$r%7qOz3gOyecL;sbXx`%JW3b&b{)AaU!%e8>7f+?pFa3;x@SE!7+ZQwUB{3la zC*(+c&-NG#sH~EJfw^$VAqF(hfuO02pASe?@IdHxVBRX>uT)~H zd-pm5cqqJPfV>NUE(-HNBB8$y>7yM2nk**hn3Pgfm@YMfbk1Zxy+{Q z@&F+iq$zvPNbnvQI!Vd#z`%i@DmOeHCqJLhHqCHaPBMV1zUKQOlMQnAuK(2na6(kn zJ_d>B$?Jg3U1i}3j2vuq6SKq>6&28p3b{3)?-4lpFfR1r*|M(~i$f&jkGOr;bcoQwwV$p!aZNN~{)Qw0f?at;S>AlWGyLr4m^!>-q#yyL z&*Me}dvqE03eC}t^(*-8F=|w~yanXx_@EXzbwc@>@sXImc)_!*IEzQAyH-}#S07)R zT3WrQrS9!<<0gf2{b)k0e4Z`|-oy|Iow!%-E8NdUM&>Tx1qDdLCcZC(cS|+H@CyPS z&-h0hXW+}<=Z|tuk|(#rAR?|o#VX&Q-U3jrQo4_5G~=vc_M}D`YHZpOwI4O8EH~E^ zK>z84xTcYRk;MLk-76aDH-8de7WSLGLHB;^sD zg;CvBY%;IlG28vG$wSR?=+9N(oPB1PS7czZx$AF7X023GaU#iJJGz=6b-|LLHqcmE zz%5BAn}*ub$Vh;d#n*DLewxRJ12i56K>a>A^9^l$ZAh)%U+Z$jcfxz87qLxB{cnisk+1?i!_+C3h%-nXMIy{Q{ z;1viqXdHhu=p4acv>r%42p#}p|54B|Vx=?E422*IkO>c9MOwZ$+O0}LhVE)mpoG|# zAUeYbE~LVkv48&uHpdjm9>Xin(524a^f9KRtLq&doJPrXOG^FFlvP7(2hDCh3{Z z@9$?iRyLefmbL?dY?L^mq;L=(0Vb);-G}|p0_0&FLH)gKoYIv1kBNI@(y`oDDbN{m zV089=>#$}P{TL3>K*4GHyR-tx958M z9<%n@UxWzFZcbj`I<2mU3UAQ%Q9L^R0S&te2J6Mc=mMl3m-MJi=J=gU215f{++b;yQXUw}!h|MMXw#Q|YPGncr3lLVtUVBW&M_ZH$Q}N!? ztvwlYL&0Y=m6-nRR^9A&)sDEoV~!gcLH44loojr;6lW%oT-5Be&W@&s<-^#f6A|1L z9F*AnHrSm_u~BdB%ZjyN~~RTG&z|*czW# zsC<(Oz>11NYT57@nhv?_b|UeOI8RG(_1@elAK-+}%xDQYrL2f31`_;ut*uYyhoL4C zZq7l$k@|{XIBTU8jWD?6jAb?iNv7b=8~0slrf{NohEIu?7ilX+F%45^MZmSO;S7ge z;VL@a9oppttkxd~b4JIp(9sHdul#CA*c52s#wDzMhAKvxGGX=Yw2}AZO)M?@4OQOa zQP?ih$R6gJ3Z`zb7{S?} zv}LNA9}-OOx3*x$8L7yRv-)1RD5>Nrc$-d1EN#w2{|Kf~$u*Mi$W%nRgQ57Q=VPv| za9LUCA#Jc2xi!ZO18o^jPfFi6V-KI$xLZfM|ExY>0~(3Bo{M?*nHZpQNq!tgyMEtU zcm4By06_A3L7M6HX^#xy1~H&7F9oSD`sGMO&-i?g`CJzcPS?e5)*-Ki7uW-W@6j;9 zh*mE~T675Z&dz=d;LwABcsR4ZfC2-&j=m&lI&$kD1du{Dgj9&Ic3nGQ(gnVNcMneY z1Z4Sk?-TlAA1Y?izHav=EKsCqwp3yW)^58XN*XQaW&H8s+kO?f+<#`}`~#1>DIdS%;D42aaNAj;^17cpskn z_?m(){j1p>+0d=t>`x-UEg`>?>rg{Y^KVO_kUZ1!Tbi3&!Mj<(BS`|-sux05`qRrd zzn7s#=QEQUiOVe4pX0O;FtEmaEK7f`nVxSh(;j@CqecKdf4;r81kPSBz{=TreY5d8 z8pe~cO^fG_3E>*Xrn3IlBQW%QubO;?fLHDrcm=-Q&`1dChWTTIn@s}`=IM2+Gye8% z&-PG2Aw1M|nvm&BY(Cxgz1e<5y{t;LxPS9!|M}<2EHGju>mz#!qDK49bKIP+CPA4y zf?*tM5=xwRa=mf!(?Ys3g8j>2-*^IBP-Jo^u{$Q=QQ-~5HK!JjxiPJRVZk#Gn5MwY zj)@tSGQWzCcdV_+E&N<2IZH#OiH(&Mc)AOTWrx@z$;VDsXuTqpyaMGcbYgh?cg{IN zae>vuHZ@Y#@LfnC86vo}c3SaXqm8zNLy-#4yH~lWQO<7+uTlmXX{G3qoG@${w)KL& z)RYqPcfz5TmQGo5g-R;xLtWJ{+Xy04Eul3FKRlLh*(iKA#bSyn*h4I2ZfU8$2~ixP z7bd~LeA-k0nl7M*1TKkWq3_okvg4;a@5j}FXS$)*lTHs4@Et~Rcbd-QcVPRUB13p4 zJj?3_ROuy81YDnvQV5!?GVgFtdU{CrXP#QR8>_yFrT?)fo$RC=XDh5qoX;Y^i0nlg z%cCt4CN6wVP)O**UrbqVjxwY(48r#sd>Worl)A9#%NGKn#YU{YgvbdJ*b@ zz!tln5*vX=677Kjs9zYUqGaiP4oQ7u0avZib>ZB>9Eth$MDc467%wA!vwiz7+qFb} zC+xKs9i$zP={FT16Fayk)&W_hAYMCAqlT>eL~jdk3$*Sie`uMa8M`|2NHuqNPhGxV zTHXZTir)7;)YnZnX1>9i^y|&uSD(~;#1kAUeiTeEeIxX-z*vb_Aj z`s-h7;eGcpp-R<~4wM8=Yd7te9=93*v2eZY zzO<*`#8Fx2dG;P6M=o2_3%Q0ukLb~#lnD!`Vf+iR%X~4x-Lr0>V?0Iy09DZbB0#;N zWgTa1e>Y;n?NH`h>H!|)F=o-jhd0orf;;Oy*G+|b{jcZs0k9EfuG+bMx-yJhEqU$D z3f)EL)1O3pLEGa7aYMJwm3^U<-QWz`Vh7-#6G1sG1<=)2p%la$!?F*kCrQzBfE4Tu z$vY7Z-J+}8VHG+}KP-c<^)@^F?>)NZeLosP9wp0sa)A|ml}2|b(*)_lx4(lD7Lddy zykACQPMP0HDAbf0pe|*ri{-ArY`B>8Td6#smnr5-t7s(1Y=I-|m6qOe&bR)-zqU4I ztkanB4Ln8hEVfg|$HgBjSZ&l*V?xez04elYxiX)f^T_TcYZRWl$7Q(MAS+|qM75U>($uyxnxvWr1!6OBXB8%hf$vkE$_zf zBmnnIri-8~c1F7=X)`fp4U>>s~hZm++t69f-evo3WMePxg?fxz>w=Y)+c^ z#fCzf+9wtyOE$a@EhBqi;@J7CeyfH^0;wr(OSN)qs<({wROaY!zj-TaVee=yG|J$Y z+h1emZ&_d59Ivkqbc67AjAvH28(PVSa=s^W-bn)!Zs2$w1?qF-tdj!frHb_Lt9Jd+ z$hfT2a`QuIEdx+dw_KY58Ez;cUYU;7WoN$M4$JuGq~)I*s7Ed4@8!aMgE-{-10yW(u?s>Q75$; z6I0V*NOIc>?xuARw4?pGnceXYZb;H>+g1P1-UObEGjPX?It+W|_lf+mZ{1*gvFiEt z%oC2HaOIlyK0Upu=z-xK`l(L`acu5J%OwZYF}^O$!uG^ZVU9im&aQ&5F# zCFslfT0}E2u#ntbUw{*!?VxQ7BnY}SImf_o znhCjw0olU-U`65wnE&<{vYj9QX-YK@?mDCcmpt3ocgsCqDa%cS2G?Wcxz?}4H*{eG zJWJIc2l&cU98Y4j#adBrwoy`7AGTN8+sdiFzviNQgv?Lvk@{I1r6 zZnVHo{F;zbDwS{dGRnl+yC(=2U9-9>r(T)6o3aJ2V)hhhRAQnE;vB-zq*iWk;_)2U zw{HUnE?GU_Uy%JaiBvaLQ)kBu>T@Ac5l>6#^MJKGY&u>q*YXJir+e%Jvc3MEp(%UvL>hvLC=1fWB zRz+JbimL%xfw^F&-_scP;e7Y8{WfyyyRz6Qm@$5&{l6bys@9hFJ@CltZ$q8xOz0Ro z&3yu_Z>X2vqubD0Gk>Layr-3X9G@YIHi4W#@&s^o<<#0}>l=YpTR4GfjU7LbFkq4< zdt*#<-d|vQK=v?>LN(qj4z*lsaR#kX-E&*gqH}6xdhZPqF2Cc^HuKvK(&&*NZJv-Z z9x^n|&CMBFT82U3Zy;4WpxVTxhys9+Ln!kC>1yifsk3Z80mPqx#yQ4RpJvNLK8<68 z+fxJJe+rF?!r5&x10^!ypAvX$z!drA_4|&6#!WVZ3oM9JC;(hZ%2U<0i@Yu%f(GZ0 zZx@-;e?B4m_97elghRJsJARY1KTFW`HrF{b$JUIWdz+xSu>yjCJuy*WpNw9Zt~Vc} zf{Y@-9nZVaXu3C30jZI7flq|_--Dy8?$e`@EzlbJ4+_=u?2oCfZPezbDb<1B13TbS z^j4fn9OFt=ItLSz$G~qt|J7H_Sif-Pim0pIz3UrkOn+bS85^LvxC&|& zLTkq{ZawsF0LbPeeQ#NBvkp|zCEu=symtubM;w}(&Yh65J#BjozW1Z*JJOOd^;^bcDAPk1DGGYs-( zS$)utU=>szB> zH5vT}#*}l7&Ypv)n&@$*C*O$oLNzVZA9fGc)^bWq*Qoa<*)A3*g{B&l$9 ztlMb?eaB~9(ERG!C86p2obK~rZhM!-PwS~byym}J#~+GuDmkaqzB6}E;FnNfhkh<( ztS?2f!uDWMNk~;Nf;NA$J`4Ze4`$|%TC_dlYN2^b=}hK?j8D2l{oqM~p|~}TyiD~3 z)(?VK4n;SVvXP~0k)5ed`jW_Nw`6%M&nd5)BevzQ>X~IuNvX%oq;+ShVoP)Hn)Jm4 zla76Z5|j(z_iwb~Q8nD?+!&=E9Z87(jQ20Ze*E?x z1k8VpJhy*cT)JCaA>sI_4DuQTF86sc}PFch#%bd{=b){!oboTFD5(%)fgA+tK(2Ah}KozFux| zW{dIKpudU%y26W+?8_4Rj89cwn`|))DlV>W8}!rz-Bf?8^GUF5wsb`HD^&cTQ1bKi zFKP2kj_dQ7Vmd7vLuDBe*jhRU8z_vJn!5Kmc}Hy)rbTrRDfN-_vz3=)Kba<8V{a_& zd~$b%rMwd9*YWhEcs*$z-j%gIV^xVcI-W z7_{T{&xVE+i|cwyB_Hriyz>GBYoz0*K9y383Pi%W{AFlR46ISI`9GEjc?bgtjU%kA zjD%VnM@@toF>))FM)$+Gm4{4(QK2NI+&)FBV8rx6po9{ka|^x&fkJ>L^_^#q#&{P* zT%qGi=c%5)&k>)lG6te6VP5)u{?asl53xg|Gld#!_NFtjchN8~MGM%eg^lS^($aUw zjcu{!W?Em!DpMgIVPo7YuXAsG{&zIn_%-`uP$^#33EaP|++PFr2>0a`O?6_3B_sB) zTg)q_&*_h{J{XpFK)HNK>2=;sy#=NXy@ zdQ9i6L%aqEclPi1^3AjS38Xog#`&Kh@XOb$FJD=RaKIt>ztcRoL2Ho2^aM=^Yq^vj zOft^qJLNlphmw4v`T6)&YfFplB5yGUI^%iBc%&uso)XGB5|8?&u{3u5)Wi1FJTgq$ zf;)!Wc7YNkesspziDeVTLHKq|QBM^}UzQJqt@8vB?@9!JxSOlIt^`7L)az8GKJ>?Y z#gm`OFm$4dLv=8%VAvX3^R-d7I!$k9rFIuo)p7T*OVQ}VKceC&216^aY4`vZr#QRT z<$z}WRDHTh%~-26e$zz@Pm3z5xGb%-w0b7DRB4f=dlf%s5-AZ%|BhZM-4B)UFPlPO z5al+!yGsW2Y~?Y0DZC2bpWs@*T5@KsbSST+GG0(R>?D%x2<UvEz!yxx~qp|Y+55+jP(&Rw3 zod$`Pj}`(&ov1X1qWjBqWcD)~E8Fs>IG_G}@7(#~V!t~|X_@f6zIoT@H&4KmY1>5IFYwuD*-;$c?S$sL zAMTcd@knEUinhI+>$nRs%mJjF9E4 z8o3*op40zmAuR%v{lpvalkpt2DO2k+?VEXHODwKVG&NBan8cXnYf&PI*d2ApKsD zh%p`6o1AJoWJ5FRyP@8Qp=DU3Z5V1TvNtLOow_=Z_S+;Dh=8r*-8gNFl!{n{g>vej zlHE(el0cG$!I{vMgIY?p_4SKNOE+!?Qe{P@d3jONQ53>%AD>4zTc$HaKDCMLvrx$> zsadn|ia;Te92D5F59Rc9Ea+8bYO`}fK9TlXY_Rx7T=p#iuKk~}^GCs`vp<(9V>oH2 z{e3V3^>PK(NqK4X3?6I@9u(&K4#pL3L2SbyV`niA&J+h@h&;KlvQ6qG#?;`23#2awR~;qo6l{?@6Y zimNoJIN{nd3SAYR4H(zbG<2?lH|CK7R$PV118BYnm~-(U%!1hiiHG=87fIg z7YA|g$Fey#!4q+$@{W)keGfZ?0!fd7km#X9zgxnp7j>@&hH#Mb->I|ZuD2*2{}{x` zR7_RqaXN3SUd9L%~E-^&I zQ#sdIxmn9%$G$-1;yJMu!D*P~d87 zYa6+{->(V4^KZ1Z2$W;U6sp0sS0~F)FJ#bLI+un6y0@6-ox&{(CDEEk#8fW(M{$G10y;W^?Mk;pa19~eP>T`a2Q z;aK=3ug?BOYMJ|f6{*%wob)SqWjZbPJcaLds1>Qe9BkIU!6QNxSq6MgjX*XMZNF|M zUha#rQdf#q+)fXF$BNY8wdGs{KMvD_!5=+s7oLw>D0*OvLvru&6MKGrBjJO5|s26Ro?imJhx+ET71` zBo$2L%P0I&;M9)94@An?qvMg`7)~K|klIdn5N~MgBk`peg8B`$j)R!>142lw(WPjt zI*=>ng_UgJC@5wsD{0_;q$4oBa1yMgT|M|)Yaw%e95eHtMqxa~YxHQ4kL*AOj5&eFG9ds_C zN5vYKh|*Flf`KOzJ3P^F&eT^WdxA~%s^J~~S(o6n0D%nE8}yubWR4--H7tr?arFi)$=S-fU0h`p)E&DPC8kyhv zYzq0FGjysd#P6ztp3>l2JJ6Gr1S_i(AUx$%_nCIT8(Wr@EGgc*JKfOT^o+KC=R70# zkcy3Dn_bZ?JrI?I*fElsX8*LO+E!yzuC7ohEtyt^Y389G559X zp1)6zO%1Bw5!KzIgxBF)wcOdAx3VEo-Ri~c-N?xJ{FFV`!PN#A9T)yr{a100ReZl` zoLhcchWJF9_KoVN5LloEp=#WBqqr*>?oWRMHW!Fl?ao;4jt>O2-0u{x`#2$FQ%f%6AadTSDCg|*WjSissa7I;-7)g& zAhp$~wmQGU6FO+xS}}K$^Vc zqUI@4Q|D4>ZNqysBUVI2!f2pyS28HYA%lD0ATqYJu_p(e`VEp_yJL|d;s0K}5nY(PP@t4m?oTC!h&m7XaqH>rTLkyPxn zFJ$v*MO6S^nl2xlgAB(q-I6Su(1=@$uAYKL($*vWFKL{I$ra+K+@!cP!7vGgg^a zPUUu;7*~7<3m3Z+Oo0rX-J6Kz|ZO z)HWrn{*cU*1)Ika;k+cEcr`6x*+P;1B#Hi&NLz@|Fh5i zQZ>hYB8Qqj%{69NK!_wLS_Z9nF*f1UxTQX017k8(fpP2I!O2NPCH+n?QSSl+73M%g z+(IYYE0Im7QQ>up4Dwu!%R^prbHCyJMMz3pklF!ss0LZ`-)(&5_cMUbUcpJ1y;%1jeK0xo+{ zA=k^PZ(#oWfL2WeqxUDf?g=7)#CrEbq8|M1Tjj6T7mePypsC|pm3(=u+B z=^8AaL4*)AYbQk8k!=>FeJWt|W_;fa?Wtye8KZF==Hvc#bBrlG8kU!LR9Ta|z0~AK z3`r0nx(6$v4uh4HNFr%nbGfc}-IJ3aJFA&w3oA<*UYaz;!lNs3Dy-W$Ih*3KB&2Xf zUg4O?iOolRXowg}{T8rEvzux-lU||Zn&31sdqTX+X;AcBiTN!riu}Atc`Zc&ZMHPF zp}XwYk|WzV%>vDNdGW|m6o*`xHz; zGN8%r$6iD0J zyW1dfCXQlB_(Fw^X~cU6jQu>o&B*OM{NG(N73{NmuQq_mW zn8wH3(|#KDv~)+5n?eyf{vVVpx>S4vIYZ~7zi*V_HNn)3vgd^S_V-(LtxrZ6r zV$o(*(PdxcY2gphSi*9+j1<tuOl~Xvc!flxz6X;Mp&-hyDuPCOPgN&hOKnP5 z51S(sBkx1W9+@?^B7MXq7A;|S5EHDl^rgFMXZ{bOJ7oks3g>mQXn`RQRYgF*Bx?Lp z91~4*k>a~8ZBfqNpYr5~)&G-cK`Zg|HWbOz(b3@c2-OF!B+OLD((b zxKRJKdMzlY&>uI?TNpE*B_uWwyBWEnWX1D#UBw=gV864}n0uuMVM!2G?9^s3Db(CD z!G-g(K3q#C$~el<4#qY&xE9DfX1vNOF^#{Nba+W-y|)LOluss*;MxkP3%tW z100X*uL4ARoAKnWIY*kVvLx9~IDE^jCWSCX?a(&x0~4qU@qhL}F1FW%E}rdohZ}(| zjLmXO)JkG^C@$wb%+YXJ6qowlX8l{&!m<%q!c2BcQ$v^(iqMGWtsE^KgKY#E!GMh3jV%811 zAk2+bn8TmIweyg5Xa%oMChH5P+*)3jjyf@>idnNZ?|Rvbc}%^fs@Y2#%aEFknI5tB zXq!SZ6krlea4d$n#F+iziIXvUMH_-V%lasp^5&c?{KGq5-kJHPTbIAz+mox7UD&Sv z?lYH8DlsX&7Kxg$lm1N+%0%>cxjm+>>36QcZKusrOJZUoaru*i5bhQY4$PWD-ub2c zdF`b8)at2I9b`-;R3v{a*MTCqg4+zS5R3T!RZ5ik!^}Gq4N}g-G0rtR z*8*D~=B|EkFbZI5$1rj$FuyFUsbtvDiu;&gUL_sJ7L+PaQxkt*P%c6_70w@r$;Dhd z944O0BIh8LhSEL!k%^uerkQCc;q!hfhRb%6hC7Tkt%;geK;0>6lgl@4_#eX5h$|g6 zycNufuN-Kj@HEX)&l%z1m~T+$F0|8>*_aZX@-KzA9*No6Wch1Q-r=_eLp@IXsWUFM zV9%;lDCUnkZi)$RoTeW~&NH}_TPI8Ab~CBwd7`zx`=afax8A`kB6w~c_aJ(B^~_3! zluR{>Mi$A?6uT3nYR*(EWKx%3@qdd0%QX~KU(|Yp^{2l1%U#ycWrm4c4X<^Z#BXq& zb>{JNH@&*mcbJ#I_1U%%;4K~aG8yGt=#@=z^t~zEd2hGe=n+k*vbEBfv4e=0!1S$9P1%4t$H24}Zls1c*K8{!+lj1p-qV04f+Qs+X+hOAR*ZZ!)Vqwbl2WW~3R(uV z{!KHU%n}VbG4jcg>HiMBv!ERH;Q+nj{LbV69&5VH{dopM3oMP0_aP-ZvD(FTqZP}_ z;Uf{anVv;g8Kc*H2%o#pc!GCqwzvM~4h4n5XM+*u>q?ng`KxLYp5BVTE+b!y|8 zXo6aO35*}17@3t1o+iQQW@a!e#~!DNVeGsr9##~nLxh{GTDe$p#IY&HbGE|rK*qzmRSAD zDJr!NIsNz`+mlLpN<65cK|Q29t!_*(zHkBw6Q<*!`cMt0GG^^uMOrD)n}xt6-+S$u z)ckAIL>&#JmUugEfBvr}aTWv>sL({`La|mdBbdn z(=;0Yky>qg0{ZhEUDddE{YbQ^XU1{)Pi67>E!zdvOAi8PfC z%dV&uOqpMXzC3&Bd^tNj88{p!*Y=#QZldKPQz1I*TbI8=#4IvYg{u7A zgu&D24K68%mx8&rCAyb0D@W9)SxcX?!r>vOr#SDIoA7khv6GYSM&X&!+B!O)Y(FV6 zA!LW7GU}wG&`)7p{qZh>XWJHQqZ3}L{}T@WCzkpAKX=R*pRE7@^N~FB4WpM&GbM#KB{;R4RZag=dJ)7&E-f|FhtBmOJXMEnzpHDHXMgt1j&$RR zT+4*m@2=E)LELY)KM^tQfsRR)L?jT9_&?RBCrjAe3||n;sN*10&1_8XwzjUfl~UIR zG8o6tJ4NZR*G%>83bt0f&EAqJ!OCYiANn5q{lB12GS}{d3fMByBYz&oCw_d4W^1BR zmYKn<_w#UK(n6kIh?CF7)QoV5U5{Dp^>)`z5H0#58@&LBc87eyP^Eumr_~?Jx+xj- z(m^^p_aN2)kB)p)K2s>x-mlMJa!gVe6MEcNX(dmjm+_?n8}6#spjLIp7TUd$CDkrS z=WYfHnQr4^m&&%7`FVsB6SwwM716PK17~(~!SKR>@e6&K7XF<-Dh`ctE=ThlEdA%;5526H<$ulrgFA%ej`c`shG!kjL1{&iPW7U@?RjmUlKiLq~RK z8zBWs@in#iUzmzspWE^byFI(PI|L%+!v-8UsD}bE=sN^RRmoL5_)hLcJ)lPheaa! z*8|23$|JuAA}LGe!hWM=^p12>8C9&5h+%}8z{=A$)yUlpr6j4+ZDZTXry7F$ykGJ? zSU&kajDGAGjUQP5XeujA?7CiE48QnV13ivoJs3vcZkZB2n`&b-9@)na4_duL&w~zH zCV+o3#iFkFRwv!S0Mk}6GS+sHy?gbe9T)lP_;Brf%ex<-eX# zn+!+*I6Z2Z9&nDOHK4;u5bRdQqX`42+ZU>|R$t3W{c9bO?y)j0H7^3K{*Vx*XT7B_ z^>Gn;A$}vQl^4Qrck_2I%p=TLv|BFHup+mVHP1tGaScIpjx46PqS?y%`6F{$GiU6t zGgFbp;OWDx0kQ?r}dPdC&WNuaHF&*k?*up{wP^!?PxJ_i+mv{OFp{L|`+T|469Qm|``<%{NC7-J zrbt?p)A4?Jw`X>eBB4`c4Q%@erpX{H!vTxWl%!pg>W@zPwMLL0B@IvYmJjeVVUG1c ztAW|W5XjC?D;pWQ<#3TIq?Dc83*`NEY-764rXW~cf)P8MwKSk1$wOr|gE5FnJ++C7 zn*MpjO9j7|bn+nCYE68ew&x#qRj%M5M^Z@sQ> z*M{q;I-0jLMgm7gGZQI=K-vvrlC`9&rD4(!pZ%jwo&Ww>if|+*)peuA^X;J*IQ>=k zz2TrZ6r6GMaHRm99>t-}0i~rm(`XT_ogTlIVUSJ?SI3Cm$;)ss2QIs*e}9 zrt;<8$9gn8bs1SH(huaL-f^wJaro+IXzNfrJ&NzOVhyCjwgz8=A7uODesTSc+D-8G zY)3g=-!VAvA_WpE(Z1Oa%};RrCoF;-iIv&BZt8A_{k@xxe1(7c+WomtTDup?@L6q=a(^1)U$!&@0l41{* z!izHjlbj-yyrZ7cz#fytA=R7bMy*BpAqVSKP+_N^gu3c1ZV$fys-bvf>_^AwkW)W1 z{LPBWG}o&3iA!BH4>e3^%DZ@REtMMr%(?e?I(- zn2em2jo;jIF{CheJ#6_@iyb5X?fc6ra`6=)lD+Hs{tHg4`C<}rnpvb(U`@z4E1&Oa zgnWiL1y?ye+oB6ykz_vp-5}m^UVnVz$XqI1jv2!}vbWzX)GkAK2{3o~zp)R}9)H-D z$}S`)-4o5e4^x*ho$0~|hbI}SBh*qo@Zf|CNsXs!bl4Agx=UZ?QM`Pt^(1PprM0WN zvRL;%UAb-K1pZ-#`6M@Xh$24%rpEK*DuKOF4Df<+AZi}Bf5$mP8WT^$%c z!Y!wUgaes+3gHzpzO2g{mu)Ir#Lnaja zX{cp~(*avn3D=vKCU@+gg6~*^>yyA5;(OL50rV&gx#4_^EF)X@pG}!#tAP(LE}!w$ zxh{(`MVaqs-G;!lpWqZLMZ~OowUR_kD&6yJzcEm&EOg0wka>$=PR?N~ydK3eGF?|A z3VMQF^{2&JuLOQ`yJWkQ2)Sm4)ZAkb%2AUDq+@!({UF;RPKmT!L~8qSZnN9wO?(?E zyWCwpiT!llX2o`6y-a$#KH;ubqXELByBQW4%A9*8GHS+&Zc_hZWCcM zPAV=KP7O$u=>KXIayyVn^$5%wL?GECeV@Id5dW{$oA;l-#FiIqM^^+M@jO<(|3^-W z_1wUpG~~c@!YKb8w_*S5&{QR|Mw|IAz@&evCnSlK7Ea{iH@8O_*^cDscgbh!iqdjq zxux;qH*{LKY{+YRKJR%7Q@DW76sU5R8%`5WuSE7PU?EC3Xu_BTjEx7&X0h+Q-gLLCau|3-lmz20zK6lJsRazueA-xxopX1a6bS%5^Z~ptJw0K zEAqpapR2;VYkjrOdsu;q5@b4$Sa1cyY(sL;sNV&6sKirpw`D&Z7bZzEa5QVt+8*51 z=|5uTkxk=}k;+Ns<&F;lB2vKodizBaQwt$Z@2p-^-`X{kNX%-D2`7}2E-$zqq*FP_ zIz(Zhv|C3(1W|VYragun1e1v=^r@vWJ()aVk z(v1oWEQYMT?w3*KY4>T<(I5Oql^vN$s}T1rBn$-l&lMS3k=xC1mEY|a^m}jTc1m-P zxu;rUN_Lku07G(};GVi3fmGlJlz3D&wzkMQPmdZ+H05HFw*GuD!O)MktBc9x#O|_C zRC;*7v>F!rp<$;TU?`Rh8z6scJ6$d;PP-zZ8uP`%-MC*wP;7ma_R%O77E zr-4nDXeMat>`Y8V@3Y`1`J0`MO%Z7Bv}SYC93t*?=F;Lz<N@6K z{rH@M9<__-B;0v$fp>B7JtoH=U*9Dmpr=8PEP!)O+(A!xPV_W%+i8H0lL-IKbHd`e z=^o-cbZZpfs?Ai$gF+@a90o_xLS3tJxB;7p{#$c?IeS!bX75!6!5PKd$+fwYgeT@|kba#IoJti|?1ZcW*v$RQvY{@A}%!%x^pX)&8{3zQU}f$t=Mt z>m}5yMnVuV<%FxoE!3QV>@-^)lJ_8vQ<*lvtjqZ+K!E%e&q;@@(LxXe;U}0ajPp(< z^ggC{Hm&)qHcqJ(-&NP9fSWhYT|c!!VdZtly1MaIDzIh8za4N|GCk3j*}&kebqq^= zRP2heIa~H$$u)B{RBv%|=d(m-e~7N|m=lAQ`J4v8-qQ2=t;ciRmRCLvUao#icrfMR z#bw%3BdetmlAl0{u<1d{Ii;a$yHbAMr;nDn-{9CI5p~A1J`i+@e~{+j&RWvRFU;aM zcg2ElzxIZ)MSNO(dP7>09!1apFz4G?pMiI=H(w1PcKg zcMZ^JaNCPF@Asd7jC01>`{Imq^{)%Y0BWtOHEY&0pIKF_o{xQdgjLIBc~yir=%H*C z*EO74lNo#Bp6sV)VWn(4B^JKwmL9H^1lcxxHl9u~t@Q_%zPmzP4TmqZz88LHmr=)m zYO=O@e}~-`jq3ml9TELBJ|Yf?z^d8Sc#6r>yETnmy?aW-0iOV;*5^;bu=^;$k7c%! zKNHJ8mk7U8^uM2)Pi%2E`O;WhDg@f1@-*hB!X@YAF1n$%jHdh5BYhY`DBfAGmwD_j zgIPoOs|Ig;AdFew$oS2qti>K<$&4N~8>_65jeqqsEhlV*yq>_0x{#-7cN6F%Y8TmBhx&UuXUTxc>Xe~7!=0^kyA!I4b zDP(!ye?3dH-!pzS7FvzWkgb>rk#2Zr^x0r2Dv_orDm_SDRWOE$E+Lu33Tym6r`6vf zab0oz`Yw1ug8o?DcwxKGO|7}kK*>K*$$-4vrAx!UJ&lPHYeF?JteI$k-?%Ae>d8Jjygyb3T^(B1>6w0{J*poN1-Sm7e$5P`= zoG$e7TKw;`7Z9A+3)GtouxtNc0*@{8vu$LkF0Q7^>F@)NdtnozCuhU{zz3|crtLw4 z?N_?@q`|jR_c(Vq+;$x1PL*lVzQ&aegagO3;w{Zn#F zyv7{**`X_Keu4%zldikVxx0%YT-nZiyy^B+#R$9o3nQqLSp(Ftvn=T6kF&Vi57AK% zSO0kd()Q;+Xy3H`%F<4qOszg}${D^V@A}0SrbtMuAfmrATz+t|L%Njs#jT;)ww~9d zwaNVE=27!45|z>bS!Bpt)Hyc=T;fg?S>-T2S3Ww%)EN>XWAogK1uf+Y8>HX5p zb?`k+qnM3J;28t(nc+RDq+jLAWL%Icn87+@AUh=N@wI~&HDcbY5!&6}zRNilDPP`B zz{`URJ>H7vF93bijj&s{X}7G;NbgJd%%AW>oIBupUxFGVl}WRu_Z4f*-gz`0Vhe)P z$Rv^f1XI?3IB#|Fe71ZfTtB3iZ@0Mth4%5ol|TbV$z@M$C2TF1BZ!us1KtX2h*6BM z2emK3EX6FuDuD>4juAh|Digx1wd2S^kaC%~>S0#FIhUtBruy|%RHOjm%Wko`rzL^2Cn1IEqpm=`4}mv#`Pw)<3fE2LET=PjA-E5oG;-NN@08%dm!k_?)2Hr$ch5xzaj z*OVwJp1uBJsPaoy8=jmPs19GN<}3tkSu?t%^Ws?>M9sm3;9eY0Np+9RM=2J$5&a9@ zA>*u;bi7V=j`gFyLTWykR^>w~o44H(MH_OSMNRcDe!ku$)s`U$*rT#OMe6qt2f>}2 zg{n?h;Pn4o$G6Xrn&|UfyuV;P(lgAW4&`Siv*?o`mZd}HeMB#hg?!vmwdB#9IgN*U z4(mD!TzRM4uTYVwC}@@!-;xLiRkzfn>;?wNK$kv+833!WJXu26AVxuRS=I}h++w_)YfALO&={~9h zy&j3zqR~kRhdS@iN1FEOq>b8ke8i<+a3K7^u>THh+NHoScuBw)D~xQ+u^0jskzDHq3e$6PBQ-Ug;uh|2P4U^9lgsiw2=Jr)W% zS~EPGlL%2X!6qQci1b$2j3fuau=5EkAjP$O_^8$wu#x&@oyOPR7we)IOHb4}kDJHRA;K z{)B-Z&2!7MZ<8$I8(B3pSqtu+i{929p^~5WP>_Y8$qr8*RiREObg$rb}?Nz-7o3NbKT#yygQD zrDiYaSh)5cIzMORNB`sq!{Sa!jPn_W;FAZKQ`I|q8O|G~yuysOvZUNsC7>pY%EnwQ z@ov#7N%K-&l=@S?+i@kmRq&*w`#F1`AFTzM3HBsu6^$Du-RX;;VV)evy)*iTv2Q1( z7j<%iFHcNey@Ovr1AqoB9()Gg>5FIXqvLU6;b2rVNmto8JnZ0!4Y)c|vB1>xOkI;G zZ@>Du5kKRceYAOMC~1JL!=AYi;eF7);@Yrvwn-j*Q=P-J9;N9k^3> zYCe}GDt)z(z^74Y^a-<5^UW`6Vd00+UWBkMQkL#_?>N~!!$VJTYW-Bu1aJ_t^xWYj zN}~{mScG&GDu})og>DtpAE@WeOZ&2KI4mY)_*N`pO?FPt=JY0=j*J(5WKyxr$pI>e z7q@ie{9leOtAX_HWlJ(BcG{O*ypw*6$cBO2w9SGFLDwCJHrC*iVBp`?uN>Af;|av-ea3HRhw7NIOvZy9Q9; z(@djEoq^Siwcq{0ch~J(rKW+yLW@Th^&9)Gl}n7PBDGlKf)=S{MR>@T0|m8Z`d#0W z$3xbdT^jGt@CXhF=8Mi9y}8}6Puwl^)%Ls?xO0}&Ir(VzMV_*^``l(^r*qNB_9|<4e>+r0tJkcPopKwnu|Xbc8bbsU2R` zwq$$-Pa;J4V+bGu1e{7F<3{$%NIE8=BMBx_ps#n7s6$wKpub%QFLJ;|;fn?|C;2)EcG z)#Am{2sm9A0)L95b2wWLdlz$6gD<2`dvAX~+klFaUd>D&TNKonT=B$^EHO{+Ys9q8 zZNuZz+Q&l$76m-ZVFu4mLXPSr>LH}ZeANpFJ{x_R!Uq?py6sk!=krTTNkaa2+uV0i zId_+)$9reM`I(&K!ya|QG{oc7qgbU8Db(AiJ2_@*wQvyWHzym7gAhZp(z5d~sCm5k zmKKxVcCC8gw!5eDcj(3-${(nMN^uyI=3C0emilHUGp>S2sgymE59Yjtk#Z%`Mh0)S z93+ZFOrBQH&b7~q)!lENA0LGNLnx!fA3S|l9{OG^c@_>abcAY(>e>sc6+J2sf1(f} z!qWHX9^6X$kbE#EvZfz@$l(Jc*oSWtUvnX_KOd_4e9}jn~ zkYxBgYr3S-qHXXPi??cB`G`1_vIlQM2&4G3+mDPHcAS*;+S2_L(&X)~cm#ArKq_7%>wKW)MBae5(|XTv4hVYhMe8`2F@H zsk;Fd5qKnLcg=!3gq_Yg$_+>xqB9r5kOy%K&Kkv1^~CwFqr)@yRCd2qy-(ty;U!3J zRQLF;%}TCZm`dLDp#Zn>>=&&iYj~o6)cWI3hVDuYF=1|l8ne&;6orN&zToW#jLir* zu@?E4fh4bQJQmyU3?l?wJv;^lxKwpSvlczh?-bAQR9+1UN-I90T&^M9NqCjssFboC zGB>w1@8}|u4WVFOY-_d^B{6BeNZ&w+nFQZPtSy9O4yP_Sk6uTLQLEv=3M%~Tn82D2 z?Vm$>qsQ^|=&84ao;T~>*BoaPn0R;f%0riq7vFwbrdeWfkhVeo^!QUWB&%_VECCTT z=e!X@C$}o_rbLsoLIjeU;U>yE#4p4c;kM0b#?WXLP*@&XPU;aHbUjuZz5EXkp+CO< z0)R!?BuL&cMGgAITjQgnm;e0S&S|>*hwj@I#f;a>SOM!R>QStRze#gkPi{z>Z~2b{ zRxhic7OmcFncjR9E)RamsbJxn>btUX*OSQJ*kJ?#Ckp${Xk@a*&02wEm(0fQ_bz_Ua2g|m@;VmUhPeHmog^x6|JpfPq zN<5cr8sT*V2+wXFB3|q7--VdA`wqJIw@iyobll2rkeUV$-wybk^LRk`>Cl8w=bce= zS|Jy!U!$EqGU9{@@P+r(AFz93IYv_URje%CX5M$$jc~qIO31liRrz(sbiKt*emv)Y+vxm-STkY`dA9qM{BOAq3I8-tT9xnbYwk z{z|=$44-m2@D%}JqZV!S+eyAz;!JLrIIIsor%B&B_sKX)6cwiur6yG?`T;TLA`vbM z6Z%>hA*3*a)$p>UKBT)-)6gY$O~>_zZ)r;N`RO52$P*y%;AaSn3Qy2rk-S|U)nn5m zo4q~LK8CgzOP-0I&XR727Vs($r;BxqxL@BqlDb}#TDf~=7If~h*m_TLnvAvC5*WOm z*6s(RBbcYbK7q_hTY?s2QUV}~L4z=E=054tdh}r@tFaQcKPEV73)|hR&W*=hz_$y=yb@w1Q(sZQv zbk51c`Gw&~xb?uqs_J;e?HxHUZW!V8{@J{^|Hzq*0u}lY`ZM`rwPKP_(T$*ARq@9| zg{qvcpC04nwVNR0zzuG%%nIueh-UWc2@vx?1K{}jE8zd>`sL^$X3$xB-ERf1;!iJw zfzzfgu1bR=Cs8eSP-E7u zao-;c4r-9~{hUqL+Zxg;t3#endl4#H#vkH7(qj+>!tCz1mAc|3(A+E?H(U(xHN#in z=&Fw#5)%;sHd!gDMLFx~kU7z_#(rP( zWpki~X4cT!yE z^L$oLn^D(-V4qxNUyYizgqz2O<4rOH+C=Cjx>Dgk<0f9J1gVi%)Z zC!t057rsvt&+&|$M0`r|PYjA1D8n3$hBy@2WeEcfxk)18Uc_IIr`BYTRJMtD&HVc7>hw~WfbAK#RF(5; zm*u);*+VLS&UWG#k$(zt`y4fyUifnkJR5mThRRBw=V@HI5TmMTrid@Q@-?kuP8vIq z`jw{|jN!zNE?+S!+FqWws3B>=u$MrdHh$jknKdJhYIj&nD;7CmLEl;rnygVg z8KS66Jc&{GgqJ&CJvt9Vq5!gWFHPVF*Vete@K#1Py>g48)Uz>Zzl*rG+$_H<3hsQX z5W(A7c<4@jc{+!2Tp6!iF7sm~y{^TlawUmda$^RT<9=K)jP+bAz)yZ!J3IK#$I-S6 z;$LUvZv0jLcMw~eV3}$0P#$DTaDucLO>NBct{d_1jVbf#mNvH#SxmU!t{(e)1A80% z&KrwiKQWWtNwdwx(cta0de>)tiprJ_mHgctgyYg9;fbWF8kj;nu})0$b|yaeclg3v z%zNZTdBfR!xw@vcA){MZs#|NWE$vT6Hga@!YufH;!esu9s;v*{J!l*C18NX2!80?C zf@9#=5D}_4@Fp7AT=C-9T-f}KDyop2xJXKu0J&v}oSL4|#Z3~Yj-mA%+`Mp5K{!s{ zXb@|wlO1BYiqOA;urp={F7qiqij@exLY=o1p(opexdktH8)7}M&ZN0O2BS=+^1WP* z;H`--enfjn_NRmO$$uMdo z7a30hUiUEnnJ`06&=F*dMrunv#>l*~A7VzmeD*1=#qU%h;ArhW6;Vxk5ntg(@9g(M z9)+fY0;?vqm4XXKe;zCklg-(k5ZjO`2)DW|{hSraSuc2adn=9O%Td$*yS&;TVT$0j zP{m!F7V6V3g5%nRO-Jxd?SxH#Piv{tGat!ld17z3)sPB$dAV`H)bMS+Q4i%HEwl`E z%NBuXZD>n=9Vu5zuPFC$tzWAHyxRh|6G$B2shxtF2h#WSvdRI%t3Ti|Z{sy`{ ztv^Ut>64`odumN_vHeMEGl}~ydEpa{`C1*GuNtZ(^KAUid;b7;qsK+F&+oq7R|;ve zp6$>C#Q5#avsnk`p83ZA^5cRdeH%8GEVnAJJ)8RS=*4nA-J4QP_tJ(B)Kp_8pER)b z&|OxR;%8}H*pitvJ)AV7;I1EwWWTktXx=eS*ex}i3I!PBb87r&I!g8pU{e$)l_Y<} zN`$OpSTb2;0>E0qzhhsOfziCWBdik0i0l+`=PDmbOOPq?R;VQC@tq1-qU|p(C8ViV zfWU0^AgW>|7DR*jgV~*O63n)cg4d`RerrdCl$2 zDx#O6608@}#kEC>E1|so?VJKU5pML4z77%exOJD(+=7Am(|-`Sd(K7vFr0^MRI+Jf zvUK#@^~apqk5{~STIh=lwmae;U6-$e?>zwaoD+1%4;&wR_i=$Ghd<~K@5<(*u*c=_ zk}4kBbKTv_mpksuo(NJ%<~Oj%w-1yyl^CpE&KrFELxw6#))&=__05_pIPW!-dm5Ha zGPcMu_s+uIAZED9s}69v{Yl*dF1>!ew<#v$JcP224r{GdlnV0}V$P9@J<#E+;zjfj z*pNbT3Z?9^9T ziuh)UuBS56FLv!pvnbUmt35^y2x@pv12B z30u8zJ(Uk^nZNi0XE+_R1!jY2_%9*5a3OGm-#zW?>v$7;rh+`rx7ytnAJSc0w5fZg zPo--m>kH4$as=O3d@#(LQ00shz$p2#UuBLNT9_(~+a@E8Yn52tcj0bMRwEI9_ zT)+1sH=Do6b64;Es?S@)=@#*b*PK7GIMEE|#{5dN{e$BqvBKaVa|Z|a29EwZXTMYq zkX#dSs>5ln`ieY4Uxm}*QMC*^MpM)3a`+mGEegNX=9UY!SF4?s+OEH5C5cGJ%yTL$ z$4*onFc?U!8c`Wlbg(Vf!oHi^WSw6y?wogZYqU$Mx zUalXLZV{58oIPTN0J(lv0Xs5i<|~WT90l1UaXI_qKuv=#@x#@mkS8$$Jv!k6_2j}6 zlJd_Bv%6~CrUZ4J&-}$-y7Su^XcCFMz<0 zwTI_Yxeaa6CcT0N0_SG0R1qzFL7c({vMsQ3&>m7Uq_Mpn##{Ib=Thb^iJr;z4Rw*O z4HLE+XMtelBTH~JX5d#N?$h4~lwt&QP2d{pV9D29RiC^>{3%eN-gJMxuU+iH`}!0) z3-I<3QicK?U=B+|VYQ=F@{1 zl5stDDrEH>Ff^e*huwZ>KO@D0_4Izp=I#($pp*|44cW{Ry2cU^%(sfURxgE;7oZWL zGBU`Gcz`=9CT!SW^lr&!?rHKBjd$&DJ}OE)GMK?Lq+*iGg5_|WJdCgdQ`Kn;`gs^Nl%cnS2r3y0V4eHk znZ)4sli;7riUo(KWX2&rZ6K*JWKhyu>@Gj_t_>-n}~Y zwWBHjl;ly$MP7$$f~{_tsZKE~o`wygU$bG<>*{(>tXEQx&=pzq%BYS2K1tM~#01w8 z()Z*+L%9h#dGQTfx{fMbffV^E|9;3aCn##*;zl9IReTtdN(^Ga84;&EDL&Je4o{!O z&GUp_mYf;TZg=)SS0IPkk#M6zV76|I_&cN`Z#_MT`^V@? zhcG#b)Tr|@2<`J@nI4M{84D_w!r_QD@sr{5rNjMOiAnS3x#}fEV{5B*^$cGu1GkCD zng}5W5o!Fwm(NRu)~_h zSAR`==lcZ7IDrRu;(2|B2z>K*&5NOwZM8BUAKJzc(hp)2@J-DHQQuzFunB_!WiDVg zzA+H@HLr#}bFVUHG?Q(g-Ck%kyBk?LX+@&p^y5gliO$^Nib0lHf^w-0n~oCQV;&7~ z7%NhOwPdYMO|nQfKGkalch)e91~DqdZaG>as))A@=bwokLMvi~z|(NH43D2b;n{{R ziF_!$buaqpe(8+ECT3g9bcK8kdEf?9etM7J(xv!Qt~ypeGTU^DuT49<2&>B|rJ1uV z8K;;u*iDiu6xtQZrA{K_l*ajKc-7)^!4{@|eSZ7hzm~w&xfwf8!V9yN;z#SF*JQmk zlt1U$Pxw57@3S>#-4+)2W}Ji_>ka2u-if%{K3>v3YnE8!;R6>B+wVX_3qZ=wq1~-; zID8ZO3OanPzf>HwujW9f7ou@}qGvcMQ;NdRU|^*ZGrH0?jnd#$Z&l`bDEstIiO zMcRD_@j$Yg$$L^Zgy{Cg^AWTUw=V8MAhBAA6x7&*$~nzCNH?wX3cGl~$%(#_2v@KX zU#9uGI*v_JpWCvJKyKBHv)&IG)Vld6M!f3FM~u#X!gVTKUWL!UzN3clC8v=Md9 zZbXd}Rog~QTT7DU@1P%(!4VrV?|VL8v=mw{DCv-U($Kpr4lB~dtlh1n?H$6>$>VB$ zS%RnOyt{NuMg1Z(Z2~2?cKF8>mMk`g?d|-cK;v?>-HeJE zp8x|N#xTgIxg)2Vr+b$##L~apcnZW(g!b!QX1;Qg@<#-4TjTTUnNS|@gDi82^pn$C zURQ?CyfNJbe_a0Evq8fQpsiK&!lCPP6r4#9%6S!Lbsk6y2LxY1Ml^mWTDsRh-_Oy* zgCkL85Y$DOQOGH%^-cZ|v(^cnGNNCEk)tD>-gjUf61}GQ!340cLn+q}a(5<4 zXlnQww3Y~x(ROk2-Pd9lA-C_7NK(lIMe!1rc`rk?mG_vQjzeG< zMK35SJRZf4KR$pS{&7;(7`HUUEIJuo`7QgelPW?B#s*?ui}yyepMAa(mEpz2+6)!& z&nF!fYxRzIV`i@GM1PnGkCG-zX^MQ|HP=fOtU=aCrp_pOw; znqQKMM}}&mwS?V`{q_^C9J2+}v0lsM!se}G>`?ybR0u*FA+4xE&T)peGkT`(Btuc4 z*+Tw3@h+HO=Hz99AIVSaTlMAR7x1;yPdNTb2JG^Fj$1pKRWiWxjf|q1)0)1gRHRhe zXyS)mFG3gb)ZDgEcces+1f|KbnpP z!F9vXM{m(wNThzY5unr6l#|W!90mnUG8EK#C)ZxCL)}Y9dAewOLd+GxB}hd7Ox|9a zJeblf1W{wIZ$nKT8_2AB9lsApr=Z#TkfQpfF2*aW++ z4GKdaCN4Da2l2j>iOQ)WX9y%lqvOMAh6pex=KEbsN`q4doZee!Ag-NA65%FGQP1IP z!y4|chw0YMgBNCRh6-Xn#Ip$jCVg`4;hsme_1@5KSGZ2;Ps*=1;z12Ll`K-V)!K&f zW8JwvCc2Akn((^sKTGviU-rhG9_M*{A3sSr>Daz=H`)|PgIZ6o8C;xdU?@Wg%Ls}J zYt>GLguK5#aq$R(uj#SpUY&6dRS?dV_+zk#+cm>vEvEPM+@0sz-1FdG5fKwz4Q>!% z-E@tINRz%#&J9>Tefk>g^i43H7bILoIrEeq`tgTl$J+xseufC@a^GA(O53Nr%AOVk z(`gq*%*qL)NL5XW-FdOt`6y} z19x+ptEQ8*0MTnRsdx2SeQ`P7b#+C+!oRm(|oKmU_n`s8OD@wd6hh@yuNn*0Wa1PW9hwwS80O(Y*-9 z`)d~XfPFc3oOYt5n~8IW`_iRhU51CUIPX(EZnZO^PokXs>I|U8H3SAAjj`$E(i%!{(%@WOPtgns@8G6(ecb|j|osy<-zk2NReny zFo~(#!u)19aHO)F*`zs+mj{$z9P!t35P!-0*5xGV(^I@f1m_ZY+$V71;2b=qfD2K0zL6%szJX>;h*DY=h897IVbOn zPJJoHG#xZQMHFDty_u35xwh1&%@@M4Cll6l4DFUxRR<>%t%>k4-r6evLz>pFtES3L zgLOH(Xll~9P4A{8!ww3Pe{^>z)Vmz$rxNpOX>i=^4BjhNFpRyP*ytc+=p^2+gvQXw zJ4R`hTIH|`@)Jf=&vsS7rzVALL?N3Q!!|0M;Y_v~U?&DG&W3t&F4jJwXAO0`B8!q2 zun++-umeJzo8#nzt}T_qkO+Z2*J7QDHinGZ?7XWu$DY@y@8yn%vQX-;u0;69bw?p$ z-a6X;z=fJqGNeg^>)Udu;$e+VQGI}Zrm2U?vL|op3|wM!D~Ef(bLX-$dndGz`vXmL z`kQw_j=S1?bQ&cdJ|c-4J5F{fbRxVWMcw{tk^Y5rS&wtO9(WDWcF z@)QmBP81Max%f27U{`-T#FVvm$<(EAcZ)OZXM|RHbz;jO`_gQMaF~3Hk8@@<{7(Bf z`)|}WFBQz_I_;6m^|iQ}gL6dVr+=c*4Sx%u`Q3g-cXJ;rsc)P|2p+bvXfK^#h@_qu zp=rHge+9z=Fm+lfhGC(ro3?>zbcc|1G&)Uz^fd^(4alEmW!2q=x2F zw8Jh>#t^o@xZhni9Hj5g6k+meLO7Cq*L4NH^P~)#?)?^OL1Hfm2{ABaQu&S}KZm5# z8t>tp_XavqCq?aNHr9pd&*weEodl@{A=T2pZCzJ#8%;WxS_<}dnZh_%F~bYsLd`v) zJ3U!>!p8Z2CBVdxGIga#pmI(j;)&t6USE_k-wS@DGUM}80TJFFliqQR2*UusKY19^ z@VOl}y)q|et#)UeU+_q=xMAn=W@yO78n1Nb(YV5!9GHs&JVWX+N4$jvu+Pr9zK85S z;e_=sDBeH8e*Zl_BSFI|xVlr~&^0|HjhJ;oefP1rC330;Z(NxnL>ZZz^V`u+nM|2& z!wkq14H_a`GoN=L862VyIT(!N^&+CNjalrj^YCJUJSU1$oLYGcRVYe0{9;1LV6x7M zoh-$KL`!x~&c^BR9ao}K7+<}J`eb2+ODlr+x+%L9Mk-SKK(5=Ups3jeYi4wP+OEQ$Wyb)0zH@^UQ+{zcH;R=|OiNpnK0iosd zoUenzoxW{j&y6T{^C()4O9}e+Lm{QUQr_wm2tLmO2yV}x}Af&ZGJ7={sn%TL|__6I3|H~)Elh1e!K8=vD9L|Qv=1|r2>K# z_Bo*@?kI2A_BX2Y=I7Dsq+5E%l~tkLmr%bR*0lCYSgVo5xE=clvP^TC38OOf^tYRk zn*TiZt|JA%+3)i$oW;?d#=4>77C?f;{wwzx6MPBbBfDL4j+hOi?%i+2{XHqL-I1yj# z}FP#HJ1(NO9FLWB>kW| z@O_mztjWF4XA~wBXV})p-sC%*J2?x!GNL|#&0YEhAhrk&e$!;0ovf2*O2icTp}@JE z3bY)ad6A1jO`Y$0 zTsbew%Hky00Ut*)j0;pPI9^4F)&~B~Wo~{xLApi8v+pj98b_-E2wGY z?q%PEa&Ei3;{+(gp1iKoITpt=%+|s(HkrHq;@2y6QEN)oV-R!vCzp%P?O#blMj;xkWd8;-IPeQ(e`;TZ5}*$x-3U<)-`OMrSGe_vyF`fU@FX;@Gx2(|98ge^@~{vE>oOpm04bBDkHhZb|X zbD;V7H%p1S$^ISO{A!eLPp_+n{O|mFRDX)*zY7@j^zIu^kp5cwe~JTN zJN%Dj{>Nwj|8s(SZhvnIqe-_-yqj`_iSciH1tOwEXM+19WyD?~^hxFet-dJw3F*!Nx0|Ntrhx9Bj=em;ve^Pw)>gBtdS^w{Od35hQJ?$hs#Hk51G&JJl<8zJVO-*T< znwofccyf)<$E<+jFMwLVeEH(2`%=Zlg&X)^O}6O3z<{EX(wN`oJ1?&<&CQ05_>6H& z2M_JDuxwms7M-S4N*);@`lzh!+mX_kq7ySvu zy!HsksI!(_fJ2$)0lihl*RON*2=Cg>c%rhzyuY-x%-j3_`0>N_e8Kx`e!g7sQzX}m zL-U6xcZK6ve2@Ea(A(QPU1LWK6k?`E3-vl!i~-aK7zi<+)v^FKpcHoQ`7G%0-L6#% zadEWvjGJ4*$q7+#FEn;G@4J=NaJ}DLn=J~|F7%)mTX3TA@+BQNH$kS5Qx~Ahmam0@ z1gOq-8oIQGSKE$${C#kf)gBqV)ZoS-^tt{`u{pN#@jcqr^_vhNBI_BeruYPz)ei`UMBaHqvJ|5bBv%!5aSsrBF z%w`ooy44M6y63f!QIe5$>@PHDZIh0Tjp^0eryN_Exw;ZwpC7;ggQ|23W~MjqH{pB& zasE<9$`Wxy8I`=|P|JF5U12k|xHDN=Y1WCdi?!w+S9 zD1-k&xA)DPH+e?#78YON{49i+W`HQI!6>Bn6X}DM*^%F!?di7w2dhq3+s3(*%OzRQ z*4Q-wDpiF6DBlVT(Yf1i`0!~Y%ZwtH4q$)`NeZ!ah%^#S;GYEI6RMB-{96p4ac|HG zquqVcv*yiLo8(`Dcc@f6I!tYpDB&!31 z=$nFgkmbH8>IKC6GBfLbSB-6G5cC!|9tgi4(rS0j3yuMUMaJ;m({OIbdjf5)edxc8 ze#D|BdfBxa%8bFH@YTp*zA4_(HAbyHKGBh|*@4+oG0`J~HM#&q^$kCzm|F7P z>0;#s1pm{Qsz66x;N>bl7&+*l*8{a9%T1w@UKNF151m~`Z(nr+LxSG?=mlRRpXlnp zO>RK#4-kERfk*3}_0WAlf8WTxyEQ|Jl1r$#$Q|fJjD7v<9?@d&hte#QlJlv?eEypq z6#jwUk)BlsQBm^C%S!-W?lU)TTYK^E1OF@;8X5v((vjKO*>3msIpHJg%tAsSV4U~; zNMSGkwzTv-=XbD+gM(~fpacMX%<9(wcD|Yh^r%L%#LfY6W!>xZ&dwKd3JR8!CE7aI zu$#-11PDZYzdaa4l!$k9bksjM*!}1CIK$nw+-fy2Ci4pl>?YS%dp~`HLSF_4-#_5c z-DB+V3P4gF7Z>-L@sizmA)TX>c;Q4l@-l#3+X%#DiGr*w>e=2* z=)}YXRt^o)b1^ZuyBlBV^YrZ{a&mIL23I-@ONJr`jlM{S%L4=g@vXdE!Pa)G0d^}f zUi4bQ#DqFPGDqW9US3|^(D2jA)~KbAk6t4+k(G_jm)u+=$()0a!PjL|w(v6js92rg zG;toD_f1XD1q20+k6cPd2~vWk+*ZTRs1(wox55F;2Oh}B&)>Vyu6eXHH#Y~Evq*4@ zAu$T@|K1+T_?w#m;Jf@s_4DpH?;XUVU5}QeZVqKz4!dwfe2>=~?lT8APcAEfsPF^? zGC6XY;*$h4wz^M7rOE}#Mc!=_0`K>jhDK`te$(ZZ+Un}6>mXhxJ3;J(65x--|4R=B_xE*RNFdCt^wNMK6jT}Zww7f#eG{Z zH#3g_RYN4|I9r)hg8SyF-Cl^OhkyYxL@Jn?yY&wdU|)Wq#)H~&KybVSrz?88-i5}% zz<}Q3NPm=ugd{v91Q{Y3Xi$pJ?4?CQLbB3{K?67eO#6P}?824v{uU57d#JCLNE$%s zEFK!w z2yOE(2AmA5+zyL*@Q@xBdEZu-Zj zfbd4cJ>{3;zvj2#;pD^r$YiOm;Zt4I5TsaY z3Hw`Lx{&Bg7oev>+$N5J;vNHlGN)5Bwp`y4r8FaU`|UC_>k?>;DbKW#5XRFt&H>paTR8whw4OG)QCdFu|8gRT-#-bd{{n=QES}F*hC7%l)cbJlZbA zbs+2lLAgOlsb+zCZcA86<~Up_PO+{$(!ucn$>P57p&0|+Tf&VGfj3<2{rvimd^wqR z0P)SiMfyOr1RCO=hIlHywqH^ z`~)1Tud#{I^8KCQp01pL_CwHc#UNB-243#JHR7cVkuo;cR5ue)7I-rGbou(qNv2Wn zL)_TFqZyl2+}8d6ipZ%Rc6RwHF+RZj7egXqcZFhZ35m~5d~EWNQu_Gt^3`{XhwyP6 z5jW)Vf43jdY{0~?KtZTc|7AUZYa|f?-54$s!JBQPlzyWE^m(U9x^+qLNmIPsUYguH zXwU3-q_K_sJ*4N}`&4Y~>;P{w&zG#Tva-qrM9qACed|iUSOXLe7W-6ITRX*_ytueH zUD$;h8yh=$FrC*DeZS>IZ3rT&*$7TzdHtoTiU^?X$}p7-Mj=7LJmA;i;o%?Fdvo;! z3kwTBUcsy7KT<+SGPK2o1^_*K}QD!u;KWmq_0L1SWN?IcXvTl zVqUl`ud}tdz5?tG4-fAL3|0EY3*?7lsi}q9u}(k=A>zFc(OKflYiJ+?o*O#4rTJWv z4ELca!1=j9JORf4LSLWUX83b~k-W1r7yP^Nr}=rK4L;yHyt_WI1hmJMVUaR)4|R2S zcLR4QE6Y6GF`5lElK1f7e=y6$M7~i38h|5uRaRI4x~B}jwX^H2w1m~Yb+xs2O!{BHeyy~hWD5YQiJ02eE`b5m_-LoJd=(Jb4RsfDZu-p( zDAO)gW+VMtpc`9Qh6QtQMt=PGu>zn=rKdkhaM2%zl0fE<$E|s5TI_W0Npa!*go`Y1qM}w&b_mp-B@fmnwy*3!yp3!3Xt^Z z>XP0BwRaj6k7ap(PQTx2Lqn2Z)7R9rylu4Z(pu_$EuarDPQVDO@QKXiVm0L|qaGU+*JPFs%e_ji8+vMHMBx zAcYTR>==;~L^D66H`v}t8BBJEiHFSc8sb_?J85-mb~|`E2(R=m;p}6@C>C#|m;~4M z6G_3*a394>boB;Ev6@U(nj+iZkin<&Ynf%`6*%ALTjDQnCe`m35!{I?NAVg2#zeE7 zPo)!#e4S^UBN-E~(%6)yl8zYNDIc<<{HM~4I2b2yG*)Tzr~{Iy2U2_*Y8q00;_oAU zwDSL7yuEommHGcaOf?x(QB+ix=pd2Mf*dMK&atGJC>;qcN@Pp2Pbox}gOrMtWl9vu zmdGAzEG-I=EnBuEJNNT_G&7&?_x}EV_kBO^`*Hs<^O&h~opZh4*LA&?=kmHdU+j1z zr*AN8AlB_FAAOJEH^1f`t3#+~*Z5+WW*ro+Nc)rH-$mN3iXuRI(>f7xz1`YWQ~rN1OO1<*%5FI>%Q0~krj)WbBU{M(mWu;N^?J@x<#XcKJJm=^ ze)7M&kmYAiYP7D=KTjzRQ}!;UWi_9ViNayHX`fIiZvJs3!6DT?p)B#_X)2~&@!cRf zZBW>L<;<99_Hu5?k!rD6!AADw5FEv?p9ojG&@;gF)=Qlo&tsLP=>#+ zPQW3~zRz5DXc?UxMZOa5d!1BFg*k8zw$JH*olEU7cA} zvHthY7cbbzON1?t9@Xd`>*}&vaB&xT^{;R3P=e?Cw{P!cWEe?BFIp-6;#)^$XjIgL z--Gd?hm33R(ro9Uv8ZQP4E_=;%dKuYjSz4|mNRVOT9?8t2_|8n9&iAqEzJqvk`aFPF+I3q2c_3ezoVBvC zDT$f+F0p0HmIEHPOBOHwER1)c%2KHhou&o{WyzP|9l;h>Ry4_#Xr~B%gdcZMR*PMF zO~&*J;;~ej!b5abtnoh*W#~;XZS4vAu3vbIt3K?@(?bv=jN+MgnxpqaK z%=Z7h>DvoMH7Y)f9XoNsmU`;csljK>(_E;V4Q1N!V&R2CZeQiv+uPR*HrLhp#^9yv zV|eTET?Pm5g@UyU7cNY%IFN3&EN}GrKvL`=w1Rce=SLqh-;{ zMKND-Oxi3@AXw!GzflU~J_A?i4%5&E#r$`T9v(c z;qC0OuMXchF)>kHS9jv6kh;1$O;`8#{x&3Kq!5$3cN)@#Bag?whHH$f4!clI+R`|g zrH-6c8%-0K<{C19DM(XrcZilH#51vf%!PBPTC8Qs?Y?0hi&Kfw{zo54)s;jzXK55x z{%ZQwIF#Y%>*KlNi#Ap4D^-HkB%^w^qVk~J_5@k3Ms=dQsCOEq?TQRXpYjFiSg>i=pkM1;JzS+iE&PU3BRG<$2eKUH- z4zXToW|~ad{OXc3$OPt|yC^GI0lGv^!vA=7h6j0W?`DJC-1a+$JUQqqgW zn5CNgai)}4o1Bm|cv@q145R-3}2Yuz+5y0 z2<;X2(~lKgMG={O=+QEg7bC+pV|~im!fshoq!TEVSW4*Q`7gyXCKXKSiC)^>%|Fi- z{U0iGv!y@dy1leHGZ7iAQ|+6kLB2!0SG}z-E{bS(RVE`_hOCpPl$~4=);hb4&B;Ow zr!LH8KYD;A{i0ksxuz@fvC%jZ9nXzUZDmH`(Ldb&xr(MpBzgrNAs_=&&6l#!QCNpliP8G9Aqxs#2IGG8OBbh?`fb=%&@$;khGu6 zE59^T?zw_qIHkr%&cSE=m(btIm1nLjyG4oW_-_k4#<4d`fR9`T+G@W2sdHCe?7o?{ zpRWvkh{C)~I<)}H)xkam3aJELV%3%h?Szk+O&~NVq!CjbD+K`#(H)-Qs{W}k6O?c0;Ul%L| zU~=o0K!-Ep{`Vh0?&jy4Web9|@b~w}BUUkjuiSn5;V1K#8fgV>LoWtKi`s_Pk5=tm z>m3~}(yGCD{*t-qj&b3g3wd*N!&!&jbr<^_K`i=y7sK84S>WXsj(*sxCG= zdllG^7c+p#~5col%@k#?movpK_vBh0CP6w&eF;peN-mZ7TNb|A^Wdtjif>ABkw`np3& z^VuGng5 z;|q_5#}(*HL<0&vknJGGN(QA8G42r$m9j19FBDzB|M_jmGhY~tg`(%+ zMd9_4`}Xg@1mH1w!Cv&KdEetVwFD{m@0C%X$2)o+-Dlod$tH`U%S$SbO|@`)z4&83 z$3qEkGP~XMne#!MG4Rnl2??dA1mmu&xNQRmql%2wXd(z4x4OQ*M|U3W1S8PjzaZ^c z=bfp2Op+c6{8biPwCX#~w{A&8@g;@%Dp4uj)WUQP8F?0+mz{1R)0q(LCD zJ1t9o)e+`f>84J~?LGoz>p8}&7}TSFG`t=9KmVjmx>l`=_&+D5 zeMKU4%Um$YY+q^Y>?%t?wNk>La>EA|;qc8b+HP^_=x!3W)9szi8SOeUb1{o%(#TXo zvWink*iI3nlOCmStpdG^byiY+p|xC+BiDPzIzRo#I$GaLvx0bFxJy?^IOVKZv`qR% zY@4%JR<#hlY9s3iQn+IGJF;mT}$zU@Z1tL|B5Ygb{`dP%xAWeMqY z_%gQ~WZYOx)-d46{Obo{2@IG;7xw3O0({ zUXz->(0J^bS3Nc<6_7 z!!1sK-AW0igbg;Pd3e5IoD2?krrj_5T|zQQCNRLgTyu*m$u{(WL-JaMBO%{QfA@`} z+?N`1iWO`Z=bXEl{=WSLXLB3o>V?z{shv(rk^v2Yfs0?=RNz*P*<>L*{r&TX=}|~9 zMK*428AcPn?MnYMgBv7#SG>Wm3iz(Get*^hLNzXAD);N|4xv_5m4ii#lq2Bp{X5$f%(0YG0?r%!7fI@C}+&+f*LAI<_h zJOuy)@VYkLn0lU>$#?bY)e0LnwwNkkAW(6MiDnaLwxv`(Josd(u<)aID@{?pSy@@_ z)X}LV3`?&eVo86$ZL+KG&23BYLy@hqJXL5i+^2@hB1<>F`_VPkKROv276w}7_oq95 z73o;B^W@2s*YUdh_-}=$cj!w@xFPu!4`22xWwomx*?%`Ox~wJau^gm*!0^N?$ z@bKl0c125gt*0n8H^RSffqc}XE3Cqnq`FI6#*-1CneJ$eLvPhCoF zTh_??L-qsDM+-ZL54z0|pe8=@a3RSi4Z!uUxD7n#yL9PNUG9vy*N(G2%MW{wA9Md+ zqs6=E#WA!Rxs;aAt?)Rm&Of&>bxJb!FP%C??<5*N1sl)bQR z-uUs`b_XPmQJ=S5UNcy&&K>*w`E;#U*dV#iNWm6g@-=G7o?br?9<^Qq)H|@!-92g( zDEfm0lrGdAcGJra)C^c@K0Pt=A@}?fhgGSmsV=kAqY=Rz-Upu?H%Ah9SyffH0gP06 z!u`p23BW?759jg+@NLR4U16ipXg8nGLKF#!LrF!^DsF=7LKL+90aChV#OS0g97#@gT4(V=ZoZ{E3X1*x&^r8=%g#HKrzrh-fMU+ zoqfxye)Kc@fheFH+t*bX6MI8cBpM}cdA?MDoZ)FRZb^P$noO~Jg@;T__KA5QhBfMb zn+v7cm6zHBbm+XCLzE{J-I;Mv$SLXSd~Hk1fm*rFY>w8@h!LFo=zm+qKOdDXjp$4O zJjQPOeG{jB98KoC)ZHXjL&LQe^>BHp6rI;lS<@w^ycL%$aw4LySLByA95wSAML)Wy z@;0@6(VW$Db-qKSk7n#5ugk~h0pQW&5uZ(wF4ax-P>DMtK$-YrwW8cOV_LkAbKAe8Qa264)3usm+))X zED$0qQbWEixf2z&8a1)Go!wxq*G(EV_wXj$td@n#`S`5gsb`X$gCE+pdv~eNr25z> z2=caKe_k{e#t+m_pO(h?*%vKQSbtlX4S)lv3m@ad=R5F4RTDB!16De%l4s0S7-#Nw&>@`RY~i5`{+ZuV26Fbm*mf9X)o;9AcY4A3j`g zXWI_`)vF6tMjJ1p!HJ(Cny3d4Hh`$uEhUsQMD?1X1_|#7N=Zr4nYFn)v|dJLH@=uE z6l@@kwJOfDj$lrL}a7o6`&F>lqe z&-1AxZw;3%n)W9^x{zy^2ugDsn~l+K{YoT!W(OXtyw*}_HY1OsAiNUsv21p#YIZGz zSb{MT5tn?0TxnA5M_ZSq`%`N3JSGR*!}P)Qs&|he=TfHzi|WBoIOEE+flV)lk|Ls# zkTgWy(6Jx{96gHhL2%9+>SAMK**iKqTz-5!5m8C9;1JRZImekXOLC5cMyMSE9uzsD zdHP}f^7>*$gS~sbA9+pp9>_R-!(o-%kFNzfo1VvQ%{$YWwQP1Qcos^oBqw)6LtgMA z4{o#7KCc2-HT2d{KQ<@FBs=g7$eH!)*YDlKcB6~b9xfdb#)D0611BRjcb+(r=r_J8 z^dxx_vG0)gLD>-ZM|}u|Cu6J5hmU{~10l0$tBzH#7?Iq#Q4{10$%-AW=}(`MrEqWy z{VrqKQh2Gtf6bBau3NTkYsF{z`T6&5xcB_I>EseKFe!VYsTgTiTWbKFghf;C`HEu% zW>a)%rxhRi>i_cf>k`5YRdK%@cdWuQBB1JiV7&e~G#%O52cHS$jJkm`2OX8(|4zm2 z$NEj$>}$clBh#9kg>Vq+z_&SC1U*H^-O=_(jbI^?`My1hfD?g>TD>dKm~shT^tTpZ zb`Lssd;8S)Ta`e>5h|yxELbWs)x2n-G$U)NXTDkY80p-6gpP=JH{#Z_EDLJEy|z!r z@^#c!-0=pgO&a-Ky}d<1iqk7aL^BC_f5k^rRga4^K#gx^@oOig?Ub+I0aKgZkk(36 zNdT5YEff2cm=SY`r{B;p?(5zs9@Mp+Ra^CLHD4kv%TLKwNd#TowETzY!jR3~+X{Xh z4Grs;OQ<)0=0!g!;9HqWH&8ykBa_mYC{FkD?Q3H{NZ<2mSm(TL_v3$bXIWdyNumj& z=;joaH3a#FD^zC$b2QRd(&#<(gMo;u2|X=^BeLbrR8aJWBeDmB{u&o+JdyLSm`g@a z{$yZW`2#La$lcB;3mG{7m6gW7d4JHN@_C$X-r*&wJ}%k^1a69o99-L?s<59stfT9e zS)oTl55bz+x54TTR4Xvdr)6cIunC4 zRd$3d)e_B7CFR$Dtedb8BO$dxc1k&@V>UDBJvX*D{a`Jb(QTYiStHVt9seKaT1z2$ zO^GSneeK(lwY+2EpWKyXzOpf8xo9Q=4MB-MC8?4;%v%`)!!VY2w*P7c)Uy7T2`@>PE(BssV@S}f_i>1tWwrZfz@jrfINdzrmiCf9%1=L2Yx;qJnd0=m$J32WyO=17h!8H}Q8A^nRhv4pe|YM9bFFQ93kN)*ZX?S9|*}XxWi7DsjKD7|BqJ*$CuKg9`=M(b4fO{G1DAN zQJdX&V9-|5_&MUFgT#sr3tK-d=frg%&A#t?U0SPi3hD^cj4FMNI&M-=7>nOnPA?P{ zXK>?M-f}Vv3XmJ__>fn{ps&nwh?ytP+-pE{wf#sH3K?mvRUsSK*XHl1D)sYNCo#^j zb$fDNx5T*xhVS5;hwEj?ILYxMt|@Wu4i{J2;gQ=n{)dkuPtAzPd*@&LWFkS=KDDkx z+wd{vw^dG)eVNu~hM&r75I|BrCY{m!Nk-OyaEOpHwkT(cRUD&;iM5pQnQ7?i2G$NC zLHhxB2Mey=u^mJLA-Bg}c9v#3BABN_#^ydd*{rpGxxt2Y(9!`L9Q#=1nZO7J6Vvkb z;}fDgK#f4ycHJ0N)9gU#=Sc9ckkg~n3n1wwF^n#{5%qiX_+RI$8yXOoEi6O{-=4gk z?MIlaco6r(i(|cJvsaMoSnxT5@N3jVByR!qEnhrxq z#(u8R-q&GOl2LMP4g~Eyk6lMjb+pkFR!AAGm6R-Z-s&ex9-g?kxFZnu9zeGgugT`M zXF5hTh8y^%iHG{RTKAU1Ursk239KzaHH)Plt26cY^ycX0&x`jO$ z8&nMok@FyJ7;EtzDiWSq3N#PEtd`cAY<;m@PO}^*$uF;O`i6!si+3r7KwLy**E1oW zm6JJT+=~|<>HO<9RBkcf9M6F2Qd-65Jn{V`b@`;aD46Ib*R zz4g$`b-{O_GPaJsH+)x0PHq{H#WI`BPsh*ZmBt+RDlIE38W^=6v7gg>Kw<#wwD`1= z>8^9N0>53Ve)sNDN{UQ}{JcNdzyzf|c@k0Scl9c%L!)lqj0n=#)ztvZO6a%VBUk!G zH53jK1iH9bBRNhJ3U;XGzm3R>E&UahiB*K0y!-{AB4`fRJcuw6)Ro?IR*XX7r+@BZ zKK@XrLc|8h=maO;pjC(MUdX}mwBpx=7il^=PiCglZ@-;?=k{Z^-IGQj1)X`cv}E}q z1oSmp1za3FqtT?MlhYQQW!NtoTe|UZibnBJq&J93+uFqJc=dJYG-H z`}7Ut(C4zQ8@l&*aI*T*3*-inqwyD@aOWi8b=PT@E*TSZ~d zY(06Q&m}8OT?W0~5JmK^|KC5Pi)hu*$?bS7(bp%=UotR}n(>HZ?Ye_&{e4`rLf(e` zpU;dlOaml4R_7mA{3b$>OPprqi$F!;X>BQt8KL{#ljLcoLn4JIMZ@rdPkDk5{jQH9 z8(I<#gVyfCzhn(5lp+c>laeY3jtkz4)8MaJi`JD?mS%{EMf0pQ6wl%zC%@J*b-;kC z_gIjdlfH|W?C9P5{(mo4jI!u-%&ddV*Y6=ATjVYcg=-aB#MG^0{x45F;1E{Yrr_Nk z%A?Oo=a$TxPZQ!CvJ<&4AreG7qdBSFGCfWn@tX$D2WV)B%GUwF{JMCvrW!Lmpn;ar zCL$tf#p+nMYz3`D7D!JB7hK76-J}?Oy><12@ETV)i<>q6$^0ZoA@p0i{^^>dv(j?89>qWSb z)hVIjE;(sM1q(R)LIxX|8fsL=*b`Mw`z@MkRDZvehyx8P*2-^$@gzTA7~D+u;^4CR z0L>6vSkKSIDQ8 zc-v^Q{BQgD!~w~i6ciQ|ToO~o@u<%F&)ug$G$Bt|t3>m-P=&~G+l)N?`9VDWy5OPe zImD08!{Y>{ti*FmNFyXCnWEkif}6$kE;WT|$YBqH6oLJ3q0_8gGvq&$J5;(R)%MeN zR+17T4a2j6ynvl+@8ESt?MEM)K3geeobr1L_~Q9ZUb7woN)8&5Tn3&$Mn^51^E@i- zWlM^=0&d#oa1sujNVr;ChFplsTYaN}@qq(=tY+l7>(g&%Z$rCQTU+}uV;$%;xK%Vf zJ@ejq9^SX_4{$7eMD=I$@wuA<1lyqUPb3Q>rIH2%#kc2_Vzg_IB$V4%TxR~9Ic;lO z>dz~YZn4kM@M7|zObbX2D@OkO^G_(rGr^J(9^`%UAzwhV9ldFGEo#VG2W))(Z*NBI zFM3j?>ed~d`L1fTufZ2dxx?9~Q%TC`;p6%sT&3KZiQW$xHaAB)FZabjZdR0Ywl^dO zfTSt65Y>;sg6E?xTTeJUZzD3sMwhm$5TpTgB%YdEDsD#ELY3Dxl7k*ro%NG}dPXTR zFHh)E?zi7E&NQbmO6&4|Y(?*|SBgFTOtUI=sF)Ww%HHjs6W_ZLY2RqGZBs_>02f^KAXH`ZiNA_rYq2p_KLqg@k-jZiv+Y1~nefaY2f=tv zIPZ8&{_v-T_dG~R*?sox)4o%Fd$CJE2hbCnL+g$g!!0s;blB$EhI*&6P;@VE-@RM) z;HnA`N$N*=Tgi`;gbY zO}oafa^dYr8-mtZQeGMA<~G{<_l-Z|M72m+#k;ee!dcGC3eM-Q`p}mkfS3wIu0xsU zGajWwL8ZCfo3Pg}Au?9rVg8a)HL#j9Y_+IpRF0yI78rZZ;jNK0zr_3JsVsgj=QrMq zk^P@7Ee6{^AG5YT1ITeo zV3O`ijWm_nt-I8CuKfI2kl7c5`-Y!gMm`Z06wng==iJ`BrAEKhM28$kp`3}!WfyUz z=$`WPv+8Nh5}>P5DX;1n00e=Vv52gk-#C@M5!%nsedrXXaqKRnUfOg{^kejF)Qj=e ztWT}>g}#`;T0G)gH8i$7wHH{m&OEyC=q}c&qZ^~i&uxap|7J6M&a$ycb#h)5l0BNY z&VAikdqxoY2A(h0GLVt|R9!4H{RPmZ@j7z@bp-k~WubCBUnJeLdCPxH)h+m|j-_0v z*-!hxUfh@TO%}*xT;-3}15nj$2>T`Ubd{H0NQjRp6PjrUrVpeWpP?X6F-M%Hag>G9 zcv?lw^SQEa9+^NUHP}e)4rjT?1NKj!Jbux%5U?6~Zb2E<&PrkEN$?fobG3 zX|1=bZrJeKPqKzo(G4OkC)O{=fAQH3wQ6=)_Y9AR0E7Y5(&73lYU?6aahmk>=?Vmx zUoWX?j}X-A_;GPkPW2*ilB0l6NSZ}LTnPxMU|>|L?j9I^l+Q0fwLhKd7;y~+S%BiH z!}{yzuqwX@uxQy?h9r zR<^e9uUvwz6mf$Y7N$wnN}Q-dcuMZPx*Vk}Q2m&1ZE5~7gc{P<&-9=}t9*HlZ5BJ5 zMsdlPDlDHkHiUw;vTq?`0XkYIB~@vn{O#K}QX~?!Smc}f`YT}kEo<&-nv4v(P_MZu z?u+~L{d=PXg&mrOM>Kk7ChGEAC3cK+vI5a=aLs5Gk|LxD*am>l!qPG#XaGqAnpvGI32Ntk4}1Y_aYv2_iB*btZ@dv)*DF@tdWhJ%W8 zf1|%mLk6S*A5CT<3n)Ydp9;532a6Fy6Yxg+{g=*8c=i?`QoZkg0p88xpwPv%}HBfv60SOjdujsXMH%&kY33 z9xlLT_D{#(bPmTtZ4QG6wbR4nxq}jXA9FzKWd{}P(Zy`X&QtkR%2{YRGLqJEonu>OstGI+U;VUV zEYfS7uhFqe1!k-L+w<<<=j7lLc(G_x?j43NLgx|iJtkz{noyoo21*vdr4*C9`251&(>^{egK6I2kT_bPN9rteG+Sig4MW>fEjzKI(p$Cd+|MJEv( z9SnYd`*!wF|KOShtGCJ*zP{p}t$2NXnZ|6ohpAr4E^X>rAXVOh+ zuP^DsnfbO+>MWpI1E?5q#MKtPH)l6tq+mTTG z#Qpn4ON+sbEZoG$Tp~`>-bM_5#RUNAAcuteZ|A_AS{Rg;PFOX-J%V99m*um#9-X11vci4XGBBIokdg%N?9P zkOeDVM_kxrRU+_8i_Ti2X~RCiTyZM8ka^Rx$hUSBx_nXN3HbLk&ai2wsoV%rtBqM> z&s5Z|2-?64HtWUhzt6kH(Rz8?PGUH@)n2H6Gr*U|v$Blm?=_kk80?Ac>t%oGcaha1 zZCU1Sd(}eVC5%AJ^x%?wDgA$Izlk)hq`*Bq?&!ru(_w^zbE_2+j~CibVd$f=b1OvybrL{n>Yd8Yrll+kJ%+$LG=cP3xC4f^G4fDcj^j~ikJDajA zeZ3pRxWp@K%qLyht6(ELY!bKUdncS7B$qVk z)d;zReF2Ld?AD*HCCLS&`b&@~r0|Alw8JH8>>%o5IK_!U-~em{=7^nRjh(8Aj9~Dz z#1z>Z#mKT_$Fpj6=8SQQ7G(TJ4-fnS%sRg9`Qc6Q45ZhK94piI^i=K7YU4th+)(0( z1|A0x5BK2UWAG-$BJ0-8$D^ppVBB(GkuyZ``sLNCj>(7cUO*`2cXj`o`nI+U}U6M@#~Gont#bm71#F>q`?{%UUu zF^rH(JR@V9b=yH85?2BO2u=cJ?Q``$fZyd|62yVR2u4Oh?y`i#x`M^|0PFV${FF32 z-`Ml78=E=EIRS;q6VHLvm3$?5Y9s(fJ2nWCO-2+Qzbq!b3#=59?+ECI&tKnsBFI6| zsZ-)b*R26~JJbM5I8b55^T!55klZ)1^ae4J)z;Mf0rsrzY2D%Z-uA)|%;9LoexmsO zc&bn==OSGFWv1soUHWCwO2&qPQ+>~;++$`&VH3=ScYmx6Dy|!C3ae&4n>|L!1yW*f zhno{^6ZU@3pVN`EZp2^SI-iX!DKW-#HE0EYt7dp%ElPf)4!)R=;{N8|>7slLEU?O4 z$B0g;>`k-)6@r|(nlz^q=VbYd-{|kSiVA!DoBlBdL8g z@DQ)x!OO!F5*c|SiL(?16~V&BrU$$pQB_|>tE_q zQ`^wzKRAS50Mv|VAQ^?^ku$kNmmQ`FSlQ?`JB_|TUv+$e{*LJ_5CmGE_556s@6~5* zRus5uGjwHOGzXypv8+omt?0A1vC%X%3^P*ULPG_v4y|nb*u=Ca*(DP&W8<7`x3-`C zjeHkMrHIKg!p*t&9%@Ex(I8~@Dgl9F@F{~eTEZVwuAD{vgoZEUy~bwT5ikh}K*??c zcV_0%avlQJwOLwP$~k_~h2`wck!10dd-vw!;?(rS5Xb6;UlyUeJA;p(uyBZbMyvYk zuiehh8Q+Eqjw~I>pUwMr)8&P(0L}x}pNIQP;cMD!YI?PzLc2pAP1M!B>*L&WfgG5d zi~P1|5#YI?pxv&nIp2o719y0#UU(ODN$Rq%uYG@`QnGyX@-#!3R$w54x*rpLz zQse*(SSK#N`+guAP&{H*C2!h>?mT&7IPStma;>K3FW_9$D`NLy=ab$K*ZJViMFU?r zA&>*ZERhJ6B_bfYP ze;DTGN!Atl=;!~DUZC-9w_5&PIcBsQ1jl#0V(9)71Di$f_VPk;aK)$mwj>YiC#{k# zSMxY!^m5{*)tu-D0$~SL4MU~>grYKfdi2cB-OM);otZlP`cItN%6oUsNw_$}lnp71 zRhg}lB0@QgC?_40x80v9_JKa_0rx54ea_YVl68iK!wY2{q2CjN*~eHG)lfp6Iz8FQaeqpI8&q_-J zT2>W>5)19*+&+Z1J9Uir@euQ{mpGkNKr|fpzq(qB^4Z(TdT^;AVDIX|RiRMm{wz~! zAwB;f{IIO^>v2MXP#Ph)6B80FKV3GUd}xb*Qe|TmPnF*==xjVZX1b(o+1+xYNp7t@s!{lPon$m5 z?3M9x{^6Q_HqdBF@;Luxx33!#OV-uJs=8;BR>K4fxoErpQ&aZ4l{c=#x7U6I?v?Av zXYnO3HN2v!=Xv00hbv}(N?A*h?+{peQ7{Gfh0rk`$aRqevT+9vlp&xx+u*?teE3kh z!==f&!G=boRi^|5p|uj&?(}Qop4$etBtw(RoFF^hV;~-g#u)T)wqWeuUqO3o5Uxh~ zr+B&Vde4tXP(S?s`|o!k3%E}{L*|`SSBC(W)a6;Tdi7p_V#qxmo;9$TkM!2H{b)F2 zoR@;6TtyvT3^7PJ0V!X+R%i}{u2Hf~R9SfqKr{v7`fCgVZuhk@n-^H#zM#Z@eZZi| z(4;Y}pDxO{rxlFj=K<^29s)ys0Sy|mg=vQi*ob=>87pT;y=D_frc6PipbZP}CF%wHT5q z5$SR7n0joa!elZbpO#@{c^jK~tYBraRiuTP&rQ9vVi8g)Hm`$teNwIKWY|~V5dT{X zK%lycW03rzyOb0bPH4ynG!{s_O2kd@vKMbyF(5VSXrtqYr`C}Us|ewPul6QT zq&(pQr?9=5-{RbW_qlvcElcOwnVp@THIO$sm|f)o5EwaFI3K6I&(t&-9T+MmLNlHd zWq0bT79r?8t-a~Nkt>ud^shO0`Ki+x+MG79TWEbtFFu~T=Ucer9$OB9U403$y z%9c-}0DN|*JhC-)BEAaP0(zsm9uh-M8v62mnP&Fv=$d9B}#co>*%ut6v2L1x-e2 zKzT5GWMmwZG@`y$>eKN*yu)U9^>0p`lTa!tjw}6L$TJ-vp7V4@PfULkMo71fs)Z?KB;;x)F&<)QU(Qj zp?kpSs{hd8gIbC^&0DuZHNVf5&9?Cb6C)$qBxAjNd(Ch9C|++ZbgWtqF-YHJ<5q*2 zhx(B^!&@<2f=SHRuob+DHmi-v{P3lMBB!gSLdlJNBi5n1c|U-X8zLfH30dD8--sO( zc7N%FOf61YA$^-!C>yJZGFFu49MtZ&l(6C}z&XFT&=HWW% zFef57dC+p|RH>L6l(ILJJ=0J#*u+vzIQ+-umw8T)wDaUTY)R3r*Gv1fgI(e8^)pv; zXJlXdUafLiEngPw7}md9eB`LTSoGf~PmcAzSjW#9=1hpiCls4!nS%O~pan5Q#PvGS zr1KV!hLjyC>8)l}wnUDkX*+O|1Ssl1yf-K)vs=G>Y?1~2c449B=DCAj=9le?wX zQ9m-={>6_Wnb458n+xK{R|7IDU&&HtI(F!7XCus4X{qS;?Gv>}6hmW>tkE*Qd2kUE z_?Ik+ee{SMQxCNJLcTqjBLbD9bEZX?{{~&;QZg{EkBG705q%%00B#@olsIpPzK^^6 z1-*CkMG&|MZ30DHd_x_qQx~L40!PYW{VOZT^ykSGiV2YzpN$#-9Ylps-SoX z!5W|n1rMn#CxgxaiU6rdUWofga@oBBEhBCvyP|#ov1yj=_2b8nQ&szP5xOI1s(yGR ztOFQ+_>r--fSg>F$lNTVw-m#|qh~*+9fL??37Mi7G&(t*)p6}mde0G2uKoM>f5$+_+!Hk{D;ZVtuO{8y_-CeT6;!V0DVk z%rx$Hk)#JB;e4%sx(Kq*iemy)m!+&JAv*NSw;>B5O(V`Co>5vR>=g)lH`%qTv`8bL zj4ENtH$OcyekOer1yBeiyZ8(vI}j$X5d73<$G#3JRx8^r8`epcYGFa*C4AH+9=g&^&-+TJ> zor$vU_W@psJq*_`A9AaljvYU40U~v*l`r4QgCWkmJXw!6BjAoKYNf|^9^w;*k>v$a zjnVgK41LS#S6@5(-2b8TCQS79@Xe^W8*rw!p6tm!_QdS+$H2tC5}gl;e1?Y8*@=}A z_B$&SMOza4J#1a`;)dTi&xXqjMdeiKKK}H`k3zr#ITguT^H#k;8w2salq1xRP`em< zq04mn!pW}R-ue%<`JOi=fu(^yvj+Aww<2Oak<#vD1{J?U+U4bi#mN%HqN3Vq*{>U@0 zo1(FT2@pAkK_~GulcUojgk+WuMY_~)X62mIE|mE_WG}bss3-~1AN!k$Z+YkLStrTp zITH%4p1pluX@T^==LsPm?WC`dD}KzM1xdRQ!^4cCnnqXv=_<(x3R$k>X#4k3WcI9s zLobEF-C(~sY^nawpULLo=IpcUhOuwUc~bjP`q&&=&P*x!-4w%;LbDXF7SEi#9ImL{ z)$HD{2?zA6v`_hZq19YO3!}P-BnUdDM2xNRBCFxY)CW`(vO>pcbXVG+zL=iGLaqOY z>-EpBml_K3^&3LS2|Atr|qPr(T?>aP`B$9XZ*XIG;bx^Rpb?3+0W-yUmO z95HU8P*2Mu`=C)Jm2a{Z7(AeJNoVy?E`zwd-?$0bR1J7T ze4=A}mMC%wChHMH0y5-NS28FMJSZ{-l#x)u08U1#$qqzx0*ZT&#N#pe=*ZV+X2Ru* zgxgsaQ+Gwvz<>wUI5DU(F3a!+Ou^uc55d+f6-kB=LR+$*+fbP^q(cxTCUjy2?Y-<1dw}QKhPbc~#lL8R#dV@`3LU zJ_%^;KrSC+l*ZA+NN?iq?$b9N4RV-}y`@nbH?Ol~TXvNTcXBv7w?;(%>+;7cvgz+> zJIb!E@s%`8IPlK0G;kGBE)vSI&6W85W~ZBH$&jFXZJXug?a*ymj(_VY?F@~H2_~~o zF)&0{m76mLy#*O(N@g>em>M4fRzKEv#w!4Q?7K1d(qQ4>NA4X`a29UauDbj;q7W^V zdX($%afe7iP*7=YybkGQ>=eHyd%zmD^4Yn8j8lAYj+Me2O*qmv7brDHHbi@SV4C2JJE>1F zltgxWFlsj>IFlHTWu_Q&Owk5k3YOCk+5p$&G9+)g^WTmp$3*2i-1JvP5+zpBNKzs~gB?fvuqOizD%s+M7977X#sK{0RKQK`o& zSGKjW-vMPHZFUAQxaK1Q6!f`b{Ej{S&$4ulU0wA^_gM+L!za*aL{{j~m$A zMTJb5JK+o>xIy5#^c;++@>6JXnU5qsY{bC_j6)Xtg1fTkArC22}6LT2A$H- zkQqV?b+6c&+11LKE^gA=(nC!zR-ezOOcCuE6OT6?Pw@@V^o+Zub35tB#!CV1S8b^L z`k7lG-4s}t9(Y2-i43B?%3~9Dyg6N0oS~xho`-wM7O_Q?bSdMZ^jOLJCv*x&?8-I2 zUFJS_nWiYL*Jn7dG>oPIf_1ll2K~ty9`PG8S*{Kb#e`O8RXdaXyo z|F=tMp=bTSUqZZxc7Y2|H^Atc^(~Z|jT?um!=>-+!o@gke9Jo1M7Q91k6wjEOgPo6xH^PF-y zSFjcGl~rVTA5NK7^9* z>C>kW|7;(v+Fh&4b_Wz_n7E#rQ~Jl@K-A{I9%J+JJd2T0sL(8u2>DJGdXt~yE7~Sc z)W*@*Z`hy)gTX#&(T)cG&Bqsu>qYqh=fqz^ok!F^4}5Wxj-+I@{)WCpL_~!B#mj~h z$f@G`(O;7$_L$n2tRN!=6LfD_yj8Wdu)qj&Z}Rj*-MYK&5NKxLS`Yt%LTC{=H?Ucm zLmPR5Hy+AZtExJ~8-H3nm(qM!I^p_^Dn%jTnz|;Zfei26zV} z%sFo8bT|TZ9)nUI*>=G_FwJdp_N^Mtkqbz#Ip8(xmODE$+0=RilkIS+fSK1%o_#vq z_~Zn)CmDK(UZQIZd6D!xe7f6Z_5-gUgFaE=bZy%i54gu$p&BWJkMAmLC^x}cA-oKH z)C+6lKDgG=7$F~_vzo$C3&Pz6_SysA#0;sWkgS*16~i^XeEDZIh6Q&u0|KkoDJbxv zATb(w+|e@wKMl!rU|)qxg9NSJW9a+=46w~yq_kPQ@M`@D43H-PsMlnsm%5_Bwksz9 zDH177((rXz^La+eCYONCpBJnmf+*_DSQ`>+QxCZT5z@^&Dk7T7&fV1KbIijO%+^bM zhhJdy`Y}c@lgZPFak%THrMIZYATMJ=zRB%?1-{(Er30R$%3quFn-!xon?04uq>uKF zj&W7NyN;yVhK-yEbV-UhxBsE3;Zm1nL16N}*hcYnpn9K5osK++|Rz zB3&309+nKKM@OBP)cOf*JsH9XW8H_YF$kbB-C#kB-G?kLG}^?C&PrYKkne1NX!5k; zfyeu~F_!P%y(>LNTyYqRQwp}rnJwblwN{AO9&aeieA)g^EKy#Z;V7vhtE&tf2VH+m!yC#)Ee3$Cbp}k^w z{vk5+`QgL#)m6A6vNrWw7+5lwwe!hSs9iA8B^AE^Hu;}kKi{XKX=UfYJMoBDYJBfb zJ$UrlfjFHHu`Q5xSoxo5=S&H3aCeM&b$1tpjyx$RC|rRYCrg|)O$gHX_=#9f?gwch zhIH-LjX|a*MV&(iS=+gxpJyE~v52#7GY?1e(2>c6n!=yQSoNIt(7)mr7Z#FRlkV~v z(_ptepHDMnhApILMH@`$Wt_nac38P>-A8Mx}RxF~XM%@CHrxRC&ZRaLEVx{Mgvys(qegX@kPml|O&_((R2uKL^|V zlA*VA`!jj%ZHjxDx=!tlcJpVV$rc`H+1zEeH`uq8nnMhYCph;`F1J3Jg2?7p#49KqMQh zugg0wP=!_m)^Ip*3$i@6!F0PhW_)qV^B?ugyu8{65Shtbp`E(Aul(|b*??T)aU%BP z)uUcSUxQHQPAu#s+&~G*#($m)K?UI@6QRj(+_B~)6%Kg0leLs27a=q=Fn?j9$(Eo? zKAO5~mqIwGgy9wr{V`X!L>n{gf5C{{tGT(k`iZrC%TPK)9@z?Q$xcN9)zK;~2fWF= zo+L}?s@hz$XgLT%{0RCa;q3NQVY>r(7X~pkFR%Q4ll?GQKpBDPW#uje4a5h$ssg)X z**L2`rYtGRNv7MQqtIH(Rz{+7?)O?9gbM^!$kmVFgP;`UAE;w~9=>G5h7Eyv!u+>} zUqc3iRqsB_ws7)Rl}EoLdV_7z!XyS_jB)ANwX2tqn^s1AE1bY(#Lo^d`0ZXF4|iCQ zK_3_F`h98xjP{t@1pAdel{IWv$)Q-0eK`Qd(luv3e>&H^HBSikdNH&WH9D&7%Y}u5 zdmrTs!9Q_*Xar!kI(Vf4!_>Uy$;n9)#4t(#aCiNBA6xvH5fv4O_eKgjw$?IeUn}fZ|=ssOF+}nQklMH-q-tg@K;wS7H zBQ{CC{IN^>=XD`n6Ra9l!-_M*Dts{q4<01kXs(1em*93ENQ9uHUK3MA=tYRS+7|3Z z#vr6Gb-X>Sbm$fcxI8FV4~TiAB)YK4qgw}%yj+TwiOFiT+LjECyEWt;B&`+(VIc4> zT$r~^Szrs9@vN06xa=0JQ8JKCG@x|nQJ~9tUUOq^~vV>^PTB; zpzFre0e)d&E?>BRtX5QE=-GmKe=KrV2z(1oSpeA0Bv&}%+CHROLbRtq21<*Rk zO}V%nO@C^XlL7`@;}%9P-zKvTH6}(;8x1{#Q3Drc2%4dojnLR3@s)0f#^b$9mfz_> zQfPqmTZDm!6?Bw_+nY*8zIJuJ7-}m^o=CP+3`DYCQ?(3!&4+wt$j=xB^mv2%g~rB4 z$7iYTJzu|Sz_h=A6`l^*ffQ8p0MVe!G@1rP!yG!m|8t&klDr?K6_IwiaoF$8VWt!7 zoAvKJtzN0s+}YMTDK>IwTjx6SM89nrLr$T&EoUQ2Mxmrsp^SKGEc-g@x0{d1XOjMk zx}67ug8%CJOFoJ@qAjkpgMrx_%D%W$zAlellzpIhFsGu;!rr=a|Fe+lVCy>@gYtxT zKW<*PT8JDTC(+%GAt_fcJBx?}rC`KleRSMpw9}iVk^xSKjn6Y+Bec&G3&>EM@Z^u( z7_A{gav{x?xyROC)>-|KfJaPlxTb5Iz@a>ePg^v!zg@1|zp{BWyV5mF+IM~7{(LPB zK3H-53ndI(|2?4OdXj_U$!nTw-!8YsO7rwP8jsi=* z8A@^K{xyDoYXNGAxE6Cn_ABYG>?)LK*-p>W$3DfKNsJzdS!W#}5voI?gu7^bBpiVM zEXtju`cm_w`yS)s9i5l(Y8V===jW?zWIs4f=;4t`}FxvkJI3FdUjj3l6ue z9F4nqnRbwDVJn!}gWQoFOEt;fng0A0Xq;?+exBcvaHr*>=F5tDHv|k!PEEPedS?AZ zKMX9pDnk%f)zDg`mOPixl|pWfI$dd>h{VF^yJ^K69g0NY*eH@IE6*QJA<7GvSK`nBa7xk~>& ztAAHPZ~8B*7@YN_ZOI|!>zIc2rhgA0%aC}TMx!;^QNuOCp~9hFDe@zKnE|s?!=z*a zBk0E&dZ(bAtbrkpu^Z-7`0_T57g!=00(e2edTu|V^T43pvoyV>6}R}Nb4wv67sBUo z)p8kTsdQNhX1j5YMh48lW76!JwwlW~xpx`)m-!(MNqpNfUS7-nm-S*+C^cnNVN76V ziWcMrP_hK}E?cqUtgUXkxtkD1G`;uOeN-SdTOa%W8QCP^ImaL^qdw7E z--doq4B$&WWfp2}Q6zpXv@M>}e%dSO>)rei+0Hj<(Q8e~1~3*gKncAdu^V%Equu7o z)>ffsvFNUi$JC3nc1APc(~iIiL71m~0t}DKMdt4CTH&!q|B62^QXa^)6bOKT^fTIt zoqBrz2X9{N6XXqL6%~88hCq(Tr`=y{0;u_*MF^T`Y9agjP@1i2om{?# zcDSQ~9Z#j|#fzm$MOYyc3`)KpR-Zp$kD+x!OVwqT; zRqmY(^z)~vRyt@>efo?W(4tn-We9*FJ+_fNtGotOMTc>!=fmFHHQ*vNP)3uWT?N;mTD-OEuTaeDk>fe*%9b2 z2@^JM*dU-a+F!~+^vY7Dyn>`8$O`Moa+*AouhHYg%LDXyd5m@Xxh87RA&v2Qt8o)Q zrHtAS5cxpRWAEvSNd;H!0jC>PC4Bedm{&xgcvROpB`r%OzkIYJJ3rR4S2-sde}UU{e1KPadG@4R(%u7EFX3^ltk7Z$DJ zp6d7IE7dpa;`{&y*0;<2ha8I|0v?}^h&@s*8zyc2VEtflK)|7%LnaqJ-i`RI+dSHG zGGJtdbM_#kd7r!f^i{hUI{W)}O*OofJv?G6D-&~kIDUIxH_y*YzEjvAAmd(o006Wd{+%mZ|yJ}S_QVOf#@W`^&M}VGQ2V@Ff)vpW_CM1W*A*ClAb4V0q7(gb#uM$1hu>MjU1CMrpK7E{;cX$2{!(1?2{D& z`GO|NGp9@|o+Mwax*ctf?vn`pwVkJ{BK?$prwF^Bl1ubqiy%fTj9J_46hJm0igFn5p&0W8`8k)Zec2;PH5sH&MPd+=X41#rW7R z3-&B`fnE(zwT^`Mn{CO$?C#`KV{0CC6tWz$HoUl#Gm#VtmzFFv1jD# zZPxg6ETYDqqb`w^&DBI2&6E$9mJi@Fhxq>FnHJu!D+RaD78IRk%AO@UE6uDh@3&fv z_mpJH*`7GooUS@$=l1V+2EqB2vCI6u@RVVI&xlEG{kGg-5qb29?x4O}e*P~nDxF~> z-!Wlm@!t5quQ9}@N5+cPaSG=KufuzH)GExMq9oS&OlL^kk~x{HN5eH2L4W;kG|r;&@D#laBEq z<1!;E%}2nI2KeG7-K0T_3BXg-LPXlkW4~407-gbx6=}) zBW&og{}DIcMM6pI;)M$|Nc-JHBo?J~EW_(&c?T$>z-H0u0{+C=S!e(LJ85cAEWpcy zmuxv(T9$I%IWR25=;z*9-$cN#sA|A!Ic^h9=>QnRJhIkdFlBh^q*EaI8X(c5%0UGK z6^u18=`_~2@I3a%zJ2poeHfavT}`Zz_TxOEZFoOT{#xQwFXq1jIRaO_|EkZ*~3EV}xo?xm%*-tkG55 z)wELo0tu-B)_#sMKahh_R?rJl`amRpc6hLNZuIo#p+6mKv&4acrWxzMH-^OwOy!(v z4p+IuBABbE(CTNv-5@Bqy}3CD74+^mx~kz+7E zW0X~pnD*&g`XBQTVaq39WD-QM=1>JkYFy$h7hxiffGm3J&Ye!P5O9n+c+t}aPBR(C zK{F8)(he~qHtyh{xhq7Y%Rhb$gato(sJf%9xrl;u@A?Bh)s9GO2e47e-fRg_JdO&< zCW|DFP6fVaSOmSy=Q{~W4~&&EjFzuEr_2ipgaT;JRu+af!Oq{_$a7Sry*6++c63we z-|i#?W^3mj$@-E@T^ZP#3HkmLC}|DIFmm7=EDF&u|9~yEd3R1Ku5puvMW93S)5`Qf ztRo5U4RCJCyD+_<`UUod3uggL1FxMEO`SM_uM2r+x{upq4MtEOBsixqnoe+bBX*#;kn~VtF3$6rlml z*s#$Ws|jK=sx(P|$VAi%Mns-l+rfprBO0qE=3Cz$No5Th~AFO`&?eoV*+wh6pCka6mK`%gP zE9z;jP843f-SY775^kS)0XmCD*PRTAzmQ1Aqd3T9C0MmmR2zwFB6%Q3>C}>l0>Dee zBXUONsO<9U-b!LAY@w?>`Gm?1(%}a!?QipQ=#cg8AahqBDP;OR<7WQ|ryZpJ_^ z%B|wGXFHE{ucur$kL&Ph?3MFkF4+0;73pzXA=RnV0?cb`y|t#nc7>5Yy)2w@Sf|9@ z=r-Tuf`UX~kRC~BHh;YcDPByn{f}q+6i*ki7OCI!otP#b<6199bRo*cxu`%bmt{1I zziS4ydBM|qjvlZ_mds$Fku)|t8UEtAN`*=cEB37u5o#G1ESM`L=bE-1QtXJImvrPq zm>5j%KLTtm7c!A8^*@7$quju*-#wtcU0V{(myzH6XN>0l_!I@j4n|shiixQ7yxT>9 zX@jc_^)%EZgr~vsGA^L(L3;P_aMNpLX2fb;FO% z;o3Y2F+uK!+-A&k&&z4jrN-LGe?BIi`u{9|(!VZg#50Opx{QI#S_E1ep}!lyp7n;! zh;)v1@Rzgn3z%&g&p2q&jY+G>peAOy;aGQSz4b+|B4p>EyRe=R9)F5^7x7bA)31eq z^P4cDt2@XRGFh*he+n@@KTyU+&zvbp#A{;DFykf&*Z=mF$SKNVHc5V_ zCT5>C9UtuA*zkY&R(!Rs%hu!$MlkMTCCNu|82L;osfuzX|6owPPmogL)^#3WxCn1PxXU?rA5p*41<4DW_oDJ zw;He2C|1NN;|w+1e4$S`kIIDvgMIb$08)RIPFZxo%X_q6b)B#b+TqnE>^BqTo8V~9 z2sm@hWG=t@3^*PD#j;=Mb7yGQ$K1 z-Y3tBjb|VVN_U zR>FK`*PYwrNe>M~0YoN9V6iZ}^b1M1gELGjzx zXh$T%b(dGY-S3zR5``~rhI~e%$qNT(_=Lw5uy3Y%>#4!Dr7%tnhbihp+Q$ICEmI;T z3YHL~xWnkohMG;hbCG}8`bYw7mt_*i`m7QP5))yIfC;Jl;IZxKhVFq;0W!@;2ThD! z{b-zs=~Q|kCTOJem#nR=4MFF}4K?e~tv!u)7DL^acLG6*#J((y?*R=8HcoPJJF{_t z%~p}Q1k%FDzgd$vNRkO zCPoj*CL^F3%u1Lv-IWiAeRYzK@3z$KwokKX0MPo1&SHDI-a z76EpuI4rnKC;1I@lAhc1%p4pPkQ&zVM3l%~7@r~!Xh_UTg_I6?gQ1C>f3B!&D^s|(c~epv)q1_$Bf24`Gr zcpmI(xoZ2WDVJoQ@SoU|b8_ORZ!qe(hoNMG3Vu)+5VUz!Nrma7%1rlK*k|tO#mF4g znIW$}Ua7IY%CGnQBH@vC-ON%u)&2v!g+E zlu|$oy>EGGe|o9E;SAtiELK#7zVJuLR}F#Fzb)rMK6CQKxln9#zn<^#W3VU(xjPui;Ly_7@oY zQB?;Amtj0zE|i6g92XY-CFrhxTEC0E?5>l}SDdCk-qSnVFtgMK132!dse^ihPt*T- z`}3ZpkalL=a?vA`(EJ6#3OW~K(IZ*aBH-C`5ki^TQzGS!&@J4sXWv3`@HujIB1c{j zlRnjqzfnY)4Z${b-DS+&!*74aZ+syjrC6mm4&+Pj1>GM*Ca0I*;_OKZ_c8FhA$R)Y zoYf3Tyu}B6BmWQA=Jyt)wluY15;W#o68M4)ifQKi^5r^?3S2fwzP4Iwj|DKBrOHqN z(; z7>PqlJOlsz6Fi$SJ|N>Mcz>|w5N3~)FC3JEUX-aY?19+Z&EVNr!dAz%z?wYi&?I7~-(g;0(G#>Du+{6R}rsJCud3 zm>dcT9b?VGzZIbZP@t`g#@|ms1eE-A-gl_HyF`f`SAet0C1nll=crI}3jjvUS};H~ zBcntJ+W`<&lFdKt8b9pkj^Z`?KA$3`0Ba6rIY0@QYh{6xPEqqqOEs)5&P=J=?X8J9 zW2P*$g!L3lLP%!sg(0lvGYUY1!kVK(4qG^=;=@ufxg*UtA#&V+O|(m5mHQjO#3#BO z-OsKTR`s80b?wlyCC*9{#tC*WDr8oI&ggCsHHct90A>2HrcGg6HU8{ItOCB8Pp&-cV>ITVC3FxchscW;9U?;y(J*)5 z0_YY$*tJ*djIqC)nVWlIr-zFGz*4X6<70=v?^|M9GE>&%`EAHfRF^N0yMMntJrFxO z*`Wp9#`B{C0q53Hb=W_ETgc(?O`22&>u>aE5TCzJ7ZaTc03Gs(gdJ)67hVO~@qCUW{z{GtlWP~|zH%JLnGxb2>aIy?WAoz zdxJQuxci)bSju>#yjK%28Ujl9+Evrj+uH{rVi7&e&U==j`ep-~wYAAXKK-ZHE^pP- z9h9vI{F}axc0%pE(^_2(nS9>Rhkjwgf^388y=Q-7;FH!kIQXc1mdV8~@G1ONiNa8h z&l$HK-YYOt;l84)OIN-_YG>bs^YIv$`DsZE9vu6U<3I|ZU^=T=iNeun32J2!l~&HU zS}Q%E%+K+gg55l5mVDO+=|n=oRJl}turLoW&qaXX)Mei{XMD2^_pcSzCEdLrUssI5 zOTsJ&j=BI|#OzGQI$_H&VUyZ!AV} zb5o7DZ~dFR_TmWK=ILSs_F0_y2i*So3XoqZYP2){iIkO!l$9wC&hz7kGB*O_aet>| zB)y&E*WU9NNEUMccT4}zun6E<(UVmvE< zUA;*tb0PldE9d^G?AX+cRz=D-tnDNqQ0~=RqF9Qq7amZI6AV`lPJf2xz%ERsol&u{ z(siQWt7wp)k=gMOPAn=)*7f=RAZDLEZ0Xs)4*q-3(EWCR-~fyQ$3%3zSL5UJxrc0W z#>KX!f*lUWa!5jdEFSG#OdG$94424c#wQXU2On|*U$DOx+7e}$Ljs2zCJSUV7}v;NmfR9i3g(JS-Y!CmK!z|JN-XTiow8U82-(gb z6pDgVd4dL%`L6VSQ@y zR9dv-9uKl%upztfqT%N9f=D&6P>4$$P4V#HRU;5J6@ zE`-z8KCuuG3emyrc$?S5bi+K4vMtHA-=mB?;9llfOq*>dxWDM zm|TQ$TLibX=Rk`WIAb+(4+;jl zTRN(;N8YN%(8ENcFbnVD?!pd>Cr1>IjdaB#>AEmo0xkkqH$Lz4C<{fJF}?Z0to|Dv z-8gGka-3PLCB*+Am$t%guDc5$Fr>_tf9*p)%xCl30j^m7=-etxufl=?_t+J}UbBfp zkHIR=LugFDK3Vw5U{WWJ1{@WjPSau0+5K}K7!G4?+VA-K5X-4$aN?r)Y7AgOKtj0? z*l(kx!hEn&3*;weZj~WCD^bHTD|N!rzydBk`m`2B=k8!Fe2!acrbD_3a_xgVYcNP* zJ*Xd2aU2N1zkv57DtDEKI^O}U01#w=25(h)dE}UIv-h8nl>j8pSr`NO7%%SB5-7WZ zdPgC8PoBUl73T}M@$G#(YG`1vv)de!8Nah#J0eOBy_y1%&q)2HS2#=kG9 z-;r0p>SVaTP7+ywnaTQJXj{rQ$W@b)fB#v1j*N_btGYn^+@@qMjZbxIE#yFQ*=#2j zWuuC47jx5Dn^RLR`7%W7oA-yc9bF0Av6>wG9I68|P|R{K&> z#gDuVSz^|`ebj`e(XGBmi4-<| zA?^1OJG#4DY{qr@n5^&>$46tAj(;$qSZ#C8r>9w6KwolAiI*;y*HNzQX>y@|?zbr{ zl_h@a#~aFBV#5|8^^#dUskRv5X%4k4sTwtJ8RGoKS@lU|GBo-leZo)Z&;_v{oO^Kn z*Triet@sI*n`&SJyPi4VG-4c&q-XuSADVqmo0uZ=ZKpS!yn(9uN^i1aM?`2QYm=Zn z&BqS%a~{OI;Ri!fWn#%IdkSiq&C+RziXXig9Odz@-s@Y`4Yz>}pLjf=8(+8vrtrk?^lbO&z;?%)jP8%phvQ>Gx+)-V6woI`t4MFv2k%SDh2eO-ysTce zpS7rwA0ra}te!{7SFs5$5CG~$c9ukRfU)GYRTv?W0R}-^h_BKD0gKN3b1*x`NTR5S zw8E8DZ;vRxeaJ!Ti+F6IJ#392rj zAp{)w>tLq#=~dooe7{8cCLb%-6AEgkE&rzWaB;O4#6kg3ml3le{St;;@M75u+pfay zTZxH}HuyXRqq`6Eb~1grMmljTbUl3_bb=HKFUoD!Dm-A1&e;u?LC5)0Pt*yF*2rLl z2na5o*xu3ic9cTE%d-b^&Qff*bd*WX9?4*>BwLq5haSFNc0V#ECKyM6IV5PRV(omf z&(czy77MI^otDwJ7R*r>`4cS$oAZ`_qupmScoAYlm#k;`+6SG&#zp$ z^6XAv?Qokb>{+Tn(kBb1RN!R+7gd`sIN%7urO%u)3-U~SWAIVgbTLcx%C6b(KyugRlGJx>wN^g zf@l#q!8ARRK$(Ode+ia!G9Z<>cS^>V(VXxAxdrULn5jndWCWDH)x~*zc%zrg2v=g% z6(>(g`Ve`h=XSH!p|@J#m*;xbpN#6dGVjBeWc?3ZgXRj&`~t>iQ!%wUvl%i1*lm-7 zqlHaCi_R|aovx=^voEP|D`w?vA7`Bx?sn4inBNB7r2nE`My**Qu94|6fsdnOS}z4x zF(0OE@%*c`&X1l~m1#XfAs7&c1a~{Cn=H()vO?sz?HHUt&KRAu$;i)f3yn}S+W8Sz z`&!y)ihOHHRKm!1y}>s}VwvD90Ip-!D8B;pUjuFtB?wCd<|6F*I~o{--*m_*tQyU9%KT>h1k}jx>&wG@Oc3 zVei2gI!n0CH6zrAal4%}*83r5#Lq`=ff7qu0t+I1WIryJZ4RB<6jOG*u-%hV)t~BM zEF{^j6kHD~fS05RVT_E)t~}Gil#5;H!{k|^57HaekGq8ZXsDXV)DHi-zS>I`Y=DzK zgZUE`n{@P2{@U%ILXLKI0$1jl%AQZ63^!dAt-@!JwNjmEH_~rfF@_40|#m%N1X(hmZx! z`aK%|x zBp~95ZNR_oY6^d^`r`tsgVen`RcPwx&X^gkHd~M_#3!$Pu)LEIbH>PSyV8$;ZpnUL z-Lacf;15SP^l*%EfJ{y(k?z48k;^#PfUTCm?^l=AgMvfOo`nW#IqdleyIp9%-xYaWEm z;iaJH7j`(}?_jPE;u}lD+}xZ7(jXad+(=i0&@7VEH0@0QS&hc;Tb}^86GDy7ygZ&~ zY?xH}O@nkl7)H5i3i8ZeaX<|BO2=~-2^4kke$zzP(4ZT^U{KC-xxoe#H!!O%vo|JZLYZZo}$AlpV2Ni0W(oI+l;(pQk2$)wtpEkA6f}Ys1>LQ!x6XMQ!N-*Gp>VWXOiWHkS1KaboM}QS?WuD^{2noHR%EG zX0kF0Lso**`^l3|(H&vnHF7`vq>-HW5~zs{p&CG=%rn#~H(g|~7ezcJ3HkAlE}fy16eyXCPu5iU2^tph#2U?VXfl3-eV}SFL>?>0WxKS zYFX>McUKz4^TFce9i-QO+1j1$H^Ejnz6@WD0Jn0(!*DhYu!;UeZ~2PnV}Bdw!3 z+P#Op){lBY1_DyW*4S{I{Q%-hqi0S(7Pb;;Z;-v9Kih~$9@6;PMK_@zBoEu<L zDTHWr{zgd!k7A_Mg(dQha9m~>LkCIx5a0o`xs;5|zWM;S?zh{6UJx1e?C%PJ@TY@# zdk;cXcZn>)VF%0I(=*Hl?mCI`96&y;dn7?l{9`QYBE6eM2etAp#|d{;K6LCgUgH6;Lt* zU!+6YNH_{+1+TfhMHv0!5OSHc_eWYgF$+eck4M5X)G>X_`U|RLVZP$ivUi{WA-L`{ zLN7?~#BKWjewSe~d ztaiAjcBf3z_!B%dS@eCRGT(}CBla1bz?{n&O^shbBKj0wiL=D5v80G94lBns_RXc!D~e4bzE>m*@N-Vx2_as7p5{=EnGGU)*{;>?gc+l9dg81 zud+{kj{b0ON397wCv=HC+ z|K>*JO@2OFbigul{&lZV@Jr&A`m%Ok4{XWed}jN27(Grx-s>b>jFRq)QuD96qDj6H z%sbp>AF19SnD08`WSIZc#oyU3%l+0AnWpkg1`z50W+8FE*E3(|^>!S;5&i2*;O(HucLfCoPMnlmocNYDX#pAJ^K#e?(@GXb?u48&Tw4LC?^Pq4aX^Zu8tLRM|IpF9>wU|Y{ z%HTdpqM?6a6)Jl^&M5{Y!prKe z5L|iaw>A%DcXf4<>mfQYw5;FoyT`x?8|NXV4L6#syIipgpg~J~RdA#N-FOfzGlXea zhToyDVlQVz0@WC@ZAS|}9uPnb$W?Ij<~2jZVcA`sorLjXYZ+9)Sp$L zatwnNo{>Sx*V*{-k~kOKgg}koMt5TyPRi^wMo7h+s(t7{iIJFeD#Au@*DeX#)`_^k zI~Jnj%TG+!zb~Yi9iY_bRP8I)eCuV7#$d>aUSr?Z5N5cn75dV1%N(Lu8kB282A{yE zZwX*Ixuqe=a>_GDjgsPG3V4N5H|U$wedqNQDvo!A0jf~>=(&-ava;j!%Inu{ZFtp6 zvF5Z?z%AOFID_rDj(ICr_FvPH?)1X|203J(LCMRD9r-E8QGrEx2MjL-kRzv}G=J&R z8FUmX8m|(wpb_T|h>LR{*uKcD%-e<<7jtaDK4C%`n)x5~0gLxSwG5##Vg>{?e}+~~ z*hn&Lbd|Ue4>0dwMZ>{`Y_om0A>-OuP9KDBb88)N7OLzWK(V4E@p=#ymvQ;%l)PGA zR)zsF$Z^vrpAHO(M#$X+ytfb-b8V1L(>DM|?+*)kRY6ZA=G`{VPSE24tl*CyjK|3G zx7`C3(LhZ=O2j{q=*%-SfTan;oV1!Fb4l(A^aeQ0U?SKif#T{!Z84-Uy3frwV+#>~ z0H7CWsaxx#DpF}*qCY-+lj_`HnXN@gxvn91S_02p$YB zsPYck7l6P4H+#`&O3`@K=S1f>IxXWbpH}8g6ls?&e(?g&T1=ifVZ~!ynUG^rRkM=QRjf>Rs2nw1RkWI<=P1 zC@;Wp3FAcZrCF{pgFeF3KXNSEQ|>j7Wd|sB> zp#MU=+dDS)?cQ5r@|64Kl`*T26dI)7IoR1cvLU%{IxHy?*${nFjwejHtEimQOvKucJF?WBN*Q zT5Gr;(FLi4IzSi4Socbe=@QVvXjNsHs50&|$}8IY4@6H-I+i8V=*cK6J2;(7vP1Xu zT(P#c9ra}b7=}w?RK*lqb@e)Z0(8<+ejd%Lww-|K+)WG(|NUxgPX2v0t-3%Nl_o9D zx5<-&I2>4NuWE`3m-I(%YPr+j7ZM*({^sC*qghVq6 zKEdh~GULnx^*S{6heqdj8!=0m{=QSz4RFmt{_PfJL1-U`_L0q8`=rcC!eZ(! zBi#o&-+zwF9``?9aZF@? zR+MS^k8$_`Sx2?EyE6a&dMY;Pb?WO0uN77tj%hzKXI`WD>I-&X)BOI+U;kP@(cq}D z@BWH8O_N(yPD053`rp18zccf#RC2g0Vi}0%#A@F-ky9djA6SJ#K?j=QQPB@_)2A)w zP9EiK`O0!m@!bc73@n1n3hhJ1Idc6+;nnlCtu6JAFW1@uttyL-*rSf|9qE{E-A6bM z(aEdyavQ$Ga>H$;p8_U=r>8;O4K@fd%77X3db)HD*bmi!SzYtQ6|9gCi-dv!OVwjv zAs_oq8ng%Ey=eM|(MRn5no?Joe;~$sE|j_=Zx+{w4~b8m#HCI=j2*t)(OrWGwlO0| z8y5*^o_E~a!;34k{yEI!1iCdDOG7QT?9TYle~7tZD^dHMURLa^u1#lU-_a5%ZO7G3wdEH8D4a5~(H zlt?jg4h}6hcX>y9Q+Ovpz{24m1svS)3gS7t!Js23xip={ar$dn)s^)0cXyP8znJT* zdJsAl|D6XpXobk2bKtT+XqN*4BEjRG^Yl_w4DLF)yFX^lp=JY~XwWTpYOWkVDclMj z0@#l2t8k0ExPhRhrLM51z=}YZ`!j~;4p=|vYFM4Jn1NA(t1=Gz$^2Kag>mhlGu~g@Qap1=tO`sh}2u`s$&O*v2mjuYTN-OVB6ioV+IsZP8=!VMbfYFE#R=YuE$@jrwtL zZ9uTlTS_dUIcrhuZjrf|AkyHV ziUC_XT=cfXei0mVSp6V>l(mSiNnE`3QR&$?n0s`yC8Hsh2ZM%nV9l)e;o#BrkZ}ww zL(Cqvua^33vu2ie4>Nu)!KJ816P$jq4PSJJI!l;-v(n2=^@{HPQFD;#!EPJO$ncN@ zIC!1z`K_7+w@}E|Ksc{6i>wKeC(AroXp=e&+0+T$%wA9ii1pK z6P$({jBitCyC%-Ir#zep>lvNgv84g3`4Po?`91kS`78EZNkol9Bs zZQgaU+TQc=HYcVrnVROJvx=she$#Qrm(nokw))j8gVnEw!u_>n($ za%P+uyLwahG&9*6%FAIMYt{Q`3-+*Fm(G!NP#{98@9*3^s@paHPR|?8^ z-0^ix9Mh3%L)VR(c7HA+qZDs87`mgMWx6{`H6v<7y~ zX`nze7HO$!2A17Z3x<$@t!JEV3x_gLJEs8gJ$r+hHi12fRU0r{{y!cFFefLNL81^G z?G($aGGM|sSes4lb@|QHL`QlZZ$6G0Zc8uwXKkVEt{slT*Oir=5a0FVvPRXB*f1sb zfZ|;4BRHV=>k-85hc7ado$=4AKDq1L&P#NFHi@y zC6VP(^UI)Q(q+Vtrlh6lZRO-F=w9-^rr4t3a9 zf=cB8g_f5e>HmWnK7TF_u#`*!8-eH}Fz6sUHIXVVBU@DiYN#;?Ra-YfCTWehMOcyToqiG@Wo#pmJOJ}MEl?$Ra4n&p0~WF5;|@)dmbI>>tG zV@tNRFvKL4^sq>3eU+hg1CeY$23nFyBrSKvM(l4cz;dNL1C5(tpYln13IWGlu0&Xp z)(WfTjYplolpJk;>Pxm9piFjc^Tm>pp6E^>y0N+&7Z2p3Z3%jTxfGXOgJI=0j^(-! zn7#&pEQ$8^yRy& zYJphw6mmTU;1D(+HyK!h0D)XEWbb}iK-z=#D{6ee-I)_zNB`qJD2V|}UE{p2s2~Bc z6%sQF4@188jBqbR%YRQ;jT+DTs+Rwa1wj7S>d6ktu=2_oqNUw{ZyMAC55 zBS-6uc?|{VP!=kj7vr|hAx9EQFSrVCL@elr%d1Y~mJ=p~2Dr_FxhU#*hM2kVtVktF z$%u=~bzOFM{Ux>Ot~G~6NJ_;~p+Hy^%_244hP@S^fSN&LR$nX^i;3~X+VBJ01Egx3 zczHn3|9weEblE|B=C}S0BRr@0uR!fJi>m7Mxem)uYkBr53N2ePUXQsQSq zRw1%PK!}+)W;k{Jc%WPiHbm3DEQz0+vgG_}3%2*a8Npyr5N2CpzlrE5TP~5+erUUN zQ{5Aw{gp|0Un{p5~MBtkvUcTi2H4{WiuojCZLdZsxIYCI9vo9m&QgUi&XH1~$yH z3%EG}vFO5WO94^o>9RK)i!mah!ojEanww%Tw+9>EU;6gL9!+pGizD)TyRU?F)@VJF zVXkb>x+gXb!YvE&#dRKbcYND9mQKsyXJBIW;InSlpyDg5QP=K6tS5|ej&#{oPF;(rU4L&}_$0Lf9W{%zV zWr+{$X<3}+Z*J?ED@iA0r4a~=xA(|2T1^=9lrmlAz#q=5#rrt)U-xVL zZq5}X$6(L}SK%c12}i_VI!KNj*tHe1!XrLNwBZ}0`@Q}!>Yg8_c>lWJ^&iIWmp1sp zmlNeV=Y5W?dEFiAWVy07BedOj`hUFmM+40DPkZhb99WE}KnB55CF{R5jeYNmwVdlw!0=8%D|CZ=;bzm_P% zce*~;m&~a7uFSeg49BI>|9ms+g}vmaq)_t};Kkp2UV-eYi{)hm*d_OKj^P0sR7BPG zOQ9_;?9I+WjEm3N7}pPal&$GM5c`20f5~O{j^IgI;oQ2VsrnXBpb~Y2{cUM#qcUUE zOXS9mhCEdd2>zDQIcX$%%T`h2`RuaelZ1qq3QPT#JKK7ldc(&# zt1h>xmBv2Y*2#9M2(GqIyJ>5koLU{sH#LWMj^ot(wtq~R%_uw0V3cV@6t?RmFiQM< zRy94}cc^pp>8%$omp{1H9xAPItxCKoRV^ndUyw92wU^J*-ym*hQp0BB8F-NT-*{#F z4A&WprG^~Z+&5Lq^?l54;Yv;E==(Gt*J_@?Treqq<`dI*pCnyms$D(OD;@7$Hrd?n zdRzYHi_@OQYLb&Mpe*GPE!fBN_(jhUyVG_;Sb|3Cc&Us_5d|5TN`Tsl5P*#m)+FV6 z`BHz&mWYbM^_Mbn4&4Za0VvaULDuFN7QsI#rO*#v%zykLi;^|X;P5a=>Ux_u&rq_= zh}STL+80rriIJrkSs??e_LnXe z-<2$af!hbp*CccDuSJj^@TiC)kYFc+6H4JHKaXjRU+{)vbTO=B$jHn&TM`U-2b;P> z$hGj&=g*CR{;hAxyc8ZzNq;Tf2jQ=T*TQT1ZlR9L%H`}A%7#DV3JsF1*%BcXLIsBY@Nl# zb27ia%l6jBuo7W!26werb?fNqojS7NaOWIU=4B3*dngXEdis!b9rU#i-+F@gISW#= zkZlj6o~zQT*2F!J7=z~a2hkK4g>GSXZGOR@W2&x%WoHy{-q{HDFnM34>fLW61>mh) zWunl7F$a?tS<0S0l3g1DI~6jMDX~VwUHzQ($KK)B#cP0`3W%xg8J_!k413!dq$6D8 zesyppCr1Z5w+T3IY?844l<5wY$=kDp)Z+YPW^J^mUM(BZu?SMSvdj=CE^ZKTFJD?- z&yZ@4{?o&N3R=Ic2o4y-M*{U-BIqTUG{@r{tWIX zU0tAJP*R}yxcKiE$ggLx^YYzS=eqlJIbZS_wi<^UL5AyIT$fdt+J><|-n-qnT&ek@ z6Mw(dp>>z|7?;+lEiPidla&22XSKe5l^lJ=(P#Cw#v4g|Ido}CK~hAVWyM2fM#Cwa zuzPK+U1yxl;=0aV3SZRFGKL}8+f&kC(H($-m4cIl^XOa zE{A07ICy@adTgpTyfEl9M9xZYrm5FOiv_{)Gh00;39Xmi9&=y!8sod*3~ZY0z`^jZ z-`)W=*}(`7cV?|^V_kn2`9PeV#UC9ih=|&Kcgq&=59UqK8t=#8j4(1*{l}F@DB#Ll zLw_!siW_}s@i^~f;p0$2W=o7xrTLfluWq)M%CQ5dNeLI6j$7bY%ER{U7i-dGF|OM+ zEj@0h^4QsTV$dPnXq)Go0*@CO)%_{M{rYU$ztcC)JTHxj(Z_~={`|o?H9;e_5T{+i z;rF!HeO`Lp&o9Y@zSu>=Ma;+PGAkZ${BA7&L1yd;3ij_h?$h;KQ={6it4B(3{v;T} znT9>z$5Wi=8GPvKyGYZNwH~w1_;`jc`qv_6-2As^pzECgEcwAQCJiMBX9f2Io;Z!B zrl!{8;Er6*lg?tq2!QPZIp2#F!BqkA(2>^I^Dg4dNM!N(^Yz@-8AGk@?M0Y2P%KxY z1w0i$Q*NoFQr^AIrb$GjzMVw6^g3+R0g8v0&4=e!=dJP{%Y{t@3T!c?))9aF(NzjtQ z?0sSK(HkS^7^pGZujHo{ghxyUZ6S;>kegutytwbRTvK+>K-pD2ib8uN${m|fRQSe? zLOo4f!sU~5VUYWz6f+2i@Md<~Ho%HO@arJ)VG;SgjaT$&W;rIuCT_Sk>^e4V-N+= zj9T_i1KHP%*xm ztVcPgB%d-_0F8uXUkQFHo0+jpU;WVc%QT+$>;Ra%Y(# zkVine{=@=4`)vR&qLhC8y*karJ@D&SJB(3-mc7|Bq5U33Aa%}8qQely1>jLu2)Xat zzALft%1qo5__%LHsjs6UGC=i;L3Jz!hPR2cxeE>M7>KBAVRpSyIcA$R0@w>0ThXRbc5z}0N4m;vgE%bLlZygG- zhajg>I`L4L0x@KE?C=9excaJ)f8IVsW~q(SZnCS_sIMqU^igA|UX54owmskK`C-IUdSEbTlF&+o&qS??J^18#M1Idm=t+0w zfc}UzxdrFeT092LAv1I)Uk5mUaZdPv4bInRhieO06r(HtjkE|ef9w)pZ&U4bn1=WLklfUFF`m$NJP2`Y;$E45+NpJX+!eC_B+B;*q zrNVNAtqY&!wG=+Pj`?ea&LL3dZnhGudCqEKqw}$*o%2rkfS&pOpW7^s+`C#gO#XE1 z@Vtp#duD|!bx+rR>D(wF70&q@x>2%BO>4Z}e_Z!1pW67fdw<<&z)I)K3W%)}TjNVh z&vc&@s)SPMr={QsQqAes@c;O0&yxJ%y}|f<)T>A#Umw|snVI17rJHUVeicXo-AhzC5*4<5ff5i1OOp%kiF* z${O-@GRJ>)7EV3o*MJkue_cCykwg071~|Nw@~8x~RI16l^1MH${mflQzv|31sW^Tk zHUG{ztFQjEdgs%3zvM1nx~XKkKIWOJwI=#5iPR=tn3_@W3?;1t1A1hU9=-c+N^d(^ zb%O;CQ&fclArTSZiw83J5zA;CtF5AP5-MuYB*&5IZRe45edZtzU^b3zXJPw{n)(Hx zE?`CiFT5?Ef`Q>&MPeyTOwI0p1)s5qw8pWc{jrrj`}FlEfTg|B%cEct_L!Onn} zFGlfKXOdYX1&`rvQLN+(_QZYF@*dIn)}GN_dpQJ~8l(@&kHlp!c#!)&>MDPBZE}h@xS!y0 zyjk@M;2hP?mhKZi**O=IZ{0F-kBw1x6_E5ET7uFrcA|jE?pYMR#C`)CVfyxK>OuFn zh80x#!u{k6{4!R1_hYW5FtGs@wkK`T;+H^6M!{%kZW3bxv-|%#NG{T5)1+$Z7g5gz zz6wzk8e8#B)W6Xd3BARJ>A{*mJ#EgfsaRj1VP{~c{-*HwYg?L{KQjHBRTU9zEY5M zb!}|oD>C7FoCk&5;i2#Qv^U4pQnQB+?H&}0jfZc-Z4#zfQpkh7D^>*O3EIUItrgWI zcyWLEz1gE)jW>M#vyn|SSpOm4eGjaeu%oLkW-6l4s6jkg?Tlc-lXY0&wgqs~`yVG#>9_c(Hd;w*8RehCyhip|_15ZVw5^2^2( zER2T&l*4Tj1j-^!SIv7QvvamA_$StOc18|UYwBONbW7to#<``$8qCU^rY`$}s}2Ya z1XJk4U*jy{#EYl7N)$;-BCu92>3ZGLy}v0ZzrX_ij%H?NDTT_E{(!z656Bje(dW;9 zLn)ZL#vt`DSKMaaKMbVY&(E)He@|*!+I*qkkh`4WlqW$S8MC}faM6*c+lTJOVfN@7${Arlrd^GYo*vqDT>oObYf!7_g#{mE#BSb5j}Qdy z{itiE(GFJ^rk(CYroW!A4^@&Xzsj%6i z*w-guiD=V@5#@45WT>Eg(|marQZwJof9O~x$87~R)W>NW8%wOqpLbO)oNa>2w|%o~ zX8MEOmXF6jj~uR>9IX_*ei>7yL9c`*Eh8Xq`f9xJmq-PRm6F2iCoMnjC(9q;&oXUW zw`a+os8-JmmnHj7MsNZ@qPciCVf%$bcBLYHVyFp!SBcC^x$?ItXR1s`kr~JD2CS_9 zYuvMur4AO>>ajy-7Due@4;AJsPFGvb%+0vA7KJlH&Bf?BnjF_4A5MhPLMlHNKi52t z;Nvd{TidUib7XU{Yo?F0jHIwqq1`2i!92{GPO^-oaG8#?(u*Ry=9Sy*8Si%~-{5Ej zR%hd-#}Oto97h+GM#dgGSC1l?^j&a!x(n-NY|w?K+>UP*?#B%uJ8Lm6Oht8BFU()y z6CWP!b8zpDspo9fV>8c9DLANMuYwnRH-7!Dr)l@A)7_hM-JkU|Ju%E>88d!INL#pm zZm~s0ap{qVJfY=2&i<{QopZA`DmCf>2IkbiWAL{GD=|gVOzX?$%|jbmyX@DRYP6>Fj!c9?f2h+ELykqYq|!&;nSpxa{+n-)G35DTf_#5OZqKp9 zjZI=V(w+H26$bNuSF#j)^7G*)`67Q>%9Vmf=e8#~HF_GU&Y?pBW07nj^YPLX1;=-$ zb~RTNT;ewkUg{hW5nw&T!d`T(Sz3JlRXtrTJV?>M9<*}Ceb^&EQ2OGyVhYb->0K6{ z(*MK#2?uYMzRgAbKK}TjjN=+i3Wc0ihJF_JCqEUKFz!dcCw_?&oEn**A9~}YXZ(;4tJlk=y~A~AQcSo z4c4qF@~J+on2Z;v#4i3_Tzpz`l|y6SWKj^&@vhy6Zzd<3ICx|0%H?#>(R-yNM(Y@+ zReYKy(#A8naNdaxVyH9O+1fUlPG6#nI44S*P8D4{#^iLkQ3tzBd;AF=EJ9?H;g3u2 zkP2!s3K3pzOSf6FH-_!VNxmx~CmZ}4!+V^_6Ft9GX6uKaz?OL(2V#)qfqIR;s~2zE zom>@w8VoU*Zu9Vsk*LVXAWO(Dk-tfB6=iQl`TF_&hF&fg(WeJ#d+fSB^5qjv{fI_4zd{mJ75bcbzK0hm!{!hewX-nTSft&#Nd0ZRl8 zctJrO_|}DYwkF(-S+h_>m9hyO%FwQNJO$H!q67wDm_60>m5cll5SW7B9XE89i3iaW`&F-;ZRalp1&6-u8uzPqR-tgKbijE=9(1(T6 zO9{sL)v^(_qs00n#&T{AVQcPLBbvzU3oPP+NQaDVjc;(eqYg!utQwz>z-J&?`|!l>D_D_BLlr;@Xu2H1fC zUo!0qzBV#FAf|kTH6Tlp3hsT{Hb_Bv2!Jo!KmK_>%M2hYWIB4&rx)h{HCupwh*>bb zSn8_X3Z0S_rAZC1Ws*Lgm$KT$JYS35PsV;%2KtJ4cPn)P{=xaP`GVNW+W%mQH#XFx z&tMxEhuvChylK9S0Ho?)m+R!R7TP`y74}tPc2$gR^?TF>+62*onE!*w0dNfe-@HyH z#3JpbGrAPhFQ;48(zOejI~OzZ=h^NOViIhX3zEWvbknKtl>eQbIi<9o;+u~g=HShk0%EJh$IK*>7s)v7x!|bY}KK=6h3=k2IHlI ze3!YgSO(*-wb&Qfs`xB4Z#*iUQm1|T-y|_mRnHNPZcs3WULQ$ zx@5d_F?Qay6`3y3u06dzL4 zHZ!&9X2h_Sy5R9k-MYGBre7w8=@X8JR!BBLq2PFH+&5>Xf4flkh78-NwLlMZdxj$J zX`f`hlf^0v`9`?wFs2id3@iHLdgCQ~p4^)3EEB5~w$bI{|L}X{eq@C7rs#+`Lt`=3 z4Z7ujp7h~tDe-`w(vtn<`V?25iJZ`9N5no#b{g$SbNm}i{`c2i_8O`Dn+x!7?_n(Q z_`vzbilx-MRkNJD1H&KeGV;?a$&_OI{gC`L6UwEC- zJINe1$8eSZ$+Pe!4MmiBAsVHaVUN$s>lxv(Y0#O>-?jNy_5Y8hVhi&Hh0{{isYT+_ z{Pm*a-J8+i<)|@2h4^}X0f{Qg7s-d|OH}g;S-!%#7Ay?_Pjk&P^M)@1!GRNaJy^ZQ z?a`f8_*X<*0JhomY9B;MILHz+nmf_CbB7pUl|zA0F?o2b!nP2}A#dJ8hnAw_h}?^Z zt2~kn8)9Mv&>U$j#ukPU=s=)iVrw8Vqk$Lna}Gs2A?(Ef3wBUfGR46FA%k5hp?j&~ zz!c3KlI3NU_L58-Zxd2vtr&~F0w>*yRf`9QkHz1-39N8~f(ZEknA2_5T8xZqQnUF5 z2Bt)nA~hy1Az`hp(=%SGXeU z1Q5SdVW;H~i4G;xy`Fv@S)~I2*3ROBNRhvBOI-_`d;z2pJ-u-WKHxr>BA^SdaSue* z(fC=WSMpalDlR(fN!de*o_NCUYI_>r=elqB9Z4~e&F#$a_ond@>?}O zC3F-1^}U1lJ^P(O+CW(#h9t?);1u~Ms@JY<+h!qdDDf>1C4vFD?SP&UOWnR5FfTdY zK%T!Kgd7Y|Yu7yHWn4`k-{ARrK@RYI)(gcnvNks3P-Gek4Y;>;D1c^5zT%6>W6P{f z5kMF5@HuK8eyXz8ox9r$)t9#;R zFfsx8pC#W$crBK2((`-VpXtAbkIN)DgU%9=)N)*KC;K2J7y+G+tgZd@DcD4-Uq68t zhmSM8wce7X1x2@sgBJkDDB46yd8Fl-kBw4;+?ChrzRm-*jcwW{taDn5!n%gdxfzYZ zui7`Jc3JH`cBqDZdhnE;Gof{R@2j7s7i`*#1~EQg=fS=Ad!lX!n5&%X8Z=Jr%8_kW zJAgFRDgTSRHxK8kUEha4nk1qSDs#q&G!a6E%oJrPGBrt==Y)z1WtJflk*CQ}$?%BG zks=j^k|`u)%J|_uZ}sf&cYlZD{k`uW?;r0z_CEHppKX2CTK8J(zOL)M&hxxbJq2uY z8tqamG-VEEZcXM9NLjzHTh@?0@7Rjw=VG7U&<;~JR1fl|+5s<&*e|5#Yml%!6Sb76 zT13DJj)(zICo!hx=lhvQOF^YBqwd{NfD}bI=Ob68e4HR(vOzFE3P{KtI@FS6$eBn^&HZl(=&%=0Fg5g^XwN2oP-UF81g zkMnax&@FyuS6pSt5igk%72NXw-CrL6-g%?$>s)c;PaD_H_y>;jjjZ# zGR581rI=1=zRZYDg)&=!zBkZx&MQP7uy3TsOI)J;|8z4xsU?@k-id!f|FHf+%9-9v zx{Xe4jQoXcWP>j_4FwB!RR8Kb?s3yp;+A9y+H$Q_qO990b_p|Z5GdgL674gpoWouD z@XTX^laUhKJVrSPIlb6tH##gC@5OVY|h(n!{ z>gl4Rk_6(%^raptn4*G1zaAue(Xf$`Y93_#=F?OPl&B#{nqiftYsbk2>kJ@nmvZZ4 z1;E?JP;DYn8Rfucr;PC@WDl1Iaef!zIzm{e#)fN<>E!=*>=UxnZ2Ql$JyH;KA22k$ zlRJ-5YN`489KdI4Z3tH#5Yi$?M8z0Jz(kmf62nf4awpdSw}|Fi%$NGRJKuIA5jzzL zOYsimx$4TwWglyPyc5pVUzmKg@Ep@GP&tvTnZ~kgq2!h1GohJD&=~O9pFUfSyZvR^F_IZI=spHs+c-3nF*Xb&jHq3KqJcC|`dK8O=WIta5aO&X&ZQK> z;e&&PLh|tTSM3=K&oeS;(NH_w+7XP&F+Lsi7DO@|-JDDq`S84acbc*VdIj{zD9$b} zG0i1-yq?dG_He+7D#w@V4c`!vtP=bJOwOf=dt=XZ0S20#Y0H;NqhTGInFR`+3WLPc zOaFp)J*#MR6@>THc<%P(`C%|Cn23#L&x(eOYGM8-WEYChU->o$E;}(|#{9c?oAy(Q zI+%+q8-Dj?P*%lM6RE?mB<@0hW=>F547iY94q9p| zR#qt=WOO_`0@nBYd;MS6k)I`@0RDOp&W)Fybp5}rJ3`saZ_ph}91PNSIO)L~bK_Y~6`BwLKgSt9e zXHa|p&JA#R1dBh-brP>9nOX`zv*_w6V6nMFm$#A8H^ifE!_uuYf%||KjDpLgezosA zb|&XY`gTr@=@qJ4oXl}AFCiON=FDXEx*N*G{w+3WLb=S_W^)(_SnXk`c6nm22+Q>QO6poI zVV=Zx)u@owZ zp%o#F3DnQVL{vu&QG{cn z249Cv*ZkHi6OxNNVxm9_M}uHbrwLhPP%%U^7t6}}K&(T|-JrDO36T6D7DECdYW3N9 z`G8((O(j91ozgX;nXfF9FmKz&=<<8r0)^OkAKX*7ck0>Oj~Ivtsm+W=oQ6c$Amyq= zGx9x*VJ9pwV^+=P8NfB~V5RgS*aj*m;*ki}o{FAwvVtfzw;4U=EA_pGp{2uGiR_aJ z0gI(-BJZe&hesE942$h*cMV(XEts}jSy_oHX7&2X!($O}_=1z+scOCs{u9XTiANAA zvEIJl~NI z%*5*XBl*?mAYR7O#N^o^_Mf40ZP;K?dlGDaaswViW<4_C0+Ij;^5N!YIc$REq@iuI zGPL(1wr}ynPgkN7UOTO$rB$}b5B%_=wRd5ED>8R;==O;WveAv-zv8!B!&<2jQmmxX|&$R{VZAhNK2{i#y zj~Qzwb5WXcup#?vs$$tSEFkoQ6`dIVe? z$Fc2Tz<}uZNJeMMHL+s@;4Wg?g&|&W&{DGXMRL;l``CVaS_9Yj`)H@atL1QSsz4fXc~qQc+&dKi47!41d3fz9sgFwDE~ied zgh>x%>HEF?TwR|wE$8Fy?cD-%Rg`Bckp2HMkrgmw1wKhKRYK16_Cu~3aUw4B%a5R*9Ig0cxs~Xu`ziD}C;!VtY zM~~eK2(e*%-6b$*_VS>ie%U;VEEXNElgw{~4MuJM*3>$o8E+g}mGe4gkW!aue70Am z)+_!u6ek*z8xX}_vk6e$*IDzpWac3gr$?VG<#-Gd(hP$A*uMpmlZ%dhA1x{DG8;Dg z#K;1|1oN9Kss^YPQ#DM_2qqS3aMQ8p-3_*<6KKofPq}w;)!6Rcphq|+jzVFq5pNv3 z@PVt>ySLUoj<-X5EnTk)|2+amr-?uNeeu>W(nF4zGly=c)7@hz;L{~4n#}wwWN>;l zsv7qHxO5Upn-|Y|_~F^)qL=che)!Wh-V0jE=~Px zetm-gbjq6Ghs?nVytm~0-U*nsEm9atc)}0-NC;LblP<~J*>k1~lUX9RWl1bNkb5)n zCT4K!-m)37l%ty#kxsQ*Yq5@QwNa7>)KgXoc62tSyS)N{)>imLiNLyk;kzTht?O+T zF@9FQjLsrhLxzW06Tv8m|9DP}<0$`_0P9l3qwm{3h-XvB*kZxP8LLfC)+|J4(JO7g zM4i~Xu8-ep-6`9KmVK!FU!JayN^eX3b%Ab`SUcRPu5wg#UF5bbf$KIm#ad}z=Cd4( zjUL1YENd7PYFv(M%n~+C( z$@h&Icv!gWtHOcLX!Nt=f5FD8cAE6hB}8=CUf<7I*6XRN8W7VZhLWvz#t9=<_m!HN z-C~OIX|Z=krfg|x`8C)^FtO{brKQrW%tnBAylWa{zqnaISImonU;{q8u=5}qKJr1a ziJ72&;_Myv`Y$rwDr@iv3J_vqxFAt9&FBk4KR za`{!4)QE4A6R|#m-i!EN*N&|7d5-EE7l3Y@qx{Wb+B%vRE@nBC-;@gI3)jJ893^vSkl0v~E|C=C ztM!NbhivRZYv&SB8=)5#Z4-B@%@MnM*ax5c1C=OyRho{j13W-Rs9%Qj0>h83(+8zocowpDp|e^G-TKZpxLv26!2c-xVeE z10|PPA9T!9!9`(0Nh~7m56(~apT?NMm(V-vVVz_)7#W1!GlPrV zAz)nt(EB$ zOTt;IC*GvXC!BU>e%lR(h=z!!S{{An{WPBh&ywGVM*n=GU?`JS=iH&kM5p@~fXcpb z!XByDH`bWhhQ1mYfG-cBG^Z0x*KqCW-4Zz}YMWT%5t(pP;XQ>8{a~A zr!(h#3zZw|BLpk>tmLkmA*@C0k3^gBOmba%_q+5H7w84nea2DFy&KQRN^BOyrhbZF zqUUX{kZ^<@0zGqWjJ5R3Uo~c03eya{PXt)wGv{=ezZPu^s;4+_XEc|1#O|Lclj$MR zbD*qHZIKcbvELxr+&K7<=Jbvc-`449WsefALYL~mY0VU~{C6rKd!U(>?^?~vD;zqI zlqR{!Ea=izp(=7^2H?L|-luO*qwNac@n%8~N1nDX+@V;*fEHw0$Zo8JW6zsxtccmL z0F~49KQG0-zFnp6QoQIA?xYvU-jhYyt!u!7RdnjvcwUn|M&i$%MUW~OtaAR8G}g~| zKPtp)Lu5*G>_!0pyO?CnC5gsD1d8>ocMM1()9ku0C;|dv@#yh=^E=3YvuJguI2(W(?LLw6`}) zDPUpn0aru6u67KqJ4Q5?u7-vhy@5x6a8m-&?`gf6k|>C$DRbg zPH=E$wks-J%DQs-GPzaMbUlMe9Mr*f0}t|!e0zPeBF|%^kG$!hfcb(V_h_f0Zjn9u zlEA2)^$_9VmmPpAe(C-r3pfM0=<(^js9NBx1k%h##Jz8u`1zLryjIiE(J>jJwKylw z4GNvwxz#srLp=x{2PybvzeoaK&iz6`uTsgzrt4$ds@^f(;vrlMOta^FOBzSm5^u_a zbK}(CC;4o7_L~D^;=*nnf*M>;9%}aMnED60M8Wd!KOyMnc+$n*gSB^y98V$ zIDHY8ebFMN9_WSRuOFs8gIh=P0HVSy21{n_T~ggu3TZ=lY-|Q6fiEz;Vn>szfz#ZS zQ_=Ldn`9x>9rxQ z(fLj`FDp%C@aUYnU#j1g3 zke=J+%a_3x1>F)zgfigUCa-P+wSd&=tYC~Gnh@Q=x6*Jo8OsMWK6K}vvh#kxf$U4D z?vVWK^O8#=knuD7&%yX+(tUo#T=-f44;KIqb3Z%EW!!(nIWqtS0q3a-iOr*=SO_V? zL4-<7Obi$DmGOgB4ngK2!ra`OiYq0}Ofsr<@o5oh4!M6M0zGH>_Ml_&nCqg<8Fgk| zBK62Bp4kk#@_qSzkYy650x@a-i^5E;Z{Qz*ivINVkxM*t6vFidPQ#$Kt{8@!S4Fmg zTf$s8(Le#FOI49Lb zZpErYUoiV>nN+`V5ACQDnfF%fYQo z@?X>o9&=@Wo#!RmyQ`UH?gzRKzkMB*vp#b4)H&bBnj5r~70iz0X;c;Qd+BiXWHR7=_|9FSnFTinPRxTA7%I2f-n*t%{CXBe`2IdV~uC-w2az8+LY^2WrN4!*kAN8 z1eJ78t^1mYS>yKu)j{SWNbvf38S0>{+ky;sNEkZXw6HZ!AKb#t z)&Fq@wNWfr3f>39GrefuvuG~nH{{w4f;Z4PX`E8lL`NK5gLCE4_R7dJ=4Pbq|FS$vQG};yfAC1%Q$Wuh(qc_+We!v z;?8&;G_vEgSAx1zozp35M`Y<5sT zdv*)H>dc*rh@)Znr({n*OA>Z`&sU({CgKXQWI}_VCSKRj36M5xVm&Azh;d@o6bh=0 za>6A_KhncpOTW=&KIgYIODHCC%U2bN+x_rbdnXg za1L8JpxqG=S*%k*c0o2i9B#6Q2gZn-fcrtYsnxo6{rVv2RBX^k>T> z6hn0Q-B+hfmm=CFG>P(F%xJ^bwqoqfRe;xz66yo6ArT5Ck>Iz4g8Pa4B(-ZbC`kSo zk*1R*duVquVh&k1V2hr5z2n<}pLm~S)q@$klr!#)RJBZBE zD@DKu+|Tx0RxeTJ0@OJR2Nd?{pf~a1s9!yXl(HZ7EcIS`ieTKe#>E$LRQe>zl9Cn7 z#V*4WoG_Wt%+6?%DI|j>upY+Hm4IH2tK}0GR{#60?GYoR8%9ZK=MtBKYOqKL1uI81 zjC247@rbZIXiXr2RO+WxcQP{6r>VblaUo)LxAp!&RL}|& z>>QvpK`newo*>;0oG&ujO?R`kI(YTm$9^V)E1sREhkV%4*|cZ zM4dj}MX%^^efDp80rD%PlJA(kT>4g(0L406o-wnXW&1o7(YWKSV1H+myFyy`-X$z! zWif7H7W?vF1x|}z2s%lL4=xX!-qQbSE-g3?OMiODV1|;S^G9P*j-d*ybT#sxN z*jXAgZ5vEKX1GMu%k{V~YCqmP3gJJe0pg|xWs_`ykvFoA9I54WOpYhg{_i}1($_KT zp-|$V8%>jkI%WE-Tv)6NU$W3vrf#T17J0FenzjjXygh$zaTPAKHdk`fDb6vg6{Nx> zmF~+7y1JzWgn(O_n|c$$C`9}95<0ci=&pmY0fSq?%DwJ9_-f^bYM(fS#WS*DQLeqI zgVTJ1OsdQTqRKaN9b2zt@0234Dsn%k=59T=g7@QJxs>jh%?N$|=hig6T&;ZnzM^dFfAiD?shpnz$5Nx^|FED_<;JN3A#gzm59rjW zb-Mp5{Vw6438rhFz9=^Kj+Ome#xlZ*p1TpLue9$xZ^xKB4Jj_P*D=;BSiS=liJhU_^~r0l&XSvIm&^W z|6`$>7?8{8FT)ew)!%O1PfHUZvGMA)g9~3q5W;ZHB0&GZH|dVjCu5HI_Utk5CtD9` zC@8^noBrWmnKEUQEa`@I$HP?{7J$v|zgFhqdzr5yvQuopk#K+iB6 zKB{1(iG&gEEIN&kWGK(Qi(yqv>_uRaiv1LE%*`!#be3lJwQg!n88&lk>zqdq9+X4Y z5P*<-`PyW^!@l{J0ZDL=Ue?vUNB`GzMZ@%1??|m!*4HGDFLZXZ)83uUxX8oTE5FLf z!YOXir87wQ*8VTI(1jcv#Fkt|cu@KI`Pqt!Q!@n6G%g-phaj;e|FP{%-cjQ33=DcH zcGnn5CFrw3Lnd)G2O}M`J4@y!2m}jJ9uq}91}>;hw*l5Vi-#QjTpSB^HPfE&HThHbqaFJ z2OlVyDO-)`yr{+J@@pWO7G^LhKcIo-IUlkZC=*3rI{?EO_UV-)oF70B1V$)qCne? zw%>E(Arme;4MP58sx$gP2(K;etlz&64Uu5g#35$%Pzj#f2(&Sao)xH$2RvH9inzYB z>>#Fd0sT2_cMaJFUM*sP(ND#UH1v)R52%5Q4MGE%d(k7MgFee<0L;7SrIn6y(PpQfyiDWz6IUKrvrvml#&Zzn;`C^GVnQQApuHe$b@$r!2@ieva8|nzeEts=KtC z`FpP=r>exCwp@bDw1xG2STGyQZ!HP(kSF`;bUYE4SQAaY|s5TT9>p$b%BS{zcplN!oE5Q#R*&h zs?b<8dug-g=JIbE1%_d|AQAuDtsA`A#N4zowdTd|4r~tq>Bk!0&#s=$o`KpVI)hQouCz8&-V4!kMrKeuW`>V5@R~on z`S7mB?&J{}dcor*AFJg5#c*0|edcd3T?RKEW6++e{@k8a@;3yIHI^kcH6c0snTy*Z zDsl%8fZ@C;#h0;aZ}*}gMl|FRYzK5uqq+(%8VWcxn&bAyY(vxJCeBFrq?ui#+4hay z&wWGf|8}qdc`yazXV${rfIM2}Vahi3MyJz%iwbwhtd2yj-lfCbwEj-qmHq#4%tRvD zj{1(1q4Uc2j2j)?_GGep&%urhlCC+jd;uh@tOwbkz%X!ZBY<*xA`hHGJ5KOr|5l+}<`fv;XQ{s&wp_0PNWUApB!} z(JmnidJ&i|m>WQhudMuv)BG^;rIM)cFz?QM;(2oj4Bx(bl+1DgomB?&b*P;v2$3x} zN!FgIi@QJsYM_zWENp3`{tXWVf(45)e&)}C2oc*V1TQWhtBoO8>({Qm0Fv;d ziAfTC!9+dOK4xnA_!u35yFii>%c{SiWGT+hGJRxP6R`lQ{A|n8i*`&t7pDZm9W%@^ zSb|&!3dXL4u7qoTC4h3FS}DKUW}ZCT*83B(NDl1JEc3JNwGJL0(xi-nn$GdDa*+@T zK(#Sq@EBS*+Fl;dgYG<}3IhC_)Q7jP^q~I0uwg9R?I-Jr?+gAg_j@}KKNMT)VvPJj ziy6P`oQt#bPo{;RO!wJ5c&}EoWh7b|gdP5!=@Tfx8U6$&{ldAW`qQ?;cI4h=2(lamYI5F*^>Lrk~{uYNTzoyRiDHedGyl-Q0xh4@>;Q>S9?%Y0%77R!Sj8=+5jaL8(Ek^b!h(lh<3wNR z03lyMYn@=u`D%Len)K_;_P(R(u?mib#=P*>frSxxTN>ggVxy2a{e zI{ef&Qq|=j6!;zq7aQxp<8F%;>?kqgg{HH#qR9E)RNiy0yT7gk$q3Xr&{@3Ec)2wG z_^eDxV$ihhxpV_}Pt6T$8ab&*ex zBF|8@|JOk&F!gbl+G`xSwM6F6b%L!d5$;?a98DGNc(tBA)MXVMO_mPHFTGk#_qKw3 zvQw{zWo{KIf07XD)53-JRxX}wk$u$4m+oaF$FvuLM|XwjSnD#nfh@6JmEZ9h+SRI& z?sc`lPx#NmeD&gu9vlpRUlg&m1Wn@v2H1E;3B1qB1k*@pDaj{5jFzCwi`V|Q=W${2 zX{t1GiQ!oWB%jp&^DH<4M|AHZH8yPZej5OhMTz|uOHZ`21ZO?ta0QV*?cX1v7&^VF zLF9w@d9&RKJ!w(TZL#VEp5`Eu-cmHbCFolD;0Gi+hEhDW#&T9hx+cag-do25W;!T0$pF5qDcS!x~8tq}>`sX2-bT3LVUyS*S zM-G`_MFi*MkB}#>IY4NH-Mbc9%|Z}U-|MTv!pIODT+CpF%#Kd!tnYVj8)EVU7|rF(;L@`CE0E5ss4Rtm0wiOZ zF*1h{b=~0s2Xshbgn+Aj&rEYWFVEwL%#&PS5TG}0+h$6$Z z0fV6y@ERPOCRXBcadBcSZd_7QC70Yn+lP(sOHz2DufTwS!+_F@<7D(;n?Np~Y*va? zUq!@mjG0egM<1a{3Vl#Vi8m#<&oR}oLdT@!<*)%ym#@N9qHHKokpGK8jWj&j1-a(C z;1md2K!73yVTOHd#YCq_{O1UniKDQR45M01f30yjj4<9z%9>CI-;C<-rhB0iOh#M07TmWaID0senKN97B&Jp!`DwI zlXJa`)Ya5%d*K|r9C&C$u!`0b&n8YXy6rxrI*!V_yqH2i|lkGS8MXG7Ic=lvavh6%4*9Vwft}$a+>g;-N{zK~Zf@ z!42o+??s>7Y#6gy&hJR3)*&Gq{nJ%Z8_so}3GEg<*ln_L%c~oLArDj&I*Wr$RxT0v zxpIx%PZ>W;Zl(QcYHjv^F}1YoA6k`?(mdSe0u);z&xs|ExFmdd|6ZnY?JLIH;NJH2 z#9B0gXvUmJ)T-xa-X$9yIuv6l?yt}4Z>eQ#dv6G{Jw79}2DMHBYrl8$me)SGlUsN0 zYV3asFYq_Fm)}0PIsM#FfjTO1{mxzF9U#romg4h&SHBd>ynR~<^?a5g%}cj!Fq>yM z`?>ItzWj)4=PqA}cLzXva@&kxR$zd5+?}7%&q8>4(ApAfguNj=XojCYbH?@bostp@ z3_5Y$ObNVoY{meZsTsVTR&m+L9JiQ0uDExd2%i5!Vl|y$y7**V{M=$Tc!v7Fy*wpy%EUGE^<;+7&)8?$145l zCE;PdQg%(^c;j@*)Zr^vBSVgp8G)DJv@mq=)azs03PC9srIbeBD>3GbTgGO8RKX-p z{G_R8@7}(%vj%3vWsd7!xm})eObX3$O*(wF9Q*s$DuV^r4m)2ft1l<{LQQ6tZ(T1! zp^;uQ^R4Gn`b`UlN_KiRk@mrWV^)zX&BC%ORT!wPwQd^>IoU!tu^G(-ifNDSG-eBy zOex{XzP4md>ai3P%e@Rysp9m#<;o?szF*Md91-9$nm&B6szGs!_&4j|O?PJ>FUO$| z;${K^Q#re&9c@_s*9h{2)HX;<@;q!Svi6F!?~r+%oEOeyTzo6-NPq8DTDgfTlR}>V za?L4zrdKZ|dB3=RfU+dByzKQWS}|!!F}XeBIx9T$UUFDa%d`~L<@5Ebe;-;)W!NM> zq0e5l^kvpRciL>#@n7z=f<^iU+s=Qu0J0{P2!bXBJckRCrR+bks3<%dci<6cwRm{# zh`$=S?)ituRCS~kalZy@YyaHs2;*%jqoFHCLnm=AZhGpwLpszl<&nVY<-hOs_c4+< zrYw{KAo(bRm;at0?>E}?`<~*1v+c2v^!fga9+c_Ixh45htCU{g1NNG>?EiMWL-Sf- z4P}^ZJ_&R*6vfw1BJz#N0F% z*KgdyNq}9+8#Rh&tTMiEXiuupI{|h-@@M5G-M`aF@1t3n&q)9KX+Qiip1pPb*U$Vm zJ`?m}RL+sFvRuE{L2PpSwe0tb<~X{*4l#xmWo_myI`5NA#C$zx&G z@3UNoGurg{p6cG;BSW87XlLnV05)=uSXv!4lStlXG&(K9tf#Hii#LRNB%KpA;f~5v}fNg zDMIlD%9K4E_8J%U0d}H8Lqip3$0sN4(CG|n1<}D6EwRqw`c=UFR9nr&Bs}Be>sw5ez!2I9gf3beVDv6FsK&OohgDTo z71XiOo>!YiL>#}f!%pHIs+q*h4*+y-aGqz^2@u0X~hL~9B@Z}o5J^G#8&SgIn^*FU=maaMW=QTCb$>Ejr1Z{ zHONa(L2h2h&Mue4(wuq}6ff*{@Z5ICpCa(}MJ3Or!8IUBhUW?+S0ieCbGWVHPK2V~ zLern*O!oOb`017+FP8>i(tE_|a4fpYc952UPaK+QR>1s22IQru*uXPfE=nOcXd;53 zBzdoGBVML36)&Eul*9xnk+;3{t5%KYgq^ROHnH|x0>xKWGWEwvSyXeKV}v%fcIyA1l1eiL-rSesZGCrCv~i5)&Ic@Gg1&DF{9nD!Lrh z)B-VqnYp(#7NGJGQf0sY^eMhi20&{r7|SFAY5FJ-`)@!Z{ekR9H4^JrZ`{>K5ON^9 zoo2^cweKT7-jsG|8m4|3#tK1DlwtFdn|946vj4y%_(4Iz?o$jzy*e{7kkmpf%h9&j zqsDsU)iLEr$hmWYFT)L0(rIJKm|V#n(uLL2?*qIHuO@9bWug!iv+H86$c5`6gr%7qj2d4WG6+ zBQrXUX(0G;9KR}cChd&d)qO>N|ZrH!n|kW)3nnH~8|(OoTi? zH~Ae4=lHz~P*uybvqw7XwTI;3DmD7lx-wW~X1V2B48XhASyS%GHrgS3kLG8G5384(p?cBA0jYwVlEd-z;Se!B7GIC3gczeO8y6=+T_na z4i7UC+UbA1Sm-mG&@j&Ru)9=wZF_TawTTR)_F%w?j{3VkISq<7{p=*rQSRhVUq8vm zVhS{r>Qka4>>Xozj`Av*=L$RZEDTO!^Xy>^uyXaZ)I2IN--~N^;S;uy9Wc<%0pve zld!Df`jR!4tkNYT;@#zCoitauhAF{e>GAwaE8r>kip5z8hmSW+AwkgACI8$>QCcZfLHyG}Reqvnt}S z5@ccvo+_S~uPSpgIW4?{-fve=NZ|Z>?GlQ3DrFf*wX3-(j=4aIxD+MpEPz6TLhn3&Y=GWN3KJMWy59I`qV+OZxUPgH zL!Alp}~N&Sk~83Xlj ze6__8y~9Lp;OxviF-)#Q)>Q-Z9vHpl!G>u>wRZ66(V^*8#a@u^%7S=Grl5}(%>!#Q zkWF%zy@9XW7Uct4boB0=WKe{S&7F)VQD)}mNk$J%Qot%!t(KCJ`3tV)1E6pQ*Z)P#ldD**z>sgJ!3(FB1gzcO={#gEKavrxyzw zT`+UdxZ00}=rr50V@DrQ#6s78gHD+*=O@iqGcX8*rog=q+kWP|^Pj5Y-XWx70#qLg z$)GOcZT7ykb%6ECfSWh^ZDi?T2jkn>MP?zw;^7-Bt>xO~ym5^XNuqK)pokV7OWvQ- zBkX{n3kDf%f^qcGu2|po+6+K0Q|Ro4Vo$Glb1y5xR94^fX>ZFWJYlFv4@G@~euyeI^;) zi*a4T5|s3Jw$yH6#kZu6@eZ}OCM?oJ#>TW;$QZ|dQl>qrSJ)`N!nQjxKePj0d5u`f z=$jkuiDmo)47#PAoqw2X*ZduB?0PFz&GM%fTBAC)9Zms_dbT zztdaJOJB*(qb97ixw(fx9}j9JBX!4<7smXIVAz*hpO=?6fE~D@1d11dP%9wA z-f%BZTBSrnKRE#wUEFCSGABK?Fj&hy$J`QcEI$%CF1aus#Y{r4&N8@nuyYrQgn0il zWX!VOA+(-tFE*EaK5o#t3t>$6p)jRg1Z5PO1Uq-_B5y~EiqkHXUZ0T$A`Mu{6&!_h zio}_{^_r+rNwe1$mxE>BcZ(S=z;8s^Q6V10Yx%7{c^DB)R79i!Rb0lL+l`wy$qLDB zxr{Eg&b8gh1zmluzn^w=B=1+LONYvlBOCSY7fO&<^&xJ4Xe)4GIbV#Vb=peR!Xg<3 z{sVM4h3sv>hJO>HB|4YvHYK7wEyC~S>iXB6JLUJT6Q6s$uNrf=@m21{r=(}2hxggo z*cc&e(^w*eOW!dHQLDc}wjh|dUK>b!jMJj0f8r(=Pk<;~}L$DJrI z)B~5S(Xkq#c@_7x)f(dqCD}T!rdU`O#RaEjikl4=2d_>E+a%`7ne&;JV)^(0<)Wz6 z7KJ#SspEXfIxS~|25YhcWvu(yIqIuBW!O*`S8Z8n=P^`}y68k7Cr=;&oe(B_1z*#b z!WfM?GSS_wI}GWs7ce)?|FS-B&d*P0x?*M37R%B7 zzn@C}G{W zcFDU}Nn)hJ+}$(NlgU#R!pPe^%j@;lAxe2eAnDeAe)9ZR z51vE52n?C{xTzm3s2}=<70xtDuP+%!Qw62R1B?jlVhLV2eea+au?F99er8;C1nf~; zh#FeZnOdzlYvrTk4uJA67dcpH)0oSpy-!+xP8qe5{lk9om%kJE|HT*9> zCMKo__wQ>hOq9$wzx5PqDXEA45mV{E!|*34%my4v{PHDKDQDe%O5)q|o;^Da3azaB zkB_StZiZbPa5LDy|2A0mwvLXD%7ZiW=lAZtK-3b0U!Kd2OnrIx?hf(MB*k7Qbg)f0 zZ!V_z9^V$f(f5~hkZo@#{>g;meD%V9U*Gxs1?q`C~HU1 zC&QR<7 zM-Jx)j?BQjPHLzGqk5D`azEd#Kd%a3SUn#f@+ii!dJBz+OU#wVW@Zl;MlI$;3~59* zCWi=W-{>1lUMNCUbZg_Y1*_@lYQH)4#m%1PDqh4%Y?J*Z3zG|(_}ANCLX9QFM{Z#} zc|M9rD};yIVdQfNR)&T&r;z!Rpy7r-q3+b|7OF3H_wPS`FdU^gKi30*oa;4q;xk4` z-B~ztb*^}%YU4~(UU53=O--#`#fp$#V03(O8Y)kLUp~{jM2Lx~rY3-A znqpViV{M1h7Dq1TY|5m><*x2r4UTD9<)*U<2K<_1HYoNxeDWvD&>*Fyu9WD z|E}6v#@kSXfbOF+?eqZH3&dl3XVF$)Ru{%$IKiu>eI}NsXY5EiNi!D)FgPhQEQzg*)wI%W2akF7uDbS|Cd4B5n%QcYh|m5;i>Da88H7Ne?1aSe*<;j|UOzpt>n zu1w&qaRjexPHA(m&Eac-N874y=+i5Qu}5Badc{uPbt7z;$vhyKuIq-Ml-Ws&JuuqE zJ1fI+waI%moa8l`&MRcGc{iTLXyWv-&N37u9C<$3PhH+l%sw(ZDaD&SU+I`cLtx0o z$3RCPrHWmBepZD&#V1%WS-h%>EXk;Cdj6?&zv}(($4YDS*F|wu(eu1qYOO}I@AB=Wc=Uo*3vNzXSB=SV-UY|)Cdhu&V*#7p%SbR;0$0= zy__kTWQoZOIzroJFQ4p+@0)@aBLgTa_K98_8iJ7Ag`!Xsb9EaqDnqdUQV8Aro}CNS zAblTYOqn+XvK09^`!9L)R_3v_9)-Z}e%i5VAgRqfbEQ1^@8QCzL8e)cqepMS(skIC zlfaHs9aj!PgVQlw4;bnlMjS~cglyy*c9@w&n<>LpHfc)6hjpX)_=ATHQT7(o4Cgr` zpMuaz3JXvgxvA|!jf&b@T2I{ujLyd_Io=brP9N7TMc> ze9g*%2?Dhq@aG|1hR7}8RiFR%kee96po+zU3#cSZ?;Xq%Az<+?hcDu`on3ihekZrB z$FvU>a!)Q+Z`4rq7B5hT?O^O3H0A{zUy`EV{2lrkS*S5{G3bao*Q*#C{lnWd+wuZr zE3iX1+FjXc)8<^T4>iX)7{2`&j{F548esV;kiBea?T;QZSz?)Uzr0VigkTyRO`AC@RqUQr3vH z>srxgB)vsCzUq=&k+@E%_|<+HWs9twJsmcUdy137f!kEnD3`3-Gk=7oaB9e~kKf?T zKNZIcySA(8H4*fo<;fr2Z%Fn3V!tA$t>1XT&a}_?)4(OKn>;tr$@ViX2n!&$Mv9ab zM-~qWF$pbSA{M1;p*$>;6N!66B44_Vo*GPgO=$7>mBd_Gt8{Jgh*r{!k8(raL z7Fk9@LB#=IoS2U&kamp8i4+S2g(HL@;#U%gLl9_jk^#>VffK4a)oW+GuMg7kL=BA$ z2C%+z;?={_^^GX*x-q)G@G9#oJ%bmU@upRlrLEXi>AbyHW-jP#B*&|?R;{%)f$75i zFLK4l#ZW=SrpsR*vM}Nv`j6Xal^GE@&rsc=FJ5_BoW40I(C4rWs<@RmaO)`QBo`yo z|NdCm>Gj2JYBT6j)xN@K&5`MnERGL+(Ck`IeZ&P0QiL)zjXd7E`!Vm`u9fGXSN#eI z)0~`gR}SuxStCqO-8??!pxf8^w|+{9M$SLQ497#KO*e$pC1e==tXoa1(poJzGsX94 z(qfPUM7&4Jb>Xoy4_y+iOh~0cr{;O$MH=>PXuYu;-Wd#h z*;^J!ACsIs>?+P?5@&XN+j^w`dO+L^QCZLmU6&pkhV~1zmEDzu!j4`I{9~@qJFpD$ zKx$?LA+{msC--WM_o-r(Iloyz!D zr}F#j(2pZMbX-ptAAb?`SL@8%FyAx`wq0?(eBy)xupa8r@OI$9TDDGjxd2ij=OY|Z=F%G1FId{i|?$}7njqI!?@);IC~mxRk*61Xzpjo2yBe9 zr&l9Yl_7d*Fh|iCpN2*esHRqEVDhB9GYdQC|0pp=zIItjqt>8`5ldP{dzog2(;PE>HPQioG^ z0hz6Se#j-+C3N{N_g%-ft>@b9tO~%9ZbLOL*bV#efMN2?!3LSMpK^l_>jriq(rAJ|d14Di98Ku0f$R=dHMObXzWDTqy@Pc72Cn%^9qkkDH>!xF z6_abrfAb@#-zVUfgK5DVe-rW{7kO?!n&Kg(F;1TxXZDHpRhrz5%7bV>Y`zP${>#?+ z7M6F@EBCagYOKxZf&*G2iy5e>J;R28Y``hbWwwhw2 zZd3QKL2`RHd+AE)vSaL1j#57;!M1ZJ+BU2;=B%V=%eh}@@&Z8N%fR!Eu6S@sfKyRd%}%} z#_{~0pR710t&qrcQ&~U3%UO=Qx#p_LsAalF!dcolhLkL0?dJ8Q$wyYe>G5NcG>fTn zoW7xXJ=I#Ct$&j>8Zun=W+#7DxGE^u=RGt#`Fpo_r&NlKzgZUK+rbhgcGEiEwfb=! zwn?+fXj=Npja3%stWiz%9wAHd&q6W3>)yNq7h)afTKzQt`tLs$qu%sl6w>Y+h?!lv zwSc^r?v##waX*&uNL=PplDgi0?Z_tU+Dn1_qFdvYC-upmxcGdr-m!sdeAF_@QCocm zIe%+rL#bEGy*N}NSepXz9{gYIy?Hd(Yx_2=QKj8L5h;yih)6Qzt27!bWXe!xl9`a9 zL0dvH7a?WLOfu78mO06sDO1Rlndi7_@B8;#&-=%_-hbX_J!}2;y4SjQZ9c>Gxvq0K zj^jMf|GxGhtknM91pDOhT_2tBfCgqoBb|NzS?_I6)qdkldeGA#xI&s81q~g~n2Q{u zZlI#CL6BP=BJhAUbO3|i=XXoA7QZ~}&JQ{{S`bxERu~|FXk8?o?mmItVMJwba;arr z0!n-Z`E_*6ho_>q-=fks932c_n4{sv1vDsh6EnpPVa_ye4<8E=vMi1bsgVs4j*Nze|7%G(a6LehfWK@t=+zBUtutN<)A=WY|Ivn`~d< zJsAJjzY%40j~<8&)R2P-ds&S_shAfp-WgUYVRZrm0={WiVJ+T}=hWUr_Y3VA7GN;E6v} zxe)3{q+|B6T&$nlMDV0O3^d@eK0g|of6$%9_bdyix*X8t|F__bFd37%C&o+nqq!gj{|&D*zwDDSEzRM3ndSxyq?!q>P>j6r;G1}pu1e9a3;NkoAT z5d{`qpCzo?Tu#Bf2+$qeyn)Opf6h$TeX%ZVcz75-&Rm!coiPx6`{JSKio7Kw-(Xs+ z<5Zlci~}>_+j{+#Ceink$AH3O*R!?q*-=**5d!)|$)u&M0>-0Wur!rqjTjlbv}f)> ziR+KbBeCeUf_PjL^ujdY72t(<*>CYh}GeQwXHe_ma8DzeYD58I`{DIKVnSkMtJzEDJFBWT(vL^ zLlbM@*!n*gK$<=YUBj9Jo({~Phj4u}(<-yWgnCe8V4<{(UGdmZrbKbv6P6j^rFC}= zbIFcIEuJ54kbKE*i$KKt79pb4y+*%1+n7ba@VEq36QwOwFtC6GgLXTKE7tLzI@?8 zzytsdHXEuHzT_)hhsG*^^;QpE<0IX=T~Y5Tx{wG64ZEV%VLKn$Q*$T#pfR`ch9U5$@=g)29-i6v|^ z74Lh`fX97TNP^7(*y$sT$$W-hx)!jKy08pQF=Rec!cMpK5uM1LJ7vEa z8^r!nz9QppH;Vo|L~2aoQ{}l^e}4GJ-p!OFCPii9-rF8-|7QWs4$t{(rbuTCJ?2f> zPP66Wa@Bt=Uh!e7r%zjJR!}LW;gc+7hI*j_aJVf;38C=LkRLLW`hdq#%o_*gXQ4T< z^aazm3!;zR_;TR!$EiMyjzjSK3LOj&P#zPl*3}GIfZ|eJf)K-`13&?f4-q-2 zwnSw61yj)I!UDTXyoLbeobvjJG^GwBT;8BTb0xZTnJ>4(B%;4P!HmxEqBhcSxH(-l z&))jU(24m81Z2&5w=*zxbkt9tfs$Acj+dxJ5!1s+Ayd9F+71q|3USzE+o+QgGH&T@ z=wX#e-oSe3i{ad)PW7dV^76^Di%y_NIm`uJoMX}wL8rZuP<_*8if%4c}W5mYOq3cY21`c@1Rh= z+rkp)+&MJlOu*j-(g`ocEfds96JuoGApa2?pGjC;g8#PIZAm^kX(LUX-OCwti={qKC9}n#-zE|Tu6qnU= z6kG$KA~0D}t;J9zK;#fSGvT8EE(=6Kw166fcyVY@YIvDY-k41bFUDrhjL!g?$_Y%g zfDIM27DEU_;1Sm|xvrs9xq4}xLJ|od6*5OgA#|!LtGKJf;1! zmM2wP7TiOZ0B8)XD)@eYK?jx`K%U{Xo@+}P^pUh&4e#eoiaAN#xe;RCOU>yZz8%Eis=B$?@}&I@d!DJ`+3?@BhKXweFX zrL>->iu*UV6=d0;YWHw23zx~w6uyDCW)$)13NEOGKUfFp?%gCco4m(r+0S^={V9-g zg2(&@S8S+ebqBudU%za?&XP#%aM4O7Y1)z_1H0+6HT6_VqaP3-#7sZFb#6Y1KHFJR z;5!5FdeM&5@gpmL8@&(2p0+JGw0=@oDR+&lSv(q`b*g_fqp&VQWYR7r)i!yH>w_Ub z!Q!+Uo>Z{_x(g<9A3~t%XwCmbMeCm$3on+irQRF zY!eNBtJ9(IfEC5D?L3tl%YR(e_49YOD+16NS|9$P`&#Y}?QmP|P6wBfKo$DAf1X=$ zd~3F*jtWHSvkxWu9sZL!!6%k}h$!~=4-WG`8}9EgvnAU(wSK>>boTk*zdvr+DO{O~ zCwEZpXHdaNf46@nuqiWSIW#gz7&p&Spj4K+1Qdrmv{H>pk8e zvEG=OUb)oufs;4B0nG#!Kh(5{uYfB4Kk1f(2hbL$2TM*;@zU7|lQcB$)HLSGf-Fsf zTzgo$|MTd@7A!0N`!6M<)i#TAYqOSa-Oc~)r31#VpXy&?Z|cXF3iO!mqr9xYn?eBIvbtAAs-$mE<< zvS(Fhb~;MUnjz#Vmg8EtxEP=uWaUcXQKa8Y_GJhy_UGJLa-l!JzWqPI9M|;7#^^_CDrb5rlh3o~@leV2yZEi=USxwN&9A#NR6&$w$D~B=eDfNbB{O=U zErynhB0L+~Ukw-^6^IB&*=zd}7Jdv5`yT-JcfceX#~Q{@w(kHB&qc_AWzFH)*xA@h z*f%Lr=q_9N=KO{ zAv@+Apr_?~=236U>o{ixPLhL%r+UC4^H()+7yQu(9V;Mj16<|+cMdS0@t zap{>S8W*wEnxl6Xq_FSUAEO2o;;tBsTs5cgXy(V`C8E`4w65p(HuFHq4HfmV-acxI zlg+JLm=pb#ok-)rB`a|FOKv-l+y%kGb1fpVxp>j5KC`!5AD}-3xyKxZZ*yOx#^LC) zr<-ni{6NB!Q*+j;H~S$wbak-%s2&cOVLbu*FXH*{bwmg_pXW7rcT54y)Ae|6}TZdkGda5KADJ zg{-da_p%}$FhW-=Eb~PxW~D@rZJ{JLVWY61)|k8wu5T2rZ9W`E;sEk`_&)uDV5O~~ zY!f>C9o0J|0shURQ_!a=4)KX#?*|5&iH(g;BnNSvZ|P7(z~wgYP#T8GRU~DWr%WTl zGDX!a2gU<#BY^~buhLxXw z&K&o@$f7o`f*=O+{2LIo|CR*b$^Vs}GytW6RO{bsuqo%z{_3UJ@`TuCrV^6o8q~gg z!8`MS6L81^19rzm0@Pq^zp4^}JP`mcvg*jX%fm47^od6`OWCNT4-;t1_Y5+GN8)|;@6DzfIj?>LShJK34k2B_A@^P|2_P*Ndf?QR4AjDo>3|4g@%L zLaMvH-f@rB*g1>p`;jl*{k1;-KUP8FrKTe5S$g^=dYjcVTZFw*+#)q+v{1H$7y5pK z`ci4N)Z0h)^r(Xx`ou84o=TnuHv=<|ROV&P?3+t2<9n@kYoDf+kVTlY+{>W&cz+bA znl0Iy7tB3Hdg!R4=n0Wh=c@@HH>0g_GfMJ$dJKVs>^P~TRp*Hcc;`7XkBhdlh zspV(q9n;KC(39+jgTx|I7iYMnjlOyL7bS4cMUu)PR5r<^LXBK(y7Er!J(2?Z^an;tT!xocY0cgcQu4Ksd}6tcX;v)jJR$XJfh3SiO+Ww1 zMXIN&i^L6a)OV*H<)yH; zzyY8@NB*}*`V&Y>p|{QM#~jQwB4^{#!cYOGYt!x%Pct=15(oML8jaRO)?SXrJib7l5yfGbAm$_lB@#pl&PpL+ZH zM0TAAdyhWN=a)rm1(}$bKDW1zJm0K@B7Df6!OmRoa==t>b-?(SA*fBZtAeNt(jZ3R zkScKgSJc%LKnJi^okzhxt`ahihOkk9jgVZ?O?-`AN+WeW3u{d)Bh?2y{!6a zG-vLClrIUaBct4=hjYoh{YpBB{%G z1d^-(9bZJU+`3h11 zZ~l}a&iOQqHzl+Q|E$AeTxP|97f? z(=f}M`T&55dk<9pW&*!Gnmh4$saJF9p!rVXV^P9IYKH#QwVCG6>eawU6e%MjfizrG zsCNP!(HuSqrI+Sp{-Y{h%%^WaE%Z4WdI{sN$)|?CBp|89NWYVy$1g0{aU%DhM7CKM z>*>nkq>PGL7w8GUvA=fvvrs60Lj_rJ)-DFnRl>GEUxYgTQu%AsrnR0$gpHa)1JgeR zH@4rO>A!KDw^7e`Xwudx2qDEudX-Fz&}%q#37|_!9Kj2=-ytPv1Y`w+J`ni8p_tEU{#vJl z@_R^--Ft;sT-*8E-I&ujJ4grpd3LF@ax}AMy4PLTTw^Ti;`LzAF1@cA@W@@Ui|UBb zZ!?ao>xRRxtm`e8^Ztw))Rc!Sobb@yMMp+vPwrkw^{alGs6@p=lnTnKtsm%VeD~8V z{`0dS&UyESEj`GuIz;cqfJMZkP+c=3Sr6A?6X+Hsq8>Q)-ry~MY2p%jbD9^4Lout> za)j~;jsA)2DC>b!a3fBEP+SYM{tv`XTlR8prXM$4F*^Cv{r|V+Ms7|O#roSDcF`{t z${8ouB>nl2+Xp70TyQ zY&Uin?HKHgRClqK9 zsspM`)-iJhC8Z%4Z+o<5rR&q49%&ow?>8&|NZ4XxQ2=1Lbz$sPJ&I52&53i#%4!-A zBwfi!As>YJtaFBSVPp;~3ccmI$tW?*VIegtkIBdu&^f8{%w&ofJQhOgd%c!WxB}b> zs5e)Bf)ZEViU1Ui;&BQG3v{rm1Wo&Zi=bTSIS#%7+Wo*KZMF$PLE??tmuF>5 z$TU$vSusgTJm$<`352cspAer&lr>maT>WHu70YI5u61}$ixao z5~0;cWrL(yrdj7b@ND;rNrNR2;r;0G0}hU8elRq9h+1ssRF?o2 z2eq_d9(>-wtcXs1mUm+)!<*UzcO!Zvl<5!wYQH%q=`Bu(EvGFlb}YAnbR&scwILQb ze>!iJ5M7e3=io{4)Zez16npNRLhB$1)xIKrg#NC^ z8%Q}w*+lXVWR!29BmT|jw4Ot*I>G==`WLZ1vEiuFT6<@M4p0$)OttK?6(0U z-?qP@LZkaA)L)Y4AO%3bjkW01HW-xOvIpDOUr=I%4hYq!`IGk@KduDH#+To60dveW zUHnEZRzqgRpgKrRM#d8&xiH(8jnFBQp)vLomB;T>9>_-5OGekf&+qU7vuW3Kun(oa zilov7ZSwmsMdhN*s*ktC6=V`>XXVgFb0hs}(zpTjPHp&@q&~(1Z~Xq7wk;PT3;?i9 z!W-!T_oX98Oi)ed0Lu!5jufIr|E|%&mNO(Ww-(Ye7veRfwnRFKym&1O4=MSACh`R@ zRU=Ud6qNwA*k)hhU?XB;kUzdfI4qehXP9{``G|?ipps(ONwt2Z?pYu_<}~l7NG90o z(!G67&scYBH|OT?iS@U+w$XeGd|+N7S&&Lg-F3Qvsrozxid3pZFPOJZR*xNeZ#e$d z0+ea#qEMZ^%qg_&FX()fsS5CuigW({f_J`}hpaFs_;uz$Vc7?}^i*3j^-sEDEcTbz z{pS{P3LyeeT)9N z;_si_!#x={hWl(ePLjT0vt02GkA&ZMW_5p{wc_`S43M>Vz}o-Ui(Hqc=X9rTU*S@w z_5ZSTK-`{k0ss100YP`Hri$CDhm4Rb{JEX%bLB!0)X|DO1aP^LyypLhhYFv#W2@5N z5rDi zTN1P7K1xNPFy*n@@zdb_y)|=f9gFH@H!E*M8QZA>u^XT}?NQ5yeVtkpzK0JHRy`z- z;;z>}08*GN-S5@^M>Ygo<1ZO`;@mNF&;ZS7^(`qSb-$RjLx~n*Il>r)$k>~9;23Jv z#a(}1y~sv;V8C=xA92}oqb#nWrR7GK zeYgGzqId;F6}7Oi@WQe69)NdT3MIRzXNP`^a99=KM*d*sLqs3;KZ2^lPNu(Ol( ztWjHdd&AUp^R9bB+wLUZU%7>f{3Fk>=EY{5;NO3s^2X8m{cl@@pg8~i&;R_@|E$FS z?1}%&jsg$qt0BCa>iCnL!c+^t5X#7*dj}-y)==}fZen`~j5IbQLkYwlFzdOR` z#lvnQ%LW6F>s(P*7L?dJmj=OkFG>x7+rX@Lk1(+K!!V`4eft_U?%*Sz*634HQ(FUy zB^69DF$R)73mGzaSb`;^y95bc%B!n4@2YbFb?w={e_~?dikzH9+p-D7ni&4z69MH7 zl(3FEynkC*n;@@+6*n2QM4fj&7=M%)Bw-r7jNZejA~VN5vfJ()xsGZ+(w~uo5Beff z4ftvq{Zh*QGR;oOw&;CCL|AxH#A+AeE97NlDoqzi$@b^wW)2Be;OhM@a48ebY5HDZ zItZEqkYBg1yW}(JB?j0*%7^g2bt^9khbSVFjqolb!cno*(Y1-z-#;-f&NUVU9PkH( zgbpO-Wo22h)nHjjQSdAGaFh!e!iycBY61ZQS(@IY+YEl0RHc1eXVMN<9DzUo+$c1y z6Z?6n^3X?B7;_H}7(zHi$4!OrMaEP3M1b-IEm3ycgO>@@w9t^{=g*(k%cgKzczMFF zfBW`=G>EbFZvXr_0{vxbmz{xVQ8u6T@hj6oy8)yW)8USSA+mA!{$PENw>hDR4>$p* zp$<3c$fGe`FofE2==s?tUdP7Pq8Y2noJiE`I7>_XS0m&m$ETh_p}^r=W`H%$hR!$L ztn)+JriKGI8mXT^_(NA$TN{qOpO}LrwF|gt6*fs*M`M}4lFD@=oU5y zyr#tdi@p?NixX(#sk-3JKT=R&f4KwdE)eH3VN11(5A?Cv1nqIO0m~Ew&0pWcEzMXl zBKpAIVFyP7`u|q{6V?U*K?r3<4Tmv@MH8FH_luL9MAE!l5SpT({rC6J=Rmalkql~} z>>kZ~2BO>rK}Iv?X_7)Aus6pS^FWf}^f0MkK&C|ilk&=tTy|wCsfzw(O;l0h)Je=I z?##-LsX-4Pz;)6wKmGH>G+GqnaXeolf6-J*RAXE$0Dp%60oL^3`~kHuh&0C+FX}rw zP#c5BVr~;TkU-b286j1m=tUdM{d$QNuqPygJ|tHqM;R0yNhr{;#&qTxrj6_pG&@Nc zpuvW?yh7e4B5Xu7uReVX#JdtmdgPzS0K@HR{CS2{aiEPmDM~s3l|V_w{V?Obf-Z5( ze#0p3%o|X@*;^b)3>?((WKx-S~uY_2@9H!J%j2*Br?7bsV$e9oB!(5Y}iN#Zqc zc{iA7Vb?o|!HER%YX41SA|(cGrV}ZeH(XmA!B?b>F9FmThCxdZalK0c7y_*oA2FkbrULn-^RCIhFX295Ji!fcH6Kau);8 z{SEX>J*9ORs&nloKDV}V8S31%NL%yb$Ni0lIjVHMp*0FhxOJ^-)I0D3J&>#Jvk-cL4#%YfPH3gf z#m#NhJ}tI5DgWhuhcXvyaB%QA3i94CD1xfx12+?RBqAf@_|V#qWuL06Zv?#8iQ7?~ zr~I-u=6jycV_{}=*7Ux1gxNKo*Judi=JpNMZ+#Ip-> zJhx1YjOI+ZjaXS(574`_CzUiTzg{kky?uCUdb-b1$Wf44vDm@hVPS@MU>Y4?A!1hj z(3z~~ro{())-`4u0r?bmj#c_jWP01fs-s({TU(!JswG~{vjRoLJs@~L+Zcdp~SfFaQK%)@QwKf$(F4f4jKN{7AV z2i&-yXz1hxdZ!NmYU&r~s9fT477=2&SzKG59PrI-Gto`nxo)-J`wMR;7jmR4bv|~Q zRaJOf_{&H{cZ@c!R>0>iUv?GWD^~xQ&X+NJG8UawewVU2oPl6RDls)=0AS9 z4FZSFy~`rP!WKq0F@0UE%$sfmoax_gO0{cwSuN-!23)C2Y2btEnbkKm_&3~^ zi{I*%V0hg0Tyr#g^twKKI@++sCMMDcRvhFGltL0;*lW?t&wV7g5`!W4VT{`P z+>X54Y!|~QdFdjrh`n75$w*Hxb6#w8X4gd5i#|u%A%KGAc9@Ali9^xA)vJ7>q9JC9 zUqn_|*w_TvtXFHbUb)Ho+INer5d~$R>VLl4Gig@F-Mi_-s@Y0PN^nlxRnZM#s;GV1 zz}otX;gd#$MgyLy*y-pWQ->ys2!kcJSXt}M!w@q%SD{R-&bw~o860i>OTdQbYH&r z3__xN{`%A_@8(V(D(|ICDz+P&m~d?8#Cf7K8AONv!!RRuDvuc4&gd8!Nv2uVX{B5M z%?E_yQIj!!K?e_vf@G6DKa>B!0EG7;x3c>#q=epvx=`b`sCvf0#7!N(XAy8{8o9Q6 z_XT=lj^5RJr+jHv-^6OoJzFkhZs9B(i>WDKz;O|x&U0R-#L{MIWao8e@PUH29tD?NL%Nb>J%wHNB$`FkvPT3|= zevOTp+bu0*oq2URD>L&69?fpN$BnOc91Uxw#KWJAn|@Aegmk#v>F8h!jzn1pwwz0O z29fC4jb>Z{$@*L=Nq%b(@N-W-#n$%rFV+j2T|cJ2c@r_<$+~ZRazMV#m*xEV^FeKQ zMhAtaxg9BY1Xu3q!HC^6IHWyL;nQw%J-B0Id2@3RnE{x0PwrW0=og1Jw1{Z9cDVjq z{jiM`nm3HhS5W6&*$(@h?=bt)AtCYkdfX;AmkZyrZY+Js)zww#ij&&;6JP0So#pZ~ zGcW0VgS)k+KKU$2)BQr#!Qq0Xxw!FV#IS#xo>7xdQI?))$GF~xshsUu4h4=2GndnT zbZ4WGgnz{auoxS`ZMb|!tVUiX)-MQF6=P4ws<=gFcZcOP83!QBp`X8gsP1sVl zQ&Y))#rHu}AO`VzHW@44D@2ypyYe`bC+lB38cs>Tn|e;i<8pq;fMC|QdRJ^HNzyFf zpwkS$bm-=VsN*N4Y;weNr$3i;8Qb3-h}77Zfs=s1%Ft_htnE&N?+tV0h@`wO-j?&bi7 z$MFF}`}C`2_?O?o*Gl(2&=5P`zONs3BbGPxY2oz!8JqFsm2ycln>6+XvOS;qbn3*m zM?*h;NZViEr763Gq2NWP@Sa*a#aQQy_WG8eSNTe_IZv&>WP0IA0(hWGE{psQ{i3L7 zyZ8%TObcqu>a0kFgPJ5nbqDd8=;`Tc+q)Ixt{@UB)Jr?Ky4e1CNQmJG;mqNIe7Iv% zmAUHBF${Yn>*om(kqY~a$1PpA9Y^%ig!;aJDyl(R{zG*V7QwH?I?sU;1{cerIXyMi zraUu&raJN5r9N&-GOO>OYODBRHUY4+sMo@+x{ofYG5Afnyy#tp`0s7gqa6&exHoRx zsQdcWR=;|=)<9vxe(n8JI5(w3v@g(TB+^F+`xx!InnP4p-MvG*q|8NcXk_Gb|Nbi7 zqk)K2%0bV_uiZH`G=$#fub7Vl8_?C#5}zJ}QhP-8iiy0uyh_^=9NCvV;~$9*6snbb z)G83;ql<~0lJ409w6&9}G3XqZBU1C@M?8A4Acg%H66`7aePKb(@_JOF5za)AT=PuF z!4}o*=xDZqqUYK7NUycN*NoP5rmi2ub9m#kBZTG82HFc`Tw3wOo*o5AlZqy<9Cz7Q z+^f_}voH^nxnE!-BjZQEgf25dJ^^v*Y2~i_Qo)t1-QC@c z^8;2kHbxwqk+_ejt=l*5C^nrFfoJ@rmj-iQHK8^o*gixZx)$%d$ND$f zPwZ`~8zq6tSLm?0H@WUNJO<@=wjN^(XcZrg9e;nkUODcKK%Zk-GsaSK?^k1iO6}^R zx;h^@QI*S=!*hK0`Ae6KK&LV`>wudi@y9F5UtGN{EoI@|yF1XOV1SjEp2`=NrJvN5 z!vX>VeuF??zWW7xGwBd(JwOR#^NZtiQ#XVt6!@fQmy?4X&WKac)y*iVAAbAU)2Es8VNKE_ z<$bFP!Y6~b>FEPOd7L~vfjH!mEUxwxaL9=`+Yz!pOJDeG+YK((1go8ZM7bS>uQ4q)_UH6|dS%F4~1EbR=SH zUh9)j*QsnkYpaPR5k;O=Ev>D7C^UYfoi;PX#G)j`yvc5|-#gc7QT6fT$AzQQ6@!DX zJv=-vaMAlFBpimQvPOCd2wVBPckj|Q&Wl!5!kGA(RrFevVR}LFd`M7GxlvPWv-+&V zw|NAKJI&Kil|1ab1-a_A%E?(W_-WltASRK&>-y{){! zey#)74QI1eUc^lb*eFud?i;wR+XL~Hl;w)2Ll6A7#(aD$KFn;6_>}ecGKXUAtzvYyVDtBJNqs3O!RRtYkmYV)?GSb!6 z)&9MkX50HQj9j+f;TWx(&MpNE7%GDw@=Mz9{V@QUG=F2I!?*s+mg zJ5U-1BtblgPbm&FdvM>N-e%2w2MOh7b3|VJRB|1UcQYNA5*@f@Z~r>$ov4V2d_bno zQKyp@WmPNCR(gHsfVEJFtZvsHCQrTi3(_{vAs6#6E0dUoAc~xI>^F?dJx_67;zGia zrH(Erm>=+6_jno#8a%?5flKvT8yg#cw${j%-ET@^$)s6#=gH+GJEV7>AvO3R>0W2{ zjV~?Uj+}$##jv4^p z#`(%doH3Q@5#&YrM{{!3UG0VP>j(UhWnu}CKh|-leDb$vR8HLPw#oJMx%20HXJ(Y0 zW_tr&jOyT&2Zn}@O83%)ZEn5dG5*Bc{#LXOG~@M~RK*!ru+=AA- zVzrZ**-u%e<%ba_**`3@3XKIg(<=1RR2aFE5Y({A=Cmbd3348=UfvSd5M&kTni*Fu zmFIsm`>tE|$_u8Ko_NC7Rd#==U$`tP;(vJR_HB&g=qqnbq_}fe{zDc&pfWd!Y$86SBt^kjmd@S1b$U%0eO%GWt)7IZvdH0T9;K!Cd z?Vm*x;nDUUxdD%gEKo&LGjL+U5=Z<=N(wLRF*;ZPRl$sn^Ho(&vBdUiP%VfD#9dtf<#O-~<-971%aV%liC)8EhU0#umF35w=9ZIWfL zfgHdh9*ifmqPc4tt_mVPDH|V8wRkzx(y>l_23*0WudEb=&Lj#`T4cqy)2hAO2^J(LI8wY4^Lau#@^w4ge4nI!`o4?s@CfGyT zk~g;?{eF9B`_(Zsj@>T?hlRFoT&oL@W%+6Q)>*IR8L!;cuURM#rG}hiQ&ZNxd8Cb- z6*aSW#ga3OL{)XQ?#-Jr03;*O4j!gd_tq^Xh$)H15~(|Ql_`2t`{*F=c`s~n54u%K zwP-MO8oq3bbhlT$Qo!^STUq@a3U?lr=$ zrjCw4WbcHku#-miC84rcLtcJA;2n%Q37bsI!ogG6@Yutnb-=BCv~yyG_C6}{bHG?w z+1QSCTvb<>$>B0a8y9)Ey%s`G8|xd32dv6*xYC1?+tv)OvPD7u_ZR9)N^Bthkl#G9 zI_VR;@=(Lgy1)EKH=-%Mq>5D$h_GH*<-_xB{GCVrUa3YzHX{wKMW&3HhJzJg}H*HtrjNx$s z6gb*Zpu}ngec4e(4FP`s3n<(N3Q=BE^Z;1fWrBGnpH}Bh#>!-gP4&pX6@6rugMEGA z`BB6NdJcZEo#yyTA83=~u3X+1I9fl$cly{=>g1(EeiqcHriNGEJz3t;yWi!C(#Mij zc?cH`jg1vvd)%_IkyC8y#gvItIIez{CsV1Q8sI&R^OQHDBZ5}Cs=rDHcrYn`CK<*fdfkI}<`S={!O2%FEaZ--*AIL$ zJZa6a>V<>objzo{3EdyFmp=X7HmF_H&>a(0-RFx>We`ddHn~uJ(>i1-sm6ua)@}A8 zCeF^9FnN&(mZ0(@FN-KYJ|RH@E<}MnWwiLOFvT)`O~6cN z@uM$9L_~aVjkZg`#Neb13=ij7GCWlRIxce%)zX@EC;M0{1e3UVD-w9*@07oV^&2Lfk-P2+SOkPJ4nd4}kjc|1sGCWYinffBoHe|nT3tl3 zoqx*qhtXL*FsDF@kur8o%<5=spKd(IkoC~CGt{F}uOiEWX<%&joy;C8@l!}`85tQR zW~UL->FMhqJDLQ5CP;@u>xMbg9(?_vIFA*+{|SCWQspKB2yy3wVb!-@W*Fd zae77hoBNd`BZP92gjoP`>9=J=`(%NQsIWN2b3&=BYHR-B{NHM07}6YbW34 z0F|qn#Ybv2$@l||27LMQ#bx3)7Akd{9##nd0q=D18-|S?vsz~HuHY4qqFEqEAYDA$ zbN3nU4*8~BCFHoYOhs|AqeF;h>IiUXgVPOSfuA8WV^P_z9$}eh*z&p z#~P@ps`_Bhr!dY@#k=Bt7s${Mu^l5nejG|`hW|xse*h^00tXDutoZUJfaJGf z%L{!wZ%Yp*zd-T-Z%zrgXt?q9u!L(zVxFCM(`lzzW5syNK5Rth1R!nwZBWc64QtoKU808# z7Y;9oGcLfTlF5Qd=Ml*WWyDnShj zcXt{Yx_;>l00%0+Et5(crQv3`W5{QQ%;PGxAk}(^n)kzWUtgbp$q5Q2^awk{Ei0=7 z1dV;D_AAsLW9C2HOr$$Zo_4CL zsMIna02ejVGa_ydP!VAz|4<#lfM6U`v$A5@hc9NxEA7Q*z*%}^dtx_`yK;7_>`hZ^ zE+tm9TNo}ms1NqC13tz+^NVzZg=EyfKEaN&gDwikI=xzk12E{-Wkqb6j@-SH8R=H+ z%R;S`=RrxDd5aAPuV=^rl)xT}EV)(iiy{augZ z-?KxjA}C7-MF6?l7~Z(y3&4nxh2>&SMKdato(E2pOio(W4HhfF{8(%n{qY0YU<|W+ zdx0|#HX~XU9rL{DJ9RVbH8A#(jshM|PELd$y4SCts*mIDXt~yLuGWBGJ0@{y-c0cM z*smhWxj4>7lev}0o2uoN#8HLi=OwYYh;~i6V!wKObtC&^L64LlRK&rB$jLvsAErR= z>ebuB8iTugme8ZK|_wx)U~yRSe*Nw zctNo^%&6qK^r{&J$K9Xz)^&D*N z#ZxjeqO-Pu#qmSqxLGiGE+4NqwYLXAhOl$}OCV3V&dVB>D|U8d@iYt>@;xK{>i~(> zoz`)5R7bY(#s4H8A1ts%t$9n}53a+94JB)^;d`NRXv{YIVp(gwb6ZzdxBX{-tyoqe z_`678mD*RX&Dx*Jc&VH zb?)safsY0K2g8~{Z?G<^6NVUX@bga&0nvO`(QNkIpA}Cqlh@l zfbR0;dv&eKMehIUS5iU(dq-mh;z`oadxweTsygiU$~`TeW~H!&={plkcAAu+h4=Sf z@e^TE?>0=enu`v$AC9G@-Ab4{w)+V6{ZqYPUygWtSAJ>0<$+#4Nj?yH$I2 z55b{P;VFX29`F1}U71;Dl2K7u=mN~$;Pu&t&)ua4{2Xy{aTd!#w1xh1v@$b$!L09$-oCk`hm3owsXg4O*4dFA@#zJKWb9#i@oTj(yU2{z zvo!>I{5<|~{-I^jv zPK*Dt4{Kf(X?X)f0e4WorWv_w^KgtW0VSxc&@iYCl-}xFxE%7Ti8?cNbv{iE4Q?pw za(+2JdPwF~^X$E}7q~#*#4O+6-;30iFevt=gL?enees8*>*zPGJ+>HihMV(FDu%f} z+d|cw)--do9)xF7ZA-Tf5$ zv+$51A^zxKEp+09AGoWXSKq$D2L{}6}jg7g6sy!Nv{PB{drUi?v z@2?#hu1itCNA<6sTUyFL^Q!jh_SW6Z%my4Zgx7gqPHk=7jUv+Qn(9UFORkz3ZgY}r= zCoe#`aLeKNGih44GY1eeB8K^35oH^;b1VDc>aU0gF9$y=(c;CzHz<@q;zUmO$UFxA z)+=P@-kz2XJz@7U=jD8R-tF<(34PoGq}riZuU@IhC?f;rtSCJ$AOLB}cAZ?yD@S%H z*sR~Q_Wt-#QUNk}gAY1UbLrMOZEF&2k2S-*0k_W;6czPXO44vc$c7dfr86^`n3zZ+ zG3GAI(=o<6=)= z$Y=@4n9oZLwjC)A$(}1QO4qaQ43y5lU7$wNV%5FEXw?M|QoCJ|APk{IA@D2d*QQg~ zW8;$ft~|qBJkIx!6TPk5PZE;Hfe)11V|L5KUxA*y5EHR(?S1ex(GG1jHD<5+;%!em ziE2NxG8PDiMXx{?)YmQ>#;OBDd0%|&e1ftmw<0YhJ;75uULs)yrmPDCeule51c6g= zNeKXwNy2|9M&=fMGZVDX7@=i(9z%Vj z>5~REW=gN-u&@(9e8ynMy45=H#z%cWqu}mW*DLgqAiyH%nls#i$Mcb#;BhDH#WPxl zp+u<9bd7o3%-5j4hsUwpnDbZl)gHajdG#^; z+8bx*bn)J(n7H-9K|v;08q_FI&9DvcwQz9YLR#$$6@~|c8+>!n3u(s=(B^AKXKj2F z6~_i6HM&xhlV1q&3{K6Ab-(GmV;jccU151EiX#%dB~&05pg!S7B&|n~Jjm7d+T={s zampOtvaXbH^#m^i#sWteF+Tg~+$%zn^jfDWa|K1VKA;DVNermj8wBNjC)2}T*TQ}`)K*lP0qARyx$odx^Y)Y9S$ zl??ca%dZWw;ZAYp5(@KScgTYq^3~Un#oEpM47aF}+#7_+|%BMH)cR~^qv4y8pIpVA6yBDMN~yi%?}v-n0=lc z*w<$l%HA?4u#R`>k@im}YDGlUs}spSltMF!3b;T47AG(0aG{YOYcGE6Z zZydT3&{+bt?KUf+20}_0;RL$bXI*{{-srs7?nqH@$QFhoZVfd#asC!v31X*E$)J(lMWm%2 z?WCYsI!cA9y{U;EIRt{MULr->xb`4G7o5FbNCQ;XBtaoY(kiGvkoD4 zB(orW@#2mAs$L48%z?tVE2%0*>UfofMEEVDZlrtjeL{Kbh6g>^rRzmkba%Tzw%@Y7!f#Bg{4QyTxGS_>yL~O}>a+H&UV*}Gn`>$$hv+XOkSXfx=GXblEGC?-$ zyC7bwxcF}(EnPb`7L_%oyB=u(1jhSuGKdZ0(a3#*IY}aUy+bfWplVBT6zk+{juGU4 zOTcUWnlE1z>ooWVLxMML-b`mBb2%;{p<%lgGvW&;#kV=9bpdht9ppg%IlXZnTjcx7 zNJLzwY`=zwZ-S${kV%E6<8UpHG(kS%+AT_i`eF%;39xFz<_(<9SMdn{D!f<}$LLPc;e8L!nKYTo&q-EeO+2}Bh~Q){cjYuaI( z+wDxXt{<6x4xVfY&2ln7!8O~#tA7ZBL`gDXbGzm(A5SEeH>8Z(C{ zC;5l#N;;h8WuD7ML+#>L(!G_$A2NC-^L?u>eE;PkXr@ohJogd8p zbihv~{du{|N%so|K`W*uHd-LyZ>&bglDDsbG*4eUnP(kWq*W$0p%1+kfSo$OZ;#Xx zkFxCAmG@<%BLH(q1`b_yd9-)X+-g%EUJ$e-4BAVD9oW6TcQit>V6p_OF%7}Q9!e^G2U55$m3OHP3@9v-^=RI;~ zxKJsgieOP!>KERh6RNJJwzgV2wq_}O)7#(WZA4=S*S0i#xm8v<_S+MBg4GV5F;Ar@9qqIr&O{&sHDPGzN>$PmM$o@`}$0O+d(9?CTSq(usFo(0UjnK#xi_*n%hzQY^wiDI8;CX|sa@iV5CE;BX`` zy|cQ2B<1V6Ua{C^q4<8@LqKS7a<>K2*o3(QDeL0ltd~qIa(jEKnl#VP=s`aZBVp}~ z`ZCWu+#^^ZxI6mtoA?+J#jn!Yf`pmN?wDWzpjQD#_iP-F(B@`Vmh*`dKOs1Ezim+q z^F(lK2s8n_J1Ya5O|Z<&8Xmfnn;VNnCR6e&vV_IH?=O6Mb4^?vkXIom4KR7_=;(dO z%EDi`%!{Z}!BYPk7_=$gS+B)qQEdBfEX7^>&juJE#l#T6)g91FM$1qWo>$DyU3vNW zX3OSFK|CQ}fy@w{lw9lF{Uhh$9A1xp2i_6KqZv6~Ec=8#49pyP>Z^11{;AeLCSYL9 zJ@#Uh5eCpFOOfz_E+d(IbhKnPdPVJ9G{kU0KXhcG5tSLE6eU|uH`!w@g$)kF_l3|Py(2^|U zoHo2ll5^*-s_K=#X2C9{K50)HRgOC9bJ%8>0X}a{W$6$1k-7|Q0hw#J8_ zULh+6!)jso#oNO}84iJfx<~f^HRN9fV*Tqlw0rn^R$$3|*bb-it0!-%FLx*AE|e0n zTdtuIjYN7GTh<|EXalG$WRYseKeskEsscZjnLCC^k_)~Fkb^CwA2BWWZPjdI_%{{R zAKzV;77j}cXvGXwJo^|UXus98;Z&Af4}zHPJFL&ZJ@ORjAhr6RJjGOTo6wl>!CZ3| zo1Lufm2Wu{inoCN_`vHlLd)5kH4=*6Y6(*`S^ zuz6SIj)4R29AwvZk1x4`toN{%4<^Ga=No{Dg3{8I6GsjM|LH%tEhCM&4uDOqYx5W> z@aUe~sbroLl-qrp!(wF&9`an5qW`D*=8s6l3ZHArtl#qz zYvL)fCe@eW!Uz^Er%1GSV}sdPubJVSPaCa(J14>A4;z^N{-i1-1hi{(cD4p<_lhHl z2NMz#YXOZA<}>^4@^{CW+S1Qmjmtl@HZ=|hK|5}AdJK$^ZQSK?q$(OgTMJkf2t)|2 zJ^BEy)IH-6I3lPKo+8JEaKZxqCAK|}!`X*Y1}6wf2#2GlmX9RKP>K1fy4)|GZX}52 zFNrJ`>)}e{?O9;21MwVy8r(%(4yV7r|4f*%N@9~JB1y)rVrwr0`3H5$1FAe2T@7`A zASOtu%`kp0y+{zGqNVsLnvE@WQ~3F=6!G)wgd zbFickyg0iAy16mPGYSUZJstH#PKJ{_YXaJl^;k)fIJ7>K`9fGoD28L9 zlTrd!AJv3nSs9u8(Ge`C!K)F7(RDU(APRV~`UaZKVJN)ypDns?Bz#NB0yTa+ z7>xHh{vS}z0|)akZ(yI(mC6obDBa$kTo>_$DRFB@Vy8I4nIyAG5CT;MX^9%h@6NApA!u=q3T+fCY6p z*bH5&=Y?HlRq5)|L5DtZor!BAl^}R4wy+ecgN7dg2bB7HLvEhxkHxJBtuN0(Xr0hs zj_L$;Hk%EOUj<8oCMJ zKhB#A(|=@aFEGeg&5@|B`KQJOVH&&MnHo^FMtX{9ph4UbXY>@8jX}c4MUyr#1 zbXmHGfqcmr&rJ_J5r?Nx2celpK3&i34ZP2q^BjK`PzYy7N1EM(e}U^pFAp@vvHn>F zGsf+yUqVpghA2dXfP~>ev1whd)<;zxAUMbK^TUIKdzpy<8E*6jCA?r}qN@jRZ+MHAdz4I?DTY_wc`0t(ANPIB20{3DJcf60`?DmJL>h`bIK!k$?NLt zFB(y4Fz|RQPQdxSUxS#A>H9khaV`@Qsl4)XX z`T7fynvZ+^iHkzaMpP5nmI!4l{~p-OwlBr#AVk*9U_jfBXycqQe3h1#)=7mz_ZIk; zCm;bNY6A*CnJq33m{4@sySH=b9DwqI9AyDZqk)x&|IU1cm8#py8tUmepRz8Uut5cb zw=*-hK+WUkeJ^okSx3-EWn<|%s3NK}^s94u&MGQWMc9jdrill^9{ee(F~}W?$Nn9w zbXv8rOX1><_6*NWi+W7|<3lfV8x?o1UdxI~m&T5^5>r@JCdAQ4TKjV3E{Bt!%pyA=!D%=>@xLzN8}7G)w1SY;th zy~$Id1SqQ9_b)6uUpF05isDvDw-k^b5i%0rD2j7PkVbx?D4}it^iy&4_&yHNLH-^v zp6VBJd1hX3>=J7G8U5M(!SJ2eG*}}^sY+hj8Zj}|eTF7}@1{j0wM?{q1^-`K&G)`k z$K;t9$ZLJs%s)gKzDe3(D3HE%Q-&o=3FD5RryI4USA5xgFJ2@#=DdkY+{GE>s7Ez) z0eRil%GCA{O=WU~>R*`GWsR>860`b0eqTIatlmJ3918wXMPA%`Mx4+LOWDA(ve$F@ zr=ohwbp!kbX-9x1H%xsGH^aORoh8(>U_nX#=UOhg4nG$OULz7#0izi{q0PSl4Yjnv zikx*4RIf*1AVWijf3#!=Ktpf~2aGTN7($t50W1UhUlk}l^%)bGsjI1}D6DFg5v17> zSpV=^`@S&#S1I>JS*X-Erwi1w*|48*f^Y;2$20gJ{p=tnbQXDtNyyd&kzmg^ywM(Q zaq^e)A9#EL4!&3aQdgEzxkK0Rxaxed3om=;=f)>a-NGgqX5@3tA3w;NYUL3X3_yBf zV36^!c85<~=3hhmB4Yc%vNYQjr)X51F!nkW3ivNjKhZGatCQ^-jadRdCwc-07r!9>g1&`elDEn6%)bE(&OE&U literal 0 HcmV?d00001 diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..ac5960d79 --- /dev/null +++ b/setup.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +"""FunASR setup script.""" + +import os + +from distutils.version import LooseVersion +from setuptools import find_packages +from setuptools import setup + + +requirements = { + "install": [ + "setuptools>=38.5.1", + "configargparse>=1.2.1", + "typeguard>=2.7.0", + "humanfriendly", + "scipy>=1.4.1", + "filelock", + "librosa>=0.8.0", + "jamo==0.4.1", # For kss + "PyYAML>=5.1.2", + "soundfile>=0.10.2", + "h5py>=2.10.0", + "kaldiio>=2.17.0", + "torch_complex", + "nltk>=3.4.5", + # ASR + "sentencepiece", + "ctc-segmentation<1.8,>=1.6.6", + # TTS + "pyworld>=0.2.10", + "pypinyin<=0.44.0", + "espnet_tts_frontend", + # ENH + "ci_sdr", + "pytorch_wpe", + "editdistance==0.5.2", + "tensorboard>=1.14", + "g2p", + ], + # train: The modules invoked when training only. + "train": [ + "pillow>=6.1.0", + "editdistance==0.5.2", + "wandb", + ], + # recipe: The modules actually are not invoked in the main module of espnet, + # but are invoked for the python scripts in each recipe + "recipe": [ + "espnet_model_zoo", + "gdown", + "resampy", + "pysptk>=0.1.17", + "morfessor", # for zeroth-korean + "youtube_dl", # for laborotv + "nnmnkwii", + "museval>=0.2.1", + "pystoi>=0.2.2", + "mir-eval>=0.6", + "fastdtw", + "nara_wpe>=0.0.5", + "sacrebleu>=1.5.1", + ], + # all: The modules should be optionally installled due to some reason. + # Please consider moving them to "install" occasionally + # NOTE(kamo): The modules in "train" and "recipe" are appended into "all" + "all": [ + # NOTE(kamo): Append modules requiring specific pytorch version or torch>1.3.0 + "torch_optimizer", + "fairscale", + "transformers", + "gtn==0.0.0", + ], + "setup": [ + "numpy<=1.21.3", + "pytest-runner", + ], + "test": [ + "pytest>=3.3.0", + "pytest-timeouts>=1.2.1", + "pytest-pythonpath>=0.7.3", + "pytest-cov>=2.7.1", + "hacking>=2.0.0", + "mock>=2.0.0", + "pycodestyle", + "jsondiff<2.0.0,>=1.2.0", + "flake8>=3.7.8", + "flake8-docstrings>=1.3.1", + "black", + ], + "doc": [ + "Jinja2<3.1", + "Sphinx==2.1.2", + "sphinx-rtd-theme>=0.2.4", + "sphinx-argparse>=0.2.5", + "commonmark==0.8.1", + "recommonmark>=0.4.0", + "nbsphinx>=0.4.2", + "sphinx-markdown-tables>=0.0.12", + ], +} +requirements["all"].extend(requirements["train"] + requirements["recipe"]) +requirements["test"].extend(requirements["train"]) + +install_requires = requirements["install"] +setup_requires = requirements["setup"] +tests_require = requirements["test"] +extras_require = { + k: v for k, v in requirements.items() if k not in ["install", "setup"] +} + +dirname = os.path.dirname(__file__) +version_file = os.path.join(dirname, "funasr", "version.txt") +with open(version_file, "r") as f: + version = f.read().strip() +setup( + name="funasr", + version=version, + url="https://github.com/alibaba-damo-academy/FunASR.git", + author="Speech Lab, Alibaba Group, China", + author_email="funasr@list.alibaba-inc.com", + description="FunASR: A Fundamental End-to-End Speech Recognition Toolkit", + long_description=open(os.path.join(dirname, "README.md"), encoding="utf-8").read(), + long_description_content_type="text/markdown", + license="The MIT License", + packages=find_packages(include=["funasr*"]), + package_data={"funasr": ["version.txt"]}, + install_requires=install_requires, + setup_requires=setup_requires, + tests_require=tests_require, + extras_require=extras_require, + python_requires=">=3.6.0", + classifiers=[ + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "Operating System :: POSIX :: Linux", + "License :: OSI Approved :: Apache Software License", + "Topic :: Software Development :: Libraries :: Python Modules", + ], +)