From c776f8afc0c24dd687b14035b94808fd1a5d48bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Tue, 28 Mar 2023 18:59:49 +0800 Subject: [PATCH 01/55] export --- funasr/export/export_model.py | 31 +- funasr/export/models/__init__.py | 6 +- funasr/export/models/e2e_vad.py | 69 ++++ funasr/export/models/encoder/fsmn_encoder.py | 297 ++++++++++++++++++ .../onnxruntime/funasr_onnx/punc_bin.py | 0 5 files changed, 389 insertions(+), 14 deletions(-) create mode 100644 funasr/export/models/e2e_vad.py create mode 100755 funasr/export/models/encoder/fsmn_encoder.py create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py diff --git a/funasr/export/export_model.py b/funasr/export/export_model.py index b1161cbf8..de57b1b13 100644 --- a/funasr/export/export_model.py +++ b/funasr/export/export_model.py @@ -161,31 +161,38 @@ class ModelExport: def export(self, tag_name: str = 'damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch', - mode: str = 'paraformer', + mode: str = None, ): model_dir = tag_name - if model_dir.startswith('damo/'): + if model_dir.startswith('damo'): from modelscope.hub.snapshot_download import snapshot_download model_dir = snapshot_download(model_dir, cache_dir=self.cache_dir) - asr_train_config = os.path.join(model_dir, 'config.yaml') - asr_model_file = os.path.join(model_dir, 'model.pb') - cmvn_file = os.path.join(model_dir, 'am.mvn') - json_file = os.path.join(model_dir, 'configuration.json') + if mode is None: import json + json_file = os.path.join(model_dir, 'configuration.json') with open(json_file, 'r') as f: config_data = json.load(f) mode = config_data['model']['model_config']['mode'] if mode.startswith('paraformer'): from funasr.tasks.asr import ASRTaskParaformer as ASRTask - elif mode.startswith('uniasr'): - from funasr.tasks.asr import ASRTaskUniASR as ASRTask + config = os.path.join(model_dir, 'config.yaml') + model_file = os.path.join(model_dir, 'model.pb') + cmvn_file = os.path.join(model_dir, 'am.mvn') + model, asr_train_args = ASRTask.build_model_from_file( + config, model_file, cmvn_file, 'cpu' + ) + self.frontend = model.frontend + elif mode.startswith('offline'): + from funasr.tasks.vad import VADTask + config = os.path.join(model_dir, 'vad.yaml') + model_file = os.path.join(model_dir, 'vad.pb') + cmvn_file = os.path.join(model_dir, 'vad.mvn') - model, asr_train_args = ASRTask.build_model_from_file( - asr_train_config, asr_model_file, cmvn_file, 'cpu' - ) - self.frontend = model.frontend + model, vad_infer_args = VADTask.build_model_from_file( + config, model_file, 'cpu' + ) self._export(model, tag_name) diff --git a/funasr/export/models/__init__.py b/funasr/export/models/__init__.py index 0012377a5..71e8f3b57 100644 --- a/funasr/export/models/__init__.py +++ b/funasr/export/models/__init__.py @@ -1,13 +1,15 @@ from funasr.models.e2e_asr_paraformer import Paraformer, BiCifParaformer from funasr.export.models.e2e_asr_paraformer import Paraformer as Paraformer_export from funasr.export.models.e2e_asr_paraformer import BiCifParaformer as BiCifParaformer_export -from funasr.models.e2e_uni_asr import UniASR - +from funasr.models.e2e_vad import E2EVadModel +from funasr.export.models.e2e_vad import E2EVadModel as E2EVadModel_export def get_model(model, export_config=None): if isinstance(model, BiCifParaformer): return BiCifParaformer_export(model, **export_config) elif isinstance(model, Paraformer): return Paraformer_export(model, **export_config) + elif isinstance(model, E2EVadModel): + return E2EVadModel_export(model, **export_config) else: raise "Funasr does not support the given model type currently." \ No newline at end of file diff --git a/funasr/export/models/e2e_vad.py b/funasr/export/models/e2e_vad.py new file mode 100644 index 000000000..0653e06ba --- /dev/null +++ b/funasr/export/models/e2e_vad.py @@ -0,0 +1,69 @@ +from enum import Enum +from typing import List, Tuple, Dict, Any + +import torch +from torch import nn +import math + +from funasr.models.encoder.fsmn_encoder import FSMN +from funasr.export.models.encoder.fsmn_encoder import FSMN as FSMN_export + +class E2EVadModel(nn.Module): + def __init__(self, model, + max_seq_len=512, + feats_dim=560, + model_name='model', + **kwargs,): + super(E2EVadModel, self).__init__() + self.feats_dim = feats_dim + self.max_seq_len = max_seq_len + self.model_name = model_name + if isinstance(model.encoder, FSMN): + self.encoder = FSMN_export(model.encoder) + else: + raise "unsupported encoder" + + + def forward(self, feats: torch.Tensor, + in_cache0: torch.Tensor, + in_cache1: torch.Tensor, + in_cache2: torch.Tensor, + in_cache3: torch.Tensor, + ): + + scores, cache0, cache1, cache2, cache3 = self.encoder(feats, + in_cache0, + in_cache1, + in_cache2, + in_cache3) # return B * T * D + return scores, cache0, cache1, cache2, cache3 + + def get_dummy_inputs(self, frame=30): + speech = torch.randn(1, frame, self.feats_dim) + in_cache0 = torch.randn(1, 128, 19, 1) + in_cache1 = torch.randn(1, 128, 19, 1) + in_cache2 = torch.randn(1, 128, 19, 1) + in_cache3 = torch.randn(1, 128, 19, 1) + + return (speech, in_cache0, in_cache1, in_cache2, in_cache3) + + # def get_dummy_inputs_txt(self, txt_file: str = "/mnt/workspace/data_fbank/0207/12345.wav.fea.txt"): + # import numpy as np + # fbank = np.loadtxt(txt_file) + # fbank_lengths = np.array([fbank.shape[0], ], dtype=np.int32) + # speech = torch.from_numpy(fbank[None, :, :].astype(np.float32)) + # speech_lengths = torch.from_numpy(fbank_lengths.astype(np.int32)) + # return (speech, speech_lengths) + + def get_input_names(self): + return ['speech', 'in_cache0', 'in_cache1', 'in_cache2', 'in_cache3'] + + def get_output_names(self): + return ['logits', 'out_cache0', 'out_cache1', 'out_cache2', 'out_cache3'] + + def get_dynamic_axes(self): + return { + 'speech': { + 1: 'feats_length' + }, + } diff --git a/funasr/export/models/encoder/fsmn_encoder.py b/funasr/export/models/encoder/fsmn_encoder.py new file mode 100755 index 000000000..bd64a6fa8 --- /dev/null +++ b/funasr/export/models/encoder/fsmn_encoder.py @@ -0,0 +1,297 @@ +from typing import Tuple, Dict +import copy + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from funasr.models.encoder.fsmn_encoder import BasicBlock + +class LinearTransform(nn.Module): + + def __init__(self, input_dim, output_dim): + super(LinearTransform, self).__init__() + self.input_dim = input_dim + self.output_dim = output_dim + self.linear = nn.Linear(input_dim, output_dim, bias=False) + + def forward(self, input): + output = self.linear(input) + + return output + + +class AffineTransform(nn.Module): + + def __init__(self, input_dim, output_dim): + super(AffineTransform, self).__init__() + self.input_dim = input_dim + self.output_dim = output_dim + self.linear = nn.Linear(input_dim, output_dim) + + def forward(self, input): + output = self.linear(input) + + return output + + +class RectifiedLinear(nn.Module): + + def __init__(self, input_dim, output_dim): + super(RectifiedLinear, self).__init__() + self.dim = input_dim + self.relu = nn.ReLU() + self.dropout = nn.Dropout(0.1) + + def forward(self, input): + out = self.relu(input) + return out + + +class FSMNBlock(nn.Module): + + def __init__( + self, + input_dim: int, + output_dim: int, + lorder=None, + rorder=None, + lstride=1, + rstride=1, + ): + super(FSMNBlock, self).__init__() + + self.dim = input_dim + + if lorder is None: + return + + self.lorder = lorder + self.rorder = rorder + self.lstride = lstride + self.rstride = rstride + + self.conv_left = nn.Conv2d( + self.dim, self.dim, [lorder, 1], dilation=[lstride, 1], groups=self.dim, bias=False) + + if self.rorder > 0: + self.conv_right = nn.Conv2d( + self.dim, self.dim, [rorder, 1], dilation=[rstride, 1], groups=self.dim, bias=False) + else: + self.conv_right = None + + def forward(self, input: torch.Tensor, cache: torch.Tensor): + x = torch.unsqueeze(input, 1) + x_per = x.permute(0, 3, 2, 1) # B D T C + + cache = cache.to(x_per.device) + y_left = torch.cat((cache, x_per), dim=2) + cache = y_left[:, :, -(self.lorder - 1) * self.lstride:, :] + y_left = self.conv_left(y_left) + out = x_per + y_left + + if self.conv_right is not None: + # maybe need to check + y_right = F.pad(x_per, [0, 0, 0, self.rorder * self.rstride]) + y_right = y_right[:, :, self.rstride:, :] + y_right = self.conv_right(y_right) + out += y_right + + out_per = out.permute(0, 3, 2, 1) + output = out_per.squeeze(1) + + return output, cache + + +class BasicBlock_export(nn.Module): + def __init__(self, + model, + ): + super(BasicBlock_export, self).__init__() + self.linear = model.linear + self.fsmn_block = model.fsmn_block + self.affine = model.affine + self.relu = model.relu + + def forward(self, input: torch.Tensor, in_cache: torch.Tensor): + x = self.linear(input) # B T D + # cache_layer_name = 'cache_layer_{}'.format(self.stack_layer) + # if cache_layer_name not in in_cache: + # in_cache[cache_layer_name] = torch.zeros(x1.shape[0], x1.shape[-1], (self.lorder - 1) * self.lstride, 1) + x, out_cache = self.fsmn_block(x, in_cache) + x = self.affine(x) + x = self.relu(x) + return x, out_cache + + +# class FsmnStack(nn.Sequential): +# def __init__(self, *args): +# super(FsmnStack, self).__init__(*args) +# +# def forward(self, input: torch.Tensor, in_cache: Dict[str, torch.Tensor]): +# x = input +# for module in self._modules.values(): +# x = module(x, in_cache) +# return x + + +''' +FSMN net for keyword spotting +input_dim: input dimension +linear_dim: fsmn input dimensionll +proj_dim: fsmn projection dimension +lorder: fsmn left order +rorder: fsmn right order +num_syn: output dimension +fsmn_layers: no. of sequential fsmn layers +''' + + +class FSMN(nn.Module): + def __init__( + self, + model, + ): + super(FSMN, self).__init__() + + # self.input_dim = input_dim + # self.input_affine_dim = input_affine_dim + # self.fsmn_layers = fsmn_layers + # self.linear_dim = linear_dim + # self.proj_dim = proj_dim + # self.output_affine_dim = output_affine_dim + # self.output_dim = output_dim + # + # self.in_linear1 = AffineTransform(input_dim, input_affine_dim) + # self.in_linear2 = AffineTransform(input_affine_dim, linear_dim) + # self.relu = RectifiedLinear(linear_dim, linear_dim) + # self.fsmn = FsmnStack(*[BasicBlock(linear_dim, proj_dim, lorder, rorder, lstride, rstride, i) for i in + # range(fsmn_layers)]) + # self.out_linear1 = AffineTransform(linear_dim, output_affine_dim) + # self.out_linear2 = AffineTransform(output_affine_dim, output_dim) + # self.softmax = nn.Softmax(dim=-1) + self.in_linear1 = model.in_linear1 + self.in_linear2 = model.in_linear2 + self.relu = model.relu + # self.fsmn = model.fsmn + self.out_linear1 = model.out_linear1 + self.out_linear2 = model.out_linear2 + self.softmax = model.softmax + + for i, d in enumerate(self.model.fsmn): + if isinstance(d, BasicBlock): + self.model.fsmn[i] = BasicBlock_export(d) + + def fuse_modules(self): + pass + + def forward( + self, + input: torch.Tensor, + *args, + ): + """ + Args: + input (torch.Tensor): Input tensor (B, T, D) + in_cache: when in_cache is not None, the forward is in streaming. The type of in_cache is a dict, egs, + {'cache_layer_1': torch.Tensor(B, T1, D)}, T1 is equal to self.lorder. It is {} for the 1st frame + """ + + x = self.in_linear1(input) + x = self.in_linear2(x) + x = self.relu(x) + # x4 = self.fsmn(x3, in_cache) # self.in_cache will update automatically in self.fsmn + out_caches = list() + for i, d in enumerate(self.model.fsmn): + in_cache = args[i] + x, out_cache = d(x, in_cache) + out_caches.append(out_cache) + x = self.out_linear1(x) + x = self.out_linear2(x) + x = self.softmax(x) + + return x, *out_caches + + +''' +one deep fsmn layer +dimproj: projection dimension, input and output dimension of memory blocks +dimlinear: dimension of mapping layer +lorder: left order +rorder: right order +lstride: left stride +rstride: right stride +''' + + +class DFSMN(nn.Module): + + def __init__(self, dimproj=64, dimlinear=128, lorder=20, rorder=1, lstride=1, rstride=1): + super(DFSMN, self).__init__() + + self.lorder = lorder + self.rorder = rorder + self.lstride = lstride + self.rstride = rstride + + self.expand = AffineTransform(dimproj, dimlinear) + self.shrink = LinearTransform(dimlinear, dimproj) + + self.conv_left = nn.Conv2d( + dimproj, dimproj, [lorder, 1], dilation=[lstride, 1], groups=dimproj, bias=False) + + if rorder > 0: + self.conv_right = nn.Conv2d( + dimproj, dimproj, [rorder, 1], dilation=[rstride, 1], groups=dimproj, bias=False) + else: + self.conv_right = None + + def forward(self, input): + f1 = F.relu(self.expand(input)) + p1 = self.shrink(f1) + + x = torch.unsqueeze(p1, 1) + x_per = x.permute(0, 3, 2, 1) + + y_left = F.pad(x_per, [0, 0, (self.lorder - 1) * self.lstride, 0]) + + if self.conv_right is not None: + y_right = F.pad(x_per, [0, 0, 0, (self.rorder) * self.rstride]) + y_right = y_right[:, :, self.rstride:, :] + out = x_per + self.conv_left(y_left) + self.conv_right(y_right) + else: + out = x_per + self.conv_left(y_left) + + out1 = out.permute(0, 3, 2, 1) + output = input + out1.squeeze(1) + + return output + + +''' +build stacked dfsmn layers +''' + + +def buildDFSMNRepeats(linear_dim=128, proj_dim=64, lorder=20, rorder=1, fsmn_layers=6): + repeats = [ + nn.Sequential( + DFSMN(proj_dim, linear_dim, lorder, rorder, 1, 1)) + for i in range(fsmn_layers) + ] + + return nn.Sequential(*repeats) + + +if __name__ == '__main__': + fsmn = FSMN(400, 140, 4, 250, 128, 10, 2, 1, 1, 140, 2599) + print(fsmn) + + num_params = sum(p.numel() for p in fsmn.parameters()) + print('the number of model params: {}'.format(num_params)) + x = torch.zeros(128, 200, 400) # batch-size * time * dim + y, _ = fsmn(x) # batch-size * time * dim + print('input shape: {}'.format(x.shape)) + print('output shape: {}'.format(y.shape)) + + print(fsmn.to_kaldi_net()) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py new file mode 100644 index 000000000..e69de29bb From 8a788ad0d922c7d1b7c597a610b131f40c93e2b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Tue, 28 Mar 2023 20:08:09 +0800 Subject: [PATCH 02/55] export --- funasr/export/export_model.py | 1 + funasr/export/models/e2e_vad.py | 4 ++-- funasr/export/models/encoder/fsmn_encoder.py | 13 ++++++------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/funasr/export/export_model.py b/funasr/export/export_model.py index de57b1b13..cad3367a3 100644 --- a/funasr/export/export_model.py +++ b/funasr/export/export_model.py @@ -193,6 +193,7 @@ class ModelExport: model, vad_infer_args = VADTask.build_model_from_file( config, model_file, 'cpu' ) + self.export_config["feats_dim"] = 400 self._export(model, tag_name) diff --git a/funasr/export/models/e2e_vad.py b/funasr/export/models/e2e_vad.py index 0653e06ba..b4236e0b5 100644 --- a/funasr/export/models/e2e_vad.py +++ b/funasr/export/models/e2e_vad.py @@ -11,7 +11,7 @@ from funasr.export.models.encoder.fsmn_encoder import FSMN as FSMN_export class E2EVadModel(nn.Module): def __init__(self, model, max_seq_len=512, - feats_dim=560, + feats_dim=400, model_name='model', **kwargs,): super(E2EVadModel, self).__init__() @@ -31,7 +31,7 @@ class E2EVadModel(nn.Module): in_cache3: torch.Tensor, ): - scores, cache0, cache1, cache2, cache3 = self.encoder(feats, + scores, (cache0, cache1, cache2, cache3) = self.encoder(feats, in_cache0, in_cache1, in_cache2, diff --git a/funasr/export/models/encoder/fsmn_encoder.py b/funasr/export/models/encoder/fsmn_encoder.py index bd64a6fa8..b8e64339c 100755 --- a/funasr/export/models/encoder/fsmn_encoder.py +++ b/funasr/export/models/encoder/fsmn_encoder.py @@ -149,8 +149,7 @@ fsmn_layers: no. of sequential fsmn layers class FSMN(nn.Module): def __init__( - self, - model, + self, model, ): super(FSMN, self).__init__() @@ -177,10 +176,10 @@ class FSMN(nn.Module): self.out_linear1 = model.out_linear1 self.out_linear2 = model.out_linear2 self.softmax = model.softmax - - for i, d in enumerate(self.model.fsmn): + self.fsmn = model.fsmn + for i, d in enumerate(model.fsmn): if isinstance(d, BasicBlock): - self.model.fsmn[i] = BasicBlock_export(d) + self.fsmn[i] = BasicBlock_export(d) def fuse_modules(self): pass @@ -202,7 +201,7 @@ class FSMN(nn.Module): x = self.relu(x) # x4 = self.fsmn(x3, in_cache) # self.in_cache will update automatically in self.fsmn out_caches = list() - for i, d in enumerate(self.model.fsmn): + for i, d in enumerate(self.fsmn): in_cache = args[i] x, out_cache = d(x, in_cache) out_caches.append(out_cache) @@ -210,7 +209,7 @@ class FSMN(nn.Module): x = self.out_linear2(x) x = self.softmax(x) - return x, *out_caches + return x, out_caches ''' From 30433b58d68b93f85e61f304af32e6c7c8ef1f11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Tue, 28 Mar 2023 20:23:01 +0800 Subject: [PATCH 03/55] export --- funasr/export/test/__init__.py | 0 funasr/export/{ => test}/test_onnx.py | 0 funasr/export/test/test_onnx_vad.py | 26 +++++++++++++++++++ funasr/export/{ => test}/test_torchscripts.py | 0 4 files changed, 26 insertions(+) create mode 100644 funasr/export/test/__init__.py rename funasr/export/{ => test}/test_onnx.py (100%) create mode 100644 funasr/export/test/test_onnx_vad.py rename funasr/export/{ => test}/test_torchscripts.py (100%) diff --git a/funasr/export/test/__init__.py b/funasr/export/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/export/test_onnx.py b/funasr/export/test/test_onnx.py similarity index 100% rename from funasr/export/test_onnx.py rename to funasr/export/test/test_onnx.py diff --git a/funasr/export/test/test_onnx_vad.py b/funasr/export/test/test_onnx_vad.py new file mode 100644 index 000000000..12f058f8e --- /dev/null +++ b/funasr/export/test/test_onnx_vad.py @@ -0,0 +1,26 @@ +import onnxruntime +import numpy as np + + +if __name__ == '__main__': + onnx_path = "/mnt/workspace/export/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/model.onnx" + sess = onnxruntime.InferenceSession(onnx_path) + input_name = [nd.name for nd in sess.get_inputs()] + output_name = [nd.name for nd in sess.get_outputs()] + + def _get_feed_dict(feats_length): + + return {'speech': np.random.rand(1, feats_length, 400).astype(np.float32), + 'in_cache0': np.random.rand(1, 128, 19, 1).astype(np.float32), + 'in_cache1': np.random.rand(1, 128, 19, 1).astype(np.float32), + 'in_cache2': np.random.rand(1, 128, 19, 1).astype(np.float32), + 'in_cache3': np.random.rand(1, 128, 19, 1).astype(np.float32), + } + + def _run(feed_dict): + output = sess.run(output_name, input_feed=feed_dict) + for name, value in zip(output_name, output): + print('{}: {}'.format(name, value.shape)) + + _run(_get_feed_dict(100)) + _run(_get_feed_dict(200)) \ No newline at end of file diff --git a/funasr/export/test_torchscripts.py b/funasr/export/test/test_torchscripts.py similarity index 100% rename from funasr/export/test_torchscripts.py rename to funasr/export/test/test_torchscripts.py From 5e59904fd49ff1fcb0d6869d297e05a59707bf58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Tue, 28 Mar 2023 20:34:53 +0800 Subject: [PATCH 04/55] export --- funasr/export/models/e2e_vad.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/funasr/export/models/e2e_vad.py b/funasr/export/models/e2e_vad.py index b4236e0b5..d3e8f30e5 100644 --- a/funasr/export/models/e2e_vad.py +++ b/funasr/export/models/e2e_vad.py @@ -24,19 +24,10 @@ class E2EVadModel(nn.Module): raise "unsupported encoder" - def forward(self, feats: torch.Tensor, - in_cache0: torch.Tensor, - in_cache1: torch.Tensor, - in_cache2: torch.Tensor, - in_cache3: torch.Tensor, - ): + def forward(self, feats: torch.Tensor, *args, ): - scores, (cache0, cache1, cache2, cache3) = self.encoder(feats, - in_cache0, - in_cache1, - in_cache2, - in_cache3) # return B * T * D - return scores, cache0, cache1, cache2, cache3 + scores, out_caches = self.encoder(feats, *args) + return scores, out_caches def get_dummy_inputs(self, frame=30): speech = torch.randn(1, frame, self.feats_dim) From 7c5fdf30f428e22fd0fdb98055834e0d2616d308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 00:27:11 +0800 Subject: [PATCH 05/55] export --- .../build/lib/funasr_onnx/__init__.py | 3 + .../build/lib/funasr_onnx/paraformer_bin.py | 187 ++++++ .../build/lib/funasr_onnx/punc_bin.py | 0 .../build/lib/funasr_onnx/utils/__init__.py | 0 .../build/lib/funasr_onnx/utils/e2e_vad.py | 607 ++++++++++++++++++ .../build/lib/funasr_onnx/utils/frontend.py | 191 ++++++ .../funasr_onnx/utils/postprocess_utils.py | 240 +++++++ .../lib/funasr_onnx/utils/timestamp_utils.py | 59 ++ .../build/lib/funasr_onnx/utils/utils.py | 257 ++++++++ .../build/lib/funasr_onnx/vad_bin.py | 166 +++++ funasr/runtime/python/onnxruntime/demo.py | 2 - funasr/runtime/python/onnxruntime/demo_vad.py | 12 + .../dist/funasr_onnx-0.0.2-py3.8.egg | Bin 0 -> 47828 bytes .../dist/funasr_onnx-0.0.3-py3.8.egg | Bin 0 -> 47828 bytes .../onnxruntime/funasr_onnx.egg-info/PKG-INFO | 80 +++ .../funasr_onnx.egg-info/SOURCES.txt | 17 + .../funasr_onnx.egg-info/dependency_links.txt | 1 + .../funasr_onnx.egg-info/requires.txt | 7 + .../funasr_onnx.egg-info/top_level.txt | 1 + .../onnxruntime/funasr_onnx/__init__.py | 1 + .../onnxruntime/funasr_onnx/utils/e2e_vad.py | 607 ++++++++++++++++++ .../onnxruntime/funasr_onnx/utils/utils.py | 2 +- .../python/onnxruntime/funasr_onnx/vad_bin.py | 134 ++++ funasr/runtime/python/onnxruntime/setup.py | 2 +- 24 files changed, 2572 insertions(+), 4 deletions(-) create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/__init__.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/paraformer_bin.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/punc_bin.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/__init__.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/e2e_vad.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/frontend.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/postprocess_utils.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/timestamp_utils.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/utils.py create mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/vad_bin.py create mode 100644 funasr/runtime/python/onnxruntime/demo_vad.py create mode 100644 funasr/runtime/python/onnxruntime/dist/funasr_onnx-0.0.2-py3.8.egg create mode 100644 funasr/runtime/python/onnxruntime/dist/funasr_onnx-0.0.3-py3.8.egg create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx/utils/e2e_vad.py create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/__init__.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/__init__.py new file mode 100644 index 000000000..475047903 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/__init__.py @@ -0,0 +1,3 @@ +# -*- encoding: utf-8 -*- +from .paraformer_bin import Paraformer +from .vad_bin import Fsmn_vad diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/paraformer_bin.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/paraformer_bin.py new file mode 100644 index 000000000..cbdb8d9e6 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/paraformer_bin.py @@ -0,0 +1,187 @@ +# -*- encoding: utf-8 -*- + +import os.path +from pathlib import Path +from typing import List, Union, Tuple + +import copy +import librosa +import numpy as np + +from .utils.utils import (CharTokenizer, Hypothesis, ONNXRuntimeError, + OrtInferSession, TokenIDConverter, get_logger, + read_yaml) +from .utils.postprocess_utils import sentence_postprocess +from .utils.frontend import WavFrontend +from .utils.timestamp_utils import time_stamp_lfr6_onnx + +logging = get_logger() + + +class Paraformer(): + def __init__(self, model_dir: Union[str, Path] = None, + batch_size: int = 1, + device_id: Union[str, int] = "-1", + plot_timestamp_to: str = "", + pred_bias: int = 1, + quantize: bool = False, + intra_op_num_threads: int = 4, + ): + + if not Path(model_dir).exists(): + raise FileNotFoundError(f'{model_dir} does not exist.') + + model_file = os.path.join(model_dir, 'model.onnx') + if quantize: + model_file = os.path.join(model_dir, 'model_quant.onnx') + config_file = os.path.join(model_dir, 'config.yaml') + cmvn_file = os.path.join(model_dir, 'am.mvn') + config = read_yaml(config_file) + + self.converter = TokenIDConverter(config['token_list']) + self.tokenizer = CharTokenizer() + self.frontend = WavFrontend( + cmvn_file=cmvn_file, + **config['frontend_conf'] + ) + self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) + self.batch_size = batch_size + self.plot_timestamp_to = plot_timestamp_to + self.pred_bias = pred_bias + + def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> List: + waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) + waveform_nums = len(waveform_list) + asr_res = [] + for beg_idx in range(0, waveform_nums, self.batch_size): + + end_idx = min(waveform_nums, beg_idx + self.batch_size) + feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx]) + try: + outputs = self.infer(feats, feats_len) + am_scores, valid_token_lens = outputs[0], outputs[1] + if len(outputs) == 4: + # for BiCifParaformer Inference + us_alphas, us_peaks = outputs[2], outputs[3] + else: + us_alphas, us_peaks = None, None + except ONNXRuntimeError: + #logging.warning(traceback.format_exc()) + logging.warning("input wav is silence or noise") + preds = [''] + else: + preds = self.decode(am_scores, valid_token_lens) + if us_peaks is None: + for pred in preds: + pred = sentence_postprocess(pred) + asr_res.append({'preds': pred}) + else: + for pred, us_peaks_ in zip(preds, us_peaks): + raw_tokens = pred + timestamp, timestamp_raw = time_stamp_lfr6_onnx(us_peaks_, copy.copy(raw_tokens)) + text_proc, timestamp_proc, _ = sentence_postprocess(raw_tokens, timestamp_raw) + # logging.warning(timestamp) + if len(self.plot_timestamp_to): + self.plot_wave_timestamp(waveform_list[0], timestamp, self.plot_timestamp_to) + asr_res.append({'preds': text_proc, 'timestamp': timestamp_proc, "raw_tokens": raw_tokens}) + return asr_res + + def plot_wave_timestamp(self, wav, text_timestamp, dest): + # TODO: Plot the wav and timestamp results with matplotlib + import matplotlib + matplotlib.use('Agg') + matplotlib.rc("font", family='Alibaba PuHuiTi') # set it to a font that your system supports + import matplotlib.pyplot as plt + fig, ax1 = plt.subplots(figsize=(11, 3.5), dpi=320) + ax2 = ax1.twinx() + ax2.set_ylim([0, 2.0]) + # plot waveform + ax1.set_ylim([-0.3, 0.3]) + time = np.arange(wav.shape[0]) / 16000 + ax1.plot(time, wav/wav.max()*0.3, color='gray', alpha=0.4) + # plot lines and text + for (char, start, end) in text_timestamp: + ax1.vlines(start, -0.3, 0.3, ls='--') + ax1.vlines(end, -0.3, 0.3, ls='--') + x_adj = 0.045 if char != '' else 0.12 + ax1.text((start + end) * 0.5 - x_adj, 0, char) + # plt.legend() + plotname = "{}/timestamp.png".format(dest) + plt.savefig(plotname, bbox_inches='tight') + + def load_data(self, + wav_content: Union[str, np.ndarray, List[str]], fs: int = None) -> List: + def load_wav(path: str) -> np.ndarray: + waveform, _ = librosa.load(path, sr=fs) + return waveform + + if isinstance(wav_content, np.ndarray): + return [wav_content] + + if isinstance(wav_content, str): + return [load_wav(wav_content)] + + if isinstance(wav_content, list): + return [load_wav(path) for path in wav_content] + + raise TypeError( + f'The type of {wav_content} is not in [str, np.ndarray, list]') + + def extract_feat(self, + waveform_list: List[np.ndarray] + ) -> Tuple[np.ndarray, np.ndarray]: + feats, feats_len = [], [] + for waveform in waveform_list: + speech, _ = self.frontend.fbank(waveform) + feat, feat_len = self.frontend.lfr_cmvn(speech) + feats.append(feat) + feats_len.append(feat_len) + + feats = self.pad_feats(feats, np.max(feats_len)) + feats_len = np.array(feats_len).astype(np.int32) + return feats, feats_len + + @staticmethod + def pad_feats(feats: List[np.ndarray], max_feat_len: int) -> np.ndarray: + def pad_feat(feat: np.ndarray, cur_len: int) -> np.ndarray: + pad_width = ((0, max_feat_len - cur_len), (0, 0)) + return np.pad(feat, pad_width, 'constant', constant_values=0) + + feat_res = [pad_feat(feat, feat.shape[0]) for feat in feats] + feats = np.array(feat_res).astype(np.float32) + return feats + + def infer(self, feats: np.ndarray, + feats_len: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + outputs = self.ort_infer([feats, feats_len]) + return outputs + + def decode(self, am_scores: np.ndarray, token_nums: int) -> List[str]: + return [self.decode_one(am_score, token_num) + for am_score, token_num in zip(am_scores, token_nums)] + + def decode_one(self, + am_score: np.ndarray, + valid_token_num: int) -> List[str]: + yseq = am_score.argmax(axis=-1) + score = am_score.max(axis=-1) + score = np.sum(score, axis=-1) + + # pad with mask tokens to ensure compatibility with sos/eos tokens + # asr_model.sos:1 asr_model.eos:2 + yseq = np.array([1] + yseq.tolist() + [2]) + hyp = Hypothesis(yseq=yseq, score=score) + + # remove sos/eos and get results + last_pos = -1 + token_int = hyp.yseq[1:last_pos].tolist() + + # remove blank symbol id, which is assumed to be 0 + token_int = list(filter(lambda x: x not in (0, 2), token_int)) + + # Change integer-ids to tokens + token = self.converter.ids2tokens(token_int) + token = token[:valid_token_num-self.pred_bias] + # texts = sentence_postprocess(token) + return token + diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/punc_bin.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/__init__.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/e2e_vad.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/e2e_vad.py new file mode 100644 index 000000000..8eed22fa4 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/e2e_vad.py @@ -0,0 +1,607 @@ +from enum import Enum +from typing import List, Tuple, Dict, Any + +import math +import numpy as np + +class VadStateMachine(Enum): + kVadInStateStartPointNotDetected = 1 + kVadInStateInSpeechSegment = 2 + kVadInStateEndPointDetected = 3 + + +class FrameState(Enum): + kFrameStateInvalid = -1 + kFrameStateSpeech = 1 + kFrameStateSil = 0 + + +# final voice/unvoice state per frame +class AudioChangeState(Enum): + kChangeStateSpeech2Speech = 0 + kChangeStateSpeech2Sil = 1 + kChangeStateSil2Sil = 2 + kChangeStateSil2Speech = 3 + kChangeStateNoBegin = 4 + kChangeStateInvalid = 5 + + +class VadDetectMode(Enum): + kVadSingleUtteranceDetectMode = 0 + kVadMutipleUtteranceDetectMode = 1 + + +class VADXOptions: + def __init__( + self, + sample_rate: int = 16000, + detect_mode: int = VadDetectMode.kVadMutipleUtteranceDetectMode.value, + snr_mode: int = 0, + max_end_silence_time: int = 800, + max_start_silence_time: int = 3000, + do_start_point_detection: bool = True, + do_end_point_detection: bool = True, + window_size_ms: int = 200, + sil_to_speech_time_thres: int = 150, + speech_to_sil_time_thres: int = 150, + speech_2_noise_ratio: float = 1.0, + do_extend: int = 1, + lookback_time_start_point: int = 200, + lookahead_time_end_point: int = 100, + max_single_segment_time: int = 60000, + nn_eval_block_size: int = 8, + dcd_block_size: int = 4, + snr_thres: int = -100.0, + noise_frame_num_used_for_snr: int = 100, + decibel_thres: int = -100.0, + speech_noise_thres: float = 0.6, + fe_prior_thres: float = 1e-4, + silence_pdf_num: int = 1, + sil_pdf_ids: List[int] = [0], + speech_noise_thresh_low: float = -0.1, + speech_noise_thresh_high: float = 0.3, + output_frame_probs: bool = False, + frame_in_ms: int = 10, + frame_length_ms: int = 25, + ): + self.sample_rate = sample_rate + self.detect_mode = detect_mode + self.snr_mode = snr_mode + self.max_end_silence_time = max_end_silence_time + self.max_start_silence_time = max_start_silence_time + self.do_start_point_detection = do_start_point_detection + self.do_end_point_detection = do_end_point_detection + self.window_size_ms = window_size_ms + self.sil_to_speech_time_thres = sil_to_speech_time_thres + self.speech_to_sil_time_thres = speech_to_sil_time_thres + self.speech_2_noise_ratio = speech_2_noise_ratio + self.do_extend = do_extend + self.lookback_time_start_point = lookback_time_start_point + self.lookahead_time_end_point = lookahead_time_end_point + self.max_single_segment_time = max_single_segment_time + self.nn_eval_block_size = nn_eval_block_size + self.dcd_block_size = dcd_block_size + self.snr_thres = snr_thres + self.noise_frame_num_used_for_snr = noise_frame_num_used_for_snr + self.decibel_thres = decibel_thres + self.speech_noise_thres = speech_noise_thres + self.fe_prior_thres = fe_prior_thres + self.silence_pdf_num = silence_pdf_num + self.sil_pdf_ids = sil_pdf_ids + self.speech_noise_thresh_low = speech_noise_thresh_low + self.speech_noise_thresh_high = speech_noise_thresh_high + self.output_frame_probs = output_frame_probs + self.frame_in_ms = frame_in_ms + self.frame_length_ms = frame_length_ms + + +class E2EVadSpeechBufWithDoa(object): + def __init__(self): + self.start_ms = 0 + self.end_ms = 0 + self.buffer = [] + self.contain_seg_start_point = False + self.contain_seg_end_point = False + self.doa = 0 + + def Reset(self): + self.start_ms = 0 + self.end_ms = 0 + self.buffer = [] + self.contain_seg_start_point = False + self.contain_seg_end_point = False + self.doa = 0 + + +class E2EVadFrameProb(object): + def __init__(self): + self.noise_prob = 0.0 + self.speech_prob = 0.0 + self.score = 0.0 + self.frame_id = 0 + self.frm_state = 0 + + +class WindowDetector(object): + def __init__(self, window_size_ms: int, sil_to_speech_time: int, + speech_to_sil_time: int, frame_size_ms: int): + self.window_size_ms = window_size_ms + self.sil_to_speech_time = sil_to_speech_time + self.speech_to_sil_time = speech_to_sil_time + self.frame_size_ms = frame_size_ms + + self.win_size_frame = int(window_size_ms / frame_size_ms) + self.win_sum = 0 + self.win_state = [0] * self.win_size_frame # 初始化窗 + + self.cur_win_pos = 0 + self.pre_frame_state = FrameState.kFrameStateSil + self.cur_frame_state = FrameState.kFrameStateSil + self.sil_to_speech_frmcnt_thres = int(sil_to_speech_time / frame_size_ms) + self.speech_to_sil_frmcnt_thres = int(speech_to_sil_time / frame_size_ms) + + self.voice_last_frame_count = 0 + self.noise_last_frame_count = 0 + self.hydre_frame_count = 0 + + def Reset(self) -> None: + self.cur_win_pos = 0 + self.win_sum = 0 + self.win_state = [0] * self.win_size_frame + self.pre_frame_state = FrameState.kFrameStateSil + self.cur_frame_state = FrameState.kFrameStateSil + self.voice_last_frame_count = 0 + self.noise_last_frame_count = 0 + self.hydre_frame_count = 0 + + def GetWinSize(self) -> int: + return int(self.win_size_frame) + + def DetectOneFrame(self, frameState: FrameState, frame_count: int) -> AudioChangeState: + cur_frame_state = FrameState.kFrameStateSil + if frameState == FrameState.kFrameStateSpeech: + cur_frame_state = 1 + elif frameState == FrameState.kFrameStateSil: + cur_frame_state = 0 + else: + return AudioChangeState.kChangeStateInvalid + self.win_sum -= self.win_state[self.cur_win_pos] + self.win_sum += cur_frame_state + self.win_state[self.cur_win_pos] = cur_frame_state + self.cur_win_pos = (self.cur_win_pos + 1) % self.win_size_frame + + if self.pre_frame_state == FrameState.kFrameStateSil and self.win_sum >= self.sil_to_speech_frmcnt_thres: + self.pre_frame_state = FrameState.kFrameStateSpeech + return AudioChangeState.kChangeStateSil2Speech + + if self.pre_frame_state == FrameState.kFrameStateSpeech and self.win_sum <= self.speech_to_sil_frmcnt_thres: + self.pre_frame_state = FrameState.kFrameStateSil + return AudioChangeState.kChangeStateSpeech2Sil + + if self.pre_frame_state == FrameState.kFrameStateSil: + return AudioChangeState.kChangeStateSil2Sil + if self.pre_frame_state == FrameState.kFrameStateSpeech: + return AudioChangeState.kChangeStateSpeech2Speech + return AudioChangeState.kChangeStateInvalid + + def FrameSizeMs(self) -> int: + return int(self.frame_size_ms) + + +class E2EVadModel(): + def __init__(self, vad_post_args: Dict[str, Any]): + super(E2EVadModel, self).__init__() + self.vad_opts = VADXOptions(**vad_post_args) + self.windows_detector = WindowDetector(self.vad_opts.window_size_ms, + self.vad_opts.sil_to_speech_time_thres, + self.vad_opts.speech_to_sil_time_thres, + self.vad_opts.frame_in_ms) + # self.encoder = encoder + # init variables + self.is_final = False + self.data_buf_start_frame = 0 + self.frm_cnt = 0 + self.latest_confirmed_speech_frame = 0 + self.lastest_confirmed_silence_frame = -1 + self.continous_silence_frame_count = 0 + self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected + self.confirmed_start_frame = -1 + self.confirmed_end_frame = -1 + self.number_end_time_detected = 0 + self.sil_frame = 0 + self.sil_pdf_ids = self.vad_opts.sil_pdf_ids + self.noise_average_decibel = -100.0 + self.pre_end_silence_detected = False + self.next_seg = True + + self.output_data_buf = [] + self.output_data_buf_offset = 0 + self.frame_probs = [] + self.max_end_sil_frame_cnt_thresh = self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres + self.speech_noise_thres = self.vad_opts.speech_noise_thres + self.scores = None + self.max_time_out = False + self.decibel = [] + self.data_buf = None + self.data_buf_all = None + self.waveform = None + self.ResetDetection() + + def AllResetDetection(self): + self.is_final = False + self.data_buf_start_frame = 0 + self.frm_cnt = 0 + self.latest_confirmed_speech_frame = 0 + self.lastest_confirmed_silence_frame = -1 + self.continous_silence_frame_count = 0 + self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected + self.confirmed_start_frame = -1 + self.confirmed_end_frame = -1 + self.number_end_time_detected = 0 + self.sil_frame = 0 + self.sil_pdf_ids = self.vad_opts.sil_pdf_ids + self.noise_average_decibel = -100.0 + self.pre_end_silence_detected = False + self.next_seg = True + + self.output_data_buf = [] + self.output_data_buf_offset = 0 + self.frame_probs = [] + self.max_end_sil_frame_cnt_thresh = self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres + self.speech_noise_thres = self.vad_opts.speech_noise_thres + self.scores = None + self.max_time_out = False + self.decibel = [] + self.data_buf = None + self.data_buf_all = None + self.waveform = None + self.ResetDetection() + + def ResetDetection(self): + self.continous_silence_frame_count = 0 + self.latest_confirmed_speech_frame = 0 + self.lastest_confirmed_silence_frame = -1 + self.confirmed_start_frame = -1 + self.confirmed_end_frame = -1 + self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected + self.windows_detector.Reset() + self.sil_frame = 0 + self.frame_probs = [] + + def ComputeDecibel(self) -> None: + frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000) + frame_shift_length = int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) + if self.data_buf_all is None: + self.data_buf_all = self.waveform[0] # self.data_buf is pointed to self.waveform[0] + self.data_buf = self.data_buf_all + else: + self.data_buf_all = np.concatenate((self.data_buf_all, self.waveform[0])) + for offset in range(0, self.waveform.shape[1] - frame_sample_length + 1, frame_shift_length): + self.decibel.append( + 10 * math.log10((self.waveform[0][offset: offset + frame_sample_length]).square().sum() + \ + 0.000001)) + + def ComputeScores(self, scores: np.ndarray) -> None: + # scores = self.encoder(feats, in_cache) # return B * T * D + self.vad_opts.nn_eval_block_size = scores.shape[1] + self.frm_cnt += scores.shape[1] # count total frames + if self.scores is None: + self.scores = scores # the first calculation + else: + self.scores = np.concatenate((self.scores, scores), axis=1) + + def PopDataBufTillFrame(self, frame_idx: int) -> None: # need check again + while self.data_buf_start_frame < frame_idx: + if len(self.data_buf) >= int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000): + self.data_buf_start_frame += 1 + self.data_buf = self.data_buf_all[self.data_buf_start_frame * int( + self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000):] + + def PopDataToOutputBuf(self, start_frm: int, frm_cnt: int, first_frm_is_start_point: bool, + last_frm_is_end_point: bool, end_point_is_sent_end: bool) -> None: + self.PopDataBufTillFrame(start_frm) + expected_sample_number = int(frm_cnt * self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000) + if last_frm_is_end_point: + extra_sample = max(0, int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 - \ + self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000)) + expected_sample_number += int(extra_sample) + if end_point_is_sent_end: + expected_sample_number = max(expected_sample_number, len(self.data_buf)) + if len(self.data_buf) < expected_sample_number: + print('error in calling pop data_buf\n') + + if len(self.output_data_buf) == 0 or first_frm_is_start_point: + self.output_data_buf.append(E2EVadSpeechBufWithDoa()) + self.output_data_buf[-1].Reset() + self.output_data_buf[-1].start_ms = start_frm * self.vad_opts.frame_in_ms + self.output_data_buf[-1].end_ms = self.output_data_buf[-1].start_ms + self.output_data_buf[-1].doa = 0 + cur_seg = self.output_data_buf[-1] + if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: + print('warning\n') + out_pos = len(cur_seg.buffer) # cur_seg.buff现在没做任何操作 + data_to_pop = 0 + if end_point_is_sent_end: + data_to_pop = expected_sample_number + else: + data_to_pop = int(frm_cnt * self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) + if data_to_pop > len(self.data_buf): + print('VAD data_to_pop is bigger than self.data_buf.size()!!!\n') + data_to_pop = len(self.data_buf) + expected_sample_number = len(self.data_buf) + + cur_seg.doa = 0 + for sample_cpy_out in range(0, data_to_pop): + # cur_seg.buffer[out_pos ++] = data_buf_.back(); + out_pos += 1 + for sample_cpy_out in range(data_to_pop, expected_sample_number): + # cur_seg.buffer[out_pos++] = data_buf_.back() + out_pos += 1 + if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: + print('Something wrong with the VAD algorithm\n') + self.data_buf_start_frame += frm_cnt + cur_seg.end_ms = (start_frm + frm_cnt) * self.vad_opts.frame_in_ms + if first_frm_is_start_point: + cur_seg.contain_seg_start_point = True + if last_frm_is_end_point: + cur_seg.contain_seg_end_point = True + + def OnSilenceDetected(self, valid_frame: int): + self.lastest_confirmed_silence_frame = valid_frame + if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + self.PopDataBufTillFrame(valid_frame) + # silence_detected_callback_ + # pass + + def OnVoiceDetected(self, valid_frame: int) -> None: + self.latest_confirmed_speech_frame = valid_frame + self.PopDataToOutputBuf(valid_frame, 1, False, False, False) + + def OnVoiceStart(self, start_frame: int, fake_result: bool = False) -> None: + if self.vad_opts.do_start_point_detection: + pass + if self.confirmed_start_frame != -1: + print('not reset vad properly\n') + else: + self.confirmed_start_frame = start_frame + + if not fake_result and self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + self.PopDataToOutputBuf(self.confirmed_start_frame, 1, True, False, False) + + def OnVoiceEnd(self, end_frame: int, fake_result: bool, is_last_frame: bool) -> None: + for t in range(self.latest_confirmed_speech_frame + 1, end_frame): + self.OnVoiceDetected(t) + if self.vad_opts.do_end_point_detection: + pass + if self.confirmed_end_frame != -1: + print('not reset vad properly\n') + else: + self.confirmed_end_frame = end_frame + if not fake_result: + self.sil_frame = 0 + self.PopDataToOutputBuf(self.confirmed_end_frame, 1, False, True, is_last_frame) + self.number_end_time_detected += 1 + + def MaybeOnVoiceEndIfLastFrame(self, is_final_frame: bool, cur_frm_idx: int) -> None: + if is_final_frame: + self.OnVoiceEnd(cur_frm_idx, False, True) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + + def GetLatency(self) -> int: + return int(self.LatencyFrmNumAtStartPoint() * self.vad_opts.frame_in_ms) + + def LatencyFrmNumAtStartPoint(self) -> int: + vad_latency = self.windows_detector.GetWinSize() + if self.vad_opts.do_extend: + vad_latency += int(self.vad_opts.lookback_time_start_point / self.vad_opts.frame_in_ms) + return vad_latency + + def GetFrameState(self, t: int) -> FrameState: + frame_state = FrameState.kFrameStateInvalid + cur_decibel = self.decibel[t] + cur_snr = cur_decibel - self.noise_average_decibel + # for each frame, calc log posterior probability of each state + if cur_decibel < self.vad_opts.decibel_thres: + frame_state = FrameState.kFrameStateSil + self.DetectOneFrame(frame_state, t, False) + return frame_state + + sum_score = 0.0 + noise_prob = 0.0 + assert len(self.sil_pdf_ids) == self.vad_opts.silence_pdf_num + if len(self.sil_pdf_ids) > 0: + assert len(self.scores) == 1 # 只支持batch_size = 1的测试 + sil_pdf_scores = [self.scores[0][t][sil_pdf_id] for sil_pdf_id in self.sil_pdf_ids] + sum_score = sum(sil_pdf_scores) + noise_prob = math.log(sum_score) * self.vad_opts.speech_2_noise_ratio + total_score = 1.0 + sum_score = total_score - sum_score + speech_prob = math.log(sum_score) + if self.vad_opts.output_frame_probs: + frame_prob = E2EVadFrameProb() + frame_prob.noise_prob = noise_prob + frame_prob.speech_prob = speech_prob + frame_prob.score = sum_score + frame_prob.frame_id = t + self.frame_probs.append(frame_prob) + if math.exp(speech_prob) >= math.exp(noise_prob) + self.speech_noise_thres: + if cur_snr >= self.vad_opts.snr_thres and cur_decibel >= self.vad_opts.decibel_thres: + frame_state = FrameState.kFrameStateSpeech + else: + frame_state = FrameState.kFrameStateSil + else: + frame_state = FrameState.kFrameStateSil + if self.noise_average_decibel < -99.9: + self.noise_average_decibel = cur_decibel + else: + self.noise_average_decibel = (cur_decibel + self.noise_average_decibel * ( + self.vad_opts.noise_frame_num_used_for_snr + - 1)) / self.vad_opts.noise_frame_num_used_for_snr + + return frame_state + + + def __call__(self, score: np.ndarray, waveform: np.ndarray, + is_final: bool = False, max_end_sil: int = 800 + ): + self.max_end_sil_frame_cnt_thresh = max_end_sil - self.vad_opts.speech_to_sil_time_thres + self.waveform = waveform # compute decibel for each frame + self.ComputeDecibel() + self.ComputeScores(score) + if not is_final: + self.DetectCommonFrames() + else: + self.DetectLastFrames() + segments = [] + for batch_num in range(0, score.shape[0]): # only support batch_size = 1 now + segment_batch = [] + if len(self.output_data_buf) > 0: + for i in range(self.output_data_buf_offset, len(self.output_data_buf)): + if not self.output_data_buf[i].contain_seg_start_point: + continue + if not self.next_seg and not self.output_data_buf[i].contain_seg_end_point: + continue + start_ms = self.output_data_buf[i].start_ms if self.next_seg else -1 + if self.output_data_buf[i].contain_seg_end_point: + end_ms = self.output_data_buf[i].end_ms + self.next_seg = True + self.output_data_buf_offset += 1 + else: + end_ms = -1 + self.next_seg = False + segment = [start_ms, end_ms] + segment_batch.append(segment) + if segment_batch: + segments.append(segment_batch) + if is_final: + # reset class variables and clear the dict for the next query + self.AllResetDetection() + return segments + + def DetectCommonFrames(self) -> int: + if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected: + return 0 + for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): + frame_state = FrameState.kFrameStateInvalid + frame_state = self.GetFrameState(self.frm_cnt - 1 - i) + self.DetectOneFrame(frame_state, self.frm_cnt - 1 - i, False) + + return 0 + + def DetectLastFrames(self) -> int: + if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected: + return 0 + for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): + frame_state = FrameState.kFrameStateInvalid + frame_state = self.GetFrameState(self.frm_cnt - 1 - i) + if i != 0: + self.DetectOneFrame(frame_state, self.frm_cnt - 1 - i, False) + else: + self.DetectOneFrame(frame_state, self.frm_cnt - 1, True) + + return 0 + + def DetectOneFrame(self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool) -> None: + tmp_cur_frm_state = FrameState.kFrameStateInvalid + if cur_frm_state == FrameState.kFrameStateSpeech: + if math.fabs(1.0) > self.vad_opts.fe_prior_thres: + tmp_cur_frm_state = FrameState.kFrameStateSpeech + else: + tmp_cur_frm_state = FrameState.kFrameStateSil + elif cur_frm_state == FrameState.kFrameStateSil: + tmp_cur_frm_state = FrameState.kFrameStateSil + state_change = self.windows_detector.DetectOneFrame(tmp_cur_frm_state, cur_frm_idx) + frm_shift_in_ms = self.vad_opts.frame_in_ms + if AudioChangeState.kChangeStateSil2Speech == state_change: + silence_frame_count = self.continous_silence_frame_count + self.continous_silence_frame_count = 0 + self.pre_end_silence_detected = False + start_frame = 0 + if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + start_frame = max(self.data_buf_start_frame, cur_frm_idx - self.LatencyFrmNumAtStartPoint()) + self.OnVoiceStart(start_frame) + self.vad_state_machine = VadStateMachine.kVadInStateInSpeechSegment + for t in range(start_frame + 1, cur_frm_idx + 1): + self.OnVoiceDetected(t) + elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + for t in range(self.latest_confirmed_speech_frame + 1, cur_frm_idx): + self.OnVoiceDetected(t) + if cur_frm_idx - self.confirmed_start_frame + 1 > \ + self.vad_opts.max_single_segment_time / frm_shift_in_ms: + self.OnVoiceEnd(cur_frm_idx, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif not is_final_frame: + self.OnVoiceDetected(cur_frm_idx) + else: + self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) + else: + pass + elif AudioChangeState.kChangeStateSpeech2Sil == state_change: + self.continous_silence_frame_count = 0 + if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + pass + elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + if cur_frm_idx - self.confirmed_start_frame + 1 > \ + self.vad_opts.max_single_segment_time / frm_shift_in_ms: + self.OnVoiceEnd(cur_frm_idx, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif not is_final_frame: + self.OnVoiceDetected(cur_frm_idx) + else: + self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) + else: + pass + elif AudioChangeState.kChangeStateSpeech2Speech == state_change: + self.continous_silence_frame_count = 0 + if self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + if cur_frm_idx - self.confirmed_start_frame + 1 > \ + self.vad_opts.max_single_segment_time / frm_shift_in_ms: + self.max_time_out = True + self.OnVoiceEnd(cur_frm_idx, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif not is_final_frame: + self.OnVoiceDetected(cur_frm_idx) + else: + self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) + else: + pass + elif AudioChangeState.kChangeStateSil2Sil == state_change: + self.continous_silence_frame_count += 1 + if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + # silence timeout, return zero length decision + if ((self.vad_opts.detect_mode == VadDetectMode.kVadSingleUtteranceDetectMode.value) and ( + self.continous_silence_frame_count * frm_shift_in_ms > self.vad_opts.max_start_silence_time)) \ + or (is_final_frame and self.number_end_time_detected == 0): + for t in range(self.lastest_confirmed_silence_frame + 1, cur_frm_idx): + self.OnSilenceDetected(t) + self.OnVoiceStart(0, True) + self.OnVoiceEnd(0, True, False); + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + else: + if cur_frm_idx >= self.LatencyFrmNumAtStartPoint(): + self.OnSilenceDetected(cur_frm_idx - self.LatencyFrmNumAtStartPoint()) + elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + if self.continous_silence_frame_count * frm_shift_in_ms >= self.max_end_sil_frame_cnt_thresh: + lookback_frame = int(self.max_end_sil_frame_cnt_thresh / frm_shift_in_ms) + if self.vad_opts.do_extend: + lookback_frame -= int(self.vad_opts.lookahead_time_end_point / frm_shift_in_ms) + lookback_frame -= 1 + lookback_frame = max(0, lookback_frame) + self.OnVoiceEnd(cur_frm_idx - lookback_frame, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif cur_frm_idx - self.confirmed_start_frame + 1 > \ + self.vad_opts.max_single_segment_time / frm_shift_in_ms: + self.OnVoiceEnd(cur_frm_idx, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif self.vad_opts.do_extend and not is_final_frame: + if self.continous_silence_frame_count <= int( + self.vad_opts.lookahead_time_end_point / frm_shift_in_ms): + self.OnVoiceDetected(cur_frm_idx) + else: + self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) + else: + pass + + if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected and \ + self.vad_opts.detect_mode == VadDetectMode.kVadMutipleUtteranceDetectMode.value: + self.ResetDetection() diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/frontend.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/frontend.py new file mode 100644 index 000000000..11a86445d --- /dev/null +++ b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/frontend.py @@ -0,0 +1,191 @@ +# -*- encoding: utf-8 -*- +from pathlib import Path +from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union + +import numpy as np +from typeguard import check_argument_types +import kaldi_native_fbank as knf + +root_dir = Path(__file__).resolve().parent + +logger_initialized = {} + + +class WavFrontend(): + """Conventional frontend structure for ASR. + """ + + def __init__( + self, + cmvn_file: str = None, + fs: int = 16000, + window: str = 'hamming', + n_mels: int = 80, + frame_length: int = 25, + frame_shift: int = 10, + lfr_m: int = 1, + lfr_n: int = 1, + dither: float = 1.0, + **kwargs, + ) -> None: + check_argument_types() + + opts = knf.FbankOptions() + opts.frame_opts.samp_freq = fs + opts.frame_opts.dither = dither + opts.frame_opts.window_type = window + opts.frame_opts.frame_shift_ms = float(frame_shift) + opts.frame_opts.frame_length_ms = float(frame_length) + opts.mel_opts.num_bins = n_mels + opts.energy_floor = 0 + opts.frame_opts.snip_edges = True + opts.mel_opts.debug_mel = False + self.opts = opts + + self.lfr_m = lfr_m + self.lfr_n = lfr_n + self.cmvn_file = cmvn_file + + if self.cmvn_file: + self.cmvn = self.load_cmvn() + self.fbank_fn = None + self.fbank_beg_idx = 0 + self.reset_status() + + def fbank(self, + waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + waveform = waveform * (1 << 15) + self.fbank_fn = knf.OnlineFbank(self.opts) + self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist()) + frames = self.fbank_fn.num_frames_ready + mat = np.empty([frames, self.opts.mel_opts.num_bins]) + for i in range(frames): + mat[i, :] = self.fbank_fn.get_frame(i) + feat = mat.astype(np.float32) + feat_len = np.array(mat.shape[0]).astype(np.int32) + return feat, feat_len + + def fbank_online(self, + waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + waveform = waveform * (1 << 15) + # self.fbank_fn = knf.OnlineFbank(self.opts) + self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist()) + frames = self.fbank_fn.num_frames_ready + mat = np.empty([frames, self.opts.mel_opts.num_bins]) + for i in range(self.fbank_beg_idx, frames): + mat[i, :] = self.fbank_fn.get_frame(i) + # self.fbank_beg_idx += (frames-self.fbank_beg_idx) + feat = mat.astype(np.float32) + feat_len = np.array(mat.shape[0]).astype(np.int32) + return feat, feat_len + + def reset_status(self): + self.fbank_fn = knf.OnlineFbank(self.opts) + self.fbank_beg_idx = 0 + + def lfr_cmvn(self, feat: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + if self.lfr_m != 1 or self.lfr_n != 1: + feat = self.apply_lfr(feat, self.lfr_m, self.lfr_n) + + if self.cmvn_file: + feat = self.apply_cmvn(feat) + + feat_len = np.array(feat.shape[0]).astype(np.int32) + return feat, feat_len + + @staticmethod + def apply_lfr(inputs: np.ndarray, lfr_m: int, lfr_n: int) -> np.ndarray: + LFR_inputs = [] + + T = inputs.shape[0] + T_lfr = int(np.ceil(T / lfr_n)) + left_padding = np.tile(inputs[0], ((lfr_m - 1) // 2, 1)) + inputs = np.vstack((left_padding, inputs)) + T = T + (lfr_m - 1) // 2 + for i in range(T_lfr): + if lfr_m <= T - i * lfr_n: + LFR_inputs.append( + (inputs[i * lfr_n:i * lfr_n + lfr_m]).reshape(1, -1)) + else: + # process last LFR frame + num_padding = lfr_m - (T - i * lfr_n) + frame = inputs[i * lfr_n:].reshape(-1) + for _ in range(num_padding): + frame = np.hstack((frame, inputs[-1])) + + LFR_inputs.append(frame) + LFR_outputs = np.vstack(LFR_inputs).astype(np.float32) + return LFR_outputs + + def apply_cmvn(self, inputs: np.ndarray) -> np.ndarray: + """ + Apply CMVN with mvn data + """ + frame, dim = inputs.shape + means = np.tile(self.cmvn[0:1, :dim], (frame, 1)) + vars = np.tile(self.cmvn[1:2, :dim], (frame, 1)) + inputs = (inputs + means) * vars + return inputs + + def load_cmvn(self,) -> np.ndarray: + with open(self.cmvn_file, 'r', encoding='utf-8') as f: + lines = f.readlines() + + means_list = [] + vars_list = [] + for i in range(len(lines)): + line_item = lines[i].split() + if line_item[0] == '': + line_item = lines[i + 1].split() + if line_item[0] == '': + add_shift_line = line_item[3:(len(line_item) - 1)] + means_list = list(add_shift_line) + continue + elif line_item[0] == '': + line_item = lines[i + 1].split() + if line_item[0] == '': + rescale_line = line_item[3:(len(line_item) - 1)] + vars_list = list(rescale_line) + continue + + means = np.array(means_list).astype(np.float64) + vars = np.array(vars_list).astype(np.float64) + cmvn = np.array([means, vars]) + return cmvn + +def load_bytes(input): + middle_data = np.frombuffer(input, dtype=np.int16) + middle_data = np.asarray(middle_data) + if middle_data.dtype.kind not in 'iu': + raise TypeError("'middle_data' must be an array of integers") + dtype = np.dtype('float32') + if dtype.kind != 'f': + raise TypeError("'dtype' must be a floating point type") + + i = np.iinfo(middle_data.dtype) + abs_max = 2 ** (i.bits - 1) + offset = i.min + abs_max + array = np.frombuffer((middle_data.astype(dtype) - offset) / abs_max, dtype=np.float32) + return array + + +def test(): + path = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav" + import librosa + cmvn_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/am.mvn" + config_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/config.yaml" + from funasr.runtime.python.onnxruntime.rapid_paraformer.utils.utils import read_yaml + config = read_yaml(config_file) + waveform, _ = librosa.load(path, sr=None) + frontend = WavFrontend( + cmvn_file=cmvn_file, + **config['frontend_conf'], + ) + speech, _ = frontend.fbank_online(waveform) #1d, (sample,), numpy + feat, feat_len = frontend.lfr_cmvn(speech) # 2d, (frame, 450), np.float32 -> torch, torch.from_numpy(), dtype, (1, frame, 450) + + frontend.reset_status() # clear cache + return feat, feat_len + +if __name__ == '__main__': + test() \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/postprocess_utils.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/postprocess_utils.py new file mode 100644 index 000000000..575fb90dd --- /dev/null +++ b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/postprocess_utils.py @@ -0,0 +1,240 @@ +# Copyright (c) Alibaba, Inc. and its affiliates. + +import string +import logging +from typing import Any, List, Union + + +def isChinese(ch: str): + if '\u4e00' <= ch <= '\u9fff' or '\u0030' <= ch <= '\u0039': + return True + return False + + +def isAllChinese(word: Union[List[Any], str]): + word_lists = [] + for i in word: + cur = i.replace(' ', '') + cur = cur.replace('', '') + cur = cur.replace('', '') + word_lists.append(cur) + + if len(word_lists) == 0: + return False + + for ch in word_lists: + if isChinese(ch) is False: + return False + return True + + +def isAllAlpha(word: Union[List[Any], str]): + word_lists = [] + for i in word: + cur = i.replace(' ', '') + cur = cur.replace('', '') + cur = cur.replace('', '') + word_lists.append(cur) + + if len(word_lists) == 0: + return False + + for ch in word_lists: + if ch.isalpha() is False and ch != "'": + return False + elif ch.isalpha() is True and isChinese(ch) is True: + return False + + return True + + +# def abbr_dispose(words: List[Any]) -> List[Any]: +def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]: + words_size = len(words) + word_lists = [] + abbr_begin = [] + abbr_end = [] + last_num = -1 + ts_lists = [] + ts_nums = [] + ts_index = 0 + for num in range(words_size): + if num <= last_num: + continue + + if len(words[num]) == 1 and words[num].encode('utf-8').isalpha(): + if num + 1 < words_size and words[ + num + 1] == ' ' and num + 2 < words_size and len( + words[num + + 2]) == 1 and words[num + + 2].encode('utf-8').isalpha(): + # found the begin of abbr + abbr_begin.append(num) + num += 2 + abbr_end.append(num) + # to find the end of abbr + while True: + num += 1 + if num < words_size and words[num] == ' ': + num += 1 + if num < words_size and len( + words[num]) == 1 and words[num].encode( + 'utf-8').isalpha(): + abbr_end.pop() + abbr_end.append(num) + last_num = num + else: + break + else: + break + + for num in range(words_size): + if words[num] == ' ': + ts_nums.append(ts_index) + else: + ts_nums.append(ts_index) + ts_index += 1 + last_num = -1 + for num in range(words_size): + if num <= last_num: + continue + + if num in abbr_begin: + if time_stamp is not None: + begin = time_stamp[ts_nums[num]][0] + word_lists.append(words[num].upper()) + num += 1 + while num < words_size: + if num in abbr_end: + word_lists.append(words[num].upper()) + last_num = num + break + else: + if words[num].encode('utf-8').isalpha(): + word_lists.append(words[num].upper()) + num += 1 + if time_stamp is not None: + end = time_stamp[ts_nums[num]][1] + ts_lists.append([begin, end]) + else: + word_lists.append(words[num]) + if time_stamp is not None and words[num] != ' ': + begin = time_stamp[ts_nums[num]][0] + end = time_stamp[ts_nums[num]][1] + ts_lists.append([begin, end]) + begin = end + + if time_stamp is not None: + return word_lists, ts_lists + else: + return word_lists + + +def sentence_postprocess(words: List[Any], time_stamp: List[List] = None): + middle_lists = [] + word_lists = [] + word_item = '' + ts_lists = [] + + # wash words lists + for i in words: + word = '' + if isinstance(i, str): + word = i + else: + word = i.decode('utf-8') + + if word in ['', '', '']: + continue + else: + middle_lists.append(word) + + # all chinese characters + if isAllChinese(middle_lists): + for i, ch in enumerate(middle_lists): + word_lists.append(ch.replace(' ', '')) + if time_stamp is not None: + ts_lists = time_stamp + + # all alpha characters + elif isAllAlpha(middle_lists): + ts_flag = True + for i, ch in enumerate(middle_lists): + if ts_flag and time_stamp is not None: + begin = time_stamp[i][0] + end = time_stamp[i][1] + word = '' + if '@@' in ch: + word = ch.replace('@@', '') + word_item += word + if time_stamp is not None: + ts_flag = False + end = time_stamp[i][1] + else: + word_item += ch + word_lists.append(word_item) + word_lists.append(' ') + word_item = '' + if time_stamp is not None: + ts_flag = True + end = time_stamp[i][1] + ts_lists.append([begin, end]) + begin = end + + # mix characters + else: + alpha_blank = False + ts_flag = True + begin = -1 + end = -1 + for i, ch in enumerate(middle_lists): + if ts_flag and time_stamp is not None: + begin = time_stamp[i][0] + end = time_stamp[i][1] + word = '' + if isAllChinese(ch): + if alpha_blank is True: + word_lists.pop() + word_lists.append(ch) + alpha_blank = False + if time_stamp is not None: + ts_flag = True + ts_lists.append([begin, end]) + begin = end + elif '@@' in ch: + word = ch.replace('@@', '') + word_item += word + alpha_blank = False + if time_stamp is not None: + ts_flag = False + end = time_stamp[i][1] + elif isAllAlpha(ch): + word_item += ch + word_lists.append(word_item) + word_lists.append(' ') + word_item = '' + alpha_blank = True + if time_stamp is not None: + ts_flag = True + end = time_stamp[i][1] + ts_lists.append([begin, end]) + begin = end + else: + raise ValueError('invalid character: {}'.format(ch)) + + if time_stamp is not None: + word_lists, ts_lists = abbr_dispose(word_lists, ts_lists) + real_word_lists = [] + for ch in word_lists: + if ch != ' ': + real_word_lists.append(ch) + sentence = ' '.join(real_word_lists).strip() + return sentence, ts_lists, real_word_lists + else: + word_lists = abbr_dispose(word_lists) + real_word_lists = [] + for ch in word_lists: + if ch != ' ': + real_word_lists.append(ch) + sentence = ''.join(word_lists).strip() + return sentence, real_word_lists diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/timestamp_utils.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/timestamp_utils.py new file mode 100644 index 000000000..3a01812e8 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/timestamp_utils.py @@ -0,0 +1,59 @@ +import numpy as np + + +def time_stamp_lfr6_onnx(us_cif_peak, char_list, begin_time=0.0, total_offset=-1.5): + if not len(char_list): + return [] + START_END_THRESHOLD = 5 + MAX_TOKEN_DURATION = 30 + TIME_RATE = 10.0 * 6 / 1000 / 3 # 3 times upsampled + cif_peak = us_cif_peak.reshape(-1) + num_frames = cif_peak.shape[-1] + if char_list[-1] == '': + char_list = char_list[:-1] + # char_list = [i for i in text] + timestamp_list = [] + new_char_list = [] + # for bicif model trained with large data, cif2 actually fires when a character starts + # so treat the frames between two peaks as the duration of the former token + fire_place = np.where(cif_peak>1.0-1e-4)[0] + total_offset # np format + num_peak = len(fire_place) + assert num_peak == len(char_list) + 1 # number of peaks is supposed to be number of tokens + 1 + # begin silence + if fire_place[0] > START_END_THRESHOLD: + # char_list.insert(0, '') + timestamp_list.append([0.0, fire_place[0]*TIME_RATE]) + new_char_list.append('') + # tokens timestamp + for i in range(len(fire_place)-1): + new_char_list.append(char_list[i]) + if i == len(fire_place)-2 or MAX_TOKEN_DURATION < 0 or fire_place[i+1] - fire_place[i] < MAX_TOKEN_DURATION: + timestamp_list.append([fire_place[i]*TIME_RATE, fire_place[i+1]*TIME_RATE]) + else: + # cut the duration to token and sil of the 0-weight frames last long + _split = fire_place[i] + MAX_TOKEN_DURATION + timestamp_list.append([fire_place[i]*TIME_RATE, _split*TIME_RATE]) + timestamp_list.append([_split*TIME_RATE, fire_place[i+1]*TIME_RATE]) + new_char_list.append('') + # tail token and end silence + if num_frames - fire_place[-1] > START_END_THRESHOLD: + _end = (num_frames + fire_place[-1]) / 2 + timestamp_list[-1][1] = _end*TIME_RATE + timestamp_list.append([_end*TIME_RATE, num_frames*TIME_RATE]) + new_char_list.append("") + else: + timestamp_list[-1][1] = num_frames*TIME_RATE + if begin_time: # add offset time in model with vad + for i in range(len(timestamp_list)): + timestamp_list[i][0] = timestamp_list[i][0] + begin_time / 1000.0 + timestamp_list[i][1] = timestamp_list[i][1] + begin_time / 1000.0 + assert len(new_char_list) == len(timestamp_list) + res_str = "" + for char, timestamp in zip(new_char_list, timestamp_list): + res_str += "{} {} {};".format(char, timestamp[0], timestamp[1]) + res = [] + for char, timestamp in zip(new_char_list, timestamp_list): + if char != '': + res.append([int(timestamp[0] * 1000), int(timestamp[1] * 1000)]) + return res_str, res + \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/utils.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/utils.py new file mode 100644 index 000000000..2edde112e --- /dev/null +++ b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/utils.py @@ -0,0 +1,257 @@ +# -*- encoding: utf-8 -*- + +import functools +import logging +import pickle +from pathlib import Path +from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union + +import numpy as np +import yaml +from onnxruntime import (GraphOptimizationLevel, InferenceSession, + SessionOptions, get_available_providers, get_device) +from typeguard import check_argument_types + +import warnings + +root_dir = Path(__file__).resolve().parent + +logger_initialized = {} + + +class TokenIDConverter(): + def __init__(self, token_list: Union[List, str], + ): + check_argument_types() + + # self.token_list = self.load_token(token_path) + self.token_list = token_list + self.unk_symbol = token_list[-1] + + # @staticmethod + # def load_token(file_path: Union[Path, str]) -> List: + # if not Path(file_path).exists(): + # raise TokenIDConverterError(f'The {file_path} does not exist.') + # + # with open(str(file_path), 'rb') as f: + # token_list = pickle.load(f) + # + # if len(token_list) != len(set(token_list)): + # raise TokenIDConverterError('The Token exists duplicated symbol.') + # return token_list + + def get_num_vocabulary_size(self) -> int: + return len(self.token_list) + + def ids2tokens(self, + integers: Union[np.ndarray, Iterable[int]]) -> List[str]: + if isinstance(integers, np.ndarray) and integers.ndim != 1: + raise TokenIDConverterError( + f"Must be 1 dim ndarray, but got {integers.ndim}") + return [self.token_list[i] for i in integers] + + def tokens2ids(self, tokens: Iterable[str]) -> List[int]: + token2id = {v: i for i, v in enumerate(self.token_list)} + if self.unk_symbol not in token2id: + raise TokenIDConverterError( + f"Unknown symbol '{self.unk_symbol}' doesn't exist in the token_list" + ) + unk_id = token2id[self.unk_symbol] + return [token2id.get(i, unk_id) for i in tokens] + + +class CharTokenizer(): + def __init__( + self, + symbol_value: Union[Path, str, Iterable[str]] = None, + space_symbol: str = "", + remove_non_linguistic_symbols: bool = False, + ): + check_argument_types() + + self.space_symbol = space_symbol + self.non_linguistic_symbols = self.load_symbols(symbol_value) + self.remove_non_linguistic_symbols = remove_non_linguistic_symbols + + @staticmethod + def load_symbols(value: Union[Path, str, Iterable[str]] = None) -> Set: + if value is None: + return set() + + if isinstance(value, Iterable[str]): + return set(value) + + file_path = Path(value) + if not file_path.exists(): + logging.warning("%s doesn't exist.", file_path) + return set() + + with file_path.open("r", encoding="utf-8") as f: + return set(line.rstrip() for line in f) + + def text2tokens(self, line: Union[str, list]) -> List[str]: + tokens = [] + while len(line) != 0: + for w in self.non_linguistic_symbols: + if line.startswith(w): + if not self.remove_non_linguistic_symbols: + tokens.append(line[: len(w)]) + line = line[len(w):] + break + else: + t = line[0] + if t == " ": + t = "" + tokens.append(t) + line = line[1:] + return tokens + + def tokens2text(self, tokens: Iterable[str]) -> str: + tokens = [t if t != self.space_symbol else " " for t in tokens] + return "".join(tokens) + + def __repr__(self): + return ( + f"{self.__class__.__name__}(" + f'space_symbol="{self.space_symbol}"' + f'non_linguistic_symbols="{self.non_linguistic_symbols}"' + f")" + ) + + + +class Hypothesis(NamedTuple): + """Hypothesis data type.""" + + yseq: np.ndarray + score: Union[float, np.ndarray] = 0 + scores: Dict[str, Union[float, np.ndarray]] = dict() + states: Dict[str, Any] = dict() + + def asdict(self) -> dict: + """Convert data to JSON-friendly dict.""" + return self._replace( + yseq=self.yseq.tolist(), + score=float(self.score), + scores={k: float(v) for k, v in self.scores.items()}, + )._asdict() + + +class TokenIDConverterError(Exception): + pass + + +class ONNXRuntimeError(Exception): + pass + + +class OrtInferSession(): + def __init__(self, model_file, device_id=-1, intra_op_num_threads=4): + device_id = str(device_id) + sess_opt = SessionOptions() + sess_opt.intra_op_num_threads = intra_op_num_threads + sess_opt.log_severity_level = 4 + sess_opt.enable_cpu_mem_arena = False + sess_opt.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_ALL + + cuda_ep = 'CUDAExecutionProvider' + cuda_provider_options = { + "device_id": device_id, + "arena_extend_strategy": "kNextPowerOfTwo", + "cudnn_conv_algo_search": "EXHAUSTIVE", + "do_copy_in_default_stream": "true", + } + cpu_ep = 'CPUExecutionProvider' + cpu_provider_options = { + "arena_extend_strategy": "kSameAsRequested", + } + + EP_list = [] + if device_id != "-1" and get_device() == 'GPU' \ + and cuda_ep in get_available_providers(): + EP_list = [(cuda_ep, cuda_provider_options)] + EP_list.append((cpu_ep, cpu_provider_options)) + + self._verify_model(model_file) + self.session = InferenceSession(model_file, + sess_options=sess_opt, + providers=EP_list) + + if device_id != "-1" and cuda_ep not in self.session.get_providers(): + warnings.warn(f'{cuda_ep} is not avaiable for current env, the inference part is automatically shifted to be executed under {cpu_ep}.\n' + 'Please ensure the installed onnxruntime-gpu version matches your cuda and cudnn version, ' + 'you can check their relations from the offical web site: ' + 'https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html', + RuntimeWarning) + + def __call__(self, + input_content: List[Union[np.ndarray, np.ndarray]]) -> np.ndarray: + input_dict = dict(zip(self.get_input_names(), input_content)) + try: + return self.session.run(None, input_dict) + except Exception as e: + raise ONNXRuntimeError('ONNXRuntime inferece failed.') from e + + def get_input_names(self, ): + return [v.name for v in self.session.get_inputs()] + + def get_output_names(self,): + return [v.name for v in self.session.get_outputs()] + + def get_character_list(self, key: str = 'character'): + return self.meta_dict[key].splitlines() + + def have_key(self, key: str = 'character') -> bool: + self.meta_dict = self.session.get_modelmeta().custom_metadata_map + if key in self.meta_dict.keys(): + return True + return False + + @staticmethod + def _verify_model(model_path): + model_path = Path(model_path) + if not model_path.exists(): + raise FileNotFoundError(f'{model_path} does not exists.') + if not model_path.is_file(): + raise FileExistsError(f'{model_path} is not a file.') + + +def read_yaml(yaml_path: Union[str, Path]) -> Dict: + if not Path(yaml_path).exists(): + raise FileExistsError(f'The {yaml_path} does not exist.') + + with open(str(yaml_path), 'rb') as f: + data = yaml.load(f, Loader=yaml.Loader) + return data + + +@functools.lru_cache() +def get_logger(name='rapdi_paraformer'): + """Initialize and get a logger by name. + If the logger has not been initialized, this method will initialize the + logger by adding one or two handlers, otherwise the initialized logger will + be directly returned. During initialization, a StreamHandler will always be + added. + Args: + name (str): Logger name. + Returns: + logging.Logger: The expected logger. + """ + logger = logging.getLogger(name) + if name in logger_initialized: + return logger + + for logger_name in logger_initialized: + if name.startswith(logger_name): + return logger + + formatter = logging.Formatter( + '[%(asctime)s] %(name)s %(levelname)s: %(message)s', + datefmt="%Y/%m/%d %H:%M:%S") + + sh = logging.StreamHandler() + sh.setFormatter(formatter) + logger.addHandler(sh) + logger_initialized[name] = True + logger.propagate = False + return logger diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/vad_bin.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/vad_bin.py new file mode 100644 index 000000000..58913bbd3 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/vad_bin.py @@ -0,0 +1,166 @@ +# -*- encoding: utf-8 -*- + +import os.path +from pathlib import Path +from typing import List, Union, Tuple + +import copy +import librosa +import numpy as np + +from .utils.utils import (CharTokenizer, Hypothesis, ONNXRuntimeError, + OrtInferSession, TokenIDConverter, get_logger, + read_yaml) +from .utils.postprocess_utils import sentence_postprocess +from .utils.frontend import WavFrontend +from .utils.timestamp_utils import time_stamp_lfr6_onnx +from .utils.e2e_vad import E2EVadModel + +logging = get_logger() + + +class Fsmn_vad(): + def __init__(self, model_dir: Union[str, Path] = None, + batch_size: int = 1, + device_id: Union[str, int] = "-1", + quantize: bool = False, + intra_op_num_threads: int = 4, + max_end_sil: int = 800, + ): + + if not Path(model_dir).exists(): + raise FileNotFoundError(f'{model_dir} does not exist.') + + model_file = os.path.join(model_dir, 'model.onnx') + if quantize: + model_file = os.path.join(model_dir, 'model_quant.onnx') + config_file = os.path.join(model_dir, 'vad.yaml') + cmvn_file = os.path.join(model_dir, 'vad.mvn') + config = read_yaml(config_file) + + self.frontend = WavFrontend( + cmvn_file=cmvn_file, + **config['frontend_conf'] + ) + self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) + self.batch_size = batch_size + self.vad_scorer = E2EVadModel(**config) + self.max_end_sil = max_end_sil + + def prepare_cache(self, in_cache: list = []): + if len(in_cache) > 0: + return in_cache + + for i in range(4): + cache = np.random.rand(1, 128, 19, 1).astype(np.float32) + in_cache.append(cache) + return in_cache + + + def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> List: + waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) + waveform_nums = len(waveform_list) + is_final = kwargs.get('kwargs', False) + + asr_res = [] + for beg_idx in range(0, waveform_nums, self.batch_size): + + end_idx = min(waveform_nums, beg_idx + self.batch_size) + waveform = waveform_list[beg_idx:end_idx] + feats, feats_len = self.extract_feat(waveform) + param_dict = kwargs.get('param_dict', dict()) + in_cache = param_dict.get('cache', list()) + in_cache = self.prepare_cache(in_cache) + try: + + scores, out_caches = self.infer(feats, *in_cache) + param_dict['cache'] = out_caches + segments = self.vad_scorer(scores, waveform, is_final=is_final, max_end_sil=self.max_end_sil) + + except ONNXRuntimeError: + # logging.warning(traceback.format_exc()) + logging.warning("input wav is silence or noise") + segments = '' + asr_res.append(segments) + # else: + # preds = self.decode(am_scores, valid_token_lens) + # + # asr_res.append({'preds': text_proc, 'timestamp': timestamp_proc, "raw_tokens": raw_tokens}) + + return asr_res + + def load_data(self, + wav_content: Union[str, np.ndarray, List[str]], fs: int = None) -> List: + def load_wav(path: str) -> np.ndarray: + waveform, _ = librosa.load(path, sr=fs) + return waveform + + if isinstance(wav_content, np.ndarray): + return [wav_content] + + if isinstance(wav_content, str): + return [load_wav(wav_content)] + + if isinstance(wav_content, list): + return [load_wav(path) for path in wav_content] + + raise TypeError( + f'The type of {wav_content} is not in [str, np.ndarray, list]') + + def extract_feat(self, + waveform_list: List[np.ndarray] + ) -> Tuple[np.ndarray, np.ndarray]: + feats, feats_len = [], [] + for waveform in waveform_list: + speech, _ = self.frontend.fbank(waveform) + feat, feat_len = self.frontend.lfr_cmvn(speech) + feats.append(feat) + feats_len.append(feat_len) + + feats = self.pad_feats(feats, np.max(feats_len)) + feats_len = np.array(feats_len).astype(np.int32) + return feats, feats_len + + @staticmethod + def pad_feats(feats: List[np.ndarray], max_feat_len: int) -> np.ndarray: + def pad_feat(feat: np.ndarray, cur_len: int) -> np.ndarray: + pad_width = ((0, max_feat_len - cur_len), (0, 0)) + return np.pad(feat, pad_width, 'constant', constant_values=0) + + feat_res = [pad_feat(feat, feat.shape[0]) for feat in feats] + feats = np.array(feat_res).astype(np.float32) + return feats + + def infer(self, feats: np.ndarray, + feats_len: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + outputs = self.ort_infer([feats, feats_len]) + return outputs + + def decode(self, am_scores: np.ndarray, token_nums: int) -> List[str]: + return [self.decode_one(am_score, token_num) + for am_score, token_num in zip(am_scores, token_nums)] + + def decode_one(self, + am_score: np.ndarray, + valid_token_num: int) -> List[str]: + yseq = am_score.argmax(axis=-1) + score = am_score.max(axis=-1) + score = np.sum(score, axis=-1) + + # pad with mask tokens to ensure compatibility with sos/eos tokens + # asr_model.sos:1 asr_model.eos:2 + yseq = np.array([1] + yseq.tolist() + [2]) + hyp = Hypothesis(yseq=yseq, score=score) + + # remove sos/eos and get results + last_pos = -1 + token_int = hyp.yseq[1:last_pos].tolist() + + # remove blank symbol id, which is assumed to be 0 + token_int = list(filter(lambda x: x not in (0, 2), token_int)) + + # Change integer-ids to tokens + token = self.converter.ids2tokens(token_int) + token = token[:valid_token_num - self.pred_bias] + # texts = sentence_postprocess(token) + return token diff --git a/funasr/runtime/python/onnxruntime/demo.py b/funasr/runtime/python/onnxruntime/demo.py index 248d2e1df..48d54e909 100644 --- a/funasr/runtime/python/onnxruntime/demo.py +++ b/funasr/runtime/python/onnxruntime/demo.py @@ -1,8 +1,6 @@ from funasr_onnx import Paraformer -#model_dir = "/Users/shixian/code/funasr/export/damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch" -#model_dir = "/Users/shixian/code/funasr/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" model_dir = "/Users/shixian/code/funasr/export/damo/speech_paraformer-tiny-commandword_asr_nat-zh-cn-16k-vocab544-pytorch" # if you use paraformer-tiny-commandword_asr_nat-zh-cn-16k-vocab544-pytorch, you should set pred_bias=0 diff --git a/funasr/runtime/python/onnxruntime/demo_vad.py b/funasr/runtime/python/onnxruntime/demo_vad.py new file mode 100644 index 000000000..ae033cc5b --- /dev/null +++ b/funasr/runtime/python/onnxruntime/demo_vad.py @@ -0,0 +1,12 @@ + +from funasr_onnx import Fsmn_vad + + +model_dir = "/Users/zhifu/Downloads/speech_fsmn_vad_zh-cn-16k-common-pytorch" + +model = Fsmn_vad(model_dir) + +wav_path = "/Users/zhifu/Downloads/speech_fsmn_vad_zh-cn-16k-common-pytorch/example/vad_example.wav" + +result = model(wav_path) +print(result) \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/dist/funasr_onnx-0.0.2-py3.8.egg b/funasr/runtime/python/onnxruntime/dist/funasr_onnx-0.0.2-py3.8.egg new file mode 100644 index 0000000000000000000000000000000000000000..b24107b404d6479b053e263ed23f22a0c30e8980 GIT binary patch literal 47828 zcmagF1CTCTvn|@TZQHhO+qStHt8Lr1ZF{wC+qV0y{qDQx?EgGfd|B~DMP<~?tcaXr zjLcGy1_nU^004jhnCkLY3#%G~4FLfFh=v9LK>T}ER9u`^Qcg^sUP1bQ{b^RSwclVv z_@35xwD21NlcAD}tYMBb6ilfROk|O0aP>pfND|T3AxR)D=NPZuQB0~f!3_bMfWat` z*yeSA*^HbHIstvj`lhQ&)S5ELl^x@(9fnDX-k(;gUgTZFyQ$}Gk*~?MyhRmC-o|E^PDp+< zO{~{hfvhzdVU|-YFX1@}F54g(VC)iu!`8Rr-T=tDw*d-O1oLU`U_|DiD&GGRrLv-{ zj8^7XpZ93-kE4Q*J5)Iq;$%6wJk0P$y3*PhuoWSDB}Yenk+!vV(P%y_+n*jQMzI(P zUZS43(5gUPFHnPh2W8&0N7M~o!hXJ9L*nPr-v4SO_3>^Qa*6#U(y=Rs^qwcK=DC-v z)efwG*xx2k!)XoHqZV>=+hs>j8m1V)!VH-?Iq}tD8=ugMb1(&$OKR<~xX+7I;8+MQ z1K>o2OHo^4aL1j39;Af_f$(BpCzHgiRAXlDyJq6)W{l*m5B7SwX=M1p29%Wo@% zC!Ipg?&&OsP%q?m=Q>&YX!Wu$x9o25NN-EKGG4~^IsyesqzpXpDy#wxrJMOGr}X&@aGe;LLBNH^+>hRUPmu3ME#CJQ+`*h$Xq(Eo^ z8uH=9G~2b|p~yZhOhypBm`P(wk4^Thg!!2ZEIP_Jh~>JvSxEAM%8+?(OEG95O2iF~~yNkS{;H!AgtEmBqF6Nu8L{y3_&hioHT;BfjXMQC;J6~;{m z)!--iy!0#B^mvCwy83hGmZKm|Hi_c~fjHCPwwOUY5haA%->!eGC@L<5vOwZ}rVxBy z%<9OL0$N-`V%bKbThXJE$7A=K?~H3YB!3gyq!K_uQzKK+58{x&E)lmOXZo}3}+T)T!ZENFa{j{e3d5;UYhQ50An;4uo0 z4ejWcvCXw6jq$K?(+9Z)L_1d3=gavnERnV|ar`yUugJ5gq28%x77d@vm){jA+}+9m z#U3IPWw0~ChtSEc@BpTC za7-|h^_+Y+;C}{~2)4om{5P<_@d4oeJ3tdt2U9x}Q#)f1eH%+VYv+F=JpcmufA>cH zx3`_W3$3T61Ff^6nd$${%XDjnP2=y~T7T_7yqruOT`iqV|IcTkj@H8oFdzWTeti#% zhKJ?LUXrLC_^}1j_I@alit5m_{&>24PZI9nY%%fD~X8`1crzvoMmjUzF(Ei$gyv^&86e!T81?9c>J*g(jkM$l{iplzR?r1B*`C;GRNatVaFS7Gi+xE12PJsjf0LA-XFYn*ab})1@G_!ZI zHFeTAvb6iVf@Dqo*o`(s-u4EC> zBDX8(`z>x^NfgSl%t-t$Cf0|{z0U)fmlPU(T1aZ-`1|`d!szi7-~Q&|0QBTCR79qV zGA`N%EfkPkCm({XPWj{D4wb&0+Mh&e)P$cTog_jJi;67FHW8K7Z_GF=Ld%~*&^hYT8*iVQ z;Dlr-m}#TPfxRH}A_=dMpZU_4tpwrFjid;)qcvKJfKDh8x50ld3D2&9?y z3;q>{DAEqgA-76^d!8Xwrt&y41Ipl7r-*N6?LFyx7;!KZ?%@Ir?Ik2%LqcIp6%3mPhnAhuO<7{Fk!Q!u<3OEmXUNd>ZaJ83LotC$U9 z&&TX@qU|P!yfbUy`P-2tGOi{c=`X{#KBVOaN`Y2!qtIZMw5nhLd0MHHbM^tLftPW= z-Z=*p0QIC5jS;1&|F^ur|Lg}LcL}O9K}M&HrQYByx=-#!rD(IVjI=!JyAV5 zFw*_<5@t<+as^W2UO112%ID``$C+o!ihjM?+?QC4`ysG?-9wU9!n7c_CQjkYjc@YR z(5haBR0ZPD~F;I z^e19YT)UZDy7{Br3P!HZL-pA3iL<76i-Uu_N+}V|I>LLS84Hm_0n+#PBO~|hA>+W5 z5-M;Kz6>toapxZCC8ThwgCq$GNgX`kK{C+vP$nfn3Sc&T&Pj&hHVAKfOiGd<^yDXc zfFTupmfa;7R>9aYY&4#@Wm>h8PPU+jQYOFyelDH{zR`Uhjx-@YdGDOxE%hm7d+9^8>R z-jqbJKYT8urRdq+d+HSUR-*gD$c{YU2BVhQ71iXM!@Ga_6*H|a3@)oMBZ763=^0gV z2j3t?lNUJnf*5-Trlo651JDT6d`d+*wYzMOBX=W8i>BSew#iPJj>2YcZzXQF1E~#=*U`~KSz+)bx>LF1^#%QEU6+uI}0=9>4 zXIaNCvfP4dO~|t!zOc}Y^_E__1RHo!0$$An4j?vz z_<6c87MDGd0$0yr*RA`k?WQR8MPJX-HZyJ5a7)W-O64gZ$*rklftLUoi)GW52>@{d zxfWyCkzW_t1-@EkGH_UdOUK#huV@%hP*Z)&SCPYlZBOyj7aj%>Jf98KN3#O!Y3b7{ zmmmQX&aLEJ0dDxQLJL?}JQ5+l<~djac5TIK^5|1{9kW`(v7=>ey6#;paEMp7FV2!P zdxJ97CeH^nYRGem?TEPH-u?_KcE&jx379gj(QfCN70 zwrp0E9qZ=zLpg)(holBVpuv&H5qTN5jVQq~b6vjZ2e6PlNo=$IIRO_S` zUhuyLiZWKZyAeIwRBVCIW5&~80FHF~rHPpChlO4vXtDI-0d86F6#Bby&897!Xy->Q zur`|JJy)xu0z^a4p^} z-h>wTJqq!!`Onq$!2|2n zrTM*>Luf1$MU@Q+-0h(j{d{c|L1Sk2sGiv)>_(7-DwD^WY0ME^6EO) zs3Bl09N1~RxtT-_Z}rB#aqz;XCsgMig{XAkPA zYTtOe>Ge+Mq z6lTD!_nc);-8fqB=oo5c+ly4m%d>&%Wez25Y*AVLJi@%-#_ zuA!sNN4aN3ZYXTdb6~Toz3?|_BwGfWwZ9(mVx?@(G5z{(L;D{%)T8ddv@nh_KU~^zFxIIPdG8spvqz z)p^rhjO2EIzfKzhmbQrA-J$V`|KD&l-r8I3FMb37`seib_@7|X!PU;@asx6Kcf(L?{dn%c@n!6>AIqX1nb z*lBSnTQ{`<1`w?xn?0!#;n|jJ;OEP99FeF;w``|`O?ZaK@t3HNXv&K8fC#BI-7_d> zhS?j|%(3(KlM*J8>$b9U8oQJmNUc^6@nGXm5cu&Ta4-K)ymex{K4>ZtF@DVN2_PNP zkHlUu5>nQu*8SBdav@g4ug&~p@j>k?K3=!`V}T`bj~S4k?DD8jhx1}ihACGuAJ3of z*4)^E+n8=usR|o8z0VKHboTbr(Sbks3MH&3FNDF zmjW01=CkA~fSxRjymW?L2Q@JiWVtr6h;`=ASiVhIabzwQW$g7QWp5t3Esu?|8!5`6 zg`n82hI@ao*Vb11AL$FZ!zp!;*v2mf*tAphBur2Zd}%-{6&O`00%=qD_1$CMsT?6{U;EV zFvJI|_6{!d*Iz|(z+r8P4WI?w>*wYPfC6($&!mUCmT(wiSFW$mS#5>#q_6lWw!Bjp znVHU1kK-`kcOejGnf;T1>X} zH1<6b*#P2m0L&CjDi+_qW5X15#L!;Z#-FtK7N|UdSf8+Q^Gq=c$tbWpzVYZxq<_X8 zc(wvxrEvt%Apy@73?)h2Z$XZ?G#D@1G>}-BBy>Y7VP|Kwwe>6h*OrGyW=COjc0~FH=Ld7X*lKLXjx!e49kI zuGNr7rKZ-xT~(;pY1;tms2^$Cwz2gQYG{%n`F51*A!nzvPv@jUeAPk0Du)PVE+|`V zJk(CzPXQnh%^mch<^D*nVruC44U7Nd_hTu?3Im_=7d1;+^|Xryn&I)6{*JA8(VC>6 zc0^(d2MaLJ*H(km zWsD1JwQMR2D@%llnO=M`9;CB)p4iiPb|xcY}BBsgdmAhISWtl_DGup4C9fNRR> zbZl%um!4^5DUW@|Or=tZ(YBCX#Q4v1T&%pYj9rzuSb1QN@j!dpp05n{VKB*lyk5)% zu1Z-#pd_s4CSRtwY>l8$C8LFoEbGFn&+#xRdh!HB z)@*@n9CKnY;8V}IMd|=JAA3ZJ!7diiCzhdc`MdbZEbrikj$s ze>RnRiC~y|9a5PglyJM0nO99Tf$W~9J@RO4r2{8$k5~N|ZgV8xn(7Q9129JWky4q{ zBn@W<9_pOiL)9brxEy|~#@!t|J6z4yf=N2BRE)qGc`dDB-L-0~Bcq|~)RV!|NB~cQ znA-{ZjU_}`i0X7;Qq~imVXy52v%umQi?AIk4?SW! zlA+--=CjkW9KaT-VNm2Z zIy;sCdkVZnse-3+EpV=0mbK-&IVSCq4D0ec^l3|nPiXB6083LScVWsS68i+a3_4ll zhMK_%wt)9w2Lk7zlK~Dr*RJ{y)P09b6xf17O^ zJV11$T+!AC|IXJjrifILVXEY+;6KPAydeGy^j1jvw+K=sQ*!vHAo#i@4;sKJ4CHlC zfDu@vMm=aaaQeTI^w2hl7cMd>h?M zIaqqeH$~jI_Ynd{ogKe|6h=^dt1}G324|t$(e+3YraUrihd@T4)23`N8c$|UP>H+}s<3h%gY4gk5W{HEBKH^tWBl<@C-U(9D|+_ME`x6(0>|c5VHYvNbPZ`z=XT> zGHnB^57HCUr#A*y0=9FPl9@=nsN_3Ugl9nuaSeFgjf00qjf4we>d3v0?Bw0JKF|_f z+8%GG9sDs>J(V?B4!(XTUI%iv{0fMSe5J85V-RGN?QI~){0o0j-*Z<3Uwg7k6FNAE zUl#D(3G`+9!>$v+kWnvFpTX&BCCwit|e97+S#P?r&|37V&qe#5$ zp1*tk)nB9kAA7&PzJrIcp|ORjzW#q}QN|7)E*AE7w9FiTMImG8qoX4ufPe>^amqV^ zcSc|=10W-C0=g7HBuW^{C{CF6D8;BK0ybqhWjbdR=C~=G$-J@q1jI6t#5*)~0V72t zEgwZYE5x`zdumyk==v#X`=w(@!|DL35%S}c)IjNY!*EF;4{82g$uOcc0#Gua4Kh?y zlah1tQe!l9a}%;M)ATj7tSpPnv$8bftju%rvhpkP|1G!8#>UP}PSE@XzSL!1!-k2O zxk?#{R^SZ<7$Ua}|CZwZNrg*oXYJu%HLnf}0D$%XBNhKc8vD$?=7$ z)1X275TAb(OTi&TOf9%{$_gSQjW0RM)fywzO&KX6Z#)HbtLuYlF;CeP1yiM?*G++! zq2wHjAyTh95|&y?g6AVm#c^tZFQ24T9Scr2w&MzW(}VXLPPi~pv~cPD_WkO)>AwD& z9tbGhz0vo11!AY89`~i1`LulPiLqxA8$g}*jF`Bsd|vE1wu&;*lE1;OnPb;&v~NeF zUSF%B^_8}h{N}Ptzt}ugeTTjl>OEKch`cuHJzjp7zE7IuuBGf6$`*%mk5PRgGIvildxNa@=p1wU!sXwxwQB#a zW{`9~;8w&CZ+E;gn1#+^XtLCL8>rUR-;8_-puhnma^L2{8alI!3zF3H^78uF;^z*V zufz9N4bwP`w2yqeP~=8%4~32S>>&Zl^%D2#4Q*yg zAdm_yL;V%VAqh@oX9zPovGat#iO9I*XeHL@A$k4+u2)Ojb2zN%vg-LPAhiIjIm`y+ zfl)K=OFs+5%dmbPf|SN&KhEO=OwG{#6hCaM=V{W`Qbp^UoriOZEu}47$>h$bX8G*GZ+(WUkC*63@C8m808>I%FVrDDzenfrR zJ-&i>_GhwM-ivp^7uS~U74%U*#i>yPuovQu^c!;&+x#$<@P}q6$Xhg{|HSD8SO4`& z;}N)?R;KI%`Dwgar~N^Me!6Ed)?@bJSD5j}-NJHBY9GWs5Zx+F`&q!gD_vF4b-nB-KI%cR)R z;`sRHrb3Kn$uBo8F16c>qNA39>PbjUt1H+e^K*Tr-=w;umogJHdY)DJI#Vh|k?Ao_ zDVP$eZ9vgn8-wd)v4JMc5erkh2zl#RTuR~ECMAPf`3mJJLP~bSirkkrh!7zqJ3Ht6 zVLIfbi@$h*Ov^#TSKz$-J7;tH`U>S_$>`=8mx3d5H^!lY`* z0e9PkkThG}O8nadAgfns9|_%)JDYj8fw;qX$PR17e`q%^0yVm#H^geQWl$Inm5M5FTXR1nVSh2C0j~xDv671g-2aAtw+~` z6f9qc>tbgYZOIcO@+eiG98NSo>1zj?aE;h;qkZS{D{qu~n2l*-+}@ z(VIUZyHyGG@K_l{c`ge`Vpm70iCeGw&)ecKPd@yf>`Ab6ik%?g_17+LZ%DRbtqja$ z&A0(>(O=schjDV=2=GoLVVe`U51KxMd^<1c=;r;D!GMLEm&G|XOS2(k5ah_6(g5f( zL<%!y5)T}uv9#K+X!^mNJBqo*_e7$VbY`W0GU8ymF70{6VO=~B%xII-e!W#?(w7Ru z+aOw@vl0b-Ku>CmK)Vu>ce) zph7%=2NSTUfgSWO8(Y)9+m$e)*H8J3q6I7ct4E;I$4h-HJZ}%?Ws~UVv6);axNa^G zy>dCy3Z+5BtwD0_ehIG-VF{73TXak6CGg|fPDQ{gQJ^vP-jG(Oy)WUtcE@=k>jlnl z&I|6pb@$igwy`w?_U%YR(OwieYtjejw24*zPHr>aTU-$M$)l#2DpcsN&RJIAG|wT4 zBeImDd+18px$xLVhfLQet7-s?t8EO?GOm;{MO8CLZO%vH=NMH@3-rufTHtb5VJjVCJ{Kal+D&eRU1Y0drXSYIWu}QDsn99ZxTznH@T{(lg<(3c=|_xm z_+acG0Qs!0_bBLFS~G^GmCiN-fhG-2ja0GJ-|I%m&slr^FuW-%=OZ*ec8ofbW&|X7?}yWRY-E48mrC5xl|zQ4}%{(MF0|+5bv(bt(@y){XK@&9)I~B*tb+p zxjM-2G6mkv0JVDziKu{MvmmcaO^6bjeOw^YRSCEx0Ofz2H0BZ$^_o(J|EQ!c46gnF zU#obJB_<-R0A&Z5h7J;n>Hm9$ay7+yg`Odq%u0W2O~s@KmrYhHEzIM{ zB-gRf`LC%S$C$=3fG9=)6iWUXdk%lN)(sZQ zWkX5E2n*`~8u`E`GsN~N4277IlI0dgo&ssoKd1ItZV3Ch{ALuPJ6FIGxGh8jAm)-w zUl|V1=oC6$U_4~xUu)&Z3f)q8%-9?v=E_ew@6)dC=974^Ks^X}{6GF=Ytu@AP#c2U zb}3)Z6_UC_E4nO`edCPne9A#_ntPO@%(FlbZpBCBhlc4=g%qJx!loshp_LG`GBe5q zRV(Agy+@c?=S%DYYejnZD$t+KQ(>P}n{!=PV**Zo(Eh&fW@UXI(^V{vgF{#nXu1-@ zrJs-0+r4lZ+F&Nlwu&IyfHwz+B(@@*1Iex60_|KJ=!4O?>usCFXRn5BnLHiOhO9jloi&=|s) zg9*l*3#Xs4VaIshuGT-KZgCFzlE^n?z~;h){Uaa$$s27irQ;Ws1DBeU2{;&cGKGF} z!>MueU`0Bg+wB}LDye_!lS-N_Wt?q|3zoqyhXS(c8Cp$gLP{yY zm$(uZ-;b#iG}qjglK!Y9U@NX>RZ7~Sz#}B#TNN-&(}rlSL-t276{=>|ORcq`Mgc!z zwCs3H`P1Ln<%dBgLi-UlJXD8Y$|08Za4(C~a~v6TGrNFzaQV54XdvBTHMu;&@%-|( zU3>j0XJ=>cJ6b>gk)P%G_9b_LO>^us9gQ~w8il}J;Co(NHjJU45MkSyIr{7tlvbK2 zFG$*v`*Q9B9kJmeh{vP4Z3+EG3prNIoO)jKPw1lB1}|2h`T+2J;M1a*AbmSwz3f>x6-8Pl9Gu1iP$+??-sV`3G@H-A7V>f?k8KmzZC^cv#Z5|hIMkLaD?xExslWHXub9DT| z6b(ZX%Qn(}w_#?dc*A3509fpgSMk&4lNi)%AXR=N(^`ejSL^W~cUKmSUAo6y(xqOX z`I;D@exKin(x$aCuPu9|+pR$vSOAYK0Os8a+o30l0Tv;jKB$ zr!!$<7F^~g9O^0#RJEu^Tk~L9&7SQT9w6SJ$LJsjxK|# zl(Y=D%I+BOU}fsXH;^wW=x~G+Li^(7_)p&MDCB1nBfM4q-|6sO*wjoV<^dsd8=E9g z1M%FK6X;tU5q9%+(kJxYH{q>Dd+=VNwx+2qo;aQN$=Nt`a&H~>uV%UU3*OR8w5qjv zrs3nf(yIlY8!ngr%jQ0+i`R^d#OJ5w<#Wu~uDyHMf0}w?G-2o3{)j41ZnI^TaD0U! zqK!0NpBc$D)sN8R>~g+qTj9`ocBcKIf;K{|oP4`F!-%}zhcKfLFKmsbJ)TT$gqz|I zVf-@;IjOJkrAFw_5&r#W=-^bJXZ8o=oqJ#M5nu6?BXNUT5~H1+r=E(I;>!0k{X`89 zqm1_y;8B;lHl)Z@9faW|BlUTC3uJe+a-bKn&&PlC>LL@*!GAiKK&tp;gqVFDX3Hg8 zCg+ej-+duikkN6q^0ILeh7Nh9(xkdKU!pn6w%*oFUpl6g$Jp2Zk9U9TsyWRPdz3A+ zalp<=qLkTaYAG+*lZ-Z))^n3-fvRlvVFKp*0jN7ByYmRxYCFL^+gF+#fP;OAj4oE-f;b813Mm zB)JGY>~&qT-B@ZBXnPTl=Fs^oaN!-xNB6ty=l!xB(|eA${i?&ZqZCiUgqdoCQ>(0+ zq0D%S&DcMZc2T?Y?e&P@@iqosS;dlW?FA%C6P5KcTYfmIbaQE(=T(A8>}SJB-Q=gi z%QuNifV8V`evT-p& zwdmyRxgOW;N9MVACa?R@C!hVhI=;;JNadUFIFe0hA`^w$?N>s(1g^SBB%H-v9n%?I zV=&7qzM0xlHx72cZLw+1>bbCIk|p%$%mXPe37ow2?6+?2e)kc3t4r-x%lMP8vI@5_ zafm_X+19T#RhpCY9AUq=5V{oD`oZ=I#k%IoK1SIBbG%?K)ZiCu>RPKJC60qqjb`J0 zCPa8B)n#7@;|Ec9TkM|jU>%`V5W(RF{#`BUS#^A_`upIjbS{5v%v_koY@Ga@(+)9g zBLVx8TT)15fiyf8@mVa-8G0tK{71!tTs}tN%9^9nY~E{FF>+E+*zqi^yL2u|TGVrRZt8fBka` zT=F;Yww7q85U=z)+9&F^`q-tu{~VcWmCi)P%i14rrOe5)<|r34=|QWfs+g4YWz|am z_~U*(-VC^dQ{N zN^eBz>A+5q-bD8y<9{dCK<6x&uLK6O3C{e0C@)=&Aip7y_*Xh2Jy6lmYi^a~k(Ql_ z@ruKhFN{rhRG+Mb2?vIiZ(F@UJhR@ zqcj!z(&tl2*O+S+T;K`>_4;dwR1Tq=BHMwS=@su92G9>*KI&m~wdU5W15S}W{M+({ za%jGW%B$Xhhk`TZf;C{1H|au8$RJY>Rb8F)q2C6PHe-rkA+TfF7l1M(?a03gK&TQ2 z{lG;ZyY^;X&$h-igP%an8oAgtGF$!rs;O2{&gcB&nOz}jTU-Jt!&thywyY;#BHsYX zIX&IgcgtrxwbLE{)t_KqHG#=t~~=YTy&!Q4@&rt9W; zH-kG2lc2?$EzAg0XuQmb*%ZgBJ+qX@<-d$;-P$i_Tapbf5d#prNyL*3upP#@Au`-K zb^K)8j7a6y@a0MBG%>-fndEo?fa3;Rmd5QVtN?Y#mxQI%`hxEG^3^e<#%$U6m{oUC z+9uFL7+unJbm&TP5Rz}ulUE%9YP+;@>$oF+{fyRPr#AB<3oG?}Rw2O&>GDyiTmcEz zp@$Ym3R0#7vq$wl*cx5QVy)Cxeci#w#0a?e?CBIP7)SC?|L zTu?O-;L0C1jvKq>9yJp{5*?N}n7?81mh1IrP;m&?dwn-{v{n}Qv~Naqh3y|Gkz(;1 zEvAc73@;sLExCgDbm@=K~G6S);?RV4BF>5*j-p@zK_L&n#&Ofd=1RdUv= zWSW;)sS6BwW2?vnrJj|oMZ9MaXvKM2*=eI(33RKd$J=knO+{Bk1oJF>d^S9jYF{m* zM01+?PDR#mvNb@Z-s6hWdbkGtrg))$EGOoklr2{B(32HLdi#8{YgWi#KbJ52)fhw- zW_0+`?fO%)FVeRWbTnKZ@UlP-BXKi%k<=lV z>LYOgUbdI43opZ&oCp4!pZC()J$e(DYcbCd4i^gp9$ID>i7K>jf?rYM!gk!=r&hHC z`AmyKoU7ugefM`mtWf~WjTiyj^+0=vS$rk{FO_KiKt19EWWQx9XQL_ZIELLjBpBK= zCS?IZ_~DFY0@H4(f1hE3e691 zWM<%sbfv6;$iDr9p@4^&LKL}n+= zNz>n=Y`0g4NinG7jJsH^S1OR5!;uiHOX~O$4zU7)h#S|J*N=QV=zM#k&f7 z!tUxQG}40E=PJ|mn)wldeXn0A7?bOmxjA<28V~Vh2JL)sVx&heVSc}tz=77hJw7ht#sOv23^ z&8G~^1_ZwzdRmGgjr&qqfS{+xSf+<4lo>$4odcQNBk!HCBD-+SO%21xZD}iQ_WW4# zae7*Kg$aTC2hp&yv)RwZS(7IU#Z4LD-EM)0G4SLXhug!bOGl%%!+mk}q+2)*JX$6I zQ>D?z>YA;Si89xU>)IBXWcuv*k_hxLJohqg>Ff%h@U@&j7YTKdMV>R8Fm`M?(3rd# zdW%-OCITR398}jBqg9n;g~kw3sa~}?z!u1Bt!h8tj~0RAv(&*vfMSW!aqYW{U~EiG zG3C(PV+Q6{?4uFr9g&J|j7T;5S=!4hLbF|Phw+=XF69?4lAf?QF3v{%pV-!I`Eq=R zO3w16%;F9;sD~Y06)ByXJ4MP}MabdVYuoAIJ~M?YR&?J>7-@1OoDkpmQ2*hxpd(s1 z@JH^_$z{fW@QkGfYP_X4WW7iP)kRObV(JpXx@YBAjPt`3>tk_Jv^QI?y1S9r=318C zw&4Z%6&DP28hg~N z-SEN5$tE66Mfn8xRm-G2obN=z!ijMc7ww{)h}-8P#aj=#Rap0`%N9nm)2CQ4$R;Y1 z9*6V&VB2fnIf&1Y(~1&1-hB>eP!C>yqqs)4%X+TUg%&y>>*Pj;A7Y~Qd8wo|` zutu1Fd*lM-^?Z&dD|y7uQ+1Uvybe>UA4m+NwMBQ09h;*J-JYa;`~Ek3`{x^gzl`U< zM#ld6faBk@U#>2eHqQUu`E~I4_uv?Ni?}jHyEhYFNb9-m*Xw=Wz$|f!6=EnB}T}^3g{ChC}^LuQdT0x|4xuDRt zAFl|%zPs^~{GCf+?`^E`Rtj&am!e~ zc7uX+%EKRJzR4$xbW}^c-zsL#ZqC|azJ*wR1+bL`q>qY2n_F}Z_tmOeS;!>$Cr+E% zX0MhALUD`_nTa{odPbpEo`gC1OK7GSf=KNMz zYQ@7@&3;!!9E)GJiMB!pjc!GqfA~Fo7fehQClJwWFi9oVOk@+HhLFF?HG6P2YZk~MsnH~}kD?Bw zGHh%GdQTF-`&cdDAipd4vz7eq1^AITxu<&vPpfDIx$xBJ+Pk3Rn7alKsx*>fIR$$S z#yXapHk~B*K*(Xq#UXMCxGmLCnQ_f(J8@A=(!**3xi_UEm@0>*%#RG4CMQ(<{$Zi`Ar>ng54nBFQ*ec#+%d{>0)m=W`xPH2|VX4O*LEhg0WVvM{5mWNcFS?mM4P`UV`_5`Ud42uPwrY zK>=V++^HHp;zC*$>@Tt-Ak0=Zs#)RAMB5$@@DG3TkJSL__&@UAs){8#mZ9%U;;nTD zBzFbHz1*vh0=^avh_7fl?oaq!-qx_Z4Aozv{+P!00#nEb5<8u)@ZQdXcbOYUv5jV~ zIwghom*$%$!$J12uiaBZ6vY_kR;_+DiZe4U6QscLJeu;~#mC_QGW(1>xCdCL$DJr8v;}I#ZUo z;zNyiLh&QF}QlINY?%ieb!K4Zyd&KwW=p}vJBs( zP+h#_u!F7<`EfAh9*?H4$YKl27$CI#l42-i-zV9r*UF`xoj?Cct_eb zY*LKSP#$2Fyuo{Lq7H;fYIqr}rb7#vqlGZ#D-7hG^D^y+bJrNE44+w{MbKxg_3gv5 z!@<8jYlR8b2oO}NH`7K{a`3hrrX{n~-JwrQ%W5c+W}Xus0@1yjYcH(2052;m|e*CVNL(XZdt%!!-I1p z>`m2-TKIZ9$R7X67G6D_VZO7!k+nRyvij=Us;~h~{pgMqY~&4XWdLs8#n_gVQHS?> z1sJxqb5Q%5*97%mah3R+BA6L?$9n!e%c~}?n!7wx%H&e69=UU?&ML?u2Gsf>y#M4E zwO`vNivgU%wyDcO2y4L&fs=iMT7NB)oOn#A{hnKw)qwHM_v)w+VSQx*EXa6G7#i+@ zHksav_%x?J(QeLr2!KKJYlXU|XG$kyVwIc;T>>9}iEPHLawS*NxiWyGT&D83)O2xn zEf;?sU8B_$TD2wFogeUFU*&7`wF86mGl37vbcB=kMjYV}BnRcBU_(l}S$gGQ`-!Cf zz#6eYqLpJ9fl7aWI4_8*&5C&y5`mBfnyuS9uPn5eywe<{wa2|2SbGt8hR;yNBhCn_ zB2S|4-IoQ9jkpRDk;>m<7eu8(G|L%X#*vc3OvJJ;Kyu_XCXbSYu>YDC2}%ml6n0we zbmSN=!-NB~ctCIjmDG$MMLF;STmX+QIpF+>er&}_eDr0cv;6DSyGjZv3rzi zi_LpzTzZ@C>_uP|>9gjQ3`-GiijaV}N-lC#grf?P3R76>T@sq{*W7=Y+%b=WD>86NMLmmFq$Pmy7%9YYQ z*_qj3@+ci<(2UJ^D(bm5xrYZ|X2lx8ygW&LXvXR9h=PS*NuHFSWw3_RS#?u!j)6syDfJ(!} z+lR`&qdzzcjsP9Yf-y#FVQw4Gyc`&6>l-CDZT1vt8eZoG#|UakZVC6$;u}>OLsuqS zA`dLN_795{{g9(g{lZd>HcMHte-z(IDr3zoviT_3|L7d#wR1lfMVCaj z1C)avypO9Jep5)zEL0WW-$mSD;=VHB zjbz*mtW^?~Cy}Z(K>mft&9lSwPD@WJ$t}m>b}ZcROaC5YW_cf_`&gJkI%h80cz|RT z`cOM+!{6V=hz(Wd1XWGd8~tR1oxP1yhj=Y1Q{v#=@=mHtJmxs7(T-?{b^gBKQCEvo`b>b$&KQR^DvCI{)r3J30|b;4DlyLl|J| zSN(;^)aDN#$!N?JV_@W=;+Jn#Q{af4EIMU)tYZKm@Ojb1W!Vj<^;MA5bS5gs%a|&p zzq2#dmQ8#)3qL*ekAW?uRGED_%AmHr@uOCqXm%Zl3(t6|5a@XRws0I~?J`P(NU(bV zwlLucYmTi463)zP6gU%8@emzZQIkxnv6>U}Gl9p@?25@=X;cs#k6utZktKL!V+55$O#WE_KMi~JBt1>QSHlgU4 zmc98YOzHVBoBi;WpO@pPmZrD)JDqrVkUZCGWQbKY6yU0e0+jREIW`J5f>ZE(CaGia z1|$Ez-gVoB@oosi_Lq@CM#>+*F0H;c-CbL)MTUzf+IX!n#yJ##PMw#CjZAs&j6ah; z-b->js;wN6;A>@neDJea#@B3=HlpjK+LO0py6Bae=&(voAShR48;SBRz z%F$AIZo&@XyA>STk{Th3vEp{Ra(kBQ9RQd zan3}rIe5a02h+f~7gkJ@+w)+?gcX<_SWVdfzF)omdcB;FnOhFI^QYve`@Wy*^L^fu z|9-CG+kjOT$n5=s><9D6J9{l8I#a>>oUq&U(~Zcb!a5==3A}i`RBGU?Oq3-I>}i+H zY_{DYN}@9V*f7B^gc8DUq^eO>dmDf_8fJ;=n2{gp8M@szjsPlrZfJcHgV-&``GImb zN1fCU0xn@<7!=lJS*&nTtn)RdlX*<$+K7d7Bs4O@UegdJP#n+rTmQ)dUm&}WEsHo2 zV{Bhuv+&HvRjWE4ok4x=J!QktTuI64jV4@Hl1?Mvj)mLtFn$mtctxeM6r#apxRp~O z?&kT_t!F3V;x`Mmcx)HG3F^uVvOpWlU7U-Ud?vc9CC+W@6(JG=C@s}J!Sy01`P9!siTU(NHVr;@s60Z2OAD3>Hz_9+ zD20XYZJeo%_7eB+lKh|>LvpYHhzkQChcz0;OaKOU4J7EqP>B7^-(2MHZOV3;)3eT+ zTJ7u&NStyJ(kV)ikfYFpdHg*T(IVYFOW}*v+nloy%ql{;E-s1VMZS11j4m$=KxOY@ z792P%NIOiHjdWf00TR!KNuAsTIE9%7sbXfd_3i{fwjbbhq;H?8O6c9;_mJD21Wo0u zoxv6xN?g`00rtGEbCz*; zBT!7r%y)R9=S*)#@mhcnVi^acQIq5!!orq%^#p@jM!wNk5~{<^j!nqZv>yO(v6>jC zxPOm2Ak0Y}d%!W2c|<*uK(3o#NQw7}S>BbTPyYt;ZRrxVM6y0TJpO@iRFptF+v3E} z%sO5>=qiqPs<;mk6xkk!?AE9g zEc?j|-}Cn){LKp27{>QraNCyY-p>KuX6E`9d*ZlJ@V)`(W?Duw2-PPTpw;RBEy2CK zAvy`HZ81#JmyzXIgTTs3Emf((0)-2;uu}EoQU|S;o8A2iz6vsaa7F7oMCf2Z$0sltJ_MCZV+zzWm{$;DSg2Ost#0mq6#0xBm|W zLw~>ykLG$}<5>?0M9jw`-YpQwvR0zC2YiUYBBC)Og(9Vp%`x)lHV5+}sPRf?e(U)| zf&XeDJ7X8-Gz#Bu6m1on%vGsW>Q=qFVf_GgW|~PEts3biz6w`aR26IL2S^`x0FQd_ z_GI_Wc=#tD3A=$`({h+D4bi*I>g`7lg~``~D*K(nun_lGwMuE$j5fe7@?u;Q8la1^sc(yrJ zT6S1Nf@=o<1VJ>lA%GLx)YP{gR&8^m)X_zhjNk?SZ}zmAB;7^kH%ab|Y(>qIB|@zk z44=BXy1n!_$U!|;LQ{1~(?%v)Lrek69rvUSIS*j;mj9KD= zWYRiyFlg3JnpUi4qt{4stAKWIcTW#4GCyTbY981AyC4eaupOV-LU+7p+`|P35i^2V zu5?dZSnTP|2xsm`KGP<)Gf#pZL}Uulo6E0(u0Dt+bY&wqX`uU_(Z{ z27@i0!q0)E5b~eSMb&C$!)Okr5C!rbn52ZUdfb(j31cm{A!{_6^~+V8_k)Lp`c%KA zj)jNbwiqEdK@v1cBI+q_(xCqJ#wagqBkhy=RgirE86a2ma6RCYI$}p+QLn3aXe;V^ zA5E1P3>zF}mg4nhu-05Q33t$(D9nP_1yv>7*d|~W8C>jD{Xb9iZ$PAi*hZ?1=#!~}P@Vh9+UA*|hP|2;S`n_ZhQQr>=d@iTPBIR)-{ zS~`jZDtNrLn8nkU5S>B6Y9z{?f^2IVj&@dT0Ae%{x*6zJnDuGct|$WrfTz^JGh_~8 zPY^(O7AsC3#OBXH-!ZvBdRKU@Daj%Qx+3TotOm4xzwM8Q;?2*Wp{IW8xnNNK?M(1u zKakeMM%wM3x#a;SBj{b&w?O5R>`MPpDrGVxJ3|N5CW{zrUt}?xB4HA)U67*X5M;{1 z3*><&zs*gkP`Q!*5g&(!GVW};fmL8bzGlPH3cB8`5hrY~XSelF613QbE}JWoD6oM$ zYuL^jS$I=7rBJ=qEVaXt%Arp?cE2EwtLwbq7ra{bm;SrrY$)m(rZzoERY#@Vp%to- z+8(3Jtj!ZQP|p@1#p*7!@!3Xl!x?Bgb$5N#tE%Q{r*Vr$qF@)nIc0Y)I{Vi%dXS^> zC_UoWWP}rgtp;K?_Q!Jn8qL}B7Bjp9YjHL?j;-S}JM_cA;NJIDU8bzV^HEpfT~_l_ zc`eAN>i~iwhsqRc>hp^b80?mqODc6X2=bV+!;qix6>B#CZ!%f_gYD5LrZAbg6w7poY68osc{?@E1c+2AF&HgiKUx>DyO3=bCPH+Cm{x${O@+B4LrU`CVjFCf;9xX5p>X+XGhb zOg!{t7jD%a`(^XUs9>dx|BGm&6MElDyK zVjmhA`;?M5B^7nqgU$bP?nM?De>ZK-cd(Bz0n4^MUsS(`uPohe)!Jv(Nqez~-Yc&i zC(l)-%uN|ITNmlwSm{CV?rzsub$C^~mFE^2B_s6AFt=zOJ|mz*%F@*OAr}kffmxN7 zPhuA#>fzwwEOFz}=jTJWN?|I}JBkBs8n&~TdnbZUs<3mG+4b}DSnhwM-H7=Qwcli^ zA|Fa?i)G*(&=`T}8y9Gt3||tXHujj`52g$g&PE=N4~ZzF3YVhFm=L?vXYKRoD~o7l zod7u65l$b#L^m8^(2d)K+iq_x8Hnq z`?$opsy<&knCDh9cZ4{7H-$2<&U~PPB7OyqXLfPne`*MwVz-3aS8_;$BX2pSw-fHP zZW{?MqmI-*`cW)!QbGx~y$*b0GQtDMU~pQSh@z&Dby`E$BuWf=x_o{zs=7nRi-DKP zZdHb3tz$X0O>!pv?{>-m7`W}H>|WV|0szGRo3@etuQM|TduJC1CwpU4XJ`HYtU&&= z@YszzWI^z|)!*;pD@NR2(Dm4>yuMH)kB>)?@0YHiVt2)Asz6?+^oBn0H@{kI!R7uI zTwjFT54#!7z?}7Y^t`eRI}=vzbLgbSVwqHW(pX^CJ7oOGHAN0y2b&io$Ubc@ZYMNi z!kE+5y$6d3Rc>XKH*MLT{AXh2Ug}=D<6o%5Tc5)C77%3?<7d`t7%-?O-gi{ienfTkB)sv<7eiFVc;T0x9hEccb(^61a`*efE>@sc_X@_^UXyc${7>(6Q zOg+D+TT^yChLreZJR-50I}xh^`2h|{m;{~hf+SVbN2V~$SVd*C)J(P1$`~yUZe&` zsY9`?m*KW0O3Yh<*5X3P;{-#P;;qq031QWt+#&-*X(Z-Qxwc8;=TjM5rum?*N+<} zvaHGrrHq2go2u;2u?)`VEh_0i({DqRjQM-Aet2QK8FQhIV6j+gh7Cu*$M5R?u z+nH!qUgptfM7Uyh-3uRC#k+7MI8`Z@V@mjTqR{vN9TOnr+L2Djn{%gkpx_VB?34VV zA?j|dP1Kpi;8KT7U5$P^du*5C2zGrKd_&$eHYH8<<2o-A*c$+Kty`Q1(KaP8q$3}l z20c>R(A><^Iu<*UmV4^=D&w)}Or|q=y9!cxiQLVrb0$wJsUJk^bU0KPgm5IFmKQO3zdzJnDAnnu)t}J-OJMH%8ZQ8OJt!%SaNbW3{q^&}+2NLOIe#N00iX zndV`BW062f5VIW?F!pTi{DWKS34~QOsf}=_^?9-m2`CS()R9?t=R22|mt-mzYNxLf zy0nW_DF~KM`m9Rj5yYQyOjEd>;po2C*1?wZo|<5+6OyFgWYBSu`ug-??hl&aZB}P{ zwz6gRlC~qej4E!IVCCq-`K9+3*r8`co5APZ@n8NZU9ckC2_feZOQ~6EMwW7F!l4%9 ziO<&v7!qnB@aDuUiujj4t-FV+7t(C32ST@xTRI|=U#w|;_4=d){6;d;KK`rkC#;L8fC<*LT z=KI}^c)j=O$ng=lU>%Yoecz6$v!&6lZVg?yl33Fj(C1y!JJ)KLQ`M1Qmy#RR{mw?5 z4{`lugFHjmC&U=06J=C|rwg5!91R@h_L}cMQ@rn~kCAamlYV2Q+iQv3!u~%gGWl3H z4#db=xfkFzBn@%^I?%o6HOf_ZlTk(-2BzWPPg1 zFI7&RpU_S5pGEa3f_mkP6{=VnT9~Y0hUX ze8ELMktgX^f^e9~*K1e!xWqrj`=YN3#a|Z3UV{Nvne-I>RG1fEMAhA61(pQ+Er(S@f zL?&LdD@)RflaK(Ss&D8HVF6tq!@+}$=;FWUnTGF$FpO;`dQ~s3IATrd(>rfd7NE*y z`!ckJ&EYEU*eh36mOXf#JMxt%Rp$BkQgB=-$iraG8@z@w8ji9d)C#JaXi(3Kpia0R z8bp&IY!_>YfFp>W(;H~#|88FGm{zZSnjn9$PI@9QPD>N@a5vv0H+vhYkXEWp@y^0b zNz!dMcCsq|HH#ef)e(nnhik;ew^f$1QiI`6;0y;>+L-eba6kBQzF7Bm9N`wIL1%*S z9u$!==G~DpPH#q2qfnc^@#jZu`C0!H5Lg(dc#CO=ksaFBf>hUIK#a*VD3#J8-F1LS z6YpVGjU7348ONF@AsZZlqUp?Zf0s`gvCl8^HrFi06tP2?B7M^Jh^?ezjUKn!B|A>r zz`(UjFlSyKCZEwB(j6GWA}|ELeOlCRbU_3CG_66PSLUp2%_S}jMz@a_xmu~@_3_Kz zZ6uELV&vNHbDEfIX6amZm)UeS!1J_`eR3Vkbz)35f}P(kKU>gR;7Pp9Xj)lNXXX@N zbgp33*!+R9#k8(rr2Fv8rt(EVBJ5xna-I@WlPuY`p`}m|Uk~C5zFueffS~7$cExBm z(ODmV|IZgEH!Z{?<%YWX?#KuFg_gi-6r68|i+uMhNmhp`J zZVvihBXfUFV&G|9L4$Rv#u zZ8e7;$t2XOR=fR0n=@fRisuhgxAev~$?~$RAhg6fB}UWLh(EX3y{PlPntPKu0U;N! zr!*5yp=H%uXyBu$Q;}Zh{cRPT)U%D^nr2-8T&V)D>Jw4ZV?Zm3^g6&5)(yy%IC&#N zf1VEHC_twckzW?{K?6k|WMtR1&H`>d5{W(q2_JQlAT@18jM9P!t%K1o6Q|0}$hX4p z2=|DJHd-*s8iy$yb zGz*b)$@E1)G~{Q=$;k~TQ;c5UgU0!~eUr?&2?bAU*@+d25t{<${`9L7#gxeqTg3n2 zG!wgqHpVJBb`s%7CH%rpntGPY{8opNzuN>35Z97)#sh7>%9M2>sT;sHYQfjdXYz(v zGVGdMgqu6KGO}Fr81YlY;vv|A>X|6Af^9Ll7qAjTvjWZkhymhpdtvUls1_ROBQevo z+v*v4_PK{}s9brf5;IODmx7^jA~mJk_HgP$ys9X8_*BqERF9d_|U8NGu@M+>>^K zM2UYPlxfw7h5htird z@V&8|6H+aS*6P=2#qcb>5IlQq#-4P|mvjwO*a>#FN~wv}V?+70-F)n_%5HvflSs3X zBaH@73|lX)hAD=~A0%gk5$tKjqQbdN!n`H`1ZEs#-i?_fI5x8pHcmIB8?3{ zJ+^w5ZjS>x2S%-?qt(ny^QxRj;OeZ`&^Nt5^GTKYPM58~8oWh~oC*;%Toz=$3wLyp z;YEA$N{mV9Hs`cQqEtzxX}h?-^WogGak33ogMM+vU$x*u9W~`LyYfc#DaTTJ_!Eqf zaLm0H#n0y;C(LifoT0u&w{lw}3YG-CmN>GdELJp3oe52Y#0S1qr z3_S>nOQ+e;s1NXl@|loU>Zq>EhKSwU311YHg*%d@Mt+1Ml?DcazoCL%s!fZM- zT~KS#cD#!?7ca^<+KCa{I3SpRyzhKYZ>Hb=SURvJ@gn?VLVm=DpyGscbB>Xh4`=Le z9!~|7q$#J@KVH{(CEz_+h!*?lXzWCtpujsd#GX2st}y~nm#*$8a@MS8h?#wG(Mrb4cfmN}ZIwueZI*V#C5K$sG*K;QOAKE%HZN}So3q(+NOSq1 z4jUk0C`$hI0wVC&B^umS3~w7G4t)WE9Ed{gJ(motuE^OoZTe`i&b*Ye!=bvhM0_Dt z5!JlYOI%kRyLp35|HvY6U{W>jBDQppd$B=tEq=bxNpoCZ{@swqnb)>Z4oF^7dMRIV z2qOWjzgIdVa0fHvdjS=rw#YHLmJQ)NPa6ske|hk$-c_sYxyLgGR3@GZ8$Cd?;Jb(O z<0m?g1Tz2Xyo&DA=< zF(PoTsguoG%RR6Y(9xzFpJCj=H)C+1gS7_)%tX(8kaFhvFoKbD*!&-sU}Vf`ZLt^6J!?Mr4i<5`jRrP`6XIE{kVAEk>MSz2&>C?6} z^*ACL#LHGJ7}?JF=kD;C8Om`!*X)IVFxbCZ(k{5Ggy%rvmkv=t42U`_&wzV5Vb~ho z)2~8)(YLQ1i$lO@jWiGgyui7&&I&YX`(TK5e*XUN?>9d~{B<83if|AkfZfyp?1!e6 z_aXyr;iaaOG{LRULfk4^9p64OlmM@;Io&k3xFAc?&yru7P$i;E`z7z5uG;cE)sYZL z+LveXd)ZCUVO}?GvEZy&CXW+!ZF%HZae{1OYa1U3w0h~#FHy=-=xu|!v0b5USI`P+ zV72o%E-K!7eT$HH55r*rM1M9%L-fK5|MG&3eheu>pN?lYzpCrAnpo=j`*3>|^?8zd zw;f}&>Db0pPPB~-w!WDbEj6iIVQzUms3>}k%0W~V7|z!w;}0qw+@VJPZYN`X{9~* zQC1og%9c?oG>Yw-Uly7eI5v{j15DQ&g>@$v=V-}lstw7k&np8ah43)*vgT$=jL%cD zqc2By1g7xBDyp@jQ98yYF7&z4(Q86gHLM-Ex=^ zrr4#pQ`zDV7v`N}vF#LT3JX7Pi&U7tu zbr#|cZv-v!h(V}54J-MhfU<|4x;K!D?nV%zjJ?gm;;Vr>JTBGiJ^tpKA)DZ8BEg#a z$KquHZ=P;t2p_tdAJD2z4+9WW+z%YiE9ugmQ)J^K?5Bph7FNh*POg}Tsj6a2R*i?emLH1(G z0*^ztd-4%7`9u-OMgp$QYjcPHjDVkoKHn7)e1mP=#?>JF7WpbHok^{wWnb(i1>x<_ ze0QOd5qJ?MKbUTDMN1S&xSDjHI0sm;2uz0pQ_pN5lS}(kdC+#xRuQAjid~)+wGWB> zQ$ANX5=1RlBv=tCME#X51JOoWDIBlgxw5EIbqhBJUm)#N4cOM=O{$ z3g&!09tr!9NL7KnOCzW*ybA15bOCt87|@%I$(=R#npXyUx|~olIb5T8GTMtr`Xw%SNb(KDQ>~tL7;@4`#guBeiJwX&EB7FF*IUt((`W%B!Mn^&xj^ojDt}1W>1G zwc%!y%|Vd#l7*G=?OLZcK@-P~Wub2fEUD%{B9(Pn%gx1A-~UsQ{|Atbm2aQY^Is*N z`S1IG1KIv7oACdaP)7_aMM?0VfI5)>0MwcN2cT{MCNtWP8Z%O05aJ-sSIN>$F49j) z%FHoB&(6rn&oQL7pddHU&PcH~Bf`(htJ6=*QqRay(Z^3uFT~GM(oC+w&&S71&rH&% zTTk$q7A|Y+D|sPXg*)CcPY~_DM{>hVDI44X~OkdfvRl!+7=$QGg?{bAN58C5T;T zckI@?lJM|yL9^Nj;UEoxGf_kvk3=CzMj7`MR3b@=69xR9;VXeu#zUnd@l`?;Q0-?p z+q#w)EO5qB6sHw;&3Cn@V(T9cMZe9J1&*`;KRBczd}(;r&3SP5#HNQ#q^ zX9T!L7UJcUxu7kBObK!dT;LWoQ-fUWv@1>sQ_PLyv6#p5n*q*&u*04>O2!CvO_1Yr z^IZ@ZNZBCIB`tUdmYS4Gd{9qS6MV1vsx~VudS3(<9lwV=Zp1d9S{A2KSJmY;(bLpI zJqg3Q)4`0SR5K)1Lnc&HC{<%BmO&Qte}nm79CC78FKgcod`)XCGt?XQRY4=5OezXL zE3~F7-T!J$vs<;qPIub+%&b_Qc0Ic(G}gm?Z`mu#3$%NW+G*V+UgxsuI*+hs@MBj2 z>SLF7G`g+mT&JgA-bSTtn=@Krvvm{QJ7Pd2%YkF7D#MLKMd)DaLHu>?Ok!Kj09c#+G{&@nWB+39t z#LgTamXk6I|=S zCY}g<6f{}Q{{RNY6)-?ZM z-T4{3us*v5cdxwfLYMJ2)bYZ(`9q)iQYKl{_^HeywmHr^R^iAfQFN*@gTS}txD{6S zVEmg$+wk563zI^Oog8dgAV(dTTK$rXtcuk}FNn)8HiLTFv!mbM&XZZ7tVc7Sd515*X$NP{cZX+=lPo8tc3k?4A-cP9cNM2;drw@UGnS|W zb%K#(ETJ20!d5~r>d1}6ouC7G;xfJ)Z-SnHGw=vjLXM~d8uEhN4{Cx}BHr?gCGj(! zhRBmRQD4M~Kau@fEgV8KQ*E(kxmLXN5W56a*UvO>T1f^M7m{1A(j*jx`&{RrdyNmU ztIzs_Dl($gX495`-eS`=jkx##9uBU(Zq%`UZKor^N=J3?cyj8;;`{WLdUDD>EN{)b zSH5V?lq-480nFiCf)gUQz~Ywb>m-wh*1v){cu3gN3JiH+5cAQ5W9IDOqUvW zUJ`?Iw2c8&OxT$av7HcmU(c`uYo-##CrByI>~NbR(1vh_%q^s2U<_=>IzxV1SA0I<5(|%`mEnB8agW_&T?On%`eLpnl8M9E6nWHQhO5 z(}r{R9zTP%*#*i*+DesF}>9>n$f1$sGf=r*`Q44^pgJK@J%hm(&=x zJ&Uf#l;&D+AMOQ|<)oz_F@#l&4r zHFfu20@D;qTHz0|(tp#n0c##;H0C4dw3<)(CyTJ+(MR{_5ia2lwbDKu$O*=xH=1LwwH}i zOvKbb-_QAv=>Pnt+$^CQE!zYcdh_5|+Y)pU#}Gd6BLnWQpop-CY82rZCkiCTT5e4C zN6QC%RNJOpNds-pVt5QWnHd%o%=zYYQ}R-<888;Dl~H%}U_%U7+Oh@rAjlDddMxQ; z)WDqUQ&I*jO=fTl#_^;5wQAE@oNv+l;15x0t}qD#et;LULj0y|pcRf*tgtm^Yc$1C zS8z6(}w@za-)AuKW=5mtWBUf*2WA>L&cJ8n}e48hKp$u4>6^QhUx2fZ2KTFV2E4Z7I2p zz}}0h*Z>fGe+H{ND0Z+aU}{oGGKn&GRwQPPYhVWYXy$R|d6Xx{51j{a?F)c;1U*R$ zlT?(*xJ17TKDnpsXE86FQwvPj#T|TNFQykC{VIJ0qD`=z(azopcyLY9dt*2B2D1?Nj?xj`M5hZ66}u$CREcd5(uqJTHSxstd?>P*!ec$=vR4Kv3ctB z!hXOHt6;I&xqk$qkqZkHoLez5^h1R(sA6f)s@IaMVFTIH6L92~jl`M`2cZ4%zx3i6 zPKeBn2H1*FVU!Ush)-|tgW-R*RsuPLI7Wu;VNdC1_mOMoisTI7OzF>-5~_U)!3m~a zolFY1X2gjbMB@mnqu_x{bKI)MujG5UDJCY=kC-me`geMRLpf%FK7kMBg!54c#B*7PbU`aGi zaPWtL1Rdau1wo|n2b-uP9@u05g(boGi*H#sCf~9dly#!-plDZS>LohGkFWou+w+Sb z;Om1=A#_~SNzRgfQ4;Y9a&8&J6d1R=nwe5<;Ny?`!Go_A?Jh~U@+L5Rq)36kUOoVw zeSqMvG#oO${U?>7d0$fQq^?zKEm@Xh*$yVR=5!-G&fp?#)@+ z@xG(6)C7eOaC{qbs%I#E5pyd+!$^8PPWqNs@|b1;_*7E$TL~7g8E~o&sQG=@=il*~ z=l>j%v?s!ka_s^&i3e$`V$*hT@{DQ?>|?gZXn;WTq=1A2f71_EyF38FCk)lp+Yo4u ztW|bX`ySES<8`%Cd6Rv+(s5n|mm8V{RxJkt*TPYS$oOW(IId&OwnZ?z8uGIfGI-m< zoqsaDD1;g}F;G!QB0n?+P+|#ip%=jGkW;O*SO(WYThS#P9{G=MU+TsKjP2<{-FKOX0@_Q>%+H9Vs#n z^;B&j)gSA6$D`x)+Gd(YwhLuoK!LJ{3gwSn4cQRhEosBs$>+0}DyHkK2et0Pxz&?` zLg}I(a!u%B<^Q>ZZ@Vi$jPdMEjKS(Sy2AmB_L*yUsMEAfwtEW3899noZ}}h)wL0j~ zV6YahhKf5U6A;YRtoOr0Kp1)CVg+yxcCaQG9tQQ{cs22ZNW|7p%sGx|ojH>s1d%aa9jiE`XvU_EaM(CHczIdqS2@v=YHqghWMqk7LgWYulPqQ(Pweebnj` z2Et3y4t}TSF1e`e`E+$&56*Y<=X-b5(V(^gcGE}Kk?*qNt1SJ-bnA+Zl2nEf7DWxN zCWJi99`4cl{jLI)7xI+AgDzNJo05JR>+-d}mXG#P6GNN`L%fm|u1TKRbm|UC!7Z!I z9&<%!n38r+Mc)@Y7jec)HwX>ysZ_j4$}r{(Ql&l(C=8S_x}hc@bH?7Zn?)L^Z$WZ2gcW6;O*~f0Uy${%4$`;vQK#7cpkP=xzIwNugd58VP@ETncD_1)ArFjt6E0=_m z5r|VzwXXwz3n`19r6vS5pjg)0)o^G>1H!OU5X=)ffT}J3?^^fazjjQt!g{sdY*ACg zuC_MQXW(BO3hfc#L1N}W@9+i5*Fr_*nDd99iy zEH=iWPWkvx1G6=;^FbA8We`*+%1T<>BR~VPyCxV|ga(A!`Ez7QqDo>f`hZCZ;$tuT z;1(eOUJMx)Nm zXs7xpxS0|Ybw&hpnPrsEIkOI*;0}a!v|MgClVHVq#Hk5frP`gU)Oe@TFHxd2{6=$j zjkLj(A8x@i=;^EgtQ`8MVT`?(Te)T$>D?9Wl`*nw~NIW%l(eFw)|1@1;%5``99 zq^`>Ts!V+^kmZ?G9 zWN`~c-Vtp+m1r+O_0OqZq7^uMaN+ueG|4xAUZahBATfm=fV@ATj&XqFb(B!)EbX5P zQpl1`z~H4m2BVM!RaDjLyF9}8M?{+`u)q1zq1L{_SxOe@)FEdEb^-1xJ%#V*6y*Xv zpPURy--}2CJhY?TBH5lELIihQA@#=sq`+>sl(}LOm@$zX3vbiGyQl^?km~4Qv<7xS z>7Q^;#(^b0D9cIr3n^tp+LE?q88TMfvNCdD1y)e#ywsppp7%N<2wIUZ>uDtlunX5w zWkh8D1REi@=!2qPzH|rlVotuEX}OF*qyID;aX2|=(ggy*GRdIiqv;0-J7MCrqIU)u zR6AwIkLB~B!F^k-fx&MQGoMrgNW*Ph)B;KA_GqEG0AQ?NtL8AM{Zyz|1Tp-#Rq?BW zb?{5{wCNv>TYDY=?Erx_?+-3vVA^(1V;?tf_RZsedl2hTt}=4FkpqMM#&wa+qnY9J ziJP4q6WmifbCm6V!E9k-Z(NyBntk)O;>O{tCHp4(DZqIWi37Fz740Ae3Ldn1)|byX z3HR`XvbgJcu>6(USGO%5=V9@gChFk9@$yftz>EWI850dpKj{Oz@yX*0Ap?Q{#cUo! z-&kHu5TAa#S)?4zi#3KlmEqu8lgkX~`BM5<*&l|ph*EHX(H;Er2Eeb7iw5i;vVxlM{FSlN^dX$t``KP^?=Tf zl$qw3mNGA7y}<_Fee6f2Z^xf%7O~CBid};n$a!KuHrwa~Y{!A60MSocQKZqg>hNt8ZGssp>Liqp{E>w+tzme$9oIS`@vM}6^V z?e`r97XC`K8{mEU%W%e(xi@mOB~7@4`YUrfSudaMK0rGTJ_;{56>ht1q28&=`*Ah5 za&>z@-o%(N^W$&MjBifyfkG~A&ZgR=2rS$mjlxc5*8PpF}iChxkhf4#s z;RatJFF7*_VUX`A)mcRO7S+X$7KiCh!bhr1)esXbcSj~R68sE|qSB-^i+~#lxYvRb zB7-8Mb}q3h7qbfC4gyPSUT0!-my^G0PShd}IM%E5g-s$KtaT?9F!O^qp)a0&WgKV^{BC|^lDj05 zilY^r*)*BY-0A&zHC^J~t3#LLElD?VMBmWU!Kwk2tCSb^L5G)nxI1uq8*4DQ3;6E% z%ThQSj8b1v9jwlM7=0!U^Mgk^ym6pMTwpLb)WwsHKpGK*MJphw}`-fC}PIw*6698d{ z=?-LXJS<5e4y%AjrH(?M7WfN){g{TEe%ACrZB>3OUzgvm&IK7T#(Xdy^6p@$B%5Qy z8@Uljuw{mB833Z66NYjLmlpchws=>?k9XINO>f|K>Kt)7N;xrFXJKQFbwd{pcC69= zy62vF6KUlXHFG5|cCqp>iHn9oXo^sZfRsDYn88D&Aq_8MJ)4TJ!F&{e3m&_Nqzbk7 zxX?%6tj|Z#<7UHy#{OVHj#$md9q9`)p=FRnMk_=I6UjdnCDn zj*9o$X$m+YK<2JT4m60rM+t{iEb$Rc>6BT6n# zxaIy|jlBbOBwP3Y9dnY2ZQHi(Oq_`(ww+9D+qP|+6Wf~D6J!2;*YjS@y}##q&swz_ ztLwARIkmgGtM)$My&c@M3?+2r8C~O=wacrq(h}6{Rcoy}N)7fYMp<{dNgsS>-6!Ou zsfKb>VK3sa0CJ3I31 z)=`*HH(HU#PQ^>?Hv0>2$&7VK1;bd~wIUALHP?r_=~a|U+NtDoZ<5`Kac6fTE8GQ1 ztBUw;KZJq8jFt#xwLQ38SmDom-J_T;7n~hq7A|njL={oAIHWr;%S=1U4;a5c&XhnE)=D?b`d&9XZpJmF~IoQ{qc6l@DuMgUv>A=0BpRbUo-%0qv z&rKF?l^-bhC$obNIll>%PoXTxzn+&fg)$@F_KqczY(jYXB)wV_W2mst%G_&d=!S{K zj0_!MFR5ihye@v^YZsAFT=|ht5wQk|?~??dq?l8RtAOy60`+Msn=P}?T6C=zJpc#{ zcDAnrcZjF;b4o3GsI}%i2y}_tXT;|E$vjQI!gPJ0D<{xn(F0mK89d{3A+#ctnxd8i z9Knd*J6Mr(ypV_-a3ja)^LTw$6Su~EH+Spu_E?J(YvaWA^};es!4-_ebM4^vJrzzK z4_2M~tlhM-%EN^W*Eu)veHm-tj7iFo)(pq*`x_Z&lb6V@I@=V{cAhRc=I9SIMbM_Y zy2&L4>1yfHbufWKYhj?x&QAT&&6}5mFlIXmVG`~dn#%xX@eYA7ts4{;m-0fG#5**x zY1vmm=`MN)539Ml)$n_?Kn*0KXaO)fkbM6p%zL;6$59kfbI-wpD& z!~$S&xkr&1YI#BpfYmJf3)Y1VVLwY!_bhk_YRe=R!p+19EOilI5*1A;=RP67hNg{$Io~nBQUqrxEtAiV41Ic`sRF` zzCWid{Q0i>kSn6-fU8CS1{`)-)c(}H>Ri^u*#yEJgJI0F%BcsHCf=*;tC_0bP$q22aYj%svTr zuZCBKps`D;8&aGvva%X-%BHc|s4O%;1O$48LO&@KDTkr57BVg16Ooz5fNfXLk@|Q7 zZ}S5$A>mHQ`Bkp+V=mi}`G8Z1O@lac37FvsNpg`1XspzWDXij}fRVZ;$-!+K2HVDU zmJatk!S`rPQDvHen;6+ZY-iy1hTb{E+m5Lr47mj+d!h#VT_R9CVTCzR=!o0l4HD8J zEfB}z-++X%E4)GtMk{KITqC0-!#lkQ^fu0-lZ-iTil9y=x*HcJnH)hEHFW$4NfOW0 zbQD@qUmP0KQL^@q;~iT#85^vd9b|(u>bN=EshWy#nJXyV;OQ_}0kgvmURnsaBbN3U zNo9O;IJ<#bu?rn?O~ucb1&YeBHaiAMph((>ML<-qssKB^XEkC>8Nq7{ zoH+)iM$OE{Hvfe_sD-T&uuy@<<%H)dE%5xoeFOL7tL1TJ<_#vU6ZAFc(I&>;(!7*c z3GHXM4@qc2`Zypi35yBtXh%40q5`-2#T!xpEpN@wj7%`zv%EoO$GAtd<)I`3Wgg%a^Pc;a=>QkK2Ttf1MKE{VCiQ_@ znTVU7s;!UIIpVl^99jg@hT&s2zRC!5<7u%4;XN%zbtZyI#h3jd3G1sVXU!r(i!C4V zo58!_tx`snm~rvBYRItdKaRvCk2|e2zvmg(k0UOmvG!nSI+tB_rKthy=Cc@p-vDDR z_j*0DTEMglfj$&Z2SCvXh0G!B23T18+chs`i; zhMaOORp~rPO~AO%v0CS0wl-GzbmSd6z++~2M8K6FEsC>lOC}B%WoZq*OY$q{;1e#X zAaskyWGm|kn80V|7yB48Z(Ep@$jNW9-(kArWTExYPt5ezN|tjqnCoq}M;m2Hl673$ z#JDvGdrt;^@TA`N8)1OIzXJEymJ`;6K^qWD%Exa4s=#0C6WB+Ww5IqehnU+dO+`Ai zWu*OiZ;YN_$7Vf9rV;r2*0%R|d^T60?`@tRovM>jK1FDYUxO^Bk$zl&;1?0K;0Yk% zKmugZJ~z?hE1s?qTx>_FNL6tE(0E)r+J4L*Q!Xed+$H0sR+`46hL^M5){s^MADix) ze^(fWD>qE@64D{Z_ltf_%rnE5EHvW!PB6CK4bED-4n!o-<4<6iH{$>r<)r(CmmPgH z)&6OVvjRHK*PAw6792zlp7(^XltR>s5+fMO>IL?y#ahda1E+ ziX)0*typ%6-KWURWWm;gym}o4PKIZV8hhYxu}L9Xp?M%wcH7z=6Hjn5o%EERkaOtO z(7HoQZ!+4FBDTjs8yh`cuDEjP-fG(Bs*IIE0R=KejMRO4$w>_h1S-BH#Y=GYLc=KL zeP+ZC8%%^;U*VU-o0O)3;3b#0oDx!9mq#0CR6o^{R7A>lFd)ZjL9Rx9gHb;=h>L<% zmcN0Jb9Atb5wARVL&(lg?Whpy!3hAuE#c0$H%@>>SiKu%@@K=4#3#Dl0nRh49;d`< zi88<%{1CRns3pZ1y`D1Mz&qq0sm76oPTH|GXg_&tefM^8Bs+NPI_u{Lp_RN;^T14@ zI&A4c&8SvdAd;S(k}}dfLh(HAqEN^hajj2{v|`+sxTnOT+@3u>0@_I6ZwpBo1(D81 zpyl@X(*}?fd1G>>+lcO{)U8{dJUc7T9g(H3Z95w-E-I67Bw%UFkFkT>R2C-sU$16UDfV#)gv4Eo5bx$H zY6-Et*A-4|NCZLiQcPiO&8doCQQU4`F3yPVCCE!}buwok+0M^P} z_QxtNP#)Zp=p5-*PLV_qP*uOVm6+Y9VBT6D4dA*fEl31n#&PO$+_n+SDer${NkjzU z^KHHv3l24e_Y;-^#Y=T01hTV&@-+|H89SPV0>Kz`6z>Ov&Itg6f&cJ5`9ohebl0bk zJ;L8ZoPW!^`+@?zT5Q)teWO}_{W84C89m6JohW(5|@W6X_~Q*AUXLB2I|NGw5Y zb3B)y3)YZb{6I>rwBwgroXtY0lpTR8h=vp-RBN*}N5{+R{lc$>X<5~ZRq;3on@Q5^LFq~Ii^}~TO~e3IL|f`W`D5T0 zZPGWL5%Q##vM$dArJ*n|)3~wL1is76+P(g0W?ufa1YV~131!-)@~+Lq&dEA4+-6d| z1Z|5B?Ijx2AN#}s@GK+m0YmEwJku%#)2Qg*OC(pkh9-Wdxo0l!P|&?BL*jVv2iG35 z%A=ra<5BsS7W$y~AVk!vOi3qKx%_G5CZLQ*-Y;maroBarWXRuAJWX zVm)c=fG`ik8de@xV0;RXf!7)@A7)^rpq&;5xq*7OqQ)@J5bt9l_mb@DcK>{G&h%vv zD5DYslctrhyL~`8RFjLUBrUZ<-3rs7aZ=89ODh+2sIrD&ybrl1_$7V8&$G&kKay}FGH%e%N=@x&c*%h) zpFQdeL`Lg8s7f=V*>5*H9`-u6_`K9BpV9c_`Mp$lU%QOFtTG>$U9ERR@$(gv400DJ z+!ytZ+(usV2%qZ+U!41mRktawLnuqnrVRVx&IBNKK?<~9{R(j^TE_arT*jm%pVTQ! z>*r80CFTq&#-antN={6i`_;m4Ps{E@%9+i9sf`BtYEJ0SW4BJ7V#Bp&-n6^cBIDQJ z8BXBy$Y~DG+pXBp@gu>rnkk1)QHZ5!p%tT#Ne4Y>bV51m6F3rdl#tFBB$%j$67YLm zGES{SKF3a)y*@7(ilOJ>A7=eRLxm1H(ykNafqZt`d5}wc)F&bj94E+V zP}?Iki0OAI*Nn$5?B3{lW0;mXvqwb=knD>Oj~`NiO@CzB1IIUaO(i*4=vGw@vEL7B zdJ4xoxUiitxi~R=&^jk@(d^epDAf)SGz4)oIE86jFE&jh)C+f5oD}t_Xmt3XCELtd z?YO6<&mmeQ=G$c%6Fc!Gad(_r#y-tWW^VVE@pwgm)K64_@wKPyV_hp(eowv<^2Y{GizTgy3aSELoEMj$S|)TEL?< zMYbibJ!0E`Ky8;>b<9JxR*Ly5(8~E`EVU9f$<%vYlaSL`QO@vWZc3`0zjiyCmmE=S zH+})PXp533?{?xW^==Ol#|$Ua)bM^_6I^yOJAo4s|1+MK3J-IDE$7=8lb}+_Or&Fx zaPhqcLmtZWni;NK-r%b{%)aS&=0$u1GLLWY!L&7AAGw~#6AJM%{TzDMH89b2<#=73 zE0t@upV`EkYU^!RFFsJ`ugyefg=U|gPBp{-eCNCiJ1ut5+1ZFuQ9^e=Xqeq0d)LKV zVPrF&;o%XHNm&_P)-h$RG&J&`oe1YaEo~o+?E|?4AMZfFhreQUYZagD;+1Qv>M)CQ ziLT4PKsz=&RhS)D)4{GIwAU|IupO&$PuW7Ccx2*oBj5u<8JhF#UiKF}7p#FS%&xE|{Z zr)Uwwd{yg*$9vR&*YXdzRh!|4%(w;}^*pKdy9Pdd<$v{oP2>FWn$FF~;<7IV_g`3d=H&Ji7+n^X^Mu zI=;SZL{^B%1}1MO>4#HnbD=m4b0!-fc4kNi^wIEv6|yp0CU&9O?bTf+L+cZwOk6LI z1{Te}b`bQw4#=dQeF^5b;eo88(?Q+qiBTSH8Q*=!s|&sM@0yW(baV;t!edMx!QzL= z$&R*y*kcf+-)LT~%ySF6)68?^ajA;w6rNGPB-1fHRdmb?BQc$)-|}#Md>@+^URFwtUKXABHsDa;m#;Te~_*ix;*egkv=d7GmB`(Zrnint7#-M5RQEOxv2%B=6 z2uIlJ>I+Y-lpx(wZ zJ5isN9k5X)s&YgB&>HWaJYl7hYeNhRheFK@LC)l+VvsST!YSLiQ367Ob%d2Fy3f54 z`8oZGt=lwkZ`(xkV@@;c^qZ=|2tji82LV^MGpFD?W`{G)W0yo2e?JCTZ`#zG`gAa_ zH>`7Rc{tRfA}H2^Hq<7rHn}`$5~_5+@>a~IJe|D=cuoD=k^n1LI|S04mu-?Ix=FM& z2bG_fp-JxmftLw2ga;(<=aNy9YP~L>)rl`zK->n}pR5{+4LFcGMq#TGf|^4Wy*R&I z43-A5aEHB#K`!X%>$5$qOh^$Da>MI_4KGF*oz#YRaAh1@eCLmIdM@9G2gY-gFzx0HlFnP+AD;jDAK+KXh`;{>aQZZEvo3;>jmf`w&sVonabsFm zQsz#Om{tgd3Y}*~j|&>k}>BNsZ8ly+7da;W!nBi+uZuyPM?|V{nbrAQ=qJmMbDdJMh78)h~$d#dm#KO@Va^- z$zlo8DM%!Pio9J0a_bb73c7+N6pc%n6SM_TWXUrsFyBSaa)6mA;7$%_nM-ffceEIV z?+O`^AD_P9poUF^| zbf@%BNdCwJtbHYS;cgbh;)I0vj|Wa@J|+TD=2+nSaEhKK=!nSkXU5= zcGc$d?eyl|l#xz6sK!USXE;(`k_T5!BsEE_CV&J54|M$;0c$A8$SU z{x&w8*%Z9=H4YPdFV;t-noY3;G7aP}c(-x0T!(n!uY%KtmOV%taKiLCp+oCAd!%Ez z_+;)LJmKIB=kG{xHUQ=`;{v+p{{?6$h zvAp$jnp@VKlaoPntIO6GmX;ef%ZZSu(HV=2EY+4U^EVvvjR(rF7 zs2^VB(p$Zv*GNOZ5jW!-^M(6945UKko-}qGb$w9O}@MA(7NTy)bzk; z$>8)SaMi>X^NwE98BWep>obv5xyi7bwsIy{N&d<9F0qNjR>Osd76C$skdvsfttRO^ zVG#bw4!Yalh63CZ9FRKUH_l+}O9jZ9DSyPu_@oV7s!i*OTwm_fDKqP=f5YEVL4uV) z;yDBpM>=Shg*;9~6I#29KdnDiK5o4?yRPv5%jrS^592@@83>4t90-W%f1WP>qdf8V zlSHTHhQpewb2mDWAQBO!RFcuS9nL_gfrEwgHQCBVizd6ZCHW>-gT=~Ik2PALD?O$IQd!}r>y_4SdX$XJH!M~YaJy5*{(klpI=+6_Y-V~rE$`2}S zm%5ru*RzCql_gsoCvA>tJ*oCmHH4|w8mWs}^re|5Rw$KQCYsfW4W=bbm8k?4f0u16 zsytp~JXNhN&>g}YTs1%w(VT4l0~>H7wDnopV!Z-=Ez|t?E6seizk8spkeAY8cCdS*te_W+ z@zgETS6>TRqo^yM^mifs;pGY9Nv`41%_pxuxjGH!UBrz-)yNqmrSumpPPOV8_4Yjx zi$9m2A)T#iydZ)m|^nQ=}JjwA-x$Ayu*x}gm$w}s<0{E)A!P{e?` zoCg}uQgV<2-v|;xS%vNCcX$|LNI2nh?WRYp6Jwm1Lr=b@55}D1<(HV#IWg8VA*J1k zZfX1Td*c$w*Lc|N%`aHkh)pYawl%pQ6iW-fRudK$=Y)*V5K>9pU)0BdDuu zlzx2!^ENLl=@YQ~5Vm~ui0$06p$XlQhGaqbkXE1l8PAnr)!}CMsSO6B3-KXAE3{OW=cg&=Zd-F)rdSAv}!-GK5lw_pOnK$rC#qa zWNJ2goC&rmBywO4nEMefE4;Zl1=@w;UIy1ZiJCk z6-_6il=>Uaerher0j+6cbw@z>P8nCl)@L03gz>&hX+${mBp$sabR7+h35ZIz_369* z5#YN>)Nb6vIpKk{kK$p`_&V2Z2)-qhqo32f+P-}V!CqxThRK^y3dE9UDSVm#DL{my zF26aXo0arg2Nwpy{Jn(U+LbYTyZwf7Wh5M(II&dy1y%i)UZrCw+UOGJ35@Os@#?1n z4Ge9wF7NHSdB^sHvpQ`z4Sp5pqo`TXr$^7Dx*DK!FZiu&alGltegkV7MuxVyWBw6G zIsTJbBs@=i3}#2Ox2KI0udn?dn>?9oKYnk$s6UiJyPD=lPkklSt~!1;ZWx**+;db% zx|2bYH;z*mm0Smvd`-Ea7OsOz(g62DHYt?Q1aGB4FgZ5O*Jk4=ObIqP<|eT}+krqT z9TbI6gR6puAoktmK--E>Cs%T*om!O!eTPr-0*5Y<7-tlQi)o)lmSsJ{7{#6_^>>5J zKk|odIA5+gSMA}#>q@qK5gfjb!}INb)rpIq^ z1J#ig^c{wlRzNvAlMyXPdXuE2_Yhmx$ttkElppCiuRR{Pw!_6$QwBdwjF*#7oXrF+ zRx=zH8^Z7^P6n-D0Dj}tiKDBf`Mle6rxK^ZGdwmOG$I08K!ONOKsukE+VZ-ZMoePU z3ljd!{#00V&hnUu&B{tgN@Q-sr4uX+$F0Eb6XY2l?C*ouzW_JG+MEu40EJ%AVXZkH5p4*wLfEa&vw5Xf=qMQAFz3 zlrWiC8BA|W*B%QJS8^IAC*8p9;e+k<0G&li4v-LT>Mt4^fq&pb;q6Jn23PDAri+Jy zN25es$8IS-oY~Cn3}f-U07_UMKaal>E4VzsoWDnvigVlUgMkW2xi7s*ZOs=E83SX0 zqur$Q!k)iHy@A}Pdt!|kI^-7Vi}mc}X2r*2p@3yX-OwgC*LfvO@dxqg^Z~vUs|aiO zM#7pJ>H-Qg%I5}Kb?VF1f+2#OBWR7S4ZS+4?%%#0Txo1)miY3jucckfRGycs(&`#B z+K2G`PlVp#OG zkZR<1V~|IWRk06MXC*xbg~b=3J$`Jgo{hz4WTFpur0wpKq=odGf#1+5I6y3BADiN zQ`r;Q3T`VsQh!0E@_oCRpkYVqqM04;?2yNP3UDy~!TS6Tgfd`yxW|QKno1D0*mkUW zY9P&z_AaJfEw(A;kPHM>G&8tDp#V7;phaQwmgphba{S2cZ!z;W-|`}egZ!m|ou3H~ zLLVv=6_e}etR$$8cIig9X9c^DI%)1I4iqS-*Uuu;R7#x>ovn08EmOk+wMo zC_wC<;N)*dOrbkvcpdTS+$n|+NR0(t14asuXMx^nwE1HHN&m~w{Q8a7`aN1Yz-6(; z(Sud{4yQhpHQHxa16N(%kxJhbpJucU+8*wQj|CatOy=J4f~<{ng?D0besQISUIY5O z&9w|n4lx)(84cppg4?9Vxkz8LsgYb2Mb$ON0pHCIOp|0?!Hl0XAi{+TP?ktbLQmk+Z&m49!l0i;H3Kf!bTI7NM&=LB#7Mfb z<*s=(r9$5L_<{T!WNX$G!*0K{5uy#P-w>hT$NK|)_3kMHCr@aI@V%?K@@VOss;RLX z2Z7=YsfIKsX=mVwBRa&vKtEg$Y3>PJ8QuM|AS(atEXhwaN0Byt{PRpMGKRg3sHSDo z528dFe{U$h>HN_fVe}J%`T5jy+rGqI%m6{4(CI1QZkO)_dbv0y4AvVz5hVj#NfCzL zjVxFBNEYHkPO5oA&?jLbWl=G>yWtakFZ^a|_&4Ra?>I1@7Jt%v3@)tQJ#)Kod+#^3 ze26WMSPNq#7(?B z#Sn+qKXV(|XRmbNnzMXB?gm=NSS5vVuqd~%kK9NMu>sccox>PmZ*+|)4!Vn_m>gl> zGqM3!;0OX!Y4+3MCAcb!PV`o!nB1b)kE* z>0nR{x;woko!CVF!R4Hcs(aK4lGB6EF}@<$t}wEPvQC`GYAO1U2m8sIEP)D|AiL(D za~!(#alf9wH^*baXm@sdUVhhF`a+9@ z8?{L*=^)gBgYN?NfQ$Xz$Ei*+o4BjZYLbm>;^7=ZooVe_6_k5F%CthmfyQU(o=pUa zZBMp%Pn08+o7UocBlm;&5#kYI<;f6JK!sX=`>SAZFrvxlgI!W-ueWb+X$FZxEi&8^ zv-PUA*)ivAG7N_~b7aOAj2aefp!#fFAQV9gJzE_Fw=mX8*Q{$mceYV;p?+xYlEa`L z*j25v%Sdq7Fu_nDlmX_$!Lkz<-Lx1kJQo4ed%?K9bELieu6PlB0fiDjj&{x4zxaI? z?`771f>5V);65UdoM6PwPK`5LNr7E9W5d}>C^ghxMlIUbG02}Ng3`2a&RG#rnxySA z2-4%DPq$E(I=W!`)H7@98Lj)F7Dc0gVxxw>S%B}W(sb?z`YGr zZPbNC!04iGDl6Oq`~Yj%KGSrrkGGKqoZIumHGiX4tHVf|5)s)oHF3@5a%JgM?d#>D z)5}Sv*5as&%-}feg#Gm>kpU9>(0*`>e|xgkdR}Yf(}yEkNo72;G|NQBa4v}<^w|;e z2v*N<>QY7zZnG*(q%N%>?Xe*ri3U4j-^xIlyb2zj$jP-uyQjFytzl4bCu{AwG z7Ni5V4~oTx;)MlTp^gxW+VXEe?z+I(_4!@ZUBa8BU2#F%Qk@>|Bbf!y5GRX{KIfLX zN$8Jv#H#Jq+*tz_G9yDYDilO3;6ga))%=#WqZ-Esbsa;(ZqcJCj(K_k?+g4IDJT;Dc<>Rj{}^upQ1Y7f4Y*s zV}$GC^yYr$=*dkUR>XD8`-vJ(2H`(^Ls5mNb6!pul)BaEq-`l+d&ojW$3WiM+Z_GA zA4FD@?5!pUZ|p`+N+*PKNMc|UU^NxVaE)TzdY~JX>y8C|Y(qOa#}7Z_<&3id84T*b z-zV-J7stY%Jsa_(ikc}?5?Q$}T5nh+XB34J^1*uEKJG9SF_{c=$_UrOQRr%bO_c$y zPD@V9GoX0dM>qG;aN-HZ;u~3W?n}00(qsYZ>QSz13 zNj>)`B6phG6#LECE?PS(=Z`sR&F)8j=mw${dP0w?+dwEKs1V82eEqu*%uxea zvv?LM{HR|A_r=*i0D1cE9YwbbY0a~`Q zv-2TMA(>`MdW<9W=n3+g=!};D?#wvX=hJBoN)-S%!d~Y?Eq4C|Jg83gQBKtG^Ou`! zBp+I;iK+APfYYofNiCVWi-UMa>Ny4FB&hMblFvO(7HzkLF%HOAp{K6a$ALD`2iBcYxYgT>Jn9upL#)V?#k5R=*QV#1)Wn4fkhVS(wmWpAaL5K zUGT6njo;DZVv}W2l-_IL9Pg)MCeC!8M_=8@EeS}<120o=5A7B-q3Dr3$s5NNz)euO z$`y{~CcE1?wlp(LzQ1B8A8vK{|`<08)|EquWWiJG7HzN3>uWXi@O?xC9!!#fn zDXTrsJ0i4c%@$eGTjG0!y_ZL5wou%<1tW8|9lTK4y|aNR|8P)f?vbpR9Mf*BuR0Qn ztdy8RnSJ9PRFWs|Sq#|Km<^9}20G7&4q12+11PjXa6~Yuguq8FoTzLQqa@Yu`6AH; zE5y*Xg^98g%`9}H@{T|_!W>kbOy3-b0|6=2x2R8wyB!T%qr>^Pv(TBUQc{AY6lt2J zDoJI^DwSm_I{X^*$6rAeq=7+DKv4j<;h=zi6Ke{o7=;T00|JT!$iX54-u=B5`2O}x zR9u`^Qcg^sUP1alPk!BjqIGv?2Z1l7vzLTNr2=CAV8iu z5E@|ny%qQ-{uSY9W2qrROJynkvqjSUoJL&>9K(6{9t-yB)!1!z7a$Q|B zYcoe(T|nW4zmDa%@(Y{p=J!d^KtSMlzs{^OAc5nr5Lo5NT`E5`fP?iuQ0Ly-WiT(S;X#D~Klu`c|#&7O=5RRAG1(=R= zz()VawET96)iD3wk-ut;{V_>@zpeb&3HuH1SlwQ|4>)u+08QlImIwUa3VaO!@BDR? ze}(@tdH>yn+2swCJ%F14*#6%CRLcKi;`d4Y?{J^0@tk^q+pz!+5#8^d2Rs(|s#E>n z@ZTr<|7QQ@Ouykk{y*m5zou&Je`o(LSNE@e$=>9tib_Uv5rzZ=bZQL*1dxgQqZRlj z{445rg}8ru_^$=cen$ZYEcI`l`eQ}MKT*HdD)|#NqV#X5e`YrQlliNh{+~=Yz&Ylx z{`8+Y`!!PfPvWn_-G37Gwf~LypJ2-0Ilp=OuX2Nba&Arjjq_K3|Nr@A`c(z+PZQmy z|I5VhNBsYG$=?R?>uR1qEocP(FAINo!hbjMYqr~;CNkpwmx+IR$bYx>YgGMDTR8w# z$bWjvUysSJp7Y-={Th(_)6!Jd|FZON;W-6qaKNw9Z&LDDK%W5h1? z_1PquaYrq(-gvZmcA=82#|*S$n`E%@uNWM*-X*tIK#rqz5V#_kPb*hrG6xl@k&h^q zB^@>FGT++b2OD5QRXl?6im4DMi<$Lt);E%kme!EXaG6_qI;!imjg`}St8vAVoFr+= zxoGGr_0-iC73yZu8mvcXv-Vx0PN*uM=j(MOeje?EuO?Do@77_LxKAP-yAnw61>zc> zd$~I8poWKo9rARXwh%pPAvd=@cJ$;Cia{*Q(Akp{KOMG-Nv(JXQ*gQDwoZ%t{CEY9 z#gK9UPDHp=wN(ap+-c|`T6ho$FXj!BY0n-rR|JO+w!kB*2zyD1oI`9uEype-Xcn`A z_A+?VY1EwFt`Z3KB5rrCll6}_FZ&A1o>q^H_Vg>`6>P5~P@p8rpg&$kHQ@SCrwo{Y z08rxP5miQDMUProX+|o6M6YK6r8ZoqUIe%rcP$BOC^j;IDz7<~)cO61$*LUsg&5Jt zM1d$=?m>&Y>XZ>}LIl`Kq>SH^Dr|TRl*NHH8eHofi=0SBd?K#S*aWi5>XDULZZa~k zS}!lRYb8e&zv-j$wzgp4WG%>tZaN4iSwVG+f4Y z#$#HJt@wBcOjDM`3>+&#{33x>vD@skj4^3gWUy6_NNTA7Iis0F%gyK5Uehs%qo@zF z*{`P|sB1?0VN$}AMf~-Jbzhig1fuSfvbL6;IHq|&rn9hk8f*{4OluNT9cl08rCfCO z0aL%}>NvEq=1KC0q?c-M?pHL4Ot;{f2lRK4Vhykf#dgf7K_#U)~lQ< z7tT~YC9Lxn)T&DFZ{FUMfGSLnWHv^M&79cQ-&4e01Eo>6M)p}s(P9vAIQ>;3w7T~S z6Q)CI@RNLA`ju>Yydxsr1G)1n(U2xv#PLHwoau1e%pjhK62cvC*FRPil@~(UAPK(H z2)-|7^<+wctuCQ)Y@;!4=+P+?aR)7T#3l~x-)QF+it`Uoi8ee5&J{ZM4@0xN?& zMj>%wodYtqdDf({9yV_JAh&>M$Ljigx!*;l(sm|}zZUovd6qQPyVT5L;Pdzjx`Tv! z+8ChNLuJBE2ru;&{-bCSD|Y17{Vi5UfB*phl(PRWTFUaOO2VSbbS}SKeBuS+0vQlQ zU$#Icx&sthMk&A!Je3RyVqCN~DFZg8S?ucTeQs#KWfE*((r6T25950zr!ZJ^gPZar zLQw<~P5e382P4I_L(uRNzpKlrhP@QQ9Rh8!rL9~%x$*xjKE;RGX6#1ks*&+HaFovz zEK4jyJH@7R*v&|XCZSzR-PR`Ks9a}pFG2mjS#w-sn&egMSG&FPyo$Lt@ zVoHa^hA>&r%l82OXMjmyt4zRu1N$2v0PepBXkzMMYG-0yy1mM#@%L`6zxE$qPNt5omQJSs*RxQ^8sG#N5CG=B zzDGnO!V6?CN!0!n90o3`P`|$%#N6zS>$4A#-oH}>$XhoLVmY^)yFrrV(tVX+(GZfD zw@LU0=O`}Y(Oc6qCvt;FSv|Mk@WYv_N?%+=1&*-78t&h8hzzP_cM zrHj5kor8z6+T8Mjs>HPPjQseNbej?dt>j`0K!jrn>d^_+DJlsXY3ULAQ7PI8x#=;b z$uZg?dI|+dzi7e41d?+6(RQTs!$E+N&cCI<*v?yR$J6RL6%qgd6z{)Y-oHQF!O+Rj z%-+e?)JfmS((dmHQZ)7BHro;XUdjyI@kvS=jc&-o2Fu}ljRc1W5!z%TK+WI zM4k(klbE(xC>Dv`>9p?hw*Fj;T!7z=Nquuy3ExkaAe|4Bq!GTo?hpNZFDFN$U;f>U z&m9D7efd_-SDB2T`ZAJha>L&)W^Y8^hOI*8#7hr!aN;gS^E2mJdgqAXaZ0EORaK)v z_KeMe6H}mIW{jc+_k+!gCA>m^7Rp|>6NSSzlOxemqT(hUyK>E+;2S6=AB+M%1mJCJ zbl(Py8YG;kNPjh!W|TwgY@JS*(hxgNN!K2%rP0g^DTOB=IZj;2UTo&57?8pQ^6vQ} zkY+V322>uRNIR^A-YNm^dxlb(%HzlkDuZL4BEFe*^k(c=I29ZWKuFm@I1ql(xK1y; z_E&gI#>~QooC>PK3mpnP<{mrQsRJl3YN#-R*jCG70E4|w!|-A()7(cV7s}%8rn7{v zVK$0AAG6Pkc9_nByxSD)qyo}uXl2#Zf1zE+9L4#S+s)7OJYo$%i+Xtov zUB>@<=Nwc3)RR^;Mi8&K>5pa|#Nb3U9gW)+u^1Z% zhzVoNEA>tV0P|;X?(T{wg#G}B=K2aaP5%iL>NO|uC=Nol(f*3(1=l$X)_yt(+o(?O ziR#ILk?xn5Fl!=|E07ZR;&}{I0Y3*j&H__*%_avyV`czh$zr;RF?xvexP&(z!CU`pX9b&HJm&MG?%{z4(`r1c1Mh z235-+!S)q0bOaSj)ulF)klH9>Mat@Hl=CLvz?N_1 zp;XVR2IgS-LL={eJLgqGd#*b#bcN-#>Ef|SG1Ztv-u75~KN_y|`gx+%2mtBO!-!*^ zn673sQX$v;E>sZFpY9!bfqa$1Dy~R>n|+LQtMEMStD?RKcOGmw-9l!Bu0YLPu~C*f zITV%P-;wj;+AZADEguzDFmm-Cs>g;;oV9)19314;N=azek=~mvScoJFkbXXojNEgF zjDyoksKCkiGPsP#UHhb$kiuyWk|Zc3_3(gyQh;WLvnT;l0dwGUPcjX6KzKW1Q?JmKv3dfIOWAMZ+(-XDMc{J;iS1dP3-e2u1MSsrD1x2c$~lxd7Igs>qBb5*4C6f8f;yp@c4e; z?Sy#%A^r8`)=(j(llA@v>xT}%c${5{6p9rHrpkorx8T&7?L$k-6Q*8#x$tUQ7d^Gf zzh%K{`eAXzoA=&KFTOx79tFnBj?+T<9q0vgNtJ_xasw3tAM+Vi4~a4}$D&lK2^uRD zusw9U$~*UvjSKU&eI>)FEshj?ZC z;w(M0Hz-$a_IyC2hCG+}ZYJH)-(UNjUu?nKsl6dHQgG~BhVJC($x#=0OQq3PLBO`+ zSQVq1SVyetua6F;u5eqMPOGbnVJxiV*>Mr?t3zHi3cwhy-xKdM4=cSVm|@5vAqVqP z{rsi$#S@icaN6wpJ*!f6bM?f`-0=sJZ}ASh!T<74dj&lf%fA2FdpG>^v%wi%=VNj{ zAc60>Et?f(=Z5+HaPCmYA*lv39JzZx#$ntItX6%FXwQ#uczcrvg)vYR->Y%Hr_&o4 z)ds1B7yPfm;>^{a9z>6J6mOK?H%%r{R~t|m{ zUR}ppH3V#hKepKhvYGFj7_WlS=D9*_wh?iw0Ig68``iQ^S9eEL={4d2a2&qaAYctW zIYTjg8d?v0k$_{d5tpBhz*-I~@xzwv z=)pFw8=O;3w-raRPBuk$llqFZA)OzNZ$|f9@+fxh;40mft*W{*RJTtp$kn>@FoA|~ z+_M_Ob6CFyf3eE+qs_UH2*^((JS!Knio$O^ijUO8)81IwUJ2^m$9}H`aY1-GuA-0g z8KZ9+3Nzp~c+N4WZ60lOb`H0(?MJEP=i5N_F^3U0ag${}7ln9}Z0o?_@_kwY5Fvu% zcz*Uf*V54zpxm<}Hx{+zJFr>RT?Cjkk*xsD*##?xC)feq??>XG>gL|(2}}U9b?7;)cs?Fee>cy2J?4i(MA&Uf{`TiHT<~+x zQgoo-?z-tIL2|pl-=GZzOJ73o>D2hd{}&uhu=ZB_iyr}i{yF_U{wJ7paJ4i3zo8uP zzu~}tFenFv4cHhs06^ehH2n{X{dXMbW@z%i0icF$+$I~+x6Kcf(Zj%kn%e4S;TWWY zqX1nL*l9@^TMxAX1`w?xn?0!#;n}up(C5ocJdvnJk8GEOO+=>0@t3HtXzHr;pa`io z-7_d>rr8_T?6LFClM*J8>yEN>I=hq`NS#(M@lexGF!=Eja3B9qf^|}YK4=;dF@CJi zB#;j2M^Yac2`TGS+riosxezPj*H*!?_>lG$AFtc}vA{C8$1KQCPDS*m!+8lO!?dfI zujkKqTVCAYZETOKRHcoa-sgv8hI^smIeKZ!A~NId*F_8pybNaW4zqHrQ=hpXR53=) zB=XgUOQ8#W%UMb_KyNlieg?yygPIr$vRu1ZYZpt9nS(w;1*#yQ?;PCdOM$kg;jdSxvK!JIsXVODmOE?U%E7#ZO?Disg(pP*G zTi)r5tSo1$#|fD4yHJR;31Ub(N#|z}*MgcGGiK2av%}c7%N`c@dkU+pM#-jE6b-&4 zEhgIr8v9;}902io0A>m%6^n1b@ev9-VrZ`%<4;kbvh3hLR-iw;;#c8jKh18b~Zm61rhkFtc@axGQG+8PYr&Hv5L> zjs-jfeFc&Lx0-ztVmGG7p=Rh!Rz7D85SY{e#nJ5FjK7MPQdHC;%GHqV1py+RP$Y`F z-lkBk>onw1sj0Pa*AyCb+BbnZ8%CRVY;1jn8k=QEz8$4{$=T`bGdQUbUv*HhDj>p` z3(MD<4z<$`QUM4=^M*WVxj#~>nHoENVDW$ae=Ox#Vc=8$qGlpST7W ztttA+m!9%MTWrsHVxRtaA&co^M}GoE<&sm%rXVQ!w#{fA{at>gmQ_kkUwiy~Sb)L) z_F9~7V_aCP6;oMQSt3l#jAA1?ub`T2A?DUJENrKuwcjkEA;EJ1QMI|@jZc+?Js=|n zT+>dcEV*$`fPPJl_( zlP4guW(#8Dm=}WqpMJ(IRtLcO5IiXHL>FX*r)+vg*W3bMIk6>zMXN81d^%nSI+F1g z4Q(L+kAr@dW8kw9ulm3_Qc?!`*&ABtyh26)n$J(n#{;Zr|3r{bBs2ulD>+f4L;J;4 z+)VHHv!&EW1jE$lkj4z5gxjsmyk?>aWcM`VkxyGE9W;r1yynkvn=AR&TyGE+h%q*R zl*XJcX*fIhQ19FkrXI=1<=~?le|PNca5Yy4Ch5FdISOawwY-jX*QTwGjE1h$Kn6=A z0Xzj_ZYStJo)~Q*s?&){*+AT}z4gpB3H~KigMHFFKsJk*idWkKa3kPdmgAV5Jj!f> zjA+{j5ojgf9;At=`EH7{sYef7kV84}9HG9(oo9Xe+zQXoxYcWhy?y}90*hlT!gi=U z{D|p{_rl)qi6^XL8r2m$87aBjm5xEjKH93)&Vs?%hR$JAf2F-fhK9#jz)r{V2ew!Z zgQB3x*|8MZQ{W|96+De=k#p^`ygkp&F?pY4M3>*8Ut2n2QtLnfSein)8&e*U*f;QH z$jKrv%nVMj6}%Ta2sj^|3~>0lZq1jV{yRdV&=wSG=88%fD~B7Q=v`hyV5llKBd9uz z)TVOu0irYYinbxdr$EP;B1%Pusfw$T|4%OA1@T{?w@NawO^_;?n#(^8!PhPMrxBdO zKwbw07=cA<%!7slXW$!24{eip@gj?Y$X7L^{01lO0{CkE*zBSbA&z0Ux~x@EfILC4 z8+~PrgQa(3OT>-)03mS9+3_n_VHCx$Cet8%Xb!pqU5_Mj+9T73m=~O`?u=n7yV9^& z7WfJbyVlIOddPx8>{ONoPZ|TsP(%!&T(TZ_%P#<==#Km54}uYkzNR~nnM2Ej%--Ufoqzwn3jJ@+*5wWqo@ zp+kcCWdYBfKwoA)?79F9Ifd-iLdZA4a%M;_wN5`o@;Y;DAbTR66LX7{tua=#KuAiEIP&SSXecPZ|lRJhc2)gAs-^Xjnx09gMIsrVn#*ncJ@L{l~%`?qUu zt{+Uj1`XPW_`;)DDh?rHTH&Qrb}$)fLg`VS);OVV>S!r>(CF?u& zb>x(G&+>O>7ukcGeNW%M9Rq1OKJ|4-9cA}$jyRNitm+GqxqFJ)8)S_~*SOOcF8{Wz zRmXP?gQW8xZbc074#yjVIp|!5W=pNN!5Urtt*Dnk3LG#Z_Z=>*;WNAVU`ahMFRzbn ze(vyvdVFuyaE-$#`>4kYMQ#N5Ad-RxtHzU6``ySQ@Bj%*f4O+ip7uq-UJ{@@FLB?# zuojj?0;!;K)L%gylHf#khA?B3yHEIAh>XjQR$@&alIJhrdUdqDha-wEYo5;nQj5Tv zBWyq(7_}3A^m9PG3>)X6Na;-W6Fk1a)C?U@2_v?8o+jerJL@##I%oLqt&2O&1_}f zkEl<3CRXvzeoxiNd+{#%;o7pjfc3uTJOVe+%9LLqKTWjgbUcXA&-5xZ~Ug2xNyD2qRl4${l^ zbRijuD0bAt3itQoF33B4{tke@5J(flC>eY?2tdbu>~TqhESZ&0~7HzqdL;Z+oOGmGTUkYq`mlH!so(Hs{Tmz=J4 znG!o%nwZ$yQi#)IijTgdV6f6E z<*b^DpY5KFIpbYLbz0uGC$xJ!bh2!1Ei-mr<>6lV2e1NPO43;wpUB`PJZnq zi)EK^ja+%O{!r$uv34Vbb`Yv%X^cgqU`J6*%*shDb*Uw`Wpe`j@;i%|xrLBXvb7Ya ziWu5B*}%`t>=Ca~Ic+6GP zdTb*^k+rcS!ZH4%7OAJ4+R|ed@n-_%y*T9#s7($7Cvw}cXyhgNtH#}ue7QPXm0{CJ z?PwzR3lQ0=^_O;spP`C0dK1<&2v4+ick;53*w@RCwI8+a#N4-rD0eKTbx}z*TV>gk z4W%v~z4;TeTeVOxkCj2R=Zb(Nc1^UJxb<4Vf-Mg7)Pv7dZ=$7B+$0HafObhoV~P!H zRZtdd<_&PG{`&3&jFa(U>=%yv2L*IQL4 zeW`G~O`^3~R+(h%!MHu3aVqB;d@&DF4iMp1r8?YKhY`?;v3RGbZc_|br;7r5qKR}K z3qX-VD#SnVU;-AkutNdm&A&bx5wgFX2@pEFm&>iyle6M1DNm=}34b3N)s^8`7He_hr1-o_H^0 zy`Y7y1;K;2o`KrDcDBZ#{#|J(+KXamP5O}BcCjj-ly>9&rA2|Cd}@m6B87pP+!X~* z^IVd6B1PE2mx~5Pq<0=_bR5f$dmI5Sxjxp8rAkVDjWv&(@ zRtYSdFHJiNG21g|5`jZY*&Iz&wJ}8h$K-_EGh-)hh9#MsN}VE&n}&%<&zib87^d^u z0mNvB55|E(kk6V1kHY@tbz^8+=^P^vXwtB>C?%^PA`SJ#e#k#5Q4{6Kn%hobmZPI7 z$VY{PldvH`=)JER;(kDfUH+NECIPL`4xSisgJ_!ai4&L+Ll<6|udw>DDmxgo2B3)u zD=57S3XLJIm&DEx=^y+jRewfc;&D_P@~}>gV(KPy5{tGd^3Sa7UASs@z;fWwcTYpO zzhnE}W(Ksu_j?cua@U_f3~$OS_z2CHGln7S!s?iX>a7Rl6?hJ;ybj`pZ$ z__r=U-|ypoJ~sQl%d(O%zyBQ9Kcg2_F*|Q!iQukE49(a@0#0yq-w}3;LL)jkQ@X$V zN-I9(AyI;l1zjLrt%DkM8kkJshkTq=+agu@S=A^?d@iucs#Rn7Oa`ix_BBwW4+ z^)FXYt_|_KOoR6@KKPssU zgR4Kl*D2m(iHS%nK-mGNql1KD2KcOUJbLfA+G+`dWO{+w9groyjD6}8)Ksw5tE50Q zmAu&2PfA?8F0GN~+y3%^Q$9&ZRl7~ukQ=5RYi{OjSJcb8F1>Bd_9^}ru>^L| z@ltuCJQ$5m_|!n9H?I!D(2&|Q9C>=d6wPAJfLrh4H#~**=WAqk!%iSN2av&x(CV9` zKuQ_4v1eCpkn~E0Ha?+=_Lgj!v#AbW2iS!`nrdLNhZC-5&jT8J2B_ zrg(J?n0d1Jo7>sL=@Y2S4!C02`of!iOqj8(MYhueT&uHHtbUbdSeGN(OYd@4Oy)2^ z5^Eso@UTz=UwC1tfQ>2KPZ8!)p@mUB@TX@yT1$W_*HD~S>KT&BtPZr*R!(_v*<`oT z!aR;naUBbfBrc5(OBX9G;zG?J5F`$4|J;WU3YkXVDujBEjwPG2GMlUiiYxqXPnEFG zCp4douZmQVoLySN$Qtg4Sf#L6^tCAyYXebRaR`~;IM+QgZryeE&a6)KvmJR^5up&Lwjgsx+i4#7LJ+bs@uBfDN)Gh>1XT&ZQ* z)hXN8$x?y>%PooJ7rG~2kkmB_&vt;tcN!PL7;539ofVhm5YJ?-^GEDi-3lxMQ(BQDTpTHbNYbghOnQ@e^wE?YZWY!+d?!D zVm_tpmErJ=PNDM!#zRK_wN8G#$Ssw}jLji(zT%YgKK<%$A(;mY)PsO0;Ny3WHmw8* zwIQf&xANtD5veP*qRR@|H_rI(ryLZgxknkw0t@uec0y!9Shy}#Xfaw9Y*Pu3||%9Ky0d z^OX=T{X(4H-i6EXCNpV{RV2|Syg4`|Id}DRjm!>pePvxD6WI+6%E1r6TnZgB5{CaM zF?UK-F*GjcNNMX=_9FLhFevwzZ<8?c^K9aP2j@s?*xEZlwG+vvY(0FkS)}HfIL(5< zrclOQOfcp=IQ`5`JI3=4wSi%Ei*v}AB)(w-HWw!BANhn&-WYo+9slTDxU}3Xz@hk) zY4npDPK}!fE7FC$9_Iv6X%&H*xU*r+G}(L4O1Bo7G}06);~Z;TuuOJ26p$^?uo_Ae zQc4NFq}A|*0Zg6X`Ih$7j7KE_TX8k3GSW^39w7<8>cA12c0_X>vfqMfP_=VjYHf|R z3iyd*<;Ua7p8>`$KMXRF+K;FaVLJR$4soo9``MhH6Ud-jIfcYSE6>$LgBcEMDHVy1 z=a;t~+8a-~ySw|}G5Q6M{4B?}FL{e>n&Y1tXuO%wCaJ)JRUXe%jh>+$Z=xk)C-!w!yoi0t6oJ<@*#Ln4vx42DDSnzw`5L@E%AjR_itvGP9EvW9(TdAY_ zEwQJTM^zJdw3x4k=Fh~fWl3eDG-=M6)RX0i2D?is)p42A)^W*JWD>3CK1yB*saA3c zN9Qk0(QqWO93$;_8)kNjH#}AbfTe*16@P6$i6Om4Qsp-?tu^QZwcdaUcV)r2<$KI! zUFwb5ugO8`_l3=9ZCWeyy7EW5y;_vPMewLXVBYO;{@n9(9#SBjk@=muQa6*KNGvK| zrM2^@g;J`@im$&Dm;W)Cd2cRKdbbF3L|y+SD#W&(6r%*Dka=r!XtP-axTBCBo0j ztIJ=*Lf_C%!N*TuC)AhELeI%Z$t+)&e~%U>Yv(IvDOZU*mOo2C{XZs{{A(xkoBq!4 zu>5V9(ET4+|Nm};{O9U@G^XP=M^R5;_(d#9Yt*_fp45>>+@PvtS0a#$Ev+y_k?fO7 z*FhQ{6v^Nwmld>XMV0WJaJBOzi7r~{w8~CPc$X+wdO%M*e#CCNTha5NALjtpzjque zzUq-7ab{r5L+y7ydUm~rUa(>^LwEUW77v<4Wk1r~PRsthBi0_?Py~~$DL7+Xdyffk z&tpEF2@|v6GB@K;S974MMK|GUZ&%5wZd=bpGX!&1hryO1vgMd@Tug=1mZ8pYIvZ|U zhCR&i9Agb)(FI4%6jrK-iVu>SOD{+0a^TfPCd^4Y#_5TI&LL1 zYi)l>H%M!-`vM&A+c538i%qL_f7*Jrj4$qZ(?c0nw9{M1f&Bu9G44$1zB}(EuNz+d zq`>%Ks_XH!Mk%}SMI!OIxLRI@-`$R0Ab6yg#GLegw$128eDd*Fmq*g7DlzN5+o*DM z8A_w1Ww=##$AAYbS1-ANd`U%zBb*dE5U;?0@^(idKa&{ctq%CkfbYhpW-2ug44vQH zB6%82;J%zh-{y$4Td0>lq3^kgXfxV}_X@K$O>6bU>AFwJ!J(6T>$HD0%fnyvmR_b+ zt;;u!nBbLOEA-rSxeQn__f=iGW@IEjKdq>kXU2Bz+sFRh+#9P2JKz3WRC#KLExVNC zD;yDRwE6nXNUphIlqPqN^Ih8tht9Jr{Wle~5n|QU+tnFH)b#;`8GS@iTMX^-R9X|< zG=C`L?-9r;eT6SI!T^ql@883JPW5@_en8&24#U(_Qt9P`w;tm1IDf{vhW-NWEgC=5j`i#p_JkNka=5r=qPh(k-5Zb zhx8`PMdD#^=#uTlQL8}Pi+HqzEnI;M?^-^(-(5c+l<%6}bG#i?AGRN*dI~1a))<^x zW!Da8B~Wa|{g$+g-dpHsKm<>)G3d@Nk#y@QBvG2IYLMCX$5Ex5Pv<>9N5$GW4=eK*m}p2Y(^8AEYj|{651nh)kPxVEa~o? z$?P77SyA!J(vH4yu=BCSrZsEe!k$f$(4#XCqP!$<^3t>4zPa=1C-zpC+N+TXAYWq@ zZe`*SgUYvUSZ%H{C+9iBes3joDYW&6?H7u3&69nMwgu*R!Cb7xFVWPsRz*sh0Hqqs z!TU^%^iZnLxe&$=rtY!WJK@1PLaQW#!wveoTGF$cggo{4p*86|{Z zV%R1E_GP!^(5OObcr4wE8Gp>gg~1=u1j|hY%~IQX*mCorZaMjBnXo)e%-vRz{fc9OEDcN1(~N+I z=Tx|qZ{Qs*(Jmog=?%0`)E)KlOZ|X(GSzCG$;y}Y-`+}DQ{^quE@sk0R!`NjsTnJ( zRRIYx)quh*qx1$fW@;Br)pzRO9ibi5<2Gzr%OM_sjdoLJ)76<7rbA`&G zsWZA&<9^IB-U>tPLSS2_n{MiC)PmcEtszahH?nb{DCMhU5y~WA&~f2JEJ^M(a>vemE@6D zoQd&DB9t$TO?OqFtb_>b>Ax_Hs_%8qDm(DN6!l*01HHW}HeY&J*d>a$Fa}=^ zUo4|F75Xz4(n!~t>l9qz3I+8BYKc@1p_`*RfSl@E`AVhI71_RE-h=o&(o+y9DY8I;U+HeWJk=7*F<&9&E>?y1ObtjgErPTU^@AwMTF{H+A+4z`M z_fXm=(L)(sGIVt4%5V@;ZqQTK90BUOwesq@BY*vj)nTW#@FEK<^?p_(!3pW|QK(!2 z3D%>B6-5bBriQRb_dVDeUCCmt)>VJt9&E^M-9*bLXe-gc7tV+~q?U2dTW})fChydg zakE@dH4@;;A2v-GyX76V5I_D4P&(~VaIV-Yu(2GY2IAUfK1 z&%p;~Necn|P(j|ILPTA7k>fg&OhrFG-lQMjPttDfYj(q7KT*Uy2;txjD! z{D#cTyFW$!JYTg;VaT%ciY#dn^#Ll7_C9IoPE}9n?xF%Q!6Pn9d0mBUe))llDUZnP zq$PRgTa@kg3Nbkrb%JpZtL;h!vTGy?Vr^L+KhhyiKoD{B`ttgbZx@}Q@4^E8q=%oTf27M>JGgOu4 zj@vV!1P55BnK_sD5vp2Pkv1;`9c(Zs@}piY;<~Ktgg@bIoHl>C>)fJo!0iGI*MLd5 zg`?$^f!ToI*F$e>F{E*SDhm+w%s9);FoiM$2)J_)lY7*?6IN6=uDPjU#DpzvmCe3C zO94)AE3Ysi@W2onR!$E4xj1XeWRbWj1H9WU@CXK;T+>KLICa@rjCO<{uAX!&r-4W7 zBw(5}`gnbdbqZ0|dP#lz5|d279bYnm9){O{=I%ABglC01;A}ZCZHV4=ud7V|==ljtTP(rpkme&Pce** zi7BQWdPnTw{HlEn0=*+r@r@Cw#sEu4MP*oy3+@Ph^Y*3u;zjZk7RSZeSilq8hAm&N z-*D+!zLZ(~p$7GcqpKpNQ%jdfg{ufTJbPUS9o%P@aOJA*dnqGLu7nff8z1UFd=_+6 z3kUwlT{@-Q_&1)h)L^Z*^royAiJ-dZNq2005?Ifi{Hk$5xMD*bPOA1++f`2w^7?%1 z^4m7ujKj!J71y8Sm&Ko|_XxL|hvL>DO>F-I?kA5d*1lSHPHs)s^a57*^JB$D1D&Q` zHETC~aB{NAM^jNg!2{KDDG%p6QLqSN+@vMDXeZ*1`6%(WLv9t;{hIQ{(VUEF77Vh< z%H+q90)N7!WN8# zqHMd|7<;R?8Y&shbtDo1z=a(E0M7rbvni9Q{$J+w zw^e+FtNWKJ-F^9j!WM;}E{-QAp-eBE_m6Xm`g1^9vC}&6BJfzc8p$S5NsjdPf}RQo z6kiGV`4{?}Do6D;Q_PhS{E)f5Gj}%WXK&|_R&aCU`-85fwm1DfnE&}bHd3u3Qny}E zXxmRzMqJN$Ln%W2Q)R_R(NySBXA`SAbPTI4 zB0fI_diZ05kyVFR|G-L(1?Myg+M!MZLKb|y%qu&ysoH4N+uJOJ`RVy2rq~OG9 zOWW+#5tgvy-X z3QMhIB)i4`s+eQx%QneY$e_usxa&8+hu@-!sp2FenhhqYq?(CrV)QWbSA}LT&ZhhV zD?U-hD-B$c48+1^*W%eys8)DF5-Yxzu;xRSOA`0~Mtp+D6c$#I;5 zeFo#5E6rO@lKUX!u;k(pxdhynYN*V(W_4Y-C?*-WQonBYrqFD1a~itw zyR=w1!vj~O*>b#B!SotUlub>KfM(VLxS31vvPO`?;b<5k&;(4NV`1ESkm1}W_T@^; zd`xN+yS=v(9T;-j$0F#eF1s$&`UzAWBW z|A*wRu%wTB?NPwbq7m^GE!X`CpUc}CmY1RCOEdt}*j`{7`H#eImn*!tv*2CU=22Xe znX67|(fy_QmdQx4J?v}Gv=BuJhPhRnf34!|Z0jT`Z~_l!3-+<;e6hEf&O8n`L2C~K zgW4}37sb;1_)_vmVYQ}XZGLQP)RBZoU^*6Vx;zw>c((lr8A(BSw%N%@MOY~gbiS_C zWv+xUBc3q)sBL+?{E13)DC6^LMzU6+bpsEfL zH>p$?FS+cX>qPz>40*?68LP6`!ZHR3t-qvL%14fRon3-(7(4NJzn1m$zQ2;n^1kmT z{l3Qu?R#sGPU|TlO*d6Mp%;gm3(_NSQ|}t|DvNq_bUaV?0Lx>bImBt4*k3MNiW%OK z_6(a9BQ=xnQI#CL?M7(HEOmG3)6=sXi=~<8g@*xbAwhW;;0D|6)*49< zn6{=;EgnxP)S8fapz2&t;My0$86L{mSyBLy3MLt7n$TW~S9C1up% zyc8CSF{u429 z@{8KPeT&5aPGQH?`QY?%GNTesDq@y++zm=Iw@RUs_ML~S@4 z?jLP3y;boUPJN=i-1krbgO=AybxqIIF2KK@eK%v`nHa%*h zv3kX%h~okG!ekU8hx+=sV^LxxLvN!vjqR*N*3&Yn7b}aUpWv!|K|dtIBNprXlQ#Zh zQTn>;cLS%W-$|;hT9}kF0laKNiQ7 zMs)yGfc|-(P&fRhkeXerF1f#pyurkUySEKq$%HY-K^N2pH-@OtTTjOIyvHW9#}7M_ zbeOo#4o2KTcuc}_X_3tFlzy%E)_)E1rflh@6j~uurzzI@0a6TF3C4G+CQ_u>x`;-s zKuI`_uo4yRE2kv^?pE+2p;9eiO!guUQ9$0!)Oj``cJr`pI?NETb=NZONV; z@3x!6sSmHh?qM}j$o`>$+c{@@PJg+>7=?D)FwBruWUS|J+)(*dI7@-Ct8r%Ma%eqf z(3*-)HiydMW*)h!K*#qu`|CHpUn3`ejBCR*uML>P2(upPq3b=Ctfvm-m)E*vPf+HNJY%>*ev+epzh2hP z$Nm>%?+_(g6l86tZQHhO+qP{RFKyelZTqEd+jeHZs()2?Rd)?~aK~re!8t2p$KGG` zR5o3GWoyTorJ%j|@c_Qsq3CzHO>2?+|M^)P`inY0n<6W3wqKop_m>@=h$L_pCY&J* zu=T6{LS$<5hmT}5W{NQ|@=)>1x2h>{L{1i+GCbBX01)`RXyUT$hST~g$Z0wg72{=0 z71H0?nQF@>zMO@hp8ChY7E-Frz8qyx+urz5t4=h#4#b6Lyi^EuJbzm_4zqR{r9mXv zJpfynaD+9-)&mJ=<~0hOiK%#qj;tuloT^t?jp#)@9RciCLUcIH4ITlJoAKC(h^rc< z40%Pi-yUJ)2f`w0h4o}POu=_~tRld83w^NYErdK|76if##u40;e(Zz%6B^KFI z+b5Inpfq=Rv%8`06X9`SilUjoV`z58WUn+T2#!ZDD4oa>JhCx@${{BIEP$Vey?TcY3c#dkYqTe)$9aRy3YHp2on90?ndy1=Cemgx-w!dG&MWV%}RwCgXe5u zH0UK0@PM|^*vyBD;)F{5aE!L0U9nBr9kG?o>)S{aR(xXm6^^oKP{>5V8@z(?&`6t5 zbWF?M{1m42e3;FC_{z`Aaa2pw+x(qQJUmFA>oqdOsu~J#RYU>GdF&h;1slOBcs`TV zF?fTKe_!vq?ZS9Bgkk&3$RH!-k6)KoUz_f(t=1yL#S?A3Rv6@tHu*Ua&HC@UNp;DonZ1f3>n)PRDGk+{vIK^;= z`7PyWDLgk}hwxpBa}%ln*-{+mqmoO7wm2rG!;Kx}p>N6pLAj)78+okh3Y{nmuBo(+ zoi7XQyPz$$!&=eqnM|{RI+Ryq?(R|;vXE{{SCG{K+d|E|5VvNBG?=};l+b#VB8BUrpfJjFk`|B%nqz3?0?^{UVpt_&d1Cxhury7^3#3aPxbje zZ^?f@SMhDYDhp)xenIwwdE}kF780GQ;C)WmZTjg(8CO;Wtv%sH(jUKpYLTM0L!_kMs=PZW~7c6+Sn#K8Zo>7UTRt zxtpU->IVUrFfj}Y>#{6XxG2{78q>)u-PIE3w)Kh-2?3Or>Ym}sIm8j? z{AtVeTC&Pp_Ou4`D+2LqiCx`}X?|LADL9cvvo9bbe^fOY6Vh z6AF~VLiaY#)JA)W`*%rx(2XHESOCO@0g%HQ4Pzz%1G@$ibYdvPe&%m3^7l4nyUgiX zXHBhk_68(Qxd`bLB}m9o=)pYx9*Ss@?w+Oa#p-R&SqNqop;uKECpXTzjUZUUUb%z{)gGunE00wCKDa5~br&r~J!?(loa?M{NG z^3~2@iwz|%Yr9SH4s1$%+uoH;ru9^(RSz7&A6J5(;jT~Jn}(fRa)`SisM7H0xY=J5 z5s8XR9GQap!xl8>>}8>@wW*fQ3#~vmq4Bge^}5z*4CS!69%w(@-m4UZQ?(wr;FX^d zFZK&5aq|cbo2DHSw4DLyGP%I=L%LV9=XIBi+~uNm6DYm}Waei5vC{@$j^iNt2=4sm znMm&X!Jc-X4#E=F^r)!8fxS9n;T?(8r9GqPmP(hE+hD9@Oni?K-QgRA=h!&;os2ol zxVsT3CS~S3ywG!|H=}qhzz4C6gVCr-@(*EQOTBu6K`kTS=qm};;bzAseq{7|J}N9!VhA%`c?H`@}5oO46r)1NpXeiCQ9ApB^6nz&9#Npq*`T z;%8zt?z6f^crAW$}Y*gjFi7fPkU8prY%Fhcwy5As! z3meft4~{%f&qIInOY0~ScbA_)@7T+1TY>MKld4^Vpen&$2(QrhX{&nk3)8A z)CrdT}yd3%%4+6rI(a8YnUqyQ$?P2fwQ^U875963sE zpEc*fFLG!<*}&-6(ci(oH1$T*Dho8qZg_@kv&!t-ZvE%NLN3VPX|wvC>8-)Pb_ zB=@G@p0PfE-)aSyotql}!v9YJjM!<&nH2&6AOQRS3&Hq50+^YTy`9THn8|;LSeo1a z2ZEtLV24L@J+bkuhXf+#V-fEb2xM6+(b@w(L|_ro7?DDeQpn~Q`E#3tc@fljr8B?v z{Gq^qwUC{$3v(KU?>CCJ3Qgv!R4R3=Ufr;MfI2hHq>NUL^b%i%t1PODHT46ek2`=z zy?1-EduBZRlaGYmz^`dJOqYh}U1s(6qld!eYeAL$PGMMx`>R@|G;2m1U>A8YE;8-1 zd^qJxEVEG|0mtNdGb`(6@ zoGL9ltRcZQgMWe`n%WS+iEV1?TMw(Yxl!uqB1%T^0{=IA+Dww}BJ-Oh_eQp&X2}wv z)(nPEU0vN?`Wxh+9xI`#x}<3%ldK^ofrc8^`|)0Q>weZEelproX~a{2dAlst6D`Ip z@jxF`66ScL>^(r%S&?|D}d1az*{CCpE{bN@eQXzn;6zwQH6*3;g%qE8GSwby#Ejk z{aq)1oWa0tmyXH1_5^f8HWsq`e9G;!ARDUd9ocdmh8)^-(p{=3ekzSIZ)xEw4(Cov z=llGk?EtJzXdQjO2r#^gOkAthjeB zU@H#XBqgW5hAc#yCEhJlD#0oHt}z^9snDZzQ;AP}Yl2t({4@bQ0>)NaOHbP{i!ZPt zBVL2S7Ej^lKvD?#Pv@d)wX$I}hf;_F`3_7{!dN}-%F2YXmfMgu8qNCUs?Gbs!$N(k z-%`iILvLG*keeV0nj{hR6gO#5|9WGTm$i}hN&PCwK7b66D|)yd@JSu9BeAI0)jPBm zb-jZw?P`i!jTQ^GF+M>~5g9YrCa_cv&KW_s=5G znHzGO>Ngx5E6m`vts)0&to5bM0itROKBoiN3@dOd`^tV_kGE`tWe&%6U^X<#jIS|? ze;lBSB;j>|jJG+wv`VLy&9yhz!%Sj=yIU~?jLi_%?zaCP9GK0nO&BR}zq|MuI^&!I z_dG2fMFJH(-dfD!X-kOCpkOr;eWz8VKDCbSupIG;CLt0RzBOYTy|% z2eBszAUul|Cl6xtXQ1zxTp+zGyw;Rtkpf*2^b1x4TEE}+$3yYv=g-hnKlNNNDF1dQ zc(ETyYhokqcF)}M0Fx2)F6>*Na!Gcj|0tC*8Iql$18S2+jI}Sam`#x|3D+)2(Q*hf z<=_SKK$G9*CRC{0NdJhBLqi#NHr>D~upwWwVQB?jZ`Oztw%4=U`X>om>_V5#6-gA> zz@0U0XN@epshd)$-fEWG;Yj7sryaXr5XaSZ-tP-uE&EIVU2!%P^$b&+o}{XyQtr?S zRY+}*(Ph@=i5sYA3y@-U7uxu2Be~%WG@ZJ;KI&Cf^R&~rMI%wLi{PBHI~Se(>lr=B z(Rh>|@oO@|iNRI_F&q11xqpr3?0JhB-hs6^n;gg1@tGa^VPJ6Y`>HNe*5Uc6tMD$X zd8xb>CT^n(nCPz3t4yW310{(NA>QjBD^q_wLe3yi;;w&pw7N0@+R+nz6~-@{jyZntXfv+AV1SVZrY z*N&6ts#4~r44SQr^lq&5Ab5ATYpgoFs@=+Si;R*HdS;kgv<{yU&>>}MYWS?AExbQzUgif(rLhUO#q`{H5oYLC~ zcUrfN1eZ}qY9IY57C0%P1lwK*J~0{L0c0>ZtxZHxQ^-24p=%N)20dLqKN(fsq2tBC z%Vf7I!?D(}oZ2Qilm2(R{VW0sFBCVBgpqlS5UFLVl`DDuTy$MANZSJt+n8C ze+#ZJLhgs%jAmfY`aF7GS%#entM)l`(qgepDm`f|u<9K$e&m`W2d{(8ixFg>HW#-O z8Zlwa>FVBtMT9E1vdWvbY)}3(v2rhUFWvDk)Zwj9VSEdSGK=vuYc&iQ)Dv`P6|Xyo z`W<*PIpp>Yl@NqI<4-NL-|5^>SG(F1>gon8PBlDPRct3BRUzp?vWQ2OWKIoWM^Y%$ zUmvXJ2EpVwD5Y@_7lDu(CzDu;1Mt<1f2SSn!&C^2cd`z+%C4D#K%Ms!XdaN( z&a((e78wx%$+$|?q^6Ylttfd|rPN3mv}VDAe~mTL-TltxLHnx7l=nk1M|E!$=D(Q5 z?vUBb9|^)KiOEd-ZO-b+(tAG%UzYF+lOV&WTlHxvD6DLjL~-_xxrJ03$yd@>%9Sk0Y?)qwl}2P8~_PIy6*s_7$B80K=0*Wgw*ny+5-17~J$ zvnoZ@m1Y1&`T;S;en2!U4O5XOu16cQsXA1hDb<@SL0DpMs>}QHWxfg_>qk!b(`Ulfq>!^i|-l zb<%Lp4LFG+VcIQ|I=kpeobRcpc%S8S{N#Us7ywFt}lB*X|mktW7NI?sqD@vl$ zDyQvCG%GLj=rbZ*F}v=CkF4TdI1-$y6w5Isd^=HSe1MJ#5OVEEr{m4J(>qY`hiCRl z{?HJ0H`XTV%wlk*EDZE7P=G8fqCzaF>qIEhPDhxt663}uN z`%@40CZXnlXlJEoDiR)bJ9N#&-MOAz?9Cga>$i+!8@^>EiPW)LSwZMET4tFxCl4(r+^8xJZ3{`Y`tgP4G6W zvprkcGJ8qekzGa=w@a{cbm9EcdkgH)GosDl^X~XBf0Qm*k?n+#^N6L?EHxubIW^%> zi}A$gYXl4lwGeo7ViraGOP|)=L)8muHr4~7+ea>TM_li3O@Bz>7`)o-W_5TUJX`$2 z8RRxH7$3ec#rTlwoySo(0`AxbsCX>Sz(1;4=Og2mCEbS=bL|SLnY^Hu_W|S7)>@PV zb}IAz?nb=c`*h^^2wboZNs+#9$JE)0KSPE3vlj&gg=_n#@=_teM8IHXCxG1Bd|L~ddKpA?yV ztQ!Yn)bepr#u+Sq%I5~gVgB{H%; z)#Mm1t8^S^g$Z%oGO**TmtN2giRL?DOtz>C4ioJQasKy;=&X) z)b?`>xffn4?EhyO4B?(VNIG-P*6YaDyt z#KBM9J8sAKlS9*+gMV1*YC;PwpHDWjPWGWv^l?9DcaZuUYvkRmb%1jJZm9;SYDK)^ zj1u;`(@;O)q5@M(3F)I*gZ;JwO`@dLK@uHVGvAbEw^fE4%rp@RWVUXP(yBS{_RK)P z%S;;WaK&N$Q9LRVhXY`y?_x`wsiM8op;{euE4iEtv?W6>2O4Y_ktwHzWQPIeRmYBy z+;OO&6e}!D9dy36?mW*g9Huy;L7yp5IW07;3{YvR7VBK40E-yw8EueQ9(n02Owdy= zKv5zSui2F)X~juM08!O9bce8ju8-m1K}K}(-}6kv_d*!PHWR(77grpyru6BZwM9=9BwDW&AuXaqU*FH^iF&x3?~$9mjZ{c0Ri=1n zVWuSMwi`QH75|z=4*Tke!?wdU;^NyXOIfMGa3^qvgDY*!`3blm{5W5%dpnMB3)G-9 zL3j^}$QbkPNExR$qp4A-P2c$QBewjk{|N{z3{$+tw8O{_ZEHcQ>oFk43YOgQn5ylTkVn^ zr)^;1+9jAXFAtN?Xbe+j^#QrCL6)dZ3{ET}Vc ziZ41>Fluc6z}RA1*D%t3_+?Z1A|MfVunRd)38_hzY}?RMD2T5I@dRJ5vwT3%b4I&j zG@IzGkH7!tixcuvdzKqOJS6mYq^h>|$M%kYX^F>cFRr*JbqjXHN7vlBpvTbnO^TW6 z6R`cCUC2drQ`~5XpF#iXL;io>^Xa5&BdvdMacv?1fPVyt|HTRQU&UAnrnUVh$Ah;Y zsJf`Z`5wiz5kF4!qmi?1=QUv#n51xEETkKxcd0)-F$((?Y zi`P?{iKfu9>Mbsn_4w;qW^pMr#sx=4_kwjxGp!GqSp=$DC8L}5hl(H#n>Q@K;ode^dXarIS#yid;Pt9oPtFV zm?WBo$hlX@S_rbVJA&J%VmD6!^q!l0tbj|Njl?!HeY4Rx{%ZjU>mjI>*h0g zLo69~O)kRC9b6e%u6d03DPr*uY(e!*6j{Nx7~Bh3iJ@76=6}Qh@wmM(cU)8pjr5V2 zY1(b|j6D0?LpW5fJXMJqCz4CS&^VEr(rtS<^&wtW6g>Pb)S)1fKl+MLoT838;hV=! zMepZuuwY~`MefM-@Q^>?gU7Yaq?S>0I3)B%ptq$7qQLt!l#`aS@P6+iAr-Wok|@;x zQRjv10NZ6{vk7J$4yH3T%j&&1=ypOt{-cZcpqk<234yW&=w#8Tk$Ao$OD!Z85Nqy9 zJ3*qvzYxl_YDB{TpbFR~w``Y)>WAh?`Cb5P0}Y_nYS$z{ZlwvhOhz@)Pf7^Z!P-M< zO&R#!Sk4KlmPBjyYqVl`mR<;+JvL)cy5>u|1}f|XJ6omH#Okr3eA;e4c3EXNzqm=H z*~pPb11N^AmsZ0RL*x&Vv%v`Vv|>@=+$LdO6957;jxq1X%n=-$*@&A<)_)@)92H2L zXG{B91guw1VlDBjzwwpLYx2%lm7OY^>lMZ;e}-h9bOEZv8GVr^Yv$3kpC}Rn&76zIMcCQ{5gKGLwx|RHblhycK|1b` zWIi}Z=RrY5X%-HMFtcm%Y;JB9(K78$H14FL(n=?&DXCEq_*?B1`E=`1Nwu+?5G#?! zhMyi=JxjO80i6S*R@2dH=B0U6&LePj)@$gS-k;Wqn68>4^S3B`dF)c@=XBAixO%|-V5IUY^|ViRFD zotZAEHE28DMVyNlWgP9qh;1AY%s<|DKBqU+Z+|Qu*phe={xKmx;zLkz!nrxe$jgT_ z_BW5G0!q@9)9W9vYrGQh9xOzQ{d6>TqE1lYof=|K9Zc64fu~DXcN95m)-%Msj+Z+L zZ?v6!rTwrAm@0Oa3CL(A>uNs>dH~G!kY&oR4 zd{BoCkT4V_|9Sxt`0El4?ka}24HAdGfItpJq4u6jhE-SOY@0THG+1X|O4;F1-C82P zkgAAk-svT-D~{c~L8gCX5jZfZns*UfI>^1)pt%-5U+APct}p*?NaM_FTPO!4FDbp0 zuQ-H}fYskCoe{W$nen}VicwqSm|V+-aGs|Pg@?a9_*L(!RrcKD83QU4Plb&hpjq(U z!};-(oVn!<&7#86MG=-X*5z@N2Z<6EuJo@AX+BPNo~}%TTpG_f^hNd)*nwX03BTrQ z9pD%dIM>w4X07EO*a_%p(~ZwC?% z2qf*xv-rL2Cg?D)8@E_+RxFdpiMqBt@~b#OHnFvh4+L7hbm*5TSG1_!&V=5=wMh08oOpBJ9)U7bLJRVdOy+-9ADhdqeYm@N@l@9JuBY(G(u|EE> z)0vTcFivfO$;0xN$UZo>y5&?lZ$h-WHr@>WY*`E0h2;_n0Z-qGbP67 zDcRANqdNjqcw!aRTG6RX3Djp|mA5T)%h%sKM%reR9x33LJIaGbG>vPWqpQt>C=TqO z*m7_lE2tWNrqbJ#Jqt}QOgM|isNJV!e2vNq~W?}Kwz#Sf!>h&If^UaV=a5a%& zP5oo>vVb>Fw=#qeUCj?@)ux95h$-#|4(FA0>CP##@sV*{F>>A4Q2w!P_*E>T2<7u$ zPEEw*rH~_L5gt62ot>{&e*$|4x;Wy`B`^PPh>s^@;<@>x`ckAEwCwM~HHyK@!kTUj zFDp{`*}0U<;-gDMZS2inwz_t~YdKlF?@xHzZ03jktfm)4ksqd?8D_?xLON_J!RjD; zF=c_rq1!$A2$_7M2xKDx*XFgk!+%D=&qAN?iU_{JHg4l;5Ppk%6_(DV*3z;s_L73| z_GiAk(8vh92$LU7x45Du3M5=jI!~MfELa4lLxHJhHjv4s{i!@?yJxG2QD((1&x+cI zME)tCD;x=;7Aq30h!mp!%9ipRu}7@&#R=T4V z%o+uAz8;T+eMqFLK;EShR2Nr~}c(YE@KJGIW74O;@J z)3n-fv&rTlNP5Y_%J_DzQ=6cPDyVHvp3v?MICnDKH3ekmjpoX(kux zrzB}v8)#>wSep^yXXVxDr)8;U7U& z>C>$zcuWhIwe^*}kgdWU@0cfu_TQrt|M4%0&!|Nb|1uIc{Qsin|7&;g-)VdQ6`S~< zVe=Yo>v&>y*PApgrAtPv7_rB-jBQQt?|4J^o~Q;`%@sXw-r-?9`-UjMk=VJvw$l>C zuCqIK>s?8Bc)6fiZG>=;hQOI9qK!wQ5G13FdkQL%B*lpWe$VigKq}**Qjz#7AquGW zGn{Q*%L^7bV=0Q$io52!-Z#9bKBrq%ZAVU_ApZD1_gpW=(usZqu-}$<{_J1eyC8!k zWNjs}Iw__;Y8z7GHI=wEm99w3BI0!y$#tzs%Ot*KmFMhIGR6_oNYd#Kty-)EHBBVN zNy#$;Tq6tda>`uLmO-WjIR!3o3!14xE_T`#Cxj{H#_?FpWBJVh=RnwDPaGvY6Q*L+o*6&AfO0*j8{!yPwbn@=r^)2OTJ@|x&r zYN4KlVcqFqMpCL7lByvSswtGJF%`=oi}}C7{4WkUIj)zr?*_i6HI^Cb4g0E~5l|)- zg`X8#)0OUjwWisvT4JX=ZGC1|tWLY0-4q(@;l8)*73Brmy+`e|ZW6C^*>s&pSTp#s zs{r+}OFJ6fR&=h@Q!j6$Qnt+*t+3g;iS8XSAQJNObZ*!-ug;i&>2S`M)NM`&_l?J5 zx4hbCxZ8H!mUMJ^H_o(N!&YskIvtuX**0esAKNv(t7l?ubdDs?Kz?y?C8SpNggxuh zeMgDF!RGqw^n& z03>2(jt@)A9nIw;S6t8lGUcl)(?BLky@x34t~B53R%urmp|B7#wy+?f3{_Fssw1;3 zKHbTfuNc>iG{=HBQO8kPwSGqgewk%4O?5F<2)gD!B>Kj6Y&3&;L%eY(dbXF69!atp z<-RFQ?9kA}m=M;>JUZ70j>P)TnKAm&sv-?_XN*nzwOZD+PPeu9Z+5mAy<~lSHoOV0 zbzl=uiOX34j`PGSK$D4Zc9A&|WRKg!$qcJ#Yf%rNloq}O!JqNG1(XJn&)JtR3M65c zPYVQ1)Z!f*+HEYElX#VOR{MD7$~K%!M^p}&1!Oe& zo$M=@?Y_)~ZZ^}Fz277S3h3$rge>`>wR_%Nv@v``67VB2e@^HQUsu6MJmUU>$v117 zf3WWS3|?5D-GaN<;PtxSjP3UMzSJ)7evth59$yO1yWeVcOKD?o>Snw$%k=RCP>vlOR4i5KmuH|=aZa~FQmZ;=;V#k}*9LhxF!*P#3%-C|v}c|npPZ)^ zfOKKb%<}Ivr+;&}G+C~2#qz^`FYBCzfCPHgbXaceoKU!}1 zVeNVhgrt2i+ivRV*Va`>sWl*e{M-DY&wMG9ENc8zW)a&QXC13>WRxg6RhdEH+j86r zt9vm1&7*C2?}CL%A;wM)wk(jN4ot0n$wgMhYNHp#P^kx0esV5=tcO&7 zQxZVd!!nm_4j}7|nID=4ntAUe(4>cFF@p`9`3fa4*6o-NS_k2>l?9;nBqc!JEtyYR z2jjAk1*G*-8Yt( z8BasxNt~!J;>4fGeytV`p_!?+ShHL!UV4aK0;=n0nm4T^gNqBvtygIhio$)a^UuA; z2iVnT{XrEO(Q314%Rg_i>6%7dd;kvz*Iqa3SiiQ@5n!dGx_3M|^<(jUdP_YyWgnKe z=G`k_v}VecJm>NkHXW{1C$S&y&whYC173cwYoE{V{pZ|p0>}2EZFkFgZBFYYwNo`e zCmjkquptSG$gwJ9$e)6MauL=^`8hR}v~esNJwO906jX@(SS!fBw62)YiSG*3O{YtO z?EG}FrFE0;{JJR>v(>usuRz482ZMe%AZR5YeH;wlS&OKzUzzm05yk9(t8|Fnybh*I zjXN)i!8zK-04gTzOo-S{h`q08*nu@uiQ*Ha6lZq0O%Z5AxI^X^(lIawwwUwfUQd=> zg|msc#wqs{MRh+E*RtKfiJS|@6t#l@5h`<#Ak9Zk3 zj$r(AJw45l7*z^lPpm7-_@abIA@Y)t!7gnm-ccC>Nu$U`JeZMP(qM(?BlK!|<+X9(+z#VOa zQn3rCc!?sV49!>sT4YQzgF+}nC2~-Q(C3IO5iBvx;RK?8*vG=kFL5D&YT;=QiB0GH z0gH^?CTxp=FGO~X%_H1eq=&H@eKsA}gxY2pRYVcQ*bID~TS3k5uN_doO7)uV zoUv)cx%+Wkovz8e1_g9+Uh{UjXBDqub4>a;TiB2q9{)(EDSSPX>)S2p0rF(cXR%%{<#L%GZ zzI&xiBaHlc{;4_7d|E^9-t69{qvbv?rVsq`;5`re(q!$_7pD#8(xhlheVx6qoL~O- zD}nh{HOVL}S`mI_1&Wb&5e+CJkRQQ@P_y?+5o_VLF`;0@9vjlID2}f~FTD3sunTYK z(Y-|zY>l-Z&9-o=q*!qnw1ag07|cDql4|I7>OXEF%T0TLSpmu~`)Sxqzd>A`1vcBu zMkpp?>YwlD{73YEep7CiP>q&tf(*TRaI9?!x`<;4pZAdg_g7Fv*h4jnaEucLl4C75 zCi|o113s#4Q?8_eHfJ$BhMdd{3kv3ZbGj*cDcB4ci`L4hJ9@AohAVB^f_o6;2thrT zbTMjR&h;rN1C}N;xCP_*(f(Ss=`7B-=zZ{qs5DoY1OY$53t1t4Q#Q~FM=Ms?8nZQ; z;;1V)8&FSYgY47wAL)pT;4|Q^SA|P2HwY&(!kELZz-@>Mr})l@ zZI}wmGs8^PJ=(TxESk7o??*u%!Ch5Jg8~XAs z@QO3ABK9O+N#xIVqnb!(b7^Rz0SJgp;(`3?6#z8XcInynVh@3S>3Kq-8`!nNo8D^` zV!x~O;SdSPATc&KWtnm{^UW8weZv=aJz#Q*dmx|){}|%|kp8`}CxB?s*@a?B?lLRj zL>jjM2jK+|1kFTLe~Z|co3Cn_H4xIVQ227k7-J7vARd;u4zdGz8tNC<*2q5A&vGOF zT2^gpvT+_OphSk0L0JOEBw@LKsxBv}e_8$b?gi9zrqC%PVzQK zLw{|L&%P1}&W6|%cLX2xNEF7*?1mjllE@Q!@+ZO*2m;U=Y2b@I5R%}FJs^_ci$1U< z8YejT!$5)#@Wp~4Quu>S)DaKtvH!x7VEo0mtQ(VW*$m1$(RWa^D>L;Ho#MyW|IzLF z#SifH!KV;9F6ty_NxvwG_yjq(jA06l+g;5}sW$NO$Nk{J*NS$RBwTqD7(P;@z+W#P zfX+TZ@K+iRnO=X}0w?bs#fRU zFh}5F?x(^7wBG;JSfs0`1Q=A=P;eX9b(3_^;vzsr-56g)QU*8By$K!|%}niP^Ss2hikEcfNW11LhR7+0zq4>_)E{ z!Ty(&FcwBn^w%Ql{}o8)9!}O_sB1E5IXPnZ0@d?}^hi`(GR&Mbg}aP+`+fql^@1<_9n()bsXK{fJOVvwL8>l+9umQ1>=ky#j3Y_5QthG zbZ9VG3s*zMos$U&=4#gaVId%lJaVxDI0rje6ATZ7`f$9O_(3FM>nG+MN3_l#TLAzs z3$u<_cYyV(2cx*E2P+pqQ4xEp5tWjBW$8U3OD9^1U@JnRqP@qlXN0wFO{pm^6aGGG z^$7#vC20r0({q2T_PJ|&|$qLsb&ult%hos<^ zRc4R5qBBfMyQiY>i=B%&W2GB}hWAt|-Xvuha|WqWp9T~L${5{H6OcJ$Z`#cw4b-@T21#zRPntRS5cxq`gIeqwlyu8EZ^o%_-}2OFR}Bew0do}1I@x4XPn z%@P(H<4~u3{HKB0n%McE3bZl^suN`;t?dz@0oh#>3@kze!tDGxG9*zYu@`;7qy+J? z7k+Sy5CE?T+>LuW$0v9fCh-qWEw|BIBd8-{5gW9vFdG=zdnRbg9427kz*#2o063#j zXJ)iheH7eGiHSNRg1O8x%IBO}hfi<^!a7WpuRsdEG{nM~T^PNVKs|~}r`-5mj{o>^C!R)dq_Qu=A;)`;xA72>G zpU#6`Y^uBOobS$3;0|G{j6+3;fLqYUmp(Moi$rliAFD8Y2(M`#;*7q7W7 z1}<*Ru`%r%WQ<$hp`rLKvt>TRa&c(}OJm>0OVnF6Iu<7sh}J~v3{wFRt)c#&4NZaL z8d|crg(B~WHlIqg7ohs*R4>sAoISX3{X&}Ln?JA7#yyajLJvURA5h0Q!0|dtsC1V0 zPX#Gt$tGa%QXhj+NP;S=YV}EK;dgBwV7bTC>2 zJD~JWI49%4k{*=hr2B=GG9qnB+p-K9D{fgCIj{mND0E(GP%F=Soe>1B$d~oB5(U_W z>!>m!GJk@NkX!UY(Jx=R19~whU(d8$MxfDunvFP|oHOYH0brSAQ1a3A1B9J0@mkS4 z0}QI2vg60{`Ox6LE!M!`H;I`~ssW_oHZE#`q;z|<&|Cm8)~{7_7}S0$)GLA*{@bef zRlz#=C3@QQkH)P%4}f-nz?%05moPAGyQi^_n>YLB@xMKYbtqREx!uTt!G7bq$mY?^ z@cG2ePL2ugshv5>cE4b@FtInTOeoF1d0TPg@YRxill>InJc-1CTK$T4kOBn{+C1ya zXPkt4ctTm+^*mVqO6{xL7LW6=_)HUZ@Zfm)r&eIb0k({ZhNqwOf!+Az@r959L4aa5 zkD+fYFD8gjzuhcSj^@Q0!=B1;aIMK@26Xa)X`sFJP_q}!AV|^jdjbFeypSy})u2@H z7mS$hxnrx~p$Z$&*4-K~C^fdC9pVb+AWY zNXtAi42oJzGFhoo3&|u(pCr|R-BrbDXzz8wl~GIUO=5BCVZWR$=W!@30cx2`Y#OE;)z_ zTZCx={3F8&bxVqp{&taa!Nsj1iq%nK=#!@44CezISlwG~oV!_})Rm$Gk-&0L8?7xz zTX*Fzi^Af_E}w-BgZA}dGV-$b&&Cn$&c?@I26H#LWY$u8e0R#UH^B_*2dhLb3GlU^bSL2>Ri%-F^+Fs@1wnhchyChK>X5EQBAB|tED*Ov(UEwJYA z!dO7=lTgtW2HK=cv+lNqcw*9n+S$y!&%Mh^9OvjIv!6VP;QbmUtJed>+m(E}zQGrr z-qQvRUV08I{xv9 zM^IpZn$3k~aF7fas(2@Gs2)HYOS(YfG&WsphqAE?$#u9yM~Sg2Yyz_^=k^&T>cpE%twf31 z$^TM>ktx-N10~CgIWUwNxTEivhxys*B|YSX4o!u0D{Vjl9_B`|yTL@64tL534ZZy$ zysAsoh!pj83_-6mF}lmiUo|Ic5eFRWRreCx=;>h90LoR$3;UqM%RSs3IK7QE7~BPX zcl>23oDD{)FQ^Vy=RS-+lZN@hBOTs2&?7D|7#!;2$wnZJ2*RQjkkoSdK5eghVG$5? z`G(sPjM@D|s=e{l!rvcnqn+<{p&d|bPls%J#@JImC!Afs z3CAtCLyde#im!w~$1nR^)wdiIe7N4>P1I9a`g^x^FSH#$+#cNBU^EJJjeHJv6~FKu zpC0dWSK&)Btjkl!F%K`v=yNihxJRCD#bb`0EJlgw+2+oawF@*0S9`>&&t@mQ4(17f zu)}l*vNs-afR7!P@OFjSJw zvEhx}h$GlCL$?e7(a#A(xr9p#{cBsitK!GI>&B)xa65I5xE!UN7_GCgvBtWg3kN&a z=zraFPrQk=a*CR{k{7#Jd6>jS!yq(8C`CZZooLM9A<~eBm$9Br#n)gy3cv-A-9u7^ z+Iw8+qi@#dBj|Cn;Xz}6Fd#>)=Hrg^1)0z?NFt*ZqJxR#pNf*|#MCZ9NnAYPZ?7Yi z-8x6b`|LCYoDd*$*CX=U=Q8+2Df&o4+CzEt9(>QkXg#cR;F5)pzU3u|NOf@Cxv^US zVRfe?HyNp)phqt-Q0MkZzX!{?7%%)_K_?KF)F0>J+eL7!$1@4o{XYWAwNRvo1V`;;S-3rm_mF49H! zF=?B2Cr=w_psJ&UvJ{=QQN$@1Et8t)c}g7dB#*-b?wNkOgt4r0pY;gx8G|Pnx8qSg z7M*%559l2-sG1Ihfa^}H&$Wo=ng`geTRAJ!hH)eAML2krs{Td z_O!*aN69#}8_}A80A^qncDk^=T|<_{V+oog?=)-rW}nMlhCuE4MBC~7D%`ZPXGk4|96!2kHxTm4^1|z zPdWf}u(v*YA^=gI%q1Ce!^e}^rGyr6*l8C#ZM4Z{M7K#9a7#kYp#-+T!k_?8!zwg! zY_lM9Hj_>6&xVr|<1)|j=MUp?^KP#Si#OwO@%~VF=me*ntJ}~T>FqbD&Axo_zU&+n zS5}7i-OA?oNc!p=7tW3`XPLfAgsBZij&8!OTxC#%m3Nf_`Q)OMs zuvOKt-Fd$y%DmXA+#2q}-G!>G{Z4VCft$D_Wuwr;;iZ82k}hC?_nvJBg8dT7`hw|! zquTbA&JMkG61Rmu)JSK;;D)-kL0S*x9Kvi7-ZoKI^9cWO#dwaYVfsWJCPOc zf}~YNe77IMKw(BpgtFQmTrRBe=e_PxOqUDJjxh@txMre?C|VrS9hhaN9pwj%>f7V? zHZ8BdHB3HO8w<~rcD9ne^8O?ay@JUeV82$kXp6 z{NU#%3%AM-6#SFfL5G~*1j?sS7UW;g%b7x%5pR3P5=k~8ynK>gt%)&I*k@(#wKR0Y z#9~H<4zQQhG9g|UKk~JUNGPuS$ft-{gT(hqf=^P+DaBPl_(_5KG?mSk*=H@fR*N10 z1O_|X*MU34)A~837CqEja~=e`#O*U;bNytVCSPH?KG2mD=&|SlEu9RWak>y%5lT%_ z%K?sHMDHD}$T?m}L=L!-WAu5vKC6jaW4@cab$NTN#fi0X;`(}FnWf+gM&h}4aQmJL zCyxiK&VANyT3O}cLWb*{8~DDAwQt5GO zn%K1LE1+~2#kX)J7k8yf1x59ob9)Zz3-L`cjACoq0ih-r9t@(`s`w%ODdY}XrKRr% z`CDQEFu2^K$PBeSp$5Qemi-0m!iKP)C8>KBJOs665)0vG?uERIOC)2%CZw79Pm3ot zbBjT)1_bvRQHB+1^3;WVomm($FPa&etwFyiKjH9ritmrxzD&upKkFw~TiFu4o9{Qr z{rEPvQeR$O@yNvW)mpY;%|ej>Uip~R&eNnaJ8;9%%5tT=Gqdleg8`m-DqT3;4YAU6 z%n*(CC+nlu{3oOMR0W~q?M!$yvX*)-yrNN<)241(AF5bS7%_`lXwb?Yqdq4i%N))F zN8b--V)s1)T$cEgBK~nr#d{9JM=6{eRxp&YvODI;f}#DW?ALbgGv+cRd#ZiyEzfL z{-KB0v$!&NpLv>3)679IOAMjxa(IIQZA=0c>E?r~{7Yrhk<3@@c;mvT*u8V1pc5%-OTdtLI34 zJb}0QftQeQC*=GpSNSoQZODAUDa58h9JvI{aD*he$OJT2>ctdRaZSKTU6bVCwhe=A z<2p-+d!FEXG^VIB&A?5JY#_EXaC<}V9O7-q)DVW;0+T&a1N|-$D4wvw94K_e?eGQ( z>5vwPWASf5!q^pFp$4NBwMDLxQIg@EUIcm@XVFQ<95+Q!CllR`3zJNapo!jp-;^d&lvPEu4%E*3AyG!5MYjob6OiMYzlr6mIZz7_5NV;RY`)1l$oz zdyJ$qJ~^D-K&{w?4!Ne{=gR^`WmuaXgCtNS?ZYA<&P5SN6-1b4I}oHkK`Tm51VK$ISqIl|*D7$jsinJu?^HAGxWm+Y)a zHb}Fy-iT?QIjxqr=4VDG81Gr$AhToKBiiy%5`i)g@QQiQ{mOKJi{1k#bc`aHJ8qNu zK&?!~O;6RZqe{%U_*^w)*!CYsVv@(5R+``QjO)h{7t&aJFf^UZuDa6HfOYd(48U)I zv6g$i9$775T7^I#il+mhXoN!M5OxDBto`kpnoulyeed0Yg0+C_MU#)y(ht^zk^1#6 z<3>YHxt6MQ9;7B<+~-)W^DtW*t9&~04jtezvpXW-%8wSsS+^wb3 zmsAkCMPss+bp%Y{GxLjm44JnrOiJYBx7hD6-Ep$edgv!+dTS-ixf;y%Hru0(vLwkm zu5Du68ic(kgFbjt@B57~z~5hi`)kVy>%yQ7h$ZFYHvv`Puk{J+qf1&-{FFn??Ukk? z9ojO|{=7Fv&#z;%9wgHU{C#WN`#V0HE712g&yP;k$ta&9w8gJM7Sl*SERYvS^>1=I<59!Q*=}n{tAUSA zcg?>m48xThrg;hJkmLJBzb59HVM`VoaeXHkTki&Etz8Eq66oiWP=p~V1Gu(z(QB!Z>tBZQjM?tI!;37g-<*3fzNb*S|!#jYYQOc-}qQ6Ih3 zSUANIMX^>ayTtBOWM;BpYe8PUjshpcvqp_Q@VD5c5UtQW5GuQE?T(2jIGIj*N>9i+ z^lE6`p`|w&ZAlT^W1x+Vo-S8hxpZ$eZF5z|%AkM(nIcB&KE33mh6Mr@Uy|Y_xO$;s z6!ShaVuuYTLawjy%i&E*(?IZ&%UezfsjkbTjWepBYDp?0Wjh#PjWYSOVMyW=-R=PAnN^Qd z;O?{ zXCu&Zd;DnwNQ%5MIn!-KcU0=uEl-}EmFJGgQrEVf5BV62@}+=}?05-Ga@Ds2m}KvV zbP+tVW-7PO)*Ls4i56;WTB`P1VfLQZ=RvZh{%a75($RSUOD)c3AymqaKovwo3KFWd*_xx{W%YjH*TS@{YS>MzKWtXj zltdxD9&?_(VW-RRI9QjmA(u9^Kn}Xi*HNp`PgZn8g-Gh!WB>6?5{M|5^es)BA)n?NWKyW@6`LofvL2 zDPDrMMThnh4eF15;sAJ-k@tY1bp@Vjm4azh^zS8-D_%nrKhxYZ7k4P=UX~$oy!V4^ z4_W0=P_^->{7Vac(0dRfYE|-d6Gl)&pRz^NC}~l~`o9=_!icH4cOFYv5&JlM@e)@~ zZ+o$xv~@t3hhYsXk1H@fg~z~ajh7EIFjCM?3xnK1y<1UZ7-xw0v5D0pFVf# zj4%O7h!TnjvfplnY0x++XS=193p!L;LonWlToe3~zToFsWyK#!I1w2)=x3#-_A|WX zz?IJ)^#vlMbskiunbGXGn;j2(9b0@}>XpxEeDeHWD!i{6xSh?rDs!y{cvXj5W64+TCaYEI2A2p{b4R+Qj$;V zl%@4^sF)IS1{Gt`0c9m8CeHn8;kTz{_aWuX=D^fOgM2k7^yjf#r%tiqS~G9jU2Bo? z>+cLF@Ok7khv)59Z0Pur;91R-L#HUj(zMWu(Z{5N9yB_k9Q6qt2|7wh=L-@{)ItgP zJuVq1Rnemp>^~ppe|Thn&8lB=xSxyUmZy;=IGL?j`T6v{1-ooP7JN$ex$>T%%jF=F zT~()hcJ}RbN!4G@#K{7u(O?5dWTtb$ZWeBDVTV92$bA}qx5bQvy8p$k6uaB285`Vg zJN9Q(sA-^N(KyC}2HjWY=@a#y`yY(J!KPfQK`~pkd~Iph3GzTbyX`#4r9J8skq3?w zWHhMl5gNqwJCtk2V;6RBbiFZ5%beMxA_YkH#fQfaDZr*bvh0E5o4clx94vIJDu>wb z2Q@u~;~iYsPMBPr7(Qs76S!#h>m!tE2M8L1xEY+nw5=DLrV;9eJ1kC$dQ>zze9)3@ z=B#$y)6(Y#CgPAy}f<|Z?@d&~Gd_Y*X~QKrbA$eGq(F+u9!J|GIF zqKFzVH(A2Jz~zJFM~S3Gr$msVs~?!=XeErM7xhbQCry0Pexl|&M4#5J*I|NPv>=o^ z9bv4yde70Uo`D13I}dZ325|0~Zd=~zHke}9ezXs?W;?zJGRtYc>0cZXV4jvt9_m$o z>wl)NAW=29s*eEG@fF{#MIFcaL5C-QU6D|eU;|o*8x(#}?0rJ;vM81;$$m#KpbIVF z(V8OL64xHF?LVNl%dI-*p;{}&d=+Tr{4$nWiJD~Uy{<{fX{;z`crrI7RnA|#9nDLQ zD7G8FfLpXh$&+_Gah7_whlpc_lWA&rKd=cdJDHupiHQFh&r5}eIlz|l?TblJDP$(n zu}HZ1UV|YI<$28vS1xbx)g5Nv^gHt+z5$uXH~3)M8n2IB&*KS&c$t0n?vTCf z;;k^U8PD+Wh{&X@j4tb#vQ`=z`Oi*-^PrZt561R^T!N2xpx?t^F}k&iPj>OjHC1(( z#koY+{2I=>=V7lOgJ!c>` zu(3NiFdsSQ2bm1%%>SG^h3tVE#u|iyr z^@UTkh+)2}^~2*m>c4CG2i&U7a6@KXgN}Nh)cRcmAHMRx`oN}f{&-F2=40{PLCp90 z0z(^Ifx`ibWS`BcZ4bGw3<~XFCXZn8 zL*!&fTS4qG2-0sfuU6){1>I@pIr6wv#dHeKs9%!ln4T&+=7o`%&eLysxIVs*&5Lr4 z_9rYW6jrcQbEoM=o7~4#&uV)`Bw!+P&CQYTZ*-7JNE6(R&pA?>yqnhb=W%o73V9@E zKe>+8hX@EJ8#DIC4+rz{dMj!REHWQ?ruUgo^&e5b-hQyca*WL8xMfhp$CS^$rh6Vo zA+D;<3Yvk&Cbtda>%A1ovPBsEDe&Pi`^Wdd7SYCnf-&T5gB9a$;Ra%_lAl%}NOI>^ z@Mj{%dsV8+g6tnS+$q$+?>%ExO(c9QyA|w}B+GMFPO1_YW;M-=7ieS9H2SDDG7N-G zIZcEkY<2a8Css<3?l4!&E6@W+sfz?Cq>#s|A@5f?PBs|Mn)M|BOT1III^4vSC$G1) z*`J-L&&m$is1jAVp?_$NcTb+MQpvR;hJ{0+=7k_0X88o0M@qWLkWnRWV2)nJ4mIs1cvE8Ce<@ExP>w* znAaQDIk!9cQdUIe_R{%uKsm8%^BY0k?w$r9Zp zTAG8(&&$xHcYwglgc`yF68Ce-C`q+mm(S|Nmn%4aEi=NFAfFRS7}Op^9Fd zUoHkqgIKu3-ozjmboBMv9#$r#2no62b-{)gBaBXJ!#lV#jxE0P$2mQh@52M*IoY%0 zIC8Pr#=QvtNQ2Ut`>LR+4DJpTbNCGAp@UMW)-x%+;YQLx$JKk*_9OMW zU>Len`X?lR3u19X!u!VqCo~@ufhcn<@O?PNSNq;nVV3TYT}sBp-4wIN z$b}Ewdze21F%y{I1Hd^6&ng=XzB9^|s|1^;GTk(E1)!nzZh@QJ`7yndf;J#~+CoSy zGJd;i^Z9mq^KQyWCmvMeBi%C`DKE)`t0t0~BvzDymN>5_nwLd&ncLzi1b-o1JR!5( z!QYfxq^jmy55nVBnB&V`dH5YGZH&zE(K&*(yYz5M(yUDYZlRK5Hcg$1$esygV zytU5z6?`hUO>^(=+ibH5Hg&bmrjdHtXrSO_zOYr|OXqWI0r&Ak?fspmAN=Iub>ok> z9)5ot8_sMBUiun`iM_@)x|@xLK}4yzp1SX+z5%qzyP>`kc_A^_)G@ zv0QvIcaNTD>jVmK1X{K8*BEXO?=@)NW{Y%>7di)bzGzE}rPD`W)lYtG5Zpt;p$>kCWE4V&de$kXVI#YOT-+2U(kk4`5Wi#ax}2Rs&B4i>Au z*+A3}FLLRvUeRl$q2Gv`@s0VyeIEu=p>ofer=UG9vP`gNWk5)l?XQ}m>NUGdq*iBnGldB~EWP6v`#9^!9!b6Jyp+m??)Yw*& z^qnvW|6~W-kV)lc>m>ep@4^RAdL(J#6}JT#PmN;7ynV7 z`1?tsQ**;%P1U&@9Y_#~h*B!aXxt8GAk@IYLi(C)<)THC-P)3TldHjE<*COSEzp&o zRDwW2*C%dJ>DYSTGER=!EZuv;F&@+Ge)Ov_etWzex7P{r^K|e%=}#fDN{?^g0!l;+ z4fKg(=q+1^Ssr}ehXN1dM*clhHs0PzZ~HWaKjq-xOs^iOUIXbBgfR5yiB)e3&=Tbb zm9|S=&86#E!o13oEsm2m$F!bQd#M`2RBMgY#Vq>LOcN`V$}JPk>cj@q5~j*j0*k-P zHWpPLFEXC0TMVU{;WkG#8&_&@vK?lcW;Vzb-F~m&th!cicf2h1s`FlTUnN~kU4ov| zct_NK`aI|kVGgbupowTsHvfSQxDnd=tZcDffxeb$e*BeYKHJ|tP*%uGX)!z4JyBNB z3&wcrmg%dng{)E36;JxRkpA%U1o0%-@aX1~SD##+hVw4sMxkotjFD3M3l^tZ^^AJ^ zo`^-GP=(H2^Oy6qwUFJu$q-nR!M$4Nak%{D5^maW7f!3wd33iF-$*Z8U%Id#8?|{} z+H6-Xn%832?AztQBAXw{9(GS;yGOwvusu_Q30Z~{e}E4|cw=O%AH<$pa#aPttXLsh zwnc<9u+{iq%QpA=j-(D@*;p<|sh>Br;IGU$rw2z8gX`nM$X(sg2A12xaTzR+ddHOYHM3Qdz7O%>v_PZb0%7XjknM3#6`DUyW1FPva3``20Ja2 zYj+5aZi$Hus6X)&$K7E+n&v#d;)HYihNl6Es7)4ahAW}QdqJL{%osk*Jwq?ID_Cgx zn3>dx$&!&6br@pB?b!a>Y%4gbxXQ~fw160|*X?&I?ui;8%>(Wrejy_Jvk#xFVK$_S zx;@M_SA0oDjQ!KKPyr*jMK_TyRr?RnXGAllBBpc2T=8l|9t~QxpI9F^y}nP%VWd*8 zcNj7?8$C}T_N{d3RXGwlu!c;4Rwya$h#CisgsJE)cqHOJGipEzeILoD{@8Gkz&ox@663WrfXNW1 z!cmvs9Ma87`mBQs17ZGNLT~NLn7!S8!?-dMj!v9ds{VqieoL>?F%)fdiSqJx?sA}Q#ix@ixztXrN`t<`CwYNGmq?5=3d6;;&mzmR9$}1PPn7z* zLFOO%!#12R*PN^NaN%_&TfPVmU&rD3_R+*R)nbVWO^mYFaO`59aRd*#(b%i!KqJ%R zH@Jc7$O`%nLrW{59G%ICmLt7MQqp^ft?OhJSYOJI^qkio4_w>f;;JcwA121j$tTWc zf)=Y84vP(8coipuRxkj+aq7g;)zW<4?YUEl)8H8%n+_Tg0WBawgeD-JPfu-mT}>k< zvFQZ~e`bFwtT|_SOvGkor6VOWx8c$W7KY31&J#;4U>~@;P&vr_IiNMq9g}M2siZ?4UND*@S*VbBw>Rq_6pO* zL&2j_BCcb%lpfA(=5~g$cwPV{ERUbZUx^i59$?Pjqe{iOZTG=I1*F`U-lVqXi-?SY zF~HGo(s^Oe-=f|??$bT7MhqQti}b~Mc5<`gjomBU4-wv)cwlhn7dDYj_u4O9E%T;N0 zjT!An=QjLQ*V;&AN{vy2{+t&2O_!jvmM$87$c@T!eoPMX+JS6 z`dUaea=S6eqsOY)hpMxZ9)rT-3(y`vHdfEZ;xjWJ#u-k((-#C3k8GlQnUe^;arg$F zX}k$Cbw}}2;FA@As-yA_?xDpfPLa_bzCil`ABfExwt3i=U>~5A@0ppWxKjw605TCw z^SY_*iEIV8l^&_Tpi=q1-AvH1BX!Zt4tI9QV?PBr82?~>eg{GsFg@Jk!ZA%Hh+1qr z);u+k=0|%M)2}c298fwJuh33_gdR)OTEp^M$joX z2b`{+ma&`KCaXxUFP}m9y1=@;O6bq03g1vy_*0usq@l%f%=EO4Ahb_JEIVp9;DOYF zte1KDT-j5RImyIMVQZUkZy;v{23l(nq3}*`5c`l@MudHb`88W~WCCF$*Rty5iA1hf zt@HAh)7Iu5psA>o`BffnUXE%TE*vOJq$Qyz@aeazbS+`f&!UmFU)l)KhSqP0Q1IjZfxde8l!22cG(`B`)m(YB^i9>& z*o}igafVbwnv=9MaKsTE;$Wa3u7@=D1g?zkepwKee|DDSCz_*3n?C+|CKnmQUPe^Y zGU*3VqKv;c6yJ3I=#4P?3Bmk)>bY%S;x1-@AW-P^6mYl8cLKd!oDv4>jh~2;fvuzn zL+?hGt9&F2aUmzwJR#_lu#mE-7~I|PiM|(pGd291a@=%=8gVZ(p=Jd>XrnW5w#*dNk!r& zUY=rzL+hWpjqI~mI&jTdJ|K4ktz)c`!Z=uzTi8c#B!<`k>-f%LjIcMlMid9##ZpX; zuX_ zJ=t_HCIBK@LFX7>5o}i&*+W?;&SSL{{l|m-WKEVp1x=7$ z^UpaBUHZ6R&)*x%m+)`IAVuHh(uNwy*iW_@F2S_4*R40#U}*N9xoxyNyFD+zYb|}D zMZ%5Rq?L3K>cGKw0eis3{_f*cr!fSeHK04&sJT!-GTh@fa$$p+}=6TUVc}+h`xYAi62M1=Ivkn zK8yD<>pwxLQ#x=T5lBuj;%2ADnXRP2E}OC8Y$cQ$YA>S}ZR;51PZU9E+BfH{h$v0c zb{Pce@zJMSD9*#v)6>D9cWt+X>;32x&8mGg$W{)gfI$~`?XE#<>{xX)I2|(uH(}u3 z2C6pdLLy*vQ8$$pZUKIPHEf@0I@ibBNCVF8`Qe(sQLEKqBu$Bk?3$Xm=5o2RbgK6C za?$DKq*7~fR7GZR9CpI~dX&fjiG654xW&Ic*=jwnHS+1h5v`;$9$A`YB4aq0#1Q)I z2zdmnXE=2!qX)NH6(&-bR*?4CkdH)z9kFj^piEu`k51&|+M?Z4T;*0W8}1b32J<7% z@uY4y@vX|9-+8ytn!V_%en{=?mR}wvOk(%e30DX5L1>W~5Fi$XVvCFVI9Kz~4YNv< zxtL(nPW~~qVTYje?D1A+Ak<;&Qb zo*)a-0ow<~Vngx50U2V4Zxb zwfn7DK7$kpHq{*$9MJe);+CrH2Epo?VOzs!G7R{rvZx?+4#{068NYFa@g4*VFAl+X zj9pxw^F5X_tl&Y0vS&EVs4^WcnMcCK$Vmw*XyM5BWm}v+q|X%Ze7DB|&e=~XE3jmmY$fbxohB;+~YvCw#HNd9I zfL5m^r{x(?Jnf^K`)D}v1Y_}ytU323TQX^~0QGTU$kd(fhv~5_e}z63xIRV(DGze5 z8G}ZQr-HvKs;2G`4p|X=oT}O5yt|8(F&-(Dq`d>|#wZ^ZA8VvKN@UG;9iO$3C9lY}QS%+YTSf`B6H8?i*eQ8HBDCa2y} zDD17X%w#3NfH&r-E|`7JwFevf*)%6L#m|QNd4ld{_$@pk*_BJ^HA_3?H3zL!5-F{; zeaxhu`xB8n&25VPW^5O&Bz3V{f0gsc9JOZmBR_Nl(F#4GN7Zd0loC{kWNN>u(6((8~9yh!{bT*rsZey(oTf!Izh`pC|!&^JIR=fa^32+Q#!GM@oYG47(TTSup z)wfWjNgba(+Hs7-NW=U9iQzT-CSi4nD#%a0AUAhq>TvYq?5~2(DTlx!i*)JDNJJ1g zZPYG!SeeG}=y9>hvM5ULHE@piQ!x`~I?to8Zse8(B;|pZskeuA3z|^$NS@@4;|ky= zs9fa=$8wY1?HpU087ALfv6BzCI(WK2#AQKB9!#o&sdi9UOECsw%ic#9$J3q@%7v}j zPr${VLMu7DJ7Z>on`^PPOlV(Qi*7 z$TocZnuC^7f=&vGe5FSMXHEg#v&gCuzpees#pwUlKl-v40=F9x{Lxo7OU)%%IG^aStlV6Zb3z>}t%0$2kL?=R=1qyodo5+8{V07*s;wqZUq7wuw=a>i2w+ z=zCa~6zW^lC&k^4hON=z{M%XROjRiIZ&d=vkQaI~@2wKQ@zvix-0UID!{f}1Qy98kTwQ#wv zu9>x&qpmKXaKc~5@>}_ZO?UJABxoQYaJ*k_o^)IMc zi@%pn|9gK(08aBal#}(Be}ViqC=Mt~h!KEgKfuKPePXnJfdI;={|nE47qqzt}0s=a<1_A=e#Qo6< zd=vf^^}9mcKRx`{f@Z&?00Wl#w@&@BqT`>aUu%{8i5gM*H`G5foBqlCRZjm;rW@cK z^H+cR&z$`lDg7t$SK;nIiTc|AM*L4O!9O{-CjZ9ytH1yM{4)Kj0{Ewi zZqxr|;`bx|f4k&wgZOnd&z}}F0{@qVKRn^ToA@=`?N1XKasSK2KRx8X+xj)C{->=R zfGXraz2&dRwb-%s(f9VHd AssI20 literal 0 HcmV?d00001 diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO new file mode 100644 index 000000000..94d2cb8fd --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO @@ -0,0 +1,80 @@ +Metadata-Version: 2.1 +Name: funasr-onnx +Version: 0.0.3 +Summary: FunASR: A Fundamental End-to-End Speech Recognition Toolkit +Home-page: https://github.com/alibaba-damo-academy/FunASR.git +Author: Speech Lab, Alibaba Group, China +Author-email: funasr@list.alibaba-inc.com +License: MIT +Keywords: funasr,asr +Platform: Any +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Description-Content-Type: text/markdown + +## Using funasr with ONNXRuntime + + +### Introduction +- Model comes from [speech_paraformer](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary). + + +### Steps: +1. Export the model. + - Command: (`Tips`: torch >= 1.11.0 is required.) + + More details ref to ([export docs](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export)) + + - `e.g.`, Export model from modelscope + ```shell + python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False + ``` + - `e.g.`, Export model from local path, the model'name must be `model.pb`. + ```shell + python -m funasr.export.export_model --model-name ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False + ``` + + +2. Install the `funasr_onnx` + +install from pip +```shell +pip install --upgrade funasr_onnx -i https://pypi.Python.org/simple +``` + +or install from source code + +```shell +git clone https://github.com/alibaba/FunASR.git && cd FunASR +cd funasr/runtime/python/funasr_onnx +python setup.py build +python setup.py install +``` + +3. Run the demo. + - Model_dir: the model path, which contains `model.onnx`, `config.yaml`, `am.mvn`. + - Input: wav formt file, support formats: `str, np.ndarray, List[str]` + - Output: `List[str]`: recognition result. + - Example: + ```python + from funasr_onnx import Paraformer + + model_dir = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" + model = Paraformer(model_dir, batch_size=1) + + wav_path = ['/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav'] + + result = model(wav_path) + print(result) + ``` + +## Performance benchmark + +Please ref to [benchmark](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/python/benchmark_onnx.md) + +## Acknowledge +1. This project is maintained by [FunASR community](https://github.com/alibaba-damo-academy/FunASR). +2. We acknowledge [SWHL](https://github.com/RapidAI/RapidASR) for contributing the onnxruntime (for paraformer model). diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt new file mode 100644 index 000000000..e759e2778 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt @@ -0,0 +1,17 @@ +README.md +setup.py +funasr_onnx/__init__.py +funasr_onnx/paraformer_bin.py +funasr_onnx/punc_bin.py +funasr_onnx/vad_bin.py +funasr_onnx.egg-info/PKG-INFO +funasr_onnx.egg-info/SOURCES.txt +funasr_onnx.egg-info/dependency_links.txt +funasr_onnx.egg-info/requires.txt +funasr_onnx.egg-info/top_level.txt +funasr_onnx/utils/__init__.py +funasr_onnx/utils/e2e_vad.py +funasr_onnx/utils/frontend.py +funasr_onnx/utils/postprocess_utils.py +funasr_onnx/utils/timestamp_utils.py +funasr_onnx/utils/utils.py \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt new file mode 100644 index 000000000..6fcb63279 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt @@ -0,0 +1,7 @@ +librosa +onnxruntime>=1.7.0 +scipy +numpy>=1.19.3 +typeguard +kaldi-native-fbank +PyYAML>=5.1.2 diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt new file mode 100644 index 000000000..de41eb90e --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt @@ -0,0 +1 @@ +funasr_onnx diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py index 647f9fadc..475047903 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py @@ -1,2 +1,3 @@ # -*- encoding: utf-8 -*- from .paraformer_bin import Paraformer +from .vad_bin import Fsmn_vad diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/e2e_vad.py b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/e2e_vad.py new file mode 100644 index 000000000..3f6c3d1f6 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/e2e_vad.py @@ -0,0 +1,607 @@ +from enum import Enum +from typing import List, Tuple, Dict, Any + +import math +import numpy as np + +class VadStateMachine(Enum): + kVadInStateStartPointNotDetected = 1 + kVadInStateInSpeechSegment = 2 + kVadInStateEndPointDetected = 3 + + +class FrameState(Enum): + kFrameStateInvalid = -1 + kFrameStateSpeech = 1 + kFrameStateSil = 0 + + +# final voice/unvoice state per frame +class AudioChangeState(Enum): + kChangeStateSpeech2Speech = 0 + kChangeStateSpeech2Sil = 1 + kChangeStateSil2Sil = 2 + kChangeStateSil2Speech = 3 + kChangeStateNoBegin = 4 + kChangeStateInvalid = 5 + + +class VadDetectMode(Enum): + kVadSingleUtteranceDetectMode = 0 + kVadMutipleUtteranceDetectMode = 1 + + +class VADXOptions: + def __init__( + self, + sample_rate: int = 16000, + detect_mode: int = VadDetectMode.kVadMutipleUtteranceDetectMode.value, + snr_mode: int = 0, + max_end_silence_time: int = 800, + max_start_silence_time: int = 3000, + do_start_point_detection: bool = True, + do_end_point_detection: bool = True, + window_size_ms: int = 200, + sil_to_speech_time_thres: int = 150, + speech_to_sil_time_thres: int = 150, + speech_2_noise_ratio: float = 1.0, + do_extend: int = 1, + lookback_time_start_point: int = 200, + lookahead_time_end_point: int = 100, + max_single_segment_time: int = 60000, + nn_eval_block_size: int = 8, + dcd_block_size: int = 4, + snr_thres: int = -100.0, + noise_frame_num_used_for_snr: int = 100, + decibel_thres: int = -100.0, + speech_noise_thres: float = 0.6, + fe_prior_thres: float = 1e-4, + silence_pdf_num: int = 1, + sil_pdf_ids: List[int] = [0], + speech_noise_thresh_low: float = -0.1, + speech_noise_thresh_high: float = 0.3, + output_frame_probs: bool = False, + frame_in_ms: int = 10, + frame_length_ms: int = 25, + ): + self.sample_rate = sample_rate + self.detect_mode = detect_mode + self.snr_mode = snr_mode + self.max_end_silence_time = max_end_silence_time + self.max_start_silence_time = max_start_silence_time + self.do_start_point_detection = do_start_point_detection + self.do_end_point_detection = do_end_point_detection + self.window_size_ms = window_size_ms + self.sil_to_speech_time_thres = sil_to_speech_time_thres + self.speech_to_sil_time_thres = speech_to_sil_time_thres + self.speech_2_noise_ratio = speech_2_noise_ratio + self.do_extend = do_extend + self.lookback_time_start_point = lookback_time_start_point + self.lookahead_time_end_point = lookahead_time_end_point + self.max_single_segment_time = max_single_segment_time + self.nn_eval_block_size = nn_eval_block_size + self.dcd_block_size = dcd_block_size + self.snr_thres = snr_thres + self.noise_frame_num_used_for_snr = noise_frame_num_used_for_snr + self.decibel_thres = decibel_thres + self.speech_noise_thres = speech_noise_thres + self.fe_prior_thres = fe_prior_thres + self.silence_pdf_num = silence_pdf_num + self.sil_pdf_ids = sil_pdf_ids + self.speech_noise_thresh_low = speech_noise_thresh_low + self.speech_noise_thresh_high = speech_noise_thresh_high + self.output_frame_probs = output_frame_probs + self.frame_in_ms = frame_in_ms + self.frame_length_ms = frame_length_ms + + +class E2EVadSpeechBufWithDoa(object): + def __init__(self): + self.start_ms = 0 + self.end_ms = 0 + self.buffer = [] + self.contain_seg_start_point = False + self.contain_seg_end_point = False + self.doa = 0 + + def Reset(self): + self.start_ms = 0 + self.end_ms = 0 + self.buffer = [] + self.contain_seg_start_point = False + self.contain_seg_end_point = False + self.doa = 0 + + +class E2EVadFrameProb(object): + def __init__(self): + self.noise_prob = 0.0 + self.speech_prob = 0.0 + self.score = 0.0 + self.frame_id = 0 + self.frm_state = 0 + + +class WindowDetector(object): + def __init__(self, window_size_ms: int, sil_to_speech_time: int, + speech_to_sil_time: int, frame_size_ms: int): + self.window_size_ms = window_size_ms + self.sil_to_speech_time = sil_to_speech_time + self.speech_to_sil_time = speech_to_sil_time + self.frame_size_ms = frame_size_ms + + self.win_size_frame = int(window_size_ms / frame_size_ms) + self.win_sum = 0 + self.win_state = [0] * self.win_size_frame # 初始化窗 + + self.cur_win_pos = 0 + self.pre_frame_state = FrameState.kFrameStateSil + self.cur_frame_state = FrameState.kFrameStateSil + self.sil_to_speech_frmcnt_thres = int(sil_to_speech_time / frame_size_ms) + self.speech_to_sil_frmcnt_thres = int(speech_to_sil_time / frame_size_ms) + + self.voice_last_frame_count = 0 + self.noise_last_frame_count = 0 + self.hydre_frame_count = 0 + + def Reset(self) -> None: + self.cur_win_pos = 0 + self.win_sum = 0 + self.win_state = [0] * self.win_size_frame + self.pre_frame_state = FrameState.kFrameStateSil + self.cur_frame_state = FrameState.kFrameStateSil + self.voice_last_frame_count = 0 + self.noise_last_frame_count = 0 + self.hydre_frame_count = 0 + + def GetWinSize(self) -> int: + return int(self.win_size_frame) + + def DetectOneFrame(self, frameState: FrameState, frame_count: int) -> AudioChangeState: + cur_frame_state = FrameState.kFrameStateSil + if frameState == FrameState.kFrameStateSpeech: + cur_frame_state = 1 + elif frameState == FrameState.kFrameStateSil: + cur_frame_state = 0 + else: + return AudioChangeState.kChangeStateInvalid + self.win_sum -= self.win_state[self.cur_win_pos] + self.win_sum += cur_frame_state + self.win_state[self.cur_win_pos] = cur_frame_state + self.cur_win_pos = (self.cur_win_pos + 1) % self.win_size_frame + + if self.pre_frame_state == FrameState.kFrameStateSil and self.win_sum >= self.sil_to_speech_frmcnt_thres: + self.pre_frame_state = FrameState.kFrameStateSpeech + return AudioChangeState.kChangeStateSil2Speech + + if self.pre_frame_state == FrameState.kFrameStateSpeech and self.win_sum <= self.speech_to_sil_frmcnt_thres: + self.pre_frame_state = FrameState.kFrameStateSil + return AudioChangeState.kChangeStateSpeech2Sil + + if self.pre_frame_state == FrameState.kFrameStateSil: + return AudioChangeState.kChangeStateSil2Sil + if self.pre_frame_state == FrameState.kFrameStateSpeech: + return AudioChangeState.kChangeStateSpeech2Speech + return AudioChangeState.kChangeStateInvalid + + def FrameSizeMs(self) -> int: + return int(self.frame_size_ms) + + +class E2EVadModel(): + def __init__(self, vad_post_args: Dict[str, Any]): + super(E2EVadModel, self).__init__() + self.vad_opts = VADXOptions(**vad_post_args) + self.windows_detector = WindowDetector(self.vad_opts.window_size_ms, + self.vad_opts.sil_to_speech_time_thres, + self.vad_opts.speech_to_sil_time_thres, + self.vad_opts.frame_in_ms) + # self.encoder = encoder + # init variables + self.is_final = False + self.data_buf_start_frame = 0 + self.frm_cnt = 0 + self.latest_confirmed_speech_frame = 0 + self.lastest_confirmed_silence_frame = -1 + self.continous_silence_frame_count = 0 + self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected + self.confirmed_start_frame = -1 + self.confirmed_end_frame = -1 + self.number_end_time_detected = 0 + self.sil_frame = 0 + self.sil_pdf_ids = self.vad_opts.sil_pdf_ids + self.noise_average_decibel = -100.0 + self.pre_end_silence_detected = False + self.next_seg = True + + self.output_data_buf = [] + self.output_data_buf_offset = 0 + self.frame_probs = [] + self.max_end_sil_frame_cnt_thresh = self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres + self.speech_noise_thres = self.vad_opts.speech_noise_thres + self.scores = None + self.max_time_out = False + self.decibel = [] + self.data_buf = None + self.data_buf_all = None + self.waveform = None + self.ResetDetection() + + def AllResetDetection(self): + self.is_final = False + self.data_buf_start_frame = 0 + self.frm_cnt = 0 + self.latest_confirmed_speech_frame = 0 + self.lastest_confirmed_silence_frame = -1 + self.continous_silence_frame_count = 0 + self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected + self.confirmed_start_frame = -1 + self.confirmed_end_frame = -1 + self.number_end_time_detected = 0 + self.sil_frame = 0 + self.sil_pdf_ids = self.vad_opts.sil_pdf_ids + self.noise_average_decibel = -100.0 + self.pre_end_silence_detected = False + self.next_seg = True + + self.output_data_buf = [] + self.output_data_buf_offset = 0 + self.frame_probs = [] + self.max_end_sil_frame_cnt_thresh = self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres + self.speech_noise_thres = self.vad_opts.speech_noise_thres + self.scores = None + self.max_time_out = False + self.decibel = [] + self.data_buf = None + self.data_buf_all = None + self.waveform = None + self.ResetDetection() + + def ResetDetection(self): + self.continous_silence_frame_count = 0 + self.latest_confirmed_speech_frame = 0 + self.lastest_confirmed_silence_frame = -1 + self.confirmed_start_frame = -1 + self.confirmed_end_frame = -1 + self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected + self.windows_detector.Reset() + self.sil_frame = 0 + self.frame_probs = [] + + def ComputeDecibel(self) -> None: + frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000) + frame_shift_length = int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) + if self.data_buf_all is None: + self.data_buf_all = self.waveform[0] # self.data_buf is pointed to self.waveform[0] + self.data_buf = self.data_buf_all + else: + self.data_buf_all = np.concatenate((self.data_buf_all, self.waveform[0])) + for offset in range(0, self.waveform.shape[1] - frame_sample_length + 1, frame_shift_length): + self.decibel.append( + 10 * math.log10(np.square((self.waveform[0][offset: offset + frame_sample_length])).sum() + \ + 0.000001)) + + def ComputeScores(self, scores: np.ndarray) -> None: + # scores = self.encoder(feats, in_cache) # return B * T * D + self.vad_opts.nn_eval_block_size = scores.shape[1] + self.frm_cnt += scores.shape[1] # count total frames + if self.scores is None: + self.scores = scores # the first calculation + else: + self.scores = np.concatenate((self.scores, scores), axis=1) + + def PopDataBufTillFrame(self, frame_idx: int) -> None: # need check again + while self.data_buf_start_frame < frame_idx: + if len(self.data_buf) >= int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000): + self.data_buf_start_frame += 1 + self.data_buf = self.data_buf_all[self.data_buf_start_frame * int( + self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000):] + + def PopDataToOutputBuf(self, start_frm: int, frm_cnt: int, first_frm_is_start_point: bool, + last_frm_is_end_point: bool, end_point_is_sent_end: bool) -> None: + self.PopDataBufTillFrame(start_frm) + expected_sample_number = int(frm_cnt * self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000) + if last_frm_is_end_point: + extra_sample = max(0, int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 - \ + self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000)) + expected_sample_number += int(extra_sample) + if end_point_is_sent_end: + expected_sample_number = max(expected_sample_number, len(self.data_buf)) + if len(self.data_buf) < expected_sample_number: + print('error in calling pop data_buf\n') + + if len(self.output_data_buf) == 0 or first_frm_is_start_point: + self.output_data_buf.append(E2EVadSpeechBufWithDoa()) + self.output_data_buf[-1].Reset() + self.output_data_buf[-1].start_ms = start_frm * self.vad_opts.frame_in_ms + self.output_data_buf[-1].end_ms = self.output_data_buf[-1].start_ms + self.output_data_buf[-1].doa = 0 + cur_seg = self.output_data_buf[-1] + if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: + print('warning\n') + out_pos = len(cur_seg.buffer) # cur_seg.buff现在没做任何操作 + data_to_pop = 0 + if end_point_is_sent_end: + data_to_pop = expected_sample_number + else: + data_to_pop = int(frm_cnt * self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) + if data_to_pop > len(self.data_buf): + print('VAD data_to_pop is bigger than self.data_buf.size()!!!\n') + data_to_pop = len(self.data_buf) + expected_sample_number = len(self.data_buf) + + cur_seg.doa = 0 + for sample_cpy_out in range(0, data_to_pop): + # cur_seg.buffer[out_pos ++] = data_buf_.back(); + out_pos += 1 + for sample_cpy_out in range(data_to_pop, expected_sample_number): + # cur_seg.buffer[out_pos++] = data_buf_.back() + out_pos += 1 + if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: + print('Something wrong with the VAD algorithm\n') + self.data_buf_start_frame += frm_cnt + cur_seg.end_ms = (start_frm + frm_cnt) * self.vad_opts.frame_in_ms + if first_frm_is_start_point: + cur_seg.contain_seg_start_point = True + if last_frm_is_end_point: + cur_seg.contain_seg_end_point = True + + def OnSilenceDetected(self, valid_frame: int): + self.lastest_confirmed_silence_frame = valid_frame + if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + self.PopDataBufTillFrame(valid_frame) + # silence_detected_callback_ + # pass + + def OnVoiceDetected(self, valid_frame: int) -> None: + self.latest_confirmed_speech_frame = valid_frame + self.PopDataToOutputBuf(valid_frame, 1, False, False, False) + + def OnVoiceStart(self, start_frame: int, fake_result: bool = False) -> None: + if self.vad_opts.do_start_point_detection: + pass + if self.confirmed_start_frame != -1: + print('not reset vad properly\n') + else: + self.confirmed_start_frame = start_frame + + if not fake_result and self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + self.PopDataToOutputBuf(self.confirmed_start_frame, 1, True, False, False) + + def OnVoiceEnd(self, end_frame: int, fake_result: bool, is_last_frame: bool) -> None: + for t in range(self.latest_confirmed_speech_frame + 1, end_frame): + self.OnVoiceDetected(t) + if self.vad_opts.do_end_point_detection: + pass + if self.confirmed_end_frame != -1: + print('not reset vad properly\n') + else: + self.confirmed_end_frame = end_frame + if not fake_result: + self.sil_frame = 0 + self.PopDataToOutputBuf(self.confirmed_end_frame, 1, False, True, is_last_frame) + self.number_end_time_detected += 1 + + def MaybeOnVoiceEndIfLastFrame(self, is_final_frame: bool, cur_frm_idx: int) -> None: + if is_final_frame: + self.OnVoiceEnd(cur_frm_idx, False, True) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + + def GetLatency(self) -> int: + return int(self.LatencyFrmNumAtStartPoint() * self.vad_opts.frame_in_ms) + + def LatencyFrmNumAtStartPoint(self) -> int: + vad_latency = self.windows_detector.GetWinSize() + if self.vad_opts.do_extend: + vad_latency += int(self.vad_opts.lookback_time_start_point / self.vad_opts.frame_in_ms) + return vad_latency + + def GetFrameState(self, t: int) -> FrameState: + frame_state = FrameState.kFrameStateInvalid + cur_decibel = self.decibel[t] + cur_snr = cur_decibel - self.noise_average_decibel + # for each frame, calc log posterior probability of each state + if cur_decibel < self.vad_opts.decibel_thres: + frame_state = FrameState.kFrameStateSil + self.DetectOneFrame(frame_state, t, False) + return frame_state + + sum_score = 0.0 + noise_prob = 0.0 + assert len(self.sil_pdf_ids) == self.vad_opts.silence_pdf_num + if len(self.sil_pdf_ids) > 0: + assert len(self.scores) == 1 # 只支持batch_size = 1的测试 + sil_pdf_scores = [self.scores[0][t][sil_pdf_id] for sil_pdf_id in self.sil_pdf_ids] + sum_score = sum(sil_pdf_scores) + noise_prob = math.log(sum_score) * self.vad_opts.speech_2_noise_ratio + total_score = 1.0 + sum_score = total_score - sum_score + speech_prob = math.log(sum_score) + if self.vad_opts.output_frame_probs: + frame_prob = E2EVadFrameProb() + frame_prob.noise_prob = noise_prob + frame_prob.speech_prob = speech_prob + frame_prob.score = sum_score + frame_prob.frame_id = t + self.frame_probs.append(frame_prob) + if math.exp(speech_prob) >= math.exp(noise_prob) + self.speech_noise_thres: + if cur_snr >= self.vad_opts.snr_thres and cur_decibel >= self.vad_opts.decibel_thres: + frame_state = FrameState.kFrameStateSpeech + else: + frame_state = FrameState.kFrameStateSil + else: + frame_state = FrameState.kFrameStateSil + if self.noise_average_decibel < -99.9: + self.noise_average_decibel = cur_decibel + else: + self.noise_average_decibel = (cur_decibel + self.noise_average_decibel * ( + self.vad_opts.noise_frame_num_used_for_snr + - 1)) / self.vad_opts.noise_frame_num_used_for_snr + + return frame_state + + + def __call__(self, score: np.ndarray, waveform: np.ndarray, + is_final: bool = False, max_end_sil: int = 800 + ): + self.max_end_sil_frame_cnt_thresh = max_end_sil - self.vad_opts.speech_to_sil_time_thres + self.waveform = waveform # compute decibel for each frame + self.ComputeDecibel() + self.ComputeScores(score) + if not is_final: + self.DetectCommonFrames() + else: + self.DetectLastFrames() + segments = [] + for batch_num in range(0, score.shape[0]): # only support batch_size = 1 now + segment_batch = [] + if len(self.output_data_buf) > 0: + for i in range(self.output_data_buf_offset, len(self.output_data_buf)): + if not self.output_data_buf[i].contain_seg_start_point: + continue + if not self.next_seg and not self.output_data_buf[i].contain_seg_end_point: + continue + start_ms = self.output_data_buf[i].start_ms if self.next_seg else -1 + if self.output_data_buf[i].contain_seg_end_point: + end_ms = self.output_data_buf[i].end_ms + self.next_seg = True + self.output_data_buf_offset += 1 + else: + end_ms = -1 + self.next_seg = False + segment = [start_ms, end_ms] + segment_batch.append(segment) + if segment_batch: + segments.append(segment_batch) + if is_final: + # reset class variables and clear the dict for the next query + self.AllResetDetection() + return segments + + def DetectCommonFrames(self) -> int: + if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected: + return 0 + for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): + frame_state = FrameState.kFrameStateInvalid + frame_state = self.GetFrameState(self.frm_cnt - 1 - i) + self.DetectOneFrame(frame_state, self.frm_cnt - 1 - i, False) + + return 0 + + def DetectLastFrames(self) -> int: + if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected: + return 0 + for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): + frame_state = FrameState.kFrameStateInvalid + frame_state = self.GetFrameState(self.frm_cnt - 1 - i) + if i != 0: + self.DetectOneFrame(frame_state, self.frm_cnt - 1 - i, False) + else: + self.DetectOneFrame(frame_state, self.frm_cnt - 1, True) + + return 0 + + def DetectOneFrame(self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool) -> None: + tmp_cur_frm_state = FrameState.kFrameStateInvalid + if cur_frm_state == FrameState.kFrameStateSpeech: + if math.fabs(1.0) > self.vad_opts.fe_prior_thres: + tmp_cur_frm_state = FrameState.kFrameStateSpeech + else: + tmp_cur_frm_state = FrameState.kFrameStateSil + elif cur_frm_state == FrameState.kFrameStateSil: + tmp_cur_frm_state = FrameState.kFrameStateSil + state_change = self.windows_detector.DetectOneFrame(tmp_cur_frm_state, cur_frm_idx) + frm_shift_in_ms = self.vad_opts.frame_in_ms + if AudioChangeState.kChangeStateSil2Speech == state_change: + silence_frame_count = self.continous_silence_frame_count + self.continous_silence_frame_count = 0 + self.pre_end_silence_detected = False + start_frame = 0 + if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + start_frame = max(self.data_buf_start_frame, cur_frm_idx - self.LatencyFrmNumAtStartPoint()) + self.OnVoiceStart(start_frame) + self.vad_state_machine = VadStateMachine.kVadInStateInSpeechSegment + for t in range(start_frame + 1, cur_frm_idx + 1): + self.OnVoiceDetected(t) + elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + for t in range(self.latest_confirmed_speech_frame + 1, cur_frm_idx): + self.OnVoiceDetected(t) + if cur_frm_idx - self.confirmed_start_frame + 1 > \ + self.vad_opts.max_single_segment_time / frm_shift_in_ms: + self.OnVoiceEnd(cur_frm_idx, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif not is_final_frame: + self.OnVoiceDetected(cur_frm_idx) + else: + self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) + else: + pass + elif AudioChangeState.kChangeStateSpeech2Sil == state_change: + self.continous_silence_frame_count = 0 + if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + pass + elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + if cur_frm_idx - self.confirmed_start_frame + 1 > \ + self.vad_opts.max_single_segment_time / frm_shift_in_ms: + self.OnVoiceEnd(cur_frm_idx, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif not is_final_frame: + self.OnVoiceDetected(cur_frm_idx) + else: + self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) + else: + pass + elif AudioChangeState.kChangeStateSpeech2Speech == state_change: + self.continous_silence_frame_count = 0 + if self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + if cur_frm_idx - self.confirmed_start_frame + 1 > \ + self.vad_opts.max_single_segment_time / frm_shift_in_ms: + self.max_time_out = True + self.OnVoiceEnd(cur_frm_idx, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif not is_final_frame: + self.OnVoiceDetected(cur_frm_idx) + else: + self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) + else: + pass + elif AudioChangeState.kChangeStateSil2Sil == state_change: + self.continous_silence_frame_count += 1 + if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: + # silence timeout, return zero length decision + if ((self.vad_opts.detect_mode == VadDetectMode.kVadSingleUtteranceDetectMode.value) and ( + self.continous_silence_frame_count * frm_shift_in_ms > self.vad_opts.max_start_silence_time)) \ + or (is_final_frame and self.number_end_time_detected == 0): + for t in range(self.lastest_confirmed_silence_frame + 1, cur_frm_idx): + self.OnSilenceDetected(t) + self.OnVoiceStart(0, True) + self.OnVoiceEnd(0, True, False); + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + else: + if cur_frm_idx >= self.LatencyFrmNumAtStartPoint(): + self.OnSilenceDetected(cur_frm_idx - self.LatencyFrmNumAtStartPoint()) + elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: + if self.continous_silence_frame_count * frm_shift_in_ms >= self.max_end_sil_frame_cnt_thresh: + lookback_frame = int(self.max_end_sil_frame_cnt_thresh / frm_shift_in_ms) + if self.vad_opts.do_extend: + lookback_frame -= int(self.vad_opts.lookahead_time_end_point / frm_shift_in_ms) + lookback_frame -= 1 + lookback_frame = max(0, lookback_frame) + self.OnVoiceEnd(cur_frm_idx - lookback_frame, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif cur_frm_idx - self.confirmed_start_frame + 1 > \ + self.vad_opts.max_single_segment_time / frm_shift_in_ms: + self.OnVoiceEnd(cur_frm_idx, False, False) + self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected + elif self.vad_opts.do_extend and not is_final_frame: + if self.continous_silence_frame_count <= int( + self.vad_opts.lookahead_time_end_point / frm_shift_in_ms): + self.OnVoiceDetected(cur_frm_idx) + else: + self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) + else: + pass + + if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected and \ + self.vad_opts.detect_mode == VadDetectMode.kVadMutipleUtteranceDetectMode.value: + self.ResetDetection() diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py index 2edde112e..fccd5a095 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py @@ -188,7 +188,7 @@ class OrtInferSession(): input_content: List[Union[np.ndarray, np.ndarray]]) -> np.ndarray: input_dict = dict(zip(self.get_input_names(), input_content)) try: - return self.session.run(None, input_dict) + return self.session.run(self.get_output_names(), input_dict) except Exception as e: raise ONNXRuntimeError('ONNXRuntime inferece failed.') from e diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py new file mode 100644 index 000000000..533b4b7fd --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -0,0 +1,134 @@ +# -*- encoding: utf-8 -*- + +import os.path +from pathlib import Path +from typing import List, Union, Tuple + +import copy +import librosa +import numpy as np + +from .utils.utils import (ONNXRuntimeError, + OrtInferSession, get_logger, + read_yaml) +from .utils.frontend import WavFrontend +from .utils.e2e_vad import E2EVadModel + +logging = get_logger() + + +class Fsmn_vad(): + def __init__(self, model_dir: Union[str, Path] = None, + batch_size: int = 1, + device_id: Union[str, int] = "-1", + quantize: bool = False, + intra_op_num_threads: int = 4, + max_end_sil: int = 800, + ): + + if not Path(model_dir).exists(): + raise FileNotFoundError(f'{model_dir} does not exist.') + + model_file = os.path.join(model_dir, 'model.onnx') + if quantize: + model_file = os.path.join(model_dir, 'model_quant.onnx') + config_file = os.path.join(model_dir, 'vad.yaml') + cmvn_file = os.path.join(model_dir, 'vad.mvn') + config = read_yaml(config_file) + + self.frontend = WavFrontend( + cmvn_file=cmvn_file, + **config['frontend_conf'] + ) + self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) + self.batch_size = batch_size + self.vad_scorer = E2EVadModel(config["vad_post_conf"]) + self.max_end_sil = max_end_sil + + def prepare_cache(self, in_cache: list = []): + if len(in_cache) > 0: + return in_cache + + for i in range(4): + cache = np.random.rand(1, 128, 19, 1).astype(np.float32) + in_cache.append(cache) + return in_cache + + + def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> List: + waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) + waveform_nums = len(waveform_list) + is_final = kwargs.get('kwargs', False) + + asr_res = [] + for beg_idx in range(0, waveform_nums, self.batch_size): + + end_idx = min(waveform_nums, beg_idx + self.batch_size) + waveform = waveform_list[beg_idx:end_idx] + feats, feats_len = self.extract_feat(waveform) + param_dict = kwargs.get('param_dict', dict()) + in_cache = param_dict.get('cache', list()) + in_cache = self.prepare_cache(in_cache) + try: + inputs = [feats] + inputs.extend(in_cache) + scores, out_caches = self.infer(inputs) + param_dict['cache'] = out_caches + segments = self.vad_scorer(scores, waveform[0][None, :], is_final=is_final, max_end_sil=self.max_end_sil) + + except ONNXRuntimeError: + # logging.warning(traceback.format_exc()) + logging.warning("input wav is silence or noise") + segments = '' + asr_res.append(segments) + + return asr_res + + def load_data(self, + wav_content: Union[str, np.ndarray, List[str]], fs: int = None) -> List: + def load_wav(path: str) -> np.ndarray: + waveform, _ = librosa.load(path, sr=fs) + return waveform + + if isinstance(wav_content, np.ndarray): + return [wav_content] + + if isinstance(wav_content, str): + return [load_wav(wav_content)] + + if isinstance(wav_content, list): + return [load_wav(path) for path in wav_content] + + raise TypeError( + f'The type of {wav_content} is not in [str, np.ndarray, list]') + + def extract_feat(self, + waveform_list: List[np.ndarray] + ) -> Tuple[np.ndarray, np.ndarray]: + feats, feats_len = [], [] + for waveform in waveform_list: + speech, _ = self.frontend.fbank(waveform) + feat, feat_len = self.frontend.lfr_cmvn(speech) + feats.append(feat) + feats_len.append(feat_len) + + feats = self.pad_feats(feats, np.max(feats_len)) + feats_len = np.array(feats_len).astype(np.int32) + return feats, feats_len + + @staticmethod + def pad_feats(feats: List[np.ndarray], max_feat_len: int) -> np.ndarray: + def pad_feat(feat: np.ndarray, cur_len: int) -> np.ndarray: + pad_width = ((0, max_feat_len - cur_len), (0, 0)) + return np.pad(feat, pad_width, 'constant', constant_values=0) + + feat_res = [pad_feat(feat, feat.shape[0]) for feat in feats] + feats = np.array(feat_res).astype(np.float32) + return feats + + def infer(self, feats: List) -> Tuple[np.ndarray, np.ndarray]: + + outputs = self.ort_infer(feats) + scores, out_caches = outputs[0], outputs[1:] + return scores, out_caches + \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/setup.py b/funasr/runtime/python/onnxruntime/setup.py index 3b9ed3bd8..1a8ed7b31 100644 --- a/funasr/runtime/python/onnxruntime/setup.py +++ b/funasr/runtime/python/onnxruntime/setup.py @@ -13,7 +13,7 @@ def get_readme(): MODULE_NAME = 'funasr_onnx' -VERSION_NUM = '0.0.2' +VERSION_NUM = '0.0.3' setuptools.setup( name=MODULE_NAME, From 6d1a4789e6348d497f8e75c4cb94bc55a2d84a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 00:27:49 +0800 Subject: [PATCH 06/55] export --- .../onnxruntime/funasr_onnx.egg-info/PKG-INFO | 80 ------------------- .../funasr_onnx.egg-info/SOURCES.txt | 17 ---- .../funasr_onnx.egg-info/dependency_links.txt | 1 - .../funasr_onnx.egg-info/requires.txt | 7 -- .../funasr_onnx.egg-info/top_level.txt | 1 - 5 files changed, 106 deletions(-) delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO deleted file mode 100644 index 94d2cb8fd..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO +++ /dev/null @@ -1,80 +0,0 @@ -Metadata-Version: 2.1 -Name: funasr-onnx -Version: 0.0.3 -Summary: FunASR: A Fundamental End-to-End Speech Recognition Toolkit -Home-page: https://github.com/alibaba-damo-academy/FunASR.git -Author: Speech Lab, Alibaba Group, China -Author-email: funasr@list.alibaba-inc.com -License: MIT -Keywords: funasr,asr -Platform: Any -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Description-Content-Type: text/markdown - -## Using funasr with ONNXRuntime - - -### Introduction -- Model comes from [speech_paraformer](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary). - - -### Steps: -1. Export the model. - - Command: (`Tips`: torch >= 1.11.0 is required.) - - More details ref to ([export docs](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export)) - - - `e.g.`, Export model from modelscope - ```shell - python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False - ``` - - `e.g.`, Export model from local path, the model'name must be `model.pb`. - ```shell - python -m funasr.export.export_model --model-name ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False - ``` - - -2. Install the `funasr_onnx` - -install from pip -```shell -pip install --upgrade funasr_onnx -i https://pypi.Python.org/simple -``` - -or install from source code - -```shell -git clone https://github.com/alibaba/FunASR.git && cd FunASR -cd funasr/runtime/python/funasr_onnx -python setup.py build -python setup.py install -``` - -3. Run the demo. - - Model_dir: the model path, which contains `model.onnx`, `config.yaml`, `am.mvn`. - - Input: wav formt file, support formats: `str, np.ndarray, List[str]` - - Output: `List[str]`: recognition result. - - Example: - ```python - from funasr_onnx import Paraformer - - model_dir = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" - model = Paraformer(model_dir, batch_size=1) - - wav_path = ['/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav'] - - result = model(wav_path) - print(result) - ``` - -## Performance benchmark - -Please ref to [benchmark](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/python/benchmark_onnx.md) - -## Acknowledge -1. This project is maintained by [FunASR community](https://github.com/alibaba-damo-academy/FunASR). -2. We acknowledge [SWHL](https://github.com/RapidAI/RapidASR) for contributing the onnxruntime (for paraformer model). diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt deleted file mode 100644 index e759e2778..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt +++ /dev/null @@ -1,17 +0,0 @@ -README.md -setup.py -funasr_onnx/__init__.py -funasr_onnx/paraformer_bin.py -funasr_onnx/punc_bin.py -funasr_onnx/vad_bin.py -funasr_onnx.egg-info/PKG-INFO -funasr_onnx.egg-info/SOURCES.txt -funasr_onnx.egg-info/dependency_links.txt -funasr_onnx.egg-info/requires.txt -funasr_onnx.egg-info/top_level.txt -funasr_onnx/utils/__init__.py -funasr_onnx/utils/e2e_vad.py -funasr_onnx/utils/frontend.py -funasr_onnx/utils/postprocess_utils.py -funasr_onnx/utils/timestamp_utils.py -funasr_onnx/utils/utils.py \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt deleted file mode 100644 index 8b1378917..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt deleted file mode 100644 index 6fcb63279..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt +++ /dev/null @@ -1,7 +0,0 @@ -librosa -onnxruntime>=1.7.0 -scipy -numpy>=1.19.3 -typeguard -kaldi-native-fbank -PyYAML>=5.1.2 diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt deleted file mode 100644 index de41eb90e..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -funasr_onnx From 242431452b682b6bf5d711506653605ed8786af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 00:30:57 +0800 Subject: [PATCH 07/55] export --- .../build/lib/funasr_onnx/__init__.py | 3 - .../build/lib/funasr_onnx/paraformer_bin.py | 187 ------ .../build/lib/funasr_onnx/punc_bin.py | 0 .../build/lib/funasr_onnx/utils/__init__.py | 0 .../build/lib/funasr_onnx/utils/e2e_vad.py | 607 ------------------ .../build/lib/funasr_onnx/utils/frontend.py | 191 ------ .../funasr_onnx/utils/postprocess_utils.py | 240 ------- .../lib/funasr_onnx/utils/timestamp_utils.py | 59 -- .../build/lib/funasr_onnx/utils/utils.py | 257 -------- .../build/lib/funasr_onnx/vad_bin.py | 166 ----- .../dist/funasr_onnx-0.0.2-py3.8.egg | Bin 47828 -> 0 bytes .../dist/funasr_onnx-0.0.3-py3.8.egg | Bin 47828 -> 0 bytes 12 files changed, 1710 deletions(-) delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/__init__.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/paraformer_bin.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/punc_bin.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/__init__.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/e2e_vad.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/frontend.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/postprocess_utils.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/timestamp_utils.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/utils.py delete mode 100644 funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/vad_bin.py delete mode 100644 funasr/runtime/python/onnxruntime/dist/funasr_onnx-0.0.2-py3.8.egg delete mode 100644 funasr/runtime/python/onnxruntime/dist/funasr_onnx-0.0.3-py3.8.egg diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/__init__.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/__init__.py deleted file mode 100644 index 475047903..000000000 --- a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -*- encoding: utf-8 -*- -from .paraformer_bin import Paraformer -from .vad_bin import Fsmn_vad diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/paraformer_bin.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/paraformer_bin.py deleted file mode 100644 index cbdb8d9e6..000000000 --- a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/paraformer_bin.py +++ /dev/null @@ -1,187 +0,0 @@ -# -*- encoding: utf-8 -*- - -import os.path -from pathlib import Path -from typing import List, Union, Tuple - -import copy -import librosa -import numpy as np - -from .utils.utils import (CharTokenizer, Hypothesis, ONNXRuntimeError, - OrtInferSession, TokenIDConverter, get_logger, - read_yaml) -from .utils.postprocess_utils import sentence_postprocess -from .utils.frontend import WavFrontend -from .utils.timestamp_utils import time_stamp_lfr6_onnx - -logging = get_logger() - - -class Paraformer(): - def __init__(self, model_dir: Union[str, Path] = None, - batch_size: int = 1, - device_id: Union[str, int] = "-1", - plot_timestamp_to: str = "", - pred_bias: int = 1, - quantize: bool = False, - intra_op_num_threads: int = 4, - ): - - if not Path(model_dir).exists(): - raise FileNotFoundError(f'{model_dir} does not exist.') - - model_file = os.path.join(model_dir, 'model.onnx') - if quantize: - model_file = os.path.join(model_dir, 'model_quant.onnx') - config_file = os.path.join(model_dir, 'config.yaml') - cmvn_file = os.path.join(model_dir, 'am.mvn') - config = read_yaml(config_file) - - self.converter = TokenIDConverter(config['token_list']) - self.tokenizer = CharTokenizer() - self.frontend = WavFrontend( - cmvn_file=cmvn_file, - **config['frontend_conf'] - ) - self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) - self.batch_size = batch_size - self.plot_timestamp_to = plot_timestamp_to - self.pred_bias = pred_bias - - def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> List: - waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) - waveform_nums = len(waveform_list) - asr_res = [] - for beg_idx in range(0, waveform_nums, self.batch_size): - - end_idx = min(waveform_nums, beg_idx + self.batch_size) - feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx]) - try: - outputs = self.infer(feats, feats_len) - am_scores, valid_token_lens = outputs[0], outputs[1] - if len(outputs) == 4: - # for BiCifParaformer Inference - us_alphas, us_peaks = outputs[2], outputs[3] - else: - us_alphas, us_peaks = None, None - except ONNXRuntimeError: - #logging.warning(traceback.format_exc()) - logging.warning("input wav is silence or noise") - preds = [''] - else: - preds = self.decode(am_scores, valid_token_lens) - if us_peaks is None: - for pred in preds: - pred = sentence_postprocess(pred) - asr_res.append({'preds': pred}) - else: - for pred, us_peaks_ in zip(preds, us_peaks): - raw_tokens = pred - timestamp, timestamp_raw = time_stamp_lfr6_onnx(us_peaks_, copy.copy(raw_tokens)) - text_proc, timestamp_proc, _ = sentence_postprocess(raw_tokens, timestamp_raw) - # logging.warning(timestamp) - if len(self.plot_timestamp_to): - self.plot_wave_timestamp(waveform_list[0], timestamp, self.plot_timestamp_to) - asr_res.append({'preds': text_proc, 'timestamp': timestamp_proc, "raw_tokens": raw_tokens}) - return asr_res - - def plot_wave_timestamp(self, wav, text_timestamp, dest): - # TODO: Plot the wav and timestamp results with matplotlib - import matplotlib - matplotlib.use('Agg') - matplotlib.rc("font", family='Alibaba PuHuiTi') # set it to a font that your system supports - import matplotlib.pyplot as plt - fig, ax1 = plt.subplots(figsize=(11, 3.5), dpi=320) - ax2 = ax1.twinx() - ax2.set_ylim([0, 2.0]) - # plot waveform - ax1.set_ylim([-0.3, 0.3]) - time = np.arange(wav.shape[0]) / 16000 - ax1.plot(time, wav/wav.max()*0.3, color='gray', alpha=0.4) - # plot lines and text - for (char, start, end) in text_timestamp: - ax1.vlines(start, -0.3, 0.3, ls='--') - ax1.vlines(end, -0.3, 0.3, ls='--') - x_adj = 0.045 if char != '' else 0.12 - ax1.text((start + end) * 0.5 - x_adj, 0, char) - # plt.legend() - plotname = "{}/timestamp.png".format(dest) - plt.savefig(plotname, bbox_inches='tight') - - def load_data(self, - wav_content: Union[str, np.ndarray, List[str]], fs: int = None) -> List: - def load_wav(path: str) -> np.ndarray: - waveform, _ = librosa.load(path, sr=fs) - return waveform - - if isinstance(wav_content, np.ndarray): - return [wav_content] - - if isinstance(wav_content, str): - return [load_wav(wav_content)] - - if isinstance(wav_content, list): - return [load_wav(path) for path in wav_content] - - raise TypeError( - f'The type of {wav_content} is not in [str, np.ndarray, list]') - - def extract_feat(self, - waveform_list: List[np.ndarray] - ) -> Tuple[np.ndarray, np.ndarray]: - feats, feats_len = [], [] - for waveform in waveform_list: - speech, _ = self.frontend.fbank(waveform) - feat, feat_len = self.frontend.lfr_cmvn(speech) - feats.append(feat) - feats_len.append(feat_len) - - feats = self.pad_feats(feats, np.max(feats_len)) - feats_len = np.array(feats_len).astype(np.int32) - return feats, feats_len - - @staticmethod - def pad_feats(feats: List[np.ndarray], max_feat_len: int) -> np.ndarray: - def pad_feat(feat: np.ndarray, cur_len: int) -> np.ndarray: - pad_width = ((0, max_feat_len - cur_len), (0, 0)) - return np.pad(feat, pad_width, 'constant', constant_values=0) - - feat_res = [pad_feat(feat, feat.shape[0]) for feat in feats] - feats = np.array(feat_res).astype(np.float32) - return feats - - def infer(self, feats: np.ndarray, - feats_len: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - outputs = self.ort_infer([feats, feats_len]) - return outputs - - def decode(self, am_scores: np.ndarray, token_nums: int) -> List[str]: - return [self.decode_one(am_score, token_num) - for am_score, token_num in zip(am_scores, token_nums)] - - def decode_one(self, - am_score: np.ndarray, - valid_token_num: int) -> List[str]: - yseq = am_score.argmax(axis=-1) - score = am_score.max(axis=-1) - score = np.sum(score, axis=-1) - - # pad with mask tokens to ensure compatibility with sos/eos tokens - # asr_model.sos:1 asr_model.eos:2 - yseq = np.array([1] + yseq.tolist() + [2]) - hyp = Hypothesis(yseq=yseq, score=score) - - # remove sos/eos and get results - last_pos = -1 - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list(filter(lambda x: x not in (0, 2), token_int)) - - # Change integer-ids to tokens - token = self.converter.ids2tokens(token_int) - token = token[:valid_token_num-self.pred_bias] - # texts = sentence_postprocess(token) - return token - diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/punc_bin.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/__init__.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/e2e_vad.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/e2e_vad.py deleted file mode 100644 index 8eed22fa4..000000000 --- a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/e2e_vad.py +++ /dev/null @@ -1,607 +0,0 @@ -from enum import Enum -from typing import List, Tuple, Dict, Any - -import math -import numpy as np - -class VadStateMachine(Enum): - kVadInStateStartPointNotDetected = 1 - kVadInStateInSpeechSegment = 2 - kVadInStateEndPointDetected = 3 - - -class FrameState(Enum): - kFrameStateInvalid = -1 - kFrameStateSpeech = 1 - kFrameStateSil = 0 - - -# final voice/unvoice state per frame -class AudioChangeState(Enum): - kChangeStateSpeech2Speech = 0 - kChangeStateSpeech2Sil = 1 - kChangeStateSil2Sil = 2 - kChangeStateSil2Speech = 3 - kChangeStateNoBegin = 4 - kChangeStateInvalid = 5 - - -class VadDetectMode(Enum): - kVadSingleUtteranceDetectMode = 0 - kVadMutipleUtteranceDetectMode = 1 - - -class VADXOptions: - def __init__( - self, - sample_rate: int = 16000, - detect_mode: int = VadDetectMode.kVadMutipleUtteranceDetectMode.value, - snr_mode: int = 0, - max_end_silence_time: int = 800, - max_start_silence_time: int = 3000, - do_start_point_detection: bool = True, - do_end_point_detection: bool = True, - window_size_ms: int = 200, - sil_to_speech_time_thres: int = 150, - speech_to_sil_time_thres: int = 150, - speech_2_noise_ratio: float = 1.0, - do_extend: int = 1, - lookback_time_start_point: int = 200, - lookahead_time_end_point: int = 100, - max_single_segment_time: int = 60000, - nn_eval_block_size: int = 8, - dcd_block_size: int = 4, - snr_thres: int = -100.0, - noise_frame_num_used_for_snr: int = 100, - decibel_thres: int = -100.0, - speech_noise_thres: float = 0.6, - fe_prior_thres: float = 1e-4, - silence_pdf_num: int = 1, - sil_pdf_ids: List[int] = [0], - speech_noise_thresh_low: float = -0.1, - speech_noise_thresh_high: float = 0.3, - output_frame_probs: bool = False, - frame_in_ms: int = 10, - frame_length_ms: int = 25, - ): - self.sample_rate = sample_rate - self.detect_mode = detect_mode - self.snr_mode = snr_mode - self.max_end_silence_time = max_end_silence_time - self.max_start_silence_time = max_start_silence_time - self.do_start_point_detection = do_start_point_detection - self.do_end_point_detection = do_end_point_detection - self.window_size_ms = window_size_ms - self.sil_to_speech_time_thres = sil_to_speech_time_thres - self.speech_to_sil_time_thres = speech_to_sil_time_thres - self.speech_2_noise_ratio = speech_2_noise_ratio - self.do_extend = do_extend - self.lookback_time_start_point = lookback_time_start_point - self.lookahead_time_end_point = lookahead_time_end_point - self.max_single_segment_time = max_single_segment_time - self.nn_eval_block_size = nn_eval_block_size - self.dcd_block_size = dcd_block_size - self.snr_thres = snr_thres - self.noise_frame_num_used_for_snr = noise_frame_num_used_for_snr - self.decibel_thres = decibel_thres - self.speech_noise_thres = speech_noise_thres - self.fe_prior_thres = fe_prior_thres - self.silence_pdf_num = silence_pdf_num - self.sil_pdf_ids = sil_pdf_ids - self.speech_noise_thresh_low = speech_noise_thresh_low - self.speech_noise_thresh_high = speech_noise_thresh_high - self.output_frame_probs = output_frame_probs - self.frame_in_ms = frame_in_ms - self.frame_length_ms = frame_length_ms - - -class E2EVadSpeechBufWithDoa(object): - def __init__(self): - self.start_ms = 0 - self.end_ms = 0 - self.buffer = [] - self.contain_seg_start_point = False - self.contain_seg_end_point = False - self.doa = 0 - - def Reset(self): - self.start_ms = 0 - self.end_ms = 0 - self.buffer = [] - self.contain_seg_start_point = False - self.contain_seg_end_point = False - self.doa = 0 - - -class E2EVadFrameProb(object): - def __init__(self): - self.noise_prob = 0.0 - self.speech_prob = 0.0 - self.score = 0.0 - self.frame_id = 0 - self.frm_state = 0 - - -class WindowDetector(object): - def __init__(self, window_size_ms: int, sil_to_speech_time: int, - speech_to_sil_time: int, frame_size_ms: int): - self.window_size_ms = window_size_ms - self.sil_to_speech_time = sil_to_speech_time - self.speech_to_sil_time = speech_to_sil_time - self.frame_size_ms = frame_size_ms - - self.win_size_frame = int(window_size_ms / frame_size_ms) - self.win_sum = 0 - self.win_state = [0] * self.win_size_frame # 初始化窗 - - self.cur_win_pos = 0 - self.pre_frame_state = FrameState.kFrameStateSil - self.cur_frame_state = FrameState.kFrameStateSil - self.sil_to_speech_frmcnt_thres = int(sil_to_speech_time / frame_size_ms) - self.speech_to_sil_frmcnt_thres = int(speech_to_sil_time / frame_size_ms) - - self.voice_last_frame_count = 0 - self.noise_last_frame_count = 0 - self.hydre_frame_count = 0 - - def Reset(self) -> None: - self.cur_win_pos = 0 - self.win_sum = 0 - self.win_state = [0] * self.win_size_frame - self.pre_frame_state = FrameState.kFrameStateSil - self.cur_frame_state = FrameState.kFrameStateSil - self.voice_last_frame_count = 0 - self.noise_last_frame_count = 0 - self.hydre_frame_count = 0 - - def GetWinSize(self) -> int: - return int(self.win_size_frame) - - def DetectOneFrame(self, frameState: FrameState, frame_count: int) -> AudioChangeState: - cur_frame_state = FrameState.kFrameStateSil - if frameState == FrameState.kFrameStateSpeech: - cur_frame_state = 1 - elif frameState == FrameState.kFrameStateSil: - cur_frame_state = 0 - else: - return AudioChangeState.kChangeStateInvalid - self.win_sum -= self.win_state[self.cur_win_pos] - self.win_sum += cur_frame_state - self.win_state[self.cur_win_pos] = cur_frame_state - self.cur_win_pos = (self.cur_win_pos + 1) % self.win_size_frame - - if self.pre_frame_state == FrameState.kFrameStateSil and self.win_sum >= self.sil_to_speech_frmcnt_thres: - self.pre_frame_state = FrameState.kFrameStateSpeech - return AudioChangeState.kChangeStateSil2Speech - - if self.pre_frame_state == FrameState.kFrameStateSpeech and self.win_sum <= self.speech_to_sil_frmcnt_thres: - self.pre_frame_state = FrameState.kFrameStateSil - return AudioChangeState.kChangeStateSpeech2Sil - - if self.pre_frame_state == FrameState.kFrameStateSil: - return AudioChangeState.kChangeStateSil2Sil - if self.pre_frame_state == FrameState.kFrameStateSpeech: - return AudioChangeState.kChangeStateSpeech2Speech - return AudioChangeState.kChangeStateInvalid - - def FrameSizeMs(self) -> int: - return int(self.frame_size_ms) - - -class E2EVadModel(): - def __init__(self, vad_post_args: Dict[str, Any]): - super(E2EVadModel, self).__init__() - self.vad_opts = VADXOptions(**vad_post_args) - self.windows_detector = WindowDetector(self.vad_opts.window_size_ms, - self.vad_opts.sil_to_speech_time_thres, - self.vad_opts.speech_to_sil_time_thres, - self.vad_opts.frame_in_ms) - # self.encoder = encoder - # init variables - self.is_final = False - self.data_buf_start_frame = 0 - self.frm_cnt = 0 - self.latest_confirmed_speech_frame = 0 - self.lastest_confirmed_silence_frame = -1 - self.continous_silence_frame_count = 0 - self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected - self.confirmed_start_frame = -1 - self.confirmed_end_frame = -1 - self.number_end_time_detected = 0 - self.sil_frame = 0 - self.sil_pdf_ids = self.vad_opts.sil_pdf_ids - self.noise_average_decibel = -100.0 - self.pre_end_silence_detected = False - self.next_seg = True - - self.output_data_buf = [] - self.output_data_buf_offset = 0 - self.frame_probs = [] - self.max_end_sil_frame_cnt_thresh = self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres - self.speech_noise_thres = self.vad_opts.speech_noise_thres - self.scores = None - self.max_time_out = False - self.decibel = [] - self.data_buf = None - self.data_buf_all = None - self.waveform = None - self.ResetDetection() - - def AllResetDetection(self): - self.is_final = False - self.data_buf_start_frame = 0 - self.frm_cnt = 0 - self.latest_confirmed_speech_frame = 0 - self.lastest_confirmed_silence_frame = -1 - self.continous_silence_frame_count = 0 - self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected - self.confirmed_start_frame = -1 - self.confirmed_end_frame = -1 - self.number_end_time_detected = 0 - self.sil_frame = 0 - self.sil_pdf_ids = self.vad_opts.sil_pdf_ids - self.noise_average_decibel = -100.0 - self.pre_end_silence_detected = False - self.next_seg = True - - self.output_data_buf = [] - self.output_data_buf_offset = 0 - self.frame_probs = [] - self.max_end_sil_frame_cnt_thresh = self.vad_opts.max_end_silence_time - self.vad_opts.speech_to_sil_time_thres - self.speech_noise_thres = self.vad_opts.speech_noise_thres - self.scores = None - self.max_time_out = False - self.decibel = [] - self.data_buf = None - self.data_buf_all = None - self.waveform = None - self.ResetDetection() - - def ResetDetection(self): - self.continous_silence_frame_count = 0 - self.latest_confirmed_speech_frame = 0 - self.lastest_confirmed_silence_frame = -1 - self.confirmed_start_frame = -1 - self.confirmed_end_frame = -1 - self.vad_state_machine = VadStateMachine.kVadInStateStartPointNotDetected - self.windows_detector.Reset() - self.sil_frame = 0 - self.frame_probs = [] - - def ComputeDecibel(self) -> None: - frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000) - frame_shift_length = int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) - if self.data_buf_all is None: - self.data_buf_all = self.waveform[0] # self.data_buf is pointed to self.waveform[0] - self.data_buf = self.data_buf_all - else: - self.data_buf_all = np.concatenate((self.data_buf_all, self.waveform[0])) - for offset in range(0, self.waveform.shape[1] - frame_sample_length + 1, frame_shift_length): - self.decibel.append( - 10 * math.log10((self.waveform[0][offset: offset + frame_sample_length]).square().sum() + \ - 0.000001)) - - def ComputeScores(self, scores: np.ndarray) -> None: - # scores = self.encoder(feats, in_cache) # return B * T * D - self.vad_opts.nn_eval_block_size = scores.shape[1] - self.frm_cnt += scores.shape[1] # count total frames - if self.scores is None: - self.scores = scores # the first calculation - else: - self.scores = np.concatenate((self.scores, scores), axis=1) - - def PopDataBufTillFrame(self, frame_idx: int) -> None: # need check again - while self.data_buf_start_frame < frame_idx: - if len(self.data_buf) >= int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000): - self.data_buf_start_frame += 1 - self.data_buf = self.data_buf_all[self.data_buf_start_frame * int( - self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000):] - - def PopDataToOutputBuf(self, start_frm: int, frm_cnt: int, first_frm_is_start_point: bool, - last_frm_is_end_point: bool, end_point_is_sent_end: bool) -> None: - self.PopDataBufTillFrame(start_frm) - expected_sample_number = int(frm_cnt * self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000) - if last_frm_is_end_point: - extra_sample = max(0, int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000 - \ - self.vad_opts.sample_rate * self.vad_opts.frame_in_ms / 1000)) - expected_sample_number += int(extra_sample) - if end_point_is_sent_end: - expected_sample_number = max(expected_sample_number, len(self.data_buf)) - if len(self.data_buf) < expected_sample_number: - print('error in calling pop data_buf\n') - - if len(self.output_data_buf) == 0 or first_frm_is_start_point: - self.output_data_buf.append(E2EVadSpeechBufWithDoa()) - self.output_data_buf[-1].Reset() - self.output_data_buf[-1].start_ms = start_frm * self.vad_opts.frame_in_ms - self.output_data_buf[-1].end_ms = self.output_data_buf[-1].start_ms - self.output_data_buf[-1].doa = 0 - cur_seg = self.output_data_buf[-1] - if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: - print('warning\n') - out_pos = len(cur_seg.buffer) # cur_seg.buff现在没做任何操作 - data_to_pop = 0 - if end_point_is_sent_end: - data_to_pop = expected_sample_number - else: - data_to_pop = int(frm_cnt * self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000) - if data_to_pop > len(self.data_buf): - print('VAD data_to_pop is bigger than self.data_buf.size()!!!\n') - data_to_pop = len(self.data_buf) - expected_sample_number = len(self.data_buf) - - cur_seg.doa = 0 - for sample_cpy_out in range(0, data_to_pop): - # cur_seg.buffer[out_pos ++] = data_buf_.back(); - out_pos += 1 - for sample_cpy_out in range(data_to_pop, expected_sample_number): - # cur_seg.buffer[out_pos++] = data_buf_.back() - out_pos += 1 - if cur_seg.end_ms != start_frm * self.vad_opts.frame_in_ms: - print('Something wrong with the VAD algorithm\n') - self.data_buf_start_frame += frm_cnt - cur_seg.end_ms = (start_frm + frm_cnt) * self.vad_opts.frame_in_ms - if first_frm_is_start_point: - cur_seg.contain_seg_start_point = True - if last_frm_is_end_point: - cur_seg.contain_seg_end_point = True - - def OnSilenceDetected(self, valid_frame: int): - self.lastest_confirmed_silence_frame = valid_frame - if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: - self.PopDataBufTillFrame(valid_frame) - # silence_detected_callback_ - # pass - - def OnVoiceDetected(self, valid_frame: int) -> None: - self.latest_confirmed_speech_frame = valid_frame - self.PopDataToOutputBuf(valid_frame, 1, False, False, False) - - def OnVoiceStart(self, start_frame: int, fake_result: bool = False) -> None: - if self.vad_opts.do_start_point_detection: - pass - if self.confirmed_start_frame != -1: - print('not reset vad properly\n') - else: - self.confirmed_start_frame = start_frame - - if not fake_result and self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: - self.PopDataToOutputBuf(self.confirmed_start_frame, 1, True, False, False) - - def OnVoiceEnd(self, end_frame: int, fake_result: bool, is_last_frame: bool) -> None: - for t in range(self.latest_confirmed_speech_frame + 1, end_frame): - self.OnVoiceDetected(t) - if self.vad_opts.do_end_point_detection: - pass - if self.confirmed_end_frame != -1: - print('not reset vad properly\n') - else: - self.confirmed_end_frame = end_frame - if not fake_result: - self.sil_frame = 0 - self.PopDataToOutputBuf(self.confirmed_end_frame, 1, False, True, is_last_frame) - self.number_end_time_detected += 1 - - def MaybeOnVoiceEndIfLastFrame(self, is_final_frame: bool, cur_frm_idx: int) -> None: - if is_final_frame: - self.OnVoiceEnd(cur_frm_idx, False, True) - self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected - - def GetLatency(self) -> int: - return int(self.LatencyFrmNumAtStartPoint() * self.vad_opts.frame_in_ms) - - def LatencyFrmNumAtStartPoint(self) -> int: - vad_latency = self.windows_detector.GetWinSize() - if self.vad_opts.do_extend: - vad_latency += int(self.vad_opts.lookback_time_start_point / self.vad_opts.frame_in_ms) - return vad_latency - - def GetFrameState(self, t: int) -> FrameState: - frame_state = FrameState.kFrameStateInvalid - cur_decibel = self.decibel[t] - cur_snr = cur_decibel - self.noise_average_decibel - # for each frame, calc log posterior probability of each state - if cur_decibel < self.vad_opts.decibel_thres: - frame_state = FrameState.kFrameStateSil - self.DetectOneFrame(frame_state, t, False) - return frame_state - - sum_score = 0.0 - noise_prob = 0.0 - assert len(self.sil_pdf_ids) == self.vad_opts.silence_pdf_num - if len(self.sil_pdf_ids) > 0: - assert len(self.scores) == 1 # 只支持batch_size = 1的测试 - sil_pdf_scores = [self.scores[0][t][sil_pdf_id] for sil_pdf_id in self.sil_pdf_ids] - sum_score = sum(sil_pdf_scores) - noise_prob = math.log(sum_score) * self.vad_opts.speech_2_noise_ratio - total_score = 1.0 - sum_score = total_score - sum_score - speech_prob = math.log(sum_score) - if self.vad_opts.output_frame_probs: - frame_prob = E2EVadFrameProb() - frame_prob.noise_prob = noise_prob - frame_prob.speech_prob = speech_prob - frame_prob.score = sum_score - frame_prob.frame_id = t - self.frame_probs.append(frame_prob) - if math.exp(speech_prob) >= math.exp(noise_prob) + self.speech_noise_thres: - if cur_snr >= self.vad_opts.snr_thres and cur_decibel >= self.vad_opts.decibel_thres: - frame_state = FrameState.kFrameStateSpeech - else: - frame_state = FrameState.kFrameStateSil - else: - frame_state = FrameState.kFrameStateSil - if self.noise_average_decibel < -99.9: - self.noise_average_decibel = cur_decibel - else: - self.noise_average_decibel = (cur_decibel + self.noise_average_decibel * ( - self.vad_opts.noise_frame_num_used_for_snr - - 1)) / self.vad_opts.noise_frame_num_used_for_snr - - return frame_state - - - def __call__(self, score: np.ndarray, waveform: np.ndarray, - is_final: bool = False, max_end_sil: int = 800 - ): - self.max_end_sil_frame_cnt_thresh = max_end_sil - self.vad_opts.speech_to_sil_time_thres - self.waveform = waveform # compute decibel for each frame - self.ComputeDecibel() - self.ComputeScores(score) - if not is_final: - self.DetectCommonFrames() - else: - self.DetectLastFrames() - segments = [] - for batch_num in range(0, score.shape[0]): # only support batch_size = 1 now - segment_batch = [] - if len(self.output_data_buf) > 0: - for i in range(self.output_data_buf_offset, len(self.output_data_buf)): - if not self.output_data_buf[i].contain_seg_start_point: - continue - if not self.next_seg and not self.output_data_buf[i].contain_seg_end_point: - continue - start_ms = self.output_data_buf[i].start_ms if self.next_seg else -1 - if self.output_data_buf[i].contain_seg_end_point: - end_ms = self.output_data_buf[i].end_ms - self.next_seg = True - self.output_data_buf_offset += 1 - else: - end_ms = -1 - self.next_seg = False - segment = [start_ms, end_ms] - segment_batch.append(segment) - if segment_batch: - segments.append(segment_batch) - if is_final: - # reset class variables and clear the dict for the next query - self.AllResetDetection() - return segments - - def DetectCommonFrames(self) -> int: - if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected: - return 0 - for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): - frame_state = FrameState.kFrameStateInvalid - frame_state = self.GetFrameState(self.frm_cnt - 1 - i) - self.DetectOneFrame(frame_state, self.frm_cnt - 1 - i, False) - - return 0 - - def DetectLastFrames(self) -> int: - if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected: - return 0 - for i in range(self.vad_opts.nn_eval_block_size - 1, -1, -1): - frame_state = FrameState.kFrameStateInvalid - frame_state = self.GetFrameState(self.frm_cnt - 1 - i) - if i != 0: - self.DetectOneFrame(frame_state, self.frm_cnt - 1 - i, False) - else: - self.DetectOneFrame(frame_state, self.frm_cnt - 1, True) - - return 0 - - def DetectOneFrame(self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool) -> None: - tmp_cur_frm_state = FrameState.kFrameStateInvalid - if cur_frm_state == FrameState.kFrameStateSpeech: - if math.fabs(1.0) > self.vad_opts.fe_prior_thres: - tmp_cur_frm_state = FrameState.kFrameStateSpeech - else: - tmp_cur_frm_state = FrameState.kFrameStateSil - elif cur_frm_state == FrameState.kFrameStateSil: - tmp_cur_frm_state = FrameState.kFrameStateSil - state_change = self.windows_detector.DetectOneFrame(tmp_cur_frm_state, cur_frm_idx) - frm_shift_in_ms = self.vad_opts.frame_in_ms - if AudioChangeState.kChangeStateSil2Speech == state_change: - silence_frame_count = self.continous_silence_frame_count - self.continous_silence_frame_count = 0 - self.pre_end_silence_detected = False - start_frame = 0 - if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: - start_frame = max(self.data_buf_start_frame, cur_frm_idx - self.LatencyFrmNumAtStartPoint()) - self.OnVoiceStart(start_frame) - self.vad_state_machine = VadStateMachine.kVadInStateInSpeechSegment - for t in range(start_frame + 1, cur_frm_idx + 1): - self.OnVoiceDetected(t) - elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: - for t in range(self.latest_confirmed_speech_frame + 1, cur_frm_idx): - self.OnVoiceDetected(t) - if cur_frm_idx - self.confirmed_start_frame + 1 > \ - self.vad_opts.max_single_segment_time / frm_shift_in_ms: - self.OnVoiceEnd(cur_frm_idx, False, False) - self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected - elif not is_final_frame: - self.OnVoiceDetected(cur_frm_idx) - else: - self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) - else: - pass - elif AudioChangeState.kChangeStateSpeech2Sil == state_change: - self.continous_silence_frame_count = 0 - if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: - pass - elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: - if cur_frm_idx - self.confirmed_start_frame + 1 > \ - self.vad_opts.max_single_segment_time / frm_shift_in_ms: - self.OnVoiceEnd(cur_frm_idx, False, False) - self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected - elif not is_final_frame: - self.OnVoiceDetected(cur_frm_idx) - else: - self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) - else: - pass - elif AudioChangeState.kChangeStateSpeech2Speech == state_change: - self.continous_silence_frame_count = 0 - if self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: - if cur_frm_idx - self.confirmed_start_frame + 1 > \ - self.vad_opts.max_single_segment_time / frm_shift_in_ms: - self.max_time_out = True - self.OnVoiceEnd(cur_frm_idx, False, False) - self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected - elif not is_final_frame: - self.OnVoiceDetected(cur_frm_idx) - else: - self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) - else: - pass - elif AudioChangeState.kChangeStateSil2Sil == state_change: - self.continous_silence_frame_count += 1 - if self.vad_state_machine == VadStateMachine.kVadInStateStartPointNotDetected: - # silence timeout, return zero length decision - if ((self.vad_opts.detect_mode == VadDetectMode.kVadSingleUtteranceDetectMode.value) and ( - self.continous_silence_frame_count * frm_shift_in_ms > self.vad_opts.max_start_silence_time)) \ - or (is_final_frame and self.number_end_time_detected == 0): - for t in range(self.lastest_confirmed_silence_frame + 1, cur_frm_idx): - self.OnSilenceDetected(t) - self.OnVoiceStart(0, True) - self.OnVoiceEnd(0, True, False); - self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected - else: - if cur_frm_idx >= self.LatencyFrmNumAtStartPoint(): - self.OnSilenceDetected(cur_frm_idx - self.LatencyFrmNumAtStartPoint()) - elif self.vad_state_machine == VadStateMachine.kVadInStateInSpeechSegment: - if self.continous_silence_frame_count * frm_shift_in_ms >= self.max_end_sil_frame_cnt_thresh: - lookback_frame = int(self.max_end_sil_frame_cnt_thresh / frm_shift_in_ms) - if self.vad_opts.do_extend: - lookback_frame -= int(self.vad_opts.lookahead_time_end_point / frm_shift_in_ms) - lookback_frame -= 1 - lookback_frame = max(0, lookback_frame) - self.OnVoiceEnd(cur_frm_idx - lookback_frame, False, False) - self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected - elif cur_frm_idx - self.confirmed_start_frame + 1 > \ - self.vad_opts.max_single_segment_time / frm_shift_in_ms: - self.OnVoiceEnd(cur_frm_idx, False, False) - self.vad_state_machine = VadStateMachine.kVadInStateEndPointDetected - elif self.vad_opts.do_extend and not is_final_frame: - if self.continous_silence_frame_count <= int( - self.vad_opts.lookahead_time_end_point / frm_shift_in_ms): - self.OnVoiceDetected(cur_frm_idx) - else: - self.MaybeOnVoiceEndIfLastFrame(is_final_frame, cur_frm_idx) - else: - pass - - if self.vad_state_machine == VadStateMachine.kVadInStateEndPointDetected and \ - self.vad_opts.detect_mode == VadDetectMode.kVadMutipleUtteranceDetectMode.value: - self.ResetDetection() diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/frontend.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/frontend.py deleted file mode 100644 index 11a86445d..000000000 --- a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/frontend.py +++ /dev/null @@ -1,191 +0,0 @@ -# -*- encoding: utf-8 -*- -from pathlib import Path -from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union - -import numpy as np -from typeguard import check_argument_types -import kaldi_native_fbank as knf - -root_dir = Path(__file__).resolve().parent - -logger_initialized = {} - - -class WavFrontend(): - """Conventional frontend structure for ASR. - """ - - def __init__( - self, - cmvn_file: str = None, - fs: int = 16000, - window: str = 'hamming', - n_mels: int = 80, - frame_length: int = 25, - frame_shift: int = 10, - lfr_m: int = 1, - lfr_n: int = 1, - dither: float = 1.0, - **kwargs, - ) -> None: - check_argument_types() - - opts = knf.FbankOptions() - opts.frame_opts.samp_freq = fs - opts.frame_opts.dither = dither - opts.frame_opts.window_type = window - opts.frame_opts.frame_shift_ms = float(frame_shift) - opts.frame_opts.frame_length_ms = float(frame_length) - opts.mel_opts.num_bins = n_mels - opts.energy_floor = 0 - opts.frame_opts.snip_edges = True - opts.mel_opts.debug_mel = False - self.opts = opts - - self.lfr_m = lfr_m - self.lfr_n = lfr_n - self.cmvn_file = cmvn_file - - if self.cmvn_file: - self.cmvn = self.load_cmvn() - self.fbank_fn = None - self.fbank_beg_idx = 0 - self.reset_status() - - def fbank(self, - waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - waveform = waveform * (1 << 15) - self.fbank_fn = knf.OnlineFbank(self.opts) - self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist()) - frames = self.fbank_fn.num_frames_ready - mat = np.empty([frames, self.opts.mel_opts.num_bins]) - for i in range(frames): - mat[i, :] = self.fbank_fn.get_frame(i) - feat = mat.astype(np.float32) - feat_len = np.array(mat.shape[0]).astype(np.int32) - return feat, feat_len - - def fbank_online(self, - waveform: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - waveform = waveform * (1 << 15) - # self.fbank_fn = knf.OnlineFbank(self.opts) - self.fbank_fn.accept_waveform(self.opts.frame_opts.samp_freq, waveform.tolist()) - frames = self.fbank_fn.num_frames_ready - mat = np.empty([frames, self.opts.mel_opts.num_bins]) - for i in range(self.fbank_beg_idx, frames): - mat[i, :] = self.fbank_fn.get_frame(i) - # self.fbank_beg_idx += (frames-self.fbank_beg_idx) - feat = mat.astype(np.float32) - feat_len = np.array(mat.shape[0]).astype(np.int32) - return feat, feat_len - - def reset_status(self): - self.fbank_fn = knf.OnlineFbank(self.opts) - self.fbank_beg_idx = 0 - - def lfr_cmvn(self, feat: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - if self.lfr_m != 1 or self.lfr_n != 1: - feat = self.apply_lfr(feat, self.lfr_m, self.lfr_n) - - if self.cmvn_file: - feat = self.apply_cmvn(feat) - - feat_len = np.array(feat.shape[0]).astype(np.int32) - return feat, feat_len - - @staticmethod - def apply_lfr(inputs: np.ndarray, lfr_m: int, lfr_n: int) -> np.ndarray: - LFR_inputs = [] - - T = inputs.shape[0] - T_lfr = int(np.ceil(T / lfr_n)) - left_padding = np.tile(inputs[0], ((lfr_m - 1) // 2, 1)) - inputs = np.vstack((left_padding, inputs)) - T = T + (lfr_m - 1) // 2 - for i in range(T_lfr): - if lfr_m <= T - i * lfr_n: - LFR_inputs.append( - (inputs[i * lfr_n:i * lfr_n + lfr_m]).reshape(1, -1)) - else: - # process last LFR frame - num_padding = lfr_m - (T - i * lfr_n) - frame = inputs[i * lfr_n:].reshape(-1) - for _ in range(num_padding): - frame = np.hstack((frame, inputs[-1])) - - LFR_inputs.append(frame) - LFR_outputs = np.vstack(LFR_inputs).astype(np.float32) - return LFR_outputs - - def apply_cmvn(self, inputs: np.ndarray) -> np.ndarray: - """ - Apply CMVN with mvn data - """ - frame, dim = inputs.shape - means = np.tile(self.cmvn[0:1, :dim], (frame, 1)) - vars = np.tile(self.cmvn[1:2, :dim], (frame, 1)) - inputs = (inputs + means) * vars - return inputs - - def load_cmvn(self,) -> np.ndarray: - with open(self.cmvn_file, 'r', encoding='utf-8') as f: - lines = f.readlines() - - means_list = [] - vars_list = [] - for i in range(len(lines)): - line_item = lines[i].split() - if line_item[0] == '': - line_item = lines[i + 1].split() - if line_item[0] == '': - add_shift_line = line_item[3:(len(line_item) - 1)] - means_list = list(add_shift_line) - continue - elif line_item[0] == '': - line_item = lines[i + 1].split() - if line_item[0] == '': - rescale_line = line_item[3:(len(line_item) - 1)] - vars_list = list(rescale_line) - continue - - means = np.array(means_list).astype(np.float64) - vars = np.array(vars_list).astype(np.float64) - cmvn = np.array([means, vars]) - return cmvn - -def load_bytes(input): - middle_data = np.frombuffer(input, dtype=np.int16) - middle_data = np.asarray(middle_data) - if middle_data.dtype.kind not in 'iu': - raise TypeError("'middle_data' must be an array of integers") - dtype = np.dtype('float32') - if dtype.kind != 'f': - raise TypeError("'dtype' must be a floating point type") - - i = np.iinfo(middle_data.dtype) - abs_max = 2 ** (i.bits - 1) - offset = i.min + abs_max - array = np.frombuffer((middle_data.astype(dtype) - offset) / abs_max, dtype=np.float32) - return array - - -def test(): - path = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav" - import librosa - cmvn_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/am.mvn" - config_file = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/config.yaml" - from funasr.runtime.python.onnxruntime.rapid_paraformer.utils.utils import read_yaml - config = read_yaml(config_file) - waveform, _ = librosa.load(path, sr=None) - frontend = WavFrontend( - cmvn_file=cmvn_file, - **config['frontend_conf'], - ) - speech, _ = frontend.fbank_online(waveform) #1d, (sample,), numpy - feat, feat_len = frontend.lfr_cmvn(speech) # 2d, (frame, 450), np.float32 -> torch, torch.from_numpy(), dtype, (1, frame, 450) - - frontend.reset_status() # clear cache - return feat, feat_len - -if __name__ == '__main__': - test() \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/postprocess_utils.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/postprocess_utils.py deleted file mode 100644 index 575fb90dd..000000000 --- a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/postprocess_utils.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright (c) Alibaba, Inc. and its affiliates. - -import string -import logging -from typing import Any, List, Union - - -def isChinese(ch: str): - if '\u4e00' <= ch <= '\u9fff' or '\u0030' <= ch <= '\u0039': - return True - return False - - -def isAllChinese(word: Union[List[Any], str]): - word_lists = [] - for i in word: - cur = i.replace(' ', '') - cur = cur.replace('', '') - cur = cur.replace('', '') - word_lists.append(cur) - - if len(word_lists) == 0: - return False - - for ch in word_lists: - if isChinese(ch) is False: - return False - return True - - -def isAllAlpha(word: Union[List[Any], str]): - word_lists = [] - for i in word: - cur = i.replace(' ', '') - cur = cur.replace('', '') - cur = cur.replace('', '') - word_lists.append(cur) - - if len(word_lists) == 0: - return False - - for ch in word_lists: - if ch.isalpha() is False and ch != "'": - return False - elif ch.isalpha() is True and isChinese(ch) is True: - return False - - return True - - -# def abbr_dispose(words: List[Any]) -> List[Any]: -def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]: - words_size = len(words) - word_lists = [] - abbr_begin = [] - abbr_end = [] - last_num = -1 - ts_lists = [] - ts_nums = [] - ts_index = 0 - for num in range(words_size): - if num <= last_num: - continue - - if len(words[num]) == 1 and words[num].encode('utf-8').isalpha(): - if num + 1 < words_size and words[ - num + 1] == ' ' and num + 2 < words_size and len( - words[num + - 2]) == 1 and words[num + - 2].encode('utf-8').isalpha(): - # found the begin of abbr - abbr_begin.append(num) - num += 2 - abbr_end.append(num) - # to find the end of abbr - while True: - num += 1 - if num < words_size and words[num] == ' ': - num += 1 - if num < words_size and len( - words[num]) == 1 and words[num].encode( - 'utf-8').isalpha(): - abbr_end.pop() - abbr_end.append(num) - last_num = num - else: - break - else: - break - - for num in range(words_size): - if words[num] == ' ': - ts_nums.append(ts_index) - else: - ts_nums.append(ts_index) - ts_index += 1 - last_num = -1 - for num in range(words_size): - if num <= last_num: - continue - - if num in abbr_begin: - if time_stamp is not None: - begin = time_stamp[ts_nums[num]][0] - word_lists.append(words[num].upper()) - num += 1 - while num < words_size: - if num in abbr_end: - word_lists.append(words[num].upper()) - last_num = num - break - else: - if words[num].encode('utf-8').isalpha(): - word_lists.append(words[num].upper()) - num += 1 - if time_stamp is not None: - end = time_stamp[ts_nums[num]][1] - ts_lists.append([begin, end]) - else: - word_lists.append(words[num]) - if time_stamp is not None and words[num] != ' ': - begin = time_stamp[ts_nums[num]][0] - end = time_stamp[ts_nums[num]][1] - ts_lists.append([begin, end]) - begin = end - - if time_stamp is not None: - return word_lists, ts_lists - else: - return word_lists - - -def sentence_postprocess(words: List[Any], time_stamp: List[List] = None): - middle_lists = [] - word_lists = [] - word_item = '' - ts_lists = [] - - # wash words lists - for i in words: - word = '' - if isinstance(i, str): - word = i - else: - word = i.decode('utf-8') - - if word in ['', '', '']: - continue - else: - middle_lists.append(word) - - # all chinese characters - if isAllChinese(middle_lists): - for i, ch in enumerate(middle_lists): - word_lists.append(ch.replace(' ', '')) - if time_stamp is not None: - ts_lists = time_stamp - - # all alpha characters - elif isAllAlpha(middle_lists): - ts_flag = True - for i, ch in enumerate(middle_lists): - if ts_flag and time_stamp is not None: - begin = time_stamp[i][0] - end = time_stamp[i][1] - word = '' - if '@@' in ch: - word = ch.replace('@@', '') - word_item += word - if time_stamp is not None: - ts_flag = False - end = time_stamp[i][1] - else: - word_item += ch - word_lists.append(word_item) - word_lists.append(' ') - word_item = '' - if time_stamp is not None: - ts_flag = True - end = time_stamp[i][1] - ts_lists.append([begin, end]) - begin = end - - # mix characters - else: - alpha_blank = False - ts_flag = True - begin = -1 - end = -1 - for i, ch in enumerate(middle_lists): - if ts_flag and time_stamp is not None: - begin = time_stamp[i][0] - end = time_stamp[i][1] - word = '' - if isAllChinese(ch): - if alpha_blank is True: - word_lists.pop() - word_lists.append(ch) - alpha_blank = False - if time_stamp is not None: - ts_flag = True - ts_lists.append([begin, end]) - begin = end - elif '@@' in ch: - word = ch.replace('@@', '') - word_item += word - alpha_blank = False - if time_stamp is not None: - ts_flag = False - end = time_stamp[i][1] - elif isAllAlpha(ch): - word_item += ch - word_lists.append(word_item) - word_lists.append(' ') - word_item = '' - alpha_blank = True - if time_stamp is not None: - ts_flag = True - end = time_stamp[i][1] - ts_lists.append([begin, end]) - begin = end - else: - raise ValueError('invalid character: {}'.format(ch)) - - if time_stamp is not None: - word_lists, ts_lists = abbr_dispose(word_lists, ts_lists) - real_word_lists = [] - for ch in word_lists: - if ch != ' ': - real_word_lists.append(ch) - sentence = ' '.join(real_word_lists).strip() - return sentence, ts_lists, real_word_lists - else: - word_lists = abbr_dispose(word_lists) - real_word_lists = [] - for ch in word_lists: - if ch != ' ': - real_word_lists.append(ch) - sentence = ''.join(word_lists).strip() - return sentence, real_word_lists diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/timestamp_utils.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/timestamp_utils.py deleted file mode 100644 index 3a01812e8..000000000 --- a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/timestamp_utils.py +++ /dev/null @@ -1,59 +0,0 @@ -import numpy as np - - -def time_stamp_lfr6_onnx(us_cif_peak, char_list, begin_time=0.0, total_offset=-1.5): - if not len(char_list): - return [] - START_END_THRESHOLD = 5 - MAX_TOKEN_DURATION = 30 - TIME_RATE = 10.0 * 6 / 1000 / 3 # 3 times upsampled - cif_peak = us_cif_peak.reshape(-1) - num_frames = cif_peak.shape[-1] - if char_list[-1] == '': - char_list = char_list[:-1] - # char_list = [i for i in text] - timestamp_list = [] - new_char_list = [] - # for bicif model trained with large data, cif2 actually fires when a character starts - # so treat the frames between two peaks as the duration of the former token - fire_place = np.where(cif_peak>1.0-1e-4)[0] + total_offset # np format - num_peak = len(fire_place) - assert num_peak == len(char_list) + 1 # number of peaks is supposed to be number of tokens + 1 - # begin silence - if fire_place[0] > START_END_THRESHOLD: - # char_list.insert(0, '') - timestamp_list.append([0.0, fire_place[0]*TIME_RATE]) - new_char_list.append('') - # tokens timestamp - for i in range(len(fire_place)-1): - new_char_list.append(char_list[i]) - if i == len(fire_place)-2 or MAX_TOKEN_DURATION < 0 or fire_place[i+1] - fire_place[i] < MAX_TOKEN_DURATION: - timestamp_list.append([fire_place[i]*TIME_RATE, fire_place[i+1]*TIME_RATE]) - else: - # cut the duration to token and sil of the 0-weight frames last long - _split = fire_place[i] + MAX_TOKEN_DURATION - timestamp_list.append([fire_place[i]*TIME_RATE, _split*TIME_RATE]) - timestamp_list.append([_split*TIME_RATE, fire_place[i+1]*TIME_RATE]) - new_char_list.append('') - # tail token and end silence - if num_frames - fire_place[-1] > START_END_THRESHOLD: - _end = (num_frames + fire_place[-1]) / 2 - timestamp_list[-1][1] = _end*TIME_RATE - timestamp_list.append([_end*TIME_RATE, num_frames*TIME_RATE]) - new_char_list.append("") - else: - timestamp_list[-1][1] = num_frames*TIME_RATE - if begin_time: # add offset time in model with vad - for i in range(len(timestamp_list)): - timestamp_list[i][0] = timestamp_list[i][0] + begin_time / 1000.0 - timestamp_list[i][1] = timestamp_list[i][1] + begin_time / 1000.0 - assert len(new_char_list) == len(timestamp_list) - res_str = "" - for char, timestamp in zip(new_char_list, timestamp_list): - res_str += "{} {} {};".format(char, timestamp[0], timestamp[1]) - res = [] - for char, timestamp in zip(new_char_list, timestamp_list): - if char != '': - res.append([int(timestamp[0] * 1000), int(timestamp[1] * 1000)]) - return res_str, res - \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/utils.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/utils.py deleted file mode 100644 index 2edde112e..000000000 --- a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/utils/utils.py +++ /dev/null @@ -1,257 +0,0 @@ -# -*- encoding: utf-8 -*- - -import functools -import logging -import pickle -from pathlib import Path -from typing import Any, Dict, Iterable, List, NamedTuple, Set, Tuple, Union - -import numpy as np -import yaml -from onnxruntime import (GraphOptimizationLevel, InferenceSession, - SessionOptions, get_available_providers, get_device) -from typeguard import check_argument_types - -import warnings - -root_dir = Path(__file__).resolve().parent - -logger_initialized = {} - - -class TokenIDConverter(): - def __init__(self, token_list: Union[List, str], - ): - check_argument_types() - - # self.token_list = self.load_token(token_path) - self.token_list = token_list - self.unk_symbol = token_list[-1] - - # @staticmethod - # def load_token(file_path: Union[Path, str]) -> List: - # if not Path(file_path).exists(): - # raise TokenIDConverterError(f'The {file_path} does not exist.') - # - # with open(str(file_path), 'rb') as f: - # token_list = pickle.load(f) - # - # if len(token_list) != len(set(token_list)): - # raise TokenIDConverterError('The Token exists duplicated symbol.') - # return token_list - - def get_num_vocabulary_size(self) -> int: - return len(self.token_list) - - def ids2tokens(self, - integers: Union[np.ndarray, Iterable[int]]) -> List[str]: - if isinstance(integers, np.ndarray) and integers.ndim != 1: - raise TokenIDConverterError( - f"Must be 1 dim ndarray, but got {integers.ndim}") - return [self.token_list[i] for i in integers] - - def tokens2ids(self, tokens: Iterable[str]) -> List[int]: - token2id = {v: i for i, v in enumerate(self.token_list)} - if self.unk_symbol not in token2id: - raise TokenIDConverterError( - f"Unknown symbol '{self.unk_symbol}' doesn't exist in the token_list" - ) - unk_id = token2id[self.unk_symbol] - return [token2id.get(i, unk_id) for i in tokens] - - -class CharTokenizer(): - def __init__( - self, - symbol_value: Union[Path, str, Iterable[str]] = None, - space_symbol: str = "", - remove_non_linguistic_symbols: bool = False, - ): - check_argument_types() - - self.space_symbol = space_symbol - self.non_linguistic_symbols = self.load_symbols(symbol_value) - self.remove_non_linguistic_symbols = remove_non_linguistic_symbols - - @staticmethod - def load_symbols(value: Union[Path, str, Iterable[str]] = None) -> Set: - if value is None: - return set() - - if isinstance(value, Iterable[str]): - return set(value) - - file_path = Path(value) - if not file_path.exists(): - logging.warning("%s doesn't exist.", file_path) - return set() - - with file_path.open("r", encoding="utf-8") as f: - return set(line.rstrip() for line in f) - - def text2tokens(self, line: Union[str, list]) -> List[str]: - tokens = [] - while len(line) != 0: - for w in self.non_linguistic_symbols: - if line.startswith(w): - if not self.remove_non_linguistic_symbols: - tokens.append(line[: len(w)]) - line = line[len(w):] - break - else: - t = line[0] - if t == " ": - t = "" - tokens.append(t) - line = line[1:] - return tokens - - def tokens2text(self, tokens: Iterable[str]) -> str: - tokens = [t if t != self.space_symbol else " " for t in tokens] - return "".join(tokens) - - def __repr__(self): - return ( - f"{self.__class__.__name__}(" - f'space_symbol="{self.space_symbol}"' - f'non_linguistic_symbols="{self.non_linguistic_symbols}"' - f")" - ) - - - -class Hypothesis(NamedTuple): - """Hypothesis data type.""" - - yseq: np.ndarray - score: Union[float, np.ndarray] = 0 - scores: Dict[str, Union[float, np.ndarray]] = dict() - states: Dict[str, Any] = dict() - - def asdict(self) -> dict: - """Convert data to JSON-friendly dict.""" - return self._replace( - yseq=self.yseq.tolist(), - score=float(self.score), - scores={k: float(v) for k, v in self.scores.items()}, - )._asdict() - - -class TokenIDConverterError(Exception): - pass - - -class ONNXRuntimeError(Exception): - pass - - -class OrtInferSession(): - def __init__(self, model_file, device_id=-1, intra_op_num_threads=4): - device_id = str(device_id) - sess_opt = SessionOptions() - sess_opt.intra_op_num_threads = intra_op_num_threads - sess_opt.log_severity_level = 4 - sess_opt.enable_cpu_mem_arena = False - sess_opt.graph_optimization_level = GraphOptimizationLevel.ORT_ENABLE_ALL - - cuda_ep = 'CUDAExecutionProvider' - cuda_provider_options = { - "device_id": device_id, - "arena_extend_strategy": "kNextPowerOfTwo", - "cudnn_conv_algo_search": "EXHAUSTIVE", - "do_copy_in_default_stream": "true", - } - cpu_ep = 'CPUExecutionProvider' - cpu_provider_options = { - "arena_extend_strategy": "kSameAsRequested", - } - - EP_list = [] - if device_id != "-1" and get_device() == 'GPU' \ - and cuda_ep in get_available_providers(): - EP_list = [(cuda_ep, cuda_provider_options)] - EP_list.append((cpu_ep, cpu_provider_options)) - - self._verify_model(model_file) - self.session = InferenceSession(model_file, - sess_options=sess_opt, - providers=EP_list) - - if device_id != "-1" and cuda_ep not in self.session.get_providers(): - warnings.warn(f'{cuda_ep} is not avaiable for current env, the inference part is automatically shifted to be executed under {cpu_ep}.\n' - 'Please ensure the installed onnxruntime-gpu version matches your cuda and cudnn version, ' - 'you can check their relations from the offical web site: ' - 'https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html', - RuntimeWarning) - - def __call__(self, - input_content: List[Union[np.ndarray, np.ndarray]]) -> np.ndarray: - input_dict = dict(zip(self.get_input_names(), input_content)) - try: - return self.session.run(None, input_dict) - except Exception as e: - raise ONNXRuntimeError('ONNXRuntime inferece failed.') from e - - def get_input_names(self, ): - return [v.name for v in self.session.get_inputs()] - - def get_output_names(self,): - return [v.name for v in self.session.get_outputs()] - - def get_character_list(self, key: str = 'character'): - return self.meta_dict[key].splitlines() - - def have_key(self, key: str = 'character') -> bool: - self.meta_dict = self.session.get_modelmeta().custom_metadata_map - if key in self.meta_dict.keys(): - return True - return False - - @staticmethod - def _verify_model(model_path): - model_path = Path(model_path) - if not model_path.exists(): - raise FileNotFoundError(f'{model_path} does not exists.') - if not model_path.is_file(): - raise FileExistsError(f'{model_path} is not a file.') - - -def read_yaml(yaml_path: Union[str, Path]) -> Dict: - if not Path(yaml_path).exists(): - raise FileExistsError(f'The {yaml_path} does not exist.') - - with open(str(yaml_path), 'rb') as f: - data = yaml.load(f, Loader=yaml.Loader) - return data - - -@functools.lru_cache() -def get_logger(name='rapdi_paraformer'): - """Initialize and get a logger by name. - If the logger has not been initialized, this method will initialize the - logger by adding one or two handlers, otherwise the initialized logger will - be directly returned. During initialization, a StreamHandler will always be - added. - Args: - name (str): Logger name. - Returns: - logging.Logger: The expected logger. - """ - logger = logging.getLogger(name) - if name in logger_initialized: - return logger - - for logger_name in logger_initialized: - if name.startswith(logger_name): - return logger - - formatter = logging.Formatter( - '[%(asctime)s] %(name)s %(levelname)s: %(message)s', - datefmt="%Y/%m/%d %H:%M:%S") - - sh = logging.StreamHandler() - sh.setFormatter(formatter) - logger.addHandler(sh) - logger_initialized[name] = True - logger.propagate = False - return logger diff --git a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/vad_bin.py b/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/vad_bin.py deleted file mode 100644 index 58913bbd3..000000000 --- a/funasr/runtime/python/onnxruntime/build/lib/funasr_onnx/vad_bin.py +++ /dev/null @@ -1,166 +0,0 @@ -# -*- encoding: utf-8 -*- - -import os.path -from pathlib import Path -from typing import List, Union, Tuple - -import copy -import librosa -import numpy as np - -from .utils.utils import (CharTokenizer, Hypothesis, ONNXRuntimeError, - OrtInferSession, TokenIDConverter, get_logger, - read_yaml) -from .utils.postprocess_utils import sentence_postprocess -from .utils.frontend import WavFrontend -from .utils.timestamp_utils import time_stamp_lfr6_onnx -from .utils.e2e_vad import E2EVadModel - -logging = get_logger() - - -class Fsmn_vad(): - def __init__(self, model_dir: Union[str, Path] = None, - batch_size: int = 1, - device_id: Union[str, int] = "-1", - quantize: bool = False, - intra_op_num_threads: int = 4, - max_end_sil: int = 800, - ): - - if not Path(model_dir).exists(): - raise FileNotFoundError(f'{model_dir} does not exist.') - - model_file = os.path.join(model_dir, 'model.onnx') - if quantize: - model_file = os.path.join(model_dir, 'model_quant.onnx') - config_file = os.path.join(model_dir, 'vad.yaml') - cmvn_file = os.path.join(model_dir, 'vad.mvn') - config = read_yaml(config_file) - - self.frontend = WavFrontend( - cmvn_file=cmvn_file, - **config['frontend_conf'] - ) - self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) - self.batch_size = batch_size - self.vad_scorer = E2EVadModel(**config) - self.max_end_sil = max_end_sil - - def prepare_cache(self, in_cache: list = []): - if len(in_cache) > 0: - return in_cache - - for i in range(4): - cache = np.random.rand(1, 128, 19, 1).astype(np.float32) - in_cache.append(cache) - return in_cache - - - def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> List: - waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) - waveform_nums = len(waveform_list) - is_final = kwargs.get('kwargs', False) - - asr_res = [] - for beg_idx in range(0, waveform_nums, self.batch_size): - - end_idx = min(waveform_nums, beg_idx + self.batch_size) - waveform = waveform_list[beg_idx:end_idx] - feats, feats_len = self.extract_feat(waveform) - param_dict = kwargs.get('param_dict', dict()) - in_cache = param_dict.get('cache', list()) - in_cache = self.prepare_cache(in_cache) - try: - - scores, out_caches = self.infer(feats, *in_cache) - param_dict['cache'] = out_caches - segments = self.vad_scorer(scores, waveform, is_final=is_final, max_end_sil=self.max_end_sil) - - except ONNXRuntimeError: - # logging.warning(traceback.format_exc()) - logging.warning("input wav is silence or noise") - segments = '' - asr_res.append(segments) - # else: - # preds = self.decode(am_scores, valid_token_lens) - # - # asr_res.append({'preds': text_proc, 'timestamp': timestamp_proc, "raw_tokens": raw_tokens}) - - return asr_res - - def load_data(self, - wav_content: Union[str, np.ndarray, List[str]], fs: int = None) -> List: - def load_wav(path: str) -> np.ndarray: - waveform, _ = librosa.load(path, sr=fs) - return waveform - - if isinstance(wav_content, np.ndarray): - return [wav_content] - - if isinstance(wav_content, str): - return [load_wav(wav_content)] - - if isinstance(wav_content, list): - return [load_wav(path) for path in wav_content] - - raise TypeError( - f'The type of {wav_content} is not in [str, np.ndarray, list]') - - def extract_feat(self, - waveform_list: List[np.ndarray] - ) -> Tuple[np.ndarray, np.ndarray]: - feats, feats_len = [], [] - for waveform in waveform_list: - speech, _ = self.frontend.fbank(waveform) - feat, feat_len = self.frontend.lfr_cmvn(speech) - feats.append(feat) - feats_len.append(feat_len) - - feats = self.pad_feats(feats, np.max(feats_len)) - feats_len = np.array(feats_len).astype(np.int32) - return feats, feats_len - - @staticmethod - def pad_feats(feats: List[np.ndarray], max_feat_len: int) -> np.ndarray: - def pad_feat(feat: np.ndarray, cur_len: int) -> np.ndarray: - pad_width = ((0, max_feat_len - cur_len), (0, 0)) - return np.pad(feat, pad_width, 'constant', constant_values=0) - - feat_res = [pad_feat(feat, feat.shape[0]) for feat in feats] - feats = np.array(feat_res).astype(np.float32) - return feats - - def infer(self, feats: np.ndarray, - feats_len: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: - outputs = self.ort_infer([feats, feats_len]) - return outputs - - def decode(self, am_scores: np.ndarray, token_nums: int) -> List[str]: - return [self.decode_one(am_score, token_num) - for am_score, token_num in zip(am_scores, token_nums)] - - def decode_one(self, - am_score: np.ndarray, - valid_token_num: int) -> List[str]: - yseq = am_score.argmax(axis=-1) - score = am_score.max(axis=-1) - score = np.sum(score, axis=-1) - - # pad with mask tokens to ensure compatibility with sos/eos tokens - # asr_model.sos:1 asr_model.eos:2 - yseq = np.array([1] + yseq.tolist() + [2]) - hyp = Hypothesis(yseq=yseq, score=score) - - # remove sos/eos and get results - last_pos = -1 - token_int = hyp.yseq[1:last_pos].tolist() - - # remove blank symbol id, which is assumed to be 0 - token_int = list(filter(lambda x: x not in (0, 2), token_int)) - - # Change integer-ids to tokens - token = self.converter.ids2tokens(token_int) - token = token[:valid_token_num - self.pred_bias] - # texts = sentence_postprocess(token) - return token diff --git a/funasr/runtime/python/onnxruntime/dist/funasr_onnx-0.0.2-py3.8.egg b/funasr/runtime/python/onnxruntime/dist/funasr_onnx-0.0.2-py3.8.egg deleted file mode 100644 index b24107b404d6479b053e263ed23f22a0c30e8980..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47828 zcmagF1CTCTvn|@TZQHhO+qStHt8Lr1ZF{wC+qV0y{qDQx?EgGfd|B~DMP<~?tcaXr zjLcGy1_nU^004jhnCkLY3#%G~4FLfFh=v9LK>T}ER9u`^Qcg^sUP1bQ{b^RSwclVv z_@35xwD21NlcAD}tYMBb6ilfROk|O0aP>pfND|T3AxR)D=NPZuQB0~f!3_bMfWat` z*yeSA*^HbHIstvj`lhQ&)S5ELl^x@(9fnDX-k(;gUgTZFyQ$}Gk*~?MyhRmC-o|E^PDp+< zO{~{hfvhzdVU|-YFX1@}F54g(VC)iu!`8Rr-T=tDw*d-O1oLU`U_|DiD&GGRrLv-{ zj8^7XpZ93-kE4Q*J5)Iq;$%6wJk0P$y3*PhuoWSDB}Yenk+!vV(P%y_+n*jQMzI(P zUZS43(5gUPFHnPh2W8&0N7M~o!hXJ9L*nPr-v4SO_3>^Qa*6#U(y=Rs^qwcK=DC-v z)efwG*xx2k!)XoHqZV>=+hs>j8m1V)!VH-?Iq}tD8=ugMb1(&$OKR<~xX+7I;8+MQ z1K>o2OHo^4aL1j39;Af_f$(BpCzHgiRAXlDyJq6)W{l*m5B7SwX=M1p29%Wo@% zC!Ipg?&&OsP%q?m=Q>&YX!Wu$x9o25NN-EKGG4~^IsyesqzpXpDy#wxrJMOGr}X&@aGe;LLBNH^+>hRUPmu3ME#CJQ+`*h$Xq(Eo^ z8uH=9G~2b|p~yZhOhypBm`P(wk4^Thg!!2ZEIP_Jh~>JvSxEAM%8+?(OEG95O2iF~~yNkS{;H!AgtEmBqF6Nu8L{y3_&hioHT;BfjXMQC;J6~;{m z)!--iy!0#B^mvCwy83hGmZKm|Hi_c~fjHCPwwOUY5haA%->!eGC@L<5vOwZ}rVxBy z%<9OL0$N-`V%bKbThXJE$7A=K?~H3YB!3gyq!K_uQzKK+58{x&E)lmOXZo}3}+T)T!ZENFa{j{e3d5;UYhQ50An;4uo0 z4ejWcvCXw6jq$K?(+9Z)L_1d3=gavnERnV|ar`yUugJ5gq28%x77d@vm){jA+}+9m z#U3IPWw0~ChtSEc@BpTC za7-|h^_+Y+;C}{~2)4om{5P<_@d4oeJ3tdt2U9x}Q#)f1eH%+VYv+F=JpcmufA>cH zx3`_W3$3T61Ff^6nd$${%XDjnP2=y~T7T_7yqruOT`iqV|IcTkj@H8oFdzWTeti#% zhKJ?LUXrLC_^}1j_I@alit5m_{&>24PZI9nY%%fD~X8`1crzvoMmjUzF(Ei$gyv^&86e!T81?9c>J*g(jkM$l{iplzR?r1B*`C;GRNatVaFS7Gi+xE12PJsjf0LA-XFYn*ab})1@G_!ZI zHFeTAvb6iVf@Dqo*o`(s-u4EC> zBDX8(`z>x^NfgSl%t-t$Cf0|{z0U)fmlPU(T1aZ-`1|`d!szi7-~Q&|0QBTCR79qV zGA`N%EfkPkCm({XPWj{D4wb&0+Mh&e)P$cTog_jJi;67FHW8K7Z_GF=Ld%~*&^hYT8*iVQ z;Dlr-m}#TPfxRH}A_=dMpZU_4tpwrFjid;)qcvKJfKDh8x50ld3D2&9?y z3;q>{DAEqgA-76^d!8Xwrt&y41Ipl7r-*N6?LFyx7;!KZ?%@Ir?Ik2%LqcIp6%3mPhnAhuO<7{Fk!Q!u<3OEmXUNd>ZaJ83LotC$U9 z&&TX@qU|P!yfbUy`P-2tGOi{c=`X{#KBVOaN`Y2!qtIZMw5nhLd0MHHbM^tLftPW= z-Z=*p0QIC5jS;1&|F^ur|Lg}LcL}O9K}M&HrQYByx=-#!rD(IVjI=!JyAV5 zFw*_<5@t<+as^W2UO112%ID``$C+o!ihjM?+?QC4`ysG?-9wU9!n7c_CQjkYjc@YR z(5haBR0ZPD~F;I z^e19YT)UZDy7{Br3P!HZL-pA3iL<76i-Uu_N+}V|I>LLS84Hm_0n+#PBO~|hA>+W5 z5-M;Kz6>toapxZCC8ThwgCq$GNgX`kK{C+vP$nfn3Sc&T&Pj&hHVAKfOiGd<^yDXc zfFTupmfa;7R>9aYY&4#@Wm>h8PPU+jQYOFyelDH{zR`Uhjx-@YdGDOxE%hm7d+9^8>R z-jqbJKYT8urRdq+d+HSUR-*gD$c{YU2BVhQ71iXM!@Ga_6*H|a3@)oMBZ763=^0gV z2j3t?lNUJnf*5-Trlo651JDT6d`d+*wYzMOBX=W8i>BSew#iPJj>2YcZzXQF1E~#=*U`~KSz+)bx>LF1^#%QEU6+uI}0=9>4 zXIaNCvfP4dO~|t!zOc}Y^_E__1RHo!0$$An4j?vz z_<6c87MDGd0$0yr*RA`k?WQR8MPJX-HZyJ5a7)W-O64gZ$*rklftLUoi)GW52>@{d zxfWyCkzW_t1-@EkGH_UdOUK#huV@%hP*Z)&SCPYlZBOyj7aj%>Jf98KN3#O!Y3b7{ zmmmQX&aLEJ0dDxQLJL?}JQ5+l<~djac5TIK^5|1{9kW`(v7=>ey6#;paEMp7FV2!P zdxJ97CeH^nYRGem?TEPH-u?_KcE&jx379gj(QfCN70 zwrp0E9qZ=zLpg)(holBVpuv&H5qTN5jVQq~b6vjZ2e6PlNo=$IIRO_S` zUhuyLiZWKZyAeIwRBVCIW5&~80FHF~rHPpChlO4vXtDI-0d86F6#Bby&897!Xy->Q zur`|JJy)xu0z^a4p^} z-h>wTJqq!!`Onq$!2|2n zrTM*>Luf1$MU@Q+-0h(j{d{c|L1Sk2sGiv)>_(7-DwD^WY0ME^6EO) zs3Bl09N1~RxtT-_Z}rB#aqz;XCsgMig{XAkPA zYTtOe>Ge+Mq z6lTD!_nc);-8fqB=oo5c+ly4m%d>&%Wez25Y*AVLJi@%-#_ zuA!sNN4aN3ZYXTdb6~Toz3?|_BwGfWwZ9(mVx?@(G5z{(L;D{%)T8ddv@nh_KU~^zFxIIPdG8spvqz z)p^rhjO2EIzfKzhmbQrA-J$V`|KD&l-r8I3FMb37`seib_@7|X!PU;@asx6Kcf(L?{dn%c@n!6>AIqX1nb z*lBSnTQ{`<1`w?xn?0!#;n|jJ;OEP99FeF;w``|`O?ZaK@t3HNXv&K8fC#BI-7_d> zhS?j|%(3(KlM*J8>$b9U8oQJmNUc^6@nGXm5cu&Ta4-K)ymex{K4>ZtF@DVN2_PNP zkHlUu5>nQu*8SBdav@g4ug&~p@j>k?K3=!`V}T`bj~S4k?DD8jhx1}ihACGuAJ3of z*4)^E+n8=usR|o8z0VKHboTbr(Sbks3MH&3FNDF zmjW01=CkA~fSxRjymW?L2Q@JiWVtr6h;`=ASiVhIabzwQW$g7QWp5t3Esu?|8!5`6 zg`n82hI@ao*Vb11AL$FZ!zp!;*v2mf*tAphBur2Zd}%-{6&O`00%=qD_1$CMsT?6{U;EV zFvJI|_6{!d*Iz|(z+r8P4WI?w>*wYPfC6($&!mUCmT(wiSFW$mS#5>#q_6lWw!Bjp znVHU1kK-`kcOejGnf;T1>X} zH1<6b*#P2m0L&CjDi+_qW5X15#L!;Z#-FtK7N|UdSf8+Q^Gq=c$tbWpzVYZxq<_X8 zc(wvxrEvt%Apy@73?)h2Z$XZ?G#D@1G>}-BBy>Y7VP|Kwwe>6h*OrGyW=COjc0~FH=Ld7X*lKLXjx!e49kI zuGNr7rKZ-xT~(;pY1;tms2^$Cwz2gQYG{%n`F51*A!nzvPv@jUeAPk0Du)PVE+|`V zJk(CzPXQnh%^mch<^D*nVruC44U7Nd_hTu?3Im_=7d1;+^|Xryn&I)6{*JA8(VC>6 zc0^(d2MaLJ*H(km zWsD1JwQMR2D@%llnO=M`9;CB)p4iiPb|xcY}BBsgdmAhISWtl_DGup4C9fNRR> zbZl%um!4^5DUW@|Or=tZ(YBCX#Q4v1T&%pYj9rzuSb1QN@j!dpp05n{VKB*lyk5)% zu1Z-#pd_s4CSRtwY>l8$C8LFoEbGFn&+#xRdh!HB z)@*@n9CKnY;8V}IMd|=JAA3ZJ!7diiCzhdc`MdbZEbrikj$s ze>RnRiC~y|9a5PglyJM0nO99Tf$W~9J@RO4r2{8$k5~N|ZgV8xn(7Q9129JWky4q{ zBn@W<9_pOiL)9brxEy|~#@!t|J6z4yf=N2BRE)qGc`dDB-L-0~Bcq|~)RV!|NB~cQ znA-{ZjU_}`i0X7;Qq~imVXy52v%umQi?AIk4?SW! zlA+--=CjkW9KaT-VNm2Z zIy;sCdkVZnse-3+EpV=0mbK-&IVSCq4D0ec^l3|nPiXB6083LScVWsS68i+a3_4ll zhMK_%wt)9w2Lk7zlK~Dr*RJ{y)P09b6xf17O^ zJV11$T+!AC|IXJjrifILVXEY+;6KPAydeGy^j1jvw+K=sQ*!vHAo#i@4;sKJ4CHlC zfDu@vMm=aaaQeTI^w2hl7cMd>h?M zIaqqeH$~jI_Ynd{ogKe|6h=^dt1}G324|t$(e+3YraUrihd@T4)23`N8c$|UP>H+}s<3h%gY4gk5W{HEBKH^tWBl<@C-U(9D|+_ME`x6(0>|c5VHYvNbPZ`z=XT> zGHnB^57HCUr#A*y0=9FPl9@=nsN_3Ugl9nuaSeFgjf00qjf4we>d3v0?Bw0JKF|_f z+8%GG9sDs>J(V?B4!(XTUI%iv{0fMSe5J85V-RGN?QI~){0o0j-*Z<3Uwg7k6FNAE zUl#D(3G`+9!>$v+kWnvFpTX&BCCwit|e97+S#P?r&|37V&qe#5$ zp1*tk)nB9kAA7&PzJrIcp|ORjzW#q}QN|7)E*AE7w9FiTMImG8qoX4ufPe>^amqV^ zcSc|=10W-C0=g7HBuW^{C{CF6D8;BK0ybqhWjbdR=C~=G$-J@q1jI6t#5*)~0V72t zEgwZYE5x`zdumyk==v#X`=w(@!|DL35%S}c)IjNY!*EF;4{82g$uOcc0#Gua4Kh?y zlah1tQe!l9a}%;M)ATj7tSpPnv$8bftju%rvhpkP|1G!8#>UP}PSE@XzSL!1!-k2O zxk?#{R^SZ<7$Ua}|CZwZNrg*oXYJu%HLnf}0D$%XBNhKc8vD$?=7$ z)1X275TAb(OTi&TOf9%{$_gSQjW0RM)fywzO&KX6Z#)HbtLuYlF;CeP1yiM?*G++! zq2wHjAyTh95|&y?g6AVm#c^tZFQ24T9Scr2w&MzW(}VXLPPi~pv~cPD_WkO)>AwD& z9tbGhz0vo11!AY89`~i1`LulPiLqxA8$g}*jF`Bsd|vE1wu&;*lE1;OnPb;&v~NeF zUSF%B^_8}h{N}Ptzt}ugeTTjl>OEKch`cuHJzjp7zE7IuuBGf6$`*%mk5PRgGIvildxNa@=p1wU!sXwxwQB#a zW{`9~;8w&CZ+E;gn1#+^XtLCL8>rUR-;8_-puhnma^L2{8alI!3zF3H^78uF;^z*V zufz9N4bwP`w2yqeP~=8%4~32S>>&Zl^%D2#4Q*yg zAdm_yL;V%VAqh@oX9zPovGat#iO9I*XeHL@A$k4+u2)Ojb2zN%vg-LPAhiIjIm`y+ zfl)K=OFs+5%dmbPf|SN&KhEO=OwG{#6hCaM=V{W`Qbp^UoriOZEu}47$>h$bX8G*GZ+(WUkC*63@C8m808>I%FVrDDzenfrR zJ-&i>_GhwM-ivp^7uS~U74%U*#i>yPuovQu^c!;&+x#$<@P}q6$Xhg{|HSD8SO4`& z;}N)?R;KI%`Dwgar~N^Me!6Ed)?@bJSD5j}-NJHBY9GWs5Zx+F`&q!gD_vF4b-nB-KI%cR)R z;`sRHrb3Kn$uBo8F16c>qNA39>PbjUt1H+e^K*Tr-=w;umogJHdY)DJI#Vh|k?Ao_ zDVP$eZ9vgn8-wd)v4JMc5erkh2zl#RTuR~ECMAPf`3mJJLP~bSirkkrh!7zqJ3Ht6 zVLIfbi@$h*Ov^#TSKz$-J7;tH`U>S_$>`=8mx3d5H^!lY`* z0e9PkkThG}O8nadAgfns9|_%)JDYj8fw;qX$PR17e`q%^0yVm#H^geQWl$Inm5M5FTXR1nVSh2C0j~xDv671g-2aAtw+~` z6f9qc>tbgYZOIcO@+eiG98NSo>1zj?aE;h;qkZS{D{qu~n2l*-+}@ z(VIUZyHyGG@K_l{c`ge`Vpm70iCeGw&)ecKPd@yf>`Ab6ik%?g_17+LZ%DRbtqja$ z&A0(>(O=schjDV=2=GoLVVe`U51KxMd^<1c=;r;D!GMLEm&G|XOS2(k5ah_6(g5f( zL<%!y5)T}uv9#K+X!^mNJBqo*_e7$VbY`W0GU8ymF70{6VO=~B%xII-e!W#?(w7Ru z+aOw@vl0b-Ku>CmK)Vu>ce) zph7%=2NSTUfgSWO8(Y)9+m$e)*H8J3q6I7ct4E;I$4h-HJZ}%?Ws~UVv6);axNa^G zy>dCy3Z+5BtwD0_ehIG-VF{73TXak6CGg|fPDQ{gQJ^vP-jG(Oy)WUtcE@=k>jlnl z&I|6pb@$igwy`w?_U%YR(OwieYtjejw24*zPHr>aTU-$M$)l#2DpcsN&RJIAG|wT4 zBeImDd+18px$xLVhfLQet7-s?t8EO?GOm;{MO8CLZO%vH=NMH@3-rufTHtb5VJjVCJ{Kal+D&eRU1Y0drXSYIWu}QDsn99ZxTznH@T{(lg<(3c=|_xm z_+acG0Qs!0_bBLFS~G^GmCiN-fhG-2ja0GJ-|I%m&slr^FuW-%=OZ*ec8ofbW&|X7?}yWRY-E48mrC5xl|zQ4}%{(MF0|+5bv(bt(@y){XK@&9)I~B*tb+p zxjM-2G6mkv0JVDziKu{MvmmcaO^6bjeOw^YRSCEx0Ofz2H0BZ$^_o(J|EQ!c46gnF zU#obJB_<-R0A&Z5h7J;n>Hm9$ay7+yg`Odq%u0W2O~s@KmrYhHEzIM{ zB-gRf`LC%S$C$=3fG9=)6iWUXdk%lN)(sZQ zWkX5E2n*`~8u`E`GsN~N4277IlI0dgo&ssoKd1ItZV3Ch{ALuPJ6FIGxGh8jAm)-w zUl|V1=oC6$U_4~xUu)&Z3f)q8%-9?v=E_ew@6)dC=974^Ks^X}{6GF=Ytu@AP#c2U zb}3)Z6_UC_E4nO`edCPne9A#_ntPO@%(FlbZpBCBhlc4=g%qJx!loshp_LG`GBe5q zRV(Agy+@c?=S%DYYejnZD$t+KQ(>P}n{!=PV**Zo(Eh&fW@UXI(^V{vgF{#nXu1-@ zrJs-0+r4lZ+F&Nlwu&IyfHwz+B(@@*1Iex60_|KJ=!4O?>usCFXRn5BnLHiOhO9jloi&=|s) zg9*l*3#Xs4VaIshuGT-KZgCFzlE^n?z~;h){Uaa$$s27irQ;Ws1DBeU2{;&cGKGF} z!>MueU`0Bg+wB}LDye_!lS-N_Wt?q|3zoqyhXS(c8Cp$gLP{yY zm$(uZ-;b#iG}qjglK!Y9U@NX>RZ7~Sz#}B#TNN-&(}rlSL-t276{=>|ORcq`Mgc!z zwCs3H`P1Ln<%dBgLi-UlJXD8Y$|08Za4(C~a~v6TGrNFzaQV54XdvBTHMu;&@%-|( zU3>j0XJ=>cJ6b>gk)P%G_9b_LO>^us9gQ~w8il}J;Co(NHjJU45MkSyIr{7tlvbK2 zFG$*v`*Q9B9kJmeh{vP4Z3+EG3prNIoO)jKPw1lB1}|2h`T+2J;M1a*AbmSwz3f>x6-8Pl9Gu1iP$+??-sV`3G@H-A7V>f?k8KmzZC^cv#Z5|hIMkLaD?xExslWHXub9DT| z6b(ZX%Qn(}w_#?dc*A3509fpgSMk&4lNi)%AXR=N(^`ejSL^W~cUKmSUAo6y(xqOX z`I;D@exKin(x$aCuPu9|+pR$vSOAYK0Os8a+o30l0Tv;jKB$ zr!!$<7F^~g9O^0#RJEu^Tk~L9&7SQT9w6SJ$LJsjxK|# zl(Y=D%I+BOU}fsXH;^wW=x~G+Li^(7_)p&MDCB1nBfM4q-|6sO*wjoV<^dsd8=E9g z1M%FK6X;tU5q9%+(kJxYH{q>Dd+=VNwx+2qo;aQN$=Nt`a&H~>uV%UU3*OR8w5qjv zrs3nf(yIlY8!ngr%jQ0+i`R^d#OJ5w<#Wu~uDyHMf0}w?G-2o3{)j41ZnI^TaD0U! zqK!0NpBc$D)sN8R>~g+qTj9`ocBcKIf;K{|oP4`F!-%}zhcKfLFKmsbJ)TT$gqz|I zVf-@;IjOJkrAFw_5&r#W=-^bJXZ8o=oqJ#M5nu6?BXNUT5~H1+r=E(I;>!0k{X`89 zqm1_y;8B;lHl)Z@9faW|BlUTC3uJe+a-bKn&&PlC>LL@*!GAiKK&tp;gqVFDX3Hg8 zCg+ej-+duikkN6q^0ILeh7Nh9(xkdKU!pn6w%*oFUpl6g$Jp2Zk9U9TsyWRPdz3A+ zalp<=qLkTaYAG+*lZ-Z))^n3-fvRlvVFKp*0jN7ByYmRxYCFL^+gF+#fP;OAj4oE-f;b813Mm zB)JGY>~&qT-B@ZBXnPTl=Fs^oaN!-xNB6ty=l!xB(|eA${i?&ZqZCiUgqdoCQ>(0+ zq0D%S&DcMZc2T?Y?e&P@@iqosS;dlW?FA%C6P5KcTYfmIbaQE(=T(A8>}SJB-Q=gi z%QuNifV8V`evT-p& zwdmyRxgOW;N9MVACa?R@C!hVhI=;;JNadUFIFe0hA`^w$?N>s(1g^SBB%H-v9n%?I zV=&7qzM0xlHx72cZLw+1>bbCIk|p%$%mXPe37ow2?6+?2e)kc3t4r-x%lMP8vI@5_ zafm_X+19T#RhpCY9AUq=5V{oD`oZ=I#k%IoK1SIBbG%?K)ZiCu>RPKJC60qqjb`J0 zCPa8B)n#7@;|Ec9TkM|jU>%`V5W(RF{#`BUS#^A_`upIjbS{5v%v_koY@Ga@(+)9g zBLVx8TT)15fiyf8@mVa-8G0tK{71!tTs}tN%9^9nY~E{FF>+E+*zqi^yL2u|TGVrRZt8fBka` zT=F;Yww7q85U=z)+9&F^`q-tu{~VcWmCi)P%i14rrOe5)<|r34=|QWfs+g4YWz|am z_~U*(-VC^dQ{N zN^eBz>A+5q-bD8y<9{dCK<6x&uLK6O3C{e0C@)=&Aip7y_*Xh2Jy6lmYi^a~k(Ql_ z@ruKhFN{rhRG+Mb2?vIiZ(F@UJhR@ zqcj!z(&tl2*O+S+T;K`>_4;dwR1Tq=BHMwS=@su92G9>*KI&m~wdU5W15S}W{M+({ za%jGW%B$Xhhk`TZf;C{1H|au8$RJY>Rb8F)q2C6PHe-rkA+TfF7l1M(?a03gK&TQ2 z{lG;ZyY^;X&$h-igP%an8oAgtGF$!rs;O2{&gcB&nOz}jTU-Jt!&thywyY;#BHsYX zIX&IgcgtrxwbLE{)t_KqHG#=t~~=YTy&!Q4@&rt9W; zH-kG2lc2?$EzAg0XuQmb*%ZgBJ+qX@<-d$;-P$i_Tapbf5d#prNyL*3upP#@Au`-K zb^K)8j7a6y@a0MBG%>-fndEo?fa3;Rmd5QVtN?Y#mxQI%`hxEG^3^e<#%$U6m{oUC z+9uFL7+unJbm&TP5Rz}ulUE%9YP+;@>$oF+{fyRPr#AB<3oG?}Rw2O&>GDyiTmcEz zp@$Ym3R0#7vq$wl*cx5QVy)Cxeci#w#0a?e?CBIP7)SC?|L zTu?O-;L0C1jvKq>9yJp{5*?N}n7?81mh1IrP;m&?dwn-{v{n}Qv~Naqh3y|Gkz(;1 zEvAc73@;sLExCgDbm@=K~G6S);?RV4BF>5*j-p@zK_L&n#&Ofd=1RdUv= zWSW;)sS6BwW2?vnrJj|oMZ9MaXvKM2*=eI(33RKd$J=knO+{Bk1oJF>d^S9jYF{m* zM01+?PDR#mvNb@Z-s6hWdbkGtrg))$EGOoklr2{B(32HLdi#8{YgWi#KbJ52)fhw- zW_0+`?fO%)FVeRWbTnKZ@UlP-BXKi%k<=lV z>LYOgUbdI43opZ&oCp4!pZC()J$e(DYcbCd4i^gp9$ID>i7K>jf?rYM!gk!=r&hHC z`AmyKoU7ugefM`mtWf~WjTiyj^+0=vS$rk{FO_KiKt19EWWQx9XQL_ZIELLjBpBK= zCS?IZ_~DFY0@H4(f1hE3e691 zWM<%sbfv6;$iDr9p@4^&LKL}n+= zNz>n=Y`0g4NinG7jJsH^S1OR5!;uiHOX~O$4zU7)h#S|J*N=QV=zM#k&f7 z!tUxQG}40E=PJ|mn)wldeXn0A7?bOmxjA<28V~Vh2JL)sVx&heVSc}tz=77hJw7ht#sOv23^ z&8G~^1_ZwzdRmGgjr&qqfS{+xSf+<4lo>$4odcQNBk!HCBD-+SO%21xZD}iQ_WW4# zae7*Kg$aTC2hp&yv)RwZS(7IU#Z4LD-EM)0G4SLXhug!bOGl%%!+mk}q+2)*JX$6I zQ>D?z>YA;Si89xU>)IBXWcuv*k_hxLJohqg>Ff%h@U@&j7YTKdMV>R8Fm`M?(3rd# zdW%-OCITR398}jBqg9n;g~kw3sa~}?z!u1Bt!h8tj~0RAv(&*vfMSW!aqYW{U~EiG zG3C(PV+Q6{?4uFr9g&J|j7T;5S=!4hLbF|Phw+=XF69?4lAf?QF3v{%pV-!I`Eq=R zO3w16%;F9;sD~Y06)ByXJ4MP}MabdVYuoAIJ~M?YR&?J>7-@1OoDkpmQ2*hxpd(s1 z@JH^_$z{fW@QkGfYP_X4WW7iP)kRObV(JpXx@YBAjPt`3>tk_Jv^QI?y1S9r=318C zw&4Z%6&DP28hg~N z-SEN5$tE66Mfn8xRm-G2obN=z!ijMc7ww{)h}-8P#aj=#Rap0`%N9nm)2CQ4$R;Y1 z9*6V&VB2fnIf&1Y(~1&1-hB>eP!C>yqqs)4%X+TUg%&y>>*Pj;A7Y~Qd8wo|` zutu1Fd*lM-^?Z&dD|y7uQ+1Uvybe>UA4m+NwMBQ09h;*J-JYa;`~Ek3`{x^gzl`U< zM#ld6faBk@U#>2eHqQUu`E~I4_uv?Ni?}jHyEhYFNb9-m*Xw=Wz$|f!6=EnB}T}^3g{ChC}^LuQdT0x|4xuDRt zAFl|%zPs^~{GCf+?`^E`Rtj&am!e~ zc7uX+%EKRJzR4$xbW}^c-zsL#ZqC|azJ*wR1+bL`q>qY2n_F}Z_tmOeS;!>$Cr+E% zX0MhALUD`_nTa{odPbpEo`gC1OK7GSf=KNMz zYQ@7@&3;!!9E)GJiMB!pjc!GqfA~Fo7fehQClJwWFi9oVOk@+HhLFF?HG6P2YZk~MsnH~}kD?Bw zGHh%GdQTF-`&cdDAipd4vz7eq1^AITxu<&vPpfDIx$xBJ+Pk3Rn7alKsx*>fIR$$S z#yXapHk~B*K*(Xq#UXMCxGmLCnQ_f(J8@A=(!**3xi_UEm@0>*%#RG4CMQ(<{$Zi`Ar>ng54nBFQ*ec#+%d{>0)m=W`xPH2|VX4O*LEhg0WVvM{5mWNcFS?mM4P`UV`_5`Ud42uPwrY zK>=V++^HHp;zC*$>@Tt-Ak0=Zs#)RAMB5$@@DG3TkJSL__&@UAs){8#mZ9%U;;nTD zBzFbHz1*vh0=^avh_7fl?oaq!-qx_Z4Aozv{+P!00#nEb5<8u)@ZQdXcbOYUv5jV~ zIwghom*$%$!$J12uiaBZ6vY_kR;_+DiZe4U6QscLJeu;~#mC_QGW(1>xCdCL$DJr8v;}I#ZUo z;zNyiLh&QF}QlINY?%ieb!K4Zyd&KwW=p}vJBs( zP+h#_u!F7<`EfAh9*?H4$YKl27$CI#l42-i-zV9r*UF`xoj?Cct_eb zY*LKSP#$2Fyuo{Lq7H;fYIqr}rb7#vqlGZ#D-7hG^D^y+bJrNE44+w{MbKxg_3gv5 z!@<8jYlR8b2oO}NH`7K{a`3hrrX{n~-JwrQ%W5c+W}Xus0@1yjYcH(2052;m|e*CVNL(XZdt%!!-I1p z>`m2-TKIZ9$R7X67G6D_VZO7!k+nRyvij=Us;~h~{pgMqY~&4XWdLs8#n_gVQHS?> z1sJxqb5Q%5*97%mah3R+BA6L?$9n!e%c~}?n!7wx%H&e69=UU?&ML?u2Gsf>y#M4E zwO`vNivgU%wyDcO2y4L&fs=iMT7NB)oOn#A{hnKw)qwHM_v)w+VSQx*EXa6G7#i+@ zHksav_%x?J(QeLr2!KKJYlXU|XG$kyVwIc;T>>9}iEPHLawS*NxiWyGT&D83)O2xn zEf;?sU8B_$TD2wFogeUFU*&7`wF86mGl37vbcB=kMjYV}BnRcBU_(l}S$gGQ`-!Cf zz#6eYqLpJ9fl7aWI4_8*&5C&y5`mBfnyuS9uPn5eywe<{wa2|2SbGt8hR;yNBhCn_ zB2S|4-IoQ9jkpRDk;>m<7eu8(G|L%X#*vc3OvJJ;Kyu_XCXbSYu>YDC2}%ml6n0we zbmSN=!-NB~ctCIjmDG$MMLF;STmX+QIpF+>er&}_eDr0cv;6DSyGjZv3rzi zi_LpzTzZ@C>_uP|>9gjQ3`-GiijaV}N-lC#grf?P3R76>T@sq{*W7=Y+%b=WD>86NMLmmFq$Pmy7%9YYQ z*_qj3@+ci<(2UJ^D(bm5xrYZ|X2lx8ygW&LXvXR9h=PS*NuHFSWw3_RS#?u!j)6syDfJ(!} z+lR`&qdzzcjsP9Yf-y#FVQw4Gyc`&6>l-CDZT1vt8eZoG#|UakZVC6$;u}>OLsuqS zA`dLN_795{{g9(g{lZd>HcMHte-z(IDr3zoviT_3|L7d#wR1lfMVCaj z1C)avypO9Jep5)zEL0WW-$mSD;=VHB zjbz*mtW^?~Cy}Z(K>mft&9lSwPD@WJ$t}m>b}ZcROaC5YW_cf_`&gJkI%h80cz|RT z`cOM+!{6V=hz(Wd1XWGd8~tR1oxP1yhj=Y1Q{v#=@=mHtJmxs7(T-?{b^gBKQCEvo`b>b$&KQR^DvCI{)r3J30|b;4DlyLl|J| zSN(;^)aDN#$!N?JV_@W=;+Jn#Q{af4EIMU)tYZKm@Ojb1W!Vj<^;MA5bS5gs%a|&p zzq2#dmQ8#)3qL*ekAW?uRGED_%AmHr@uOCqXm%Zl3(t6|5a@XRws0I~?J`P(NU(bV zwlLucYmTi463)zP6gU%8@emzZQIkxnv6>U}Gl9p@?25@=X;cs#k6utZktKL!V+55$O#WE_KMi~JBt1>QSHlgU4 zmc98YOzHVBoBi;WpO@pPmZrD)JDqrVkUZCGWQbKY6yU0e0+jREIW`J5f>ZE(CaGia z1|$Ez-gVoB@oosi_Lq@CM#>+*F0H;c-CbL)MTUzf+IX!n#yJ##PMw#CjZAs&j6ah; z-b->js;wN6;A>@neDJea#@B3=HlpjK+LO0py6Bae=&(voAShR48;SBRz z%F$AIZo&@XyA>STk{Th3vEp{Ra(kBQ9RQd zan3}rIe5a02h+f~7gkJ@+w)+?gcX<_SWVdfzF)omdcB;FnOhFI^QYve`@Wy*^L^fu z|9-CG+kjOT$n5=s><9D6J9{l8I#a>>oUq&U(~Zcb!a5==3A}i`RBGU?Oq3-I>}i+H zY_{DYN}@9V*f7B^gc8DUq^eO>dmDf_8fJ;=n2{gp8M@szjsPlrZfJcHgV-&``GImb zN1fCU0xn@<7!=lJS*&nTtn)RdlX*<$+K7d7Bs4O@UegdJP#n+rTmQ)dUm&}WEsHo2 zV{Bhuv+&HvRjWE4ok4x=J!QktTuI64jV4@Hl1?Mvj)mLtFn$mtctxeM6r#apxRp~O z?&kT_t!F3V;x`Mmcx)HG3F^uVvOpWlU7U-Ud?vc9CC+W@6(JG=C@s}J!Sy01`P9!siTU(NHVr;@s60Z2OAD3>Hz_9+ zD20XYZJeo%_7eB+lKh|>LvpYHhzkQChcz0;OaKOU4J7EqP>B7^-(2MHZOV3;)3eT+ zTJ7u&NStyJ(kV)ikfYFpdHg*T(IVYFOW}*v+nloy%ql{;E-s1VMZS11j4m$=KxOY@ z792P%NIOiHjdWf00TR!KNuAsTIE9%7sbXfd_3i{fwjbbhq;H?8O6c9;_mJD21Wo0u zoxv6xN?g`00rtGEbCz*; zBT!7r%y)R9=S*)#@mhcnVi^acQIq5!!orq%^#p@jM!wNk5~{<^j!nqZv>yO(v6>jC zxPOm2Ak0Y}d%!W2c|<*uK(3o#NQw7}S>BbTPyYt;ZRrxVM6y0TJpO@iRFptF+v3E} z%sO5>=qiqPs<;mk6xkk!?AE9g zEc?j|-}Cn){LKp27{>QraNCyY-p>KuX6E`9d*ZlJ@V)`(W?Duw2-PPTpw;RBEy2CK zAvy`HZ81#JmyzXIgTTs3Emf((0)-2;uu}EoQU|S;o8A2iz6vsaa7F7oMCf2Z$0sltJ_MCZV+zzWm{$;DSg2Ost#0mq6#0xBm|W zLw~>ykLG$}<5>?0M9jw`-YpQwvR0zC2YiUYBBC)Og(9Vp%`x)lHV5+}sPRf?e(U)| zf&XeDJ7X8-Gz#Bu6m1on%vGsW>Q=qFVf_GgW|~PEts3biz6w`aR26IL2S^`x0FQd_ z_GI_Wc=#tD3A=$`({h+D4bi*I>g`7lg~``~D*K(nun_lGwMuE$j5fe7@?u;Q8la1^sc(yrJ zT6S1Nf@=o<1VJ>lA%GLx)YP{gR&8^m)X_zhjNk?SZ}zmAB;7^kH%ab|Y(>qIB|@zk z44=BXy1n!_$U!|;LQ{1~(?%v)Lrek69rvUSIS*j;mj9KD= zWYRiyFlg3JnpUi4qt{4stAKWIcTW#4GCyTbY981AyC4eaupOV-LU+7p+`|P35i^2V zu5?dZSnTP|2xsm`KGP<)Gf#pZL}Uulo6E0(u0Dt+bY&wqX`uU_(Z{ z27@i0!q0)E5b~eSMb&C$!)Okr5C!rbn52ZUdfb(j31cm{A!{_6^~+V8_k)Lp`c%KA zj)jNbwiqEdK@v1cBI+q_(xCqJ#wagqBkhy=RgirE86a2ma6RCYI$}p+QLn3aXe;V^ zA5E1P3>zF}mg4nhu-05Q33t$(D9nP_1yv>7*d|~W8C>jD{Xb9iZ$PAi*hZ?1=#!~}P@Vh9+UA*|hP|2;S`n_ZhQQr>=d@iTPBIR)-{ zS~`jZDtNrLn8nkU5S>B6Y9z{?f^2IVj&@dT0Ae%{x*6zJnDuGct|$WrfTz^JGh_~8 zPY^(O7AsC3#OBXH-!ZvBdRKU@Daj%Qx+3TotOm4xzwM8Q;?2*Wp{IW8xnNNK?M(1u zKakeMM%wM3x#a;SBj{b&w?O5R>`MPpDrGVxJ3|N5CW{zrUt}?xB4HA)U67*X5M;{1 z3*><&zs*gkP`Q!*5g&(!GVW};fmL8bzGlPH3cB8`5hrY~XSelF613QbE}JWoD6oM$ zYuL^jS$I=7rBJ=qEVaXt%Arp?cE2EwtLwbq7ra{bm;SrrY$)m(rZzoERY#@Vp%to- z+8(3Jtj!ZQP|p@1#p*7!@!3Xl!x?Bgb$5N#tE%Q{r*Vr$qF@)nIc0Y)I{Vi%dXS^> zC_UoWWP}rgtp;K?_Q!Jn8qL}B7Bjp9YjHL?j;-S}JM_cA;NJIDU8bzV^HEpfT~_l_ zc`eAN>i~iwhsqRc>hp^b80?mqODc6X2=bV+!;qix6>B#CZ!%f_gYD5LrZAbg6w7poY68osc{?@E1c+2AF&HgiKUx>DyO3=bCPH+Cm{x${O@+B4LrU`CVjFCf;9xX5p>X+XGhb zOg!{t7jD%a`(^XUs9>dx|BGm&6MElDyK zVjmhA`;?M5B^7nqgU$bP?nM?De>ZK-cd(Bz0n4^MUsS(`uPohe)!Jv(Nqez~-Yc&i zC(l)-%uN|ITNmlwSm{CV?rzsub$C^~mFE^2B_s6AFt=zOJ|mz*%F@*OAr}kffmxN7 zPhuA#>fzwwEOFz}=jTJWN?|I}JBkBs8n&~TdnbZUs<3mG+4b}DSnhwM-H7=Qwcli^ zA|Fa?i)G*(&=`T}8y9Gt3||tXHujj`52g$g&PE=N4~ZzF3YVhFm=L?vXYKRoD~o7l zod7u65l$b#L^m8^(2d)K+iq_x8Hnq z`?$opsy<&knCDh9cZ4{7H-$2<&U~PPB7OyqXLfPne`*MwVz-3aS8_;$BX2pSw-fHP zZW{?MqmI-*`cW)!QbGx~y$*b0GQtDMU~pQSh@z&Dby`E$BuWf=x_o{zs=7nRi-DKP zZdHb3tz$X0O>!pv?{>-m7`W}H>|WV|0szGRo3@etuQM|TduJC1CwpU4XJ`HYtU&&= z@YszzWI^z|)!*;pD@NR2(Dm4>yuMH)kB>)?@0YHiVt2)Asz6?+^oBn0H@{kI!R7uI zTwjFT54#!7z?}7Y^t`eRI}=vzbLgbSVwqHW(pX^CJ7oOGHAN0y2b&io$Ubc@ZYMNi z!kE+5y$6d3Rc>XKH*MLT{AXh2Ug}=D<6o%5Tc5)C77%3?<7d`t7%-?O-gi{ienfTkB)sv<7eiFVc;T0x9hEccb(^61a`*efE>@sc_X@_^UXyc${7>(6Q zOg+D+TT^yChLreZJR-50I}xh^`2h|{m;{~hf+SVbN2V~$SVd*C)J(P1$`~yUZe&` zsY9`?m*KW0O3Yh<*5X3P;{-#P;;qq031QWt+#&-*X(Z-Qxwc8;=TjM5rum?*N+<} zvaHGrrHq2go2u;2u?)`VEh_0i({DqRjQM-Aet2QK8FQhIV6j+gh7Cu*$M5R?u z+nH!qUgptfM7Uyh-3uRC#k+7MI8`Z@V@mjTqR{vN9TOnr+L2Djn{%gkpx_VB?34VV zA?j|dP1Kpi;8KT7U5$P^du*5C2zGrKd_&$eHYH8<<2o-A*c$+Kty`Q1(KaP8q$3}l z20c>R(A><^Iu<*UmV4^=D&w)}Or|q=y9!cxiQLVrb0$wJsUJk^bU0KPgm5IFmKQO3zdzJnDAnnu)t}J-OJMH%8ZQ8OJt!%SaNbW3{q^&}+2NLOIe#N00iX zndV`BW062f5VIW?F!pTi{DWKS34~QOsf}=_^?9-m2`CS()R9?t=R22|mt-mzYNxLf zy0nW_DF~KM`m9Rj5yYQyOjEd>;po2C*1?wZo|<5+6OyFgWYBSu`ug-??hl&aZB}P{ zwz6gRlC~qej4E!IVCCq-`K9+3*r8`co5APZ@n8NZU9ckC2_feZOQ~6EMwW7F!l4%9 ziO<&v7!qnB@aDuUiujj4t-FV+7t(C32ST@xTRI|=U#w|;_4=d){6;d;KK`rkC#;L8fC<*LT z=KI}^c)j=O$ng=lU>%Yoecz6$v!&6lZVg?yl33Fj(C1y!JJ)KLQ`M1Qmy#RR{mw?5 z4{`lugFHjmC&U=06J=C|rwg5!91R@h_L}cMQ@rn~kCAamlYV2Q+iQv3!u~%gGWl3H z4#db=xfkFzBn@%^I?%o6HOf_ZlTk(-2BzWPPg1 zFI7&RpU_S5pGEa3f_mkP6{=VnT9~Y0hUX ze8ELMktgX^f^e9~*K1e!xWqrj`=YN3#a|Z3UV{Nvne-I>RG1fEMAhA61(pQ+Er(S@f zL?&LdD@)RflaK(Ss&D8HVF6tq!@+}$=;FWUnTGF$FpO;`dQ~s3IATrd(>rfd7NE*y z`!ckJ&EYEU*eh36mOXf#JMxt%Rp$BkQgB=-$iraG8@z@w8ji9d)C#JaXi(3Kpia0R z8bp&IY!_>YfFp>W(;H~#|88FGm{zZSnjn9$PI@9QPD>N@a5vv0H+vhYkXEWp@y^0b zNz!dMcCsq|HH#ef)e(nnhik;ew^f$1QiI`6;0y;>+L-eba6kBQzF7Bm9N`wIL1%*S z9u$!==G~DpPH#q2qfnc^@#jZu`C0!H5Lg(dc#CO=ksaFBf>hUIK#a*VD3#J8-F1LS z6YpVGjU7348ONF@AsZZlqUp?Zf0s`gvCl8^HrFi06tP2?B7M^Jh^?ezjUKn!B|A>r zz`(UjFlSyKCZEwB(j6GWA}|ELeOlCRbU_3CG_66PSLUp2%_S}jMz@a_xmu~@_3_Kz zZ6uELV&vNHbDEfIX6amZm)UeS!1J_`eR3Vkbz)35f}P(kKU>gR;7Pp9Xj)lNXXX@N zbgp33*!+R9#k8(rr2Fv8rt(EVBJ5xna-I@WlPuY`p`}m|Uk~C5zFueffS~7$cExBm z(ODmV|IZgEH!Z{?<%YWX?#KuFg_gi-6r68|i+uMhNmhp`J zZVvihBXfUFV&G|9L4$Rv#u zZ8e7;$t2XOR=fR0n=@fRisuhgxAev~$?~$RAhg6fB}UWLh(EX3y{PlPntPKu0U;N! zr!*5yp=H%uXyBu$Q;}Zh{cRPT)U%D^nr2-8T&V)D>Jw4ZV?Zm3^g6&5)(yy%IC&#N zf1VEHC_twckzW?{K?6k|WMtR1&H`>d5{W(q2_JQlAT@18jM9P!t%K1o6Q|0}$hX4p z2=|DJHd-*s8iy$yb zGz*b)$@E1)G~{Q=$;k~TQ;c5UgU0!~eUr?&2?bAU*@+d25t{<${`9L7#gxeqTg3n2 zG!wgqHpVJBb`s%7CH%rpntGPY{8opNzuN>35Z97)#sh7>%9M2>sT;sHYQfjdXYz(v zGVGdMgqu6KGO}Fr81YlY;vv|A>X|6Af^9Ll7qAjTvjWZkhymhpdtvUls1_ROBQevo z+v*v4_PK{}s9brf5;IODmx7^jA~mJk_HgP$ys9X8_*BqERF9d_|U8NGu@M+>>^K zM2UYPlxfw7h5htird z@V&8|6H+aS*6P=2#qcb>5IlQq#-4P|mvjwO*a>#FN~wv}V?+70-F)n_%5HvflSs3X zBaH@73|lX)hAD=~A0%gk5$tKjqQbdN!n`H`1ZEs#-i?_fI5x8pHcmIB8?3{ zJ+^w5ZjS>x2S%-?qt(ny^QxRj;OeZ`&^Nt5^GTKYPM58~8oWh~oC*;%Toz=$3wLyp z;YEA$N{mV9Hs`cQqEtzxX}h?-^WogGak33ogMM+vU$x*u9W~`LyYfc#DaTTJ_!Eqf zaLm0H#n0y;C(LifoT0u&w{lw}3YG-CmN>GdELJp3oe52Y#0S1qr z3_S>nOQ+e;s1NXl@|loU>Zq>EhKSwU311YHg*%d@Mt+1Ml?DcazoCL%s!fZM- zT~KS#cD#!?7ca^<+KCa{I3SpRyzhKYZ>Hb=SURvJ@gn?VLVm=DpyGscbB>Xh4`=Le z9!~|7q$#J@KVH{(CEz_+h!*?lXzWCtpujsd#GX2st}y~nm#*$8a@MS8h?#wG(Mrb4cfmN}ZIwueZI*V#C5K$sG*K;QOAKE%HZN}So3q(+NOSq1 z4jUk0C`$hI0wVC&B^umS3~w7G4t)WE9Ed{gJ(motuE^OoZTe`i&b*Ye!=bvhM0_Dt z5!JlYOI%kRyLp35|HvY6U{W>jBDQppd$B=tEq=bxNpoCZ{@swqnb)>Z4oF^7dMRIV z2qOWjzgIdVa0fHvdjS=rw#YHLmJQ)NPa6ske|hk$-c_sYxyLgGR3@GZ8$Cd?;Jb(O z<0m?g1Tz2Xyo&DA=< zF(PoTsguoG%RR6Y(9xzFpJCj=H)C+1gS7_)%tX(8kaFhvFoKbD*!&-sU}Vf`ZLt^6J!?Mr4i<5`jRrP`6XIE{kVAEk>MSz2&>C?6} z^*ACL#LHGJ7}?JF=kD;C8Om`!*X)IVFxbCZ(k{5Ggy%rvmkv=t42U`_&wzV5Vb~ho z)2~8)(YLQ1i$lO@jWiGgyui7&&I&YX`(TK5e*XUN?>9d~{B<83if|AkfZfyp?1!e6 z_aXyr;iaaOG{LRULfk4^9p64OlmM@;Io&k3xFAc?&yru7P$i;E`z7z5uG;cE)sYZL z+LveXd)ZCUVO}?GvEZy&CXW+!ZF%HZae{1OYa1U3w0h~#FHy=-=xu|!v0b5USI`P+ zV72o%E-K!7eT$HH55r*rM1M9%L-fK5|MG&3eheu>pN?lYzpCrAnpo=j`*3>|^?8zd zw;f}&>Db0pPPB~-w!WDbEj6iIVQzUms3>}k%0W~V7|z!w;}0qw+@VJPZYN`X{9~* zQC1og%9c?oG>Yw-Uly7eI5v{j15DQ&g>@$v=V-}lstw7k&np8ah43)*vgT$=jL%cD zqc2By1g7xBDyp@jQ98yYF7&z4(Q86gHLM-Ex=^ zrr4#pQ`zDV7v`N}vF#LT3JX7Pi&U7tu zbr#|cZv-v!h(V}54J-MhfU<|4x;K!D?nV%zjJ?gm;;Vr>JTBGiJ^tpKA)DZ8BEg#a z$KquHZ=P;t2p_tdAJD2z4+9WW+z%YiE9ugmQ)J^K?5Bph7FNh*POg}Tsj6a2R*i?emLH1(G z0*^ztd-4%7`9u-OMgp$QYjcPHjDVkoKHn7)e1mP=#?>JF7WpbHok^{wWnb(i1>x<_ ze0QOd5qJ?MKbUTDMN1S&xSDjHI0sm;2uz0pQ_pN5lS}(kdC+#xRuQAjid~)+wGWB> zQ$ANX5=1RlBv=tCME#X51JOoWDIBlgxw5EIbqhBJUm)#N4cOM=O{$ z3g&!09tr!9NL7KnOCzW*ybA15bOCt87|@%I$(=R#npXyUx|~olIb5T8GTMtr`Xw%SNb(KDQ>~tL7;@4`#guBeiJwX&EB7FF*IUt((`W%B!Mn^&xj^ojDt}1W>1G zwc%!y%|Vd#l7*G=?OLZcK@-P~Wub2fEUD%{B9(Pn%gx1A-~UsQ{|Atbm2aQY^Is*N z`S1IG1KIv7oACdaP)7_aMM?0VfI5)>0MwcN2cT{MCNtWP8Z%O05aJ-sSIN>$F49j) z%FHoB&(6rn&oQL7pddHU&PcH~Bf`(htJ6=*QqRay(Z^3uFT~GM(oC+w&&S71&rH&% zTTk$q7A|Y+D|sPXg*)CcPY~_DM{>hVDI44X~OkdfvRl!+7=$QGg?{bAN58C5T;T zckI@?lJM|yL9^Nj;UEoxGf_kvk3=CzMj7`MR3b@=69xR9;VXeu#zUnd@l`?;Q0-?p z+q#w)EO5qB6sHw;&3Cn@V(T9cMZe9J1&*`;KRBczd}(;r&3SP5#HNQ#q^ zX9T!L7UJcUxu7kBObK!dT;LWoQ-fUWv@1>sQ_PLyv6#p5n*q*&u*04>O2!CvO_1Yr z^IZ@ZNZBCIB`tUdmYS4Gd{9qS6MV1vsx~VudS3(<9lwV=Zp1d9S{A2KSJmY;(bLpI zJqg3Q)4`0SR5K)1Lnc&HC{<%BmO&Qte}nm79CC78FKgcod`)XCGt?XQRY4=5OezXL zE3~F7-T!J$vs<;qPIub+%&b_Qc0Ic(G}gm?Z`mu#3$%NW+G*V+UgxsuI*+hs@MBj2 z>SLF7G`g+mT&JgA-bSTtn=@Krvvm{QJ7Pd2%YkF7D#MLKMd)DaLHu>?Ok!Kj09c#+G{&@nWB+39t z#LgTamXk6I|=S zCY}g<6f{}Q{{RNY6)-?ZM z-T4{3us*v5cdxwfLYMJ2)bYZ(`9q)iQYKl{_^HeywmHr^R^iAfQFN*@gTS}txD{6S zVEmg$+wk563zI^Oog8dgAV(dTTK$rXtcuk}FNn)8HiLTFv!mbM&XZZ7tVc7Sd515*X$NP{cZX+=lPo8tc3k?4A-cP9cNM2;drw@UGnS|W zb%K#(ETJ20!d5~r>d1}6ouC7G;xfJ)Z-SnHGw=vjLXM~d8uEhN4{Cx}BHr?gCGj(! zhRBmRQD4M~Kau@fEgV8KQ*E(kxmLXN5W56a*UvO>T1f^M7m{1A(j*jx`&{RrdyNmU ztIzs_Dl($gX495`-eS`=jkx##9uBU(Zq%`UZKor^N=J3?cyj8;;`{WLdUDD>EN{)b zSH5V?lq-480nFiCf)gUQz~Ywb>m-wh*1v){cu3gN3JiH+5cAQ5W9IDOqUvW zUJ`?Iw2c8&OxT$av7HcmU(c`uYo-##CrByI>~NbR(1vh_%q^s2U<_=>IzxV1SA0I<5(|%`mEnB8agW_&T?On%`eLpnl8M9E6nWHQhO5 z(}r{R9zTP%*#*i*+DesF}>9>n$f1$sGf=r*`Q44^pgJK@J%hm(&=x zJ&Uf#l;&D+AMOQ|<)oz_F@#l&4r zHFfu20@D;qTHz0|(tp#n0c##;H0C4dw3<)(CyTJ+(MR{_5ia2lwbDKu$O*=xH=1LwwH}i zOvKbb-_QAv=>Pnt+$^CQE!zYcdh_5|+Y)pU#}Gd6BLnWQpop-CY82rZCkiCTT5e4C zN6QC%RNJOpNds-pVt5QWnHd%o%=zYYQ}R-<888;Dl~H%}U_%U7+Oh@rAjlDddMxQ; z)WDqUQ&I*jO=fTl#_^;5wQAE@oNv+l;15x0t}qD#et;LULj0y|pcRf*tgtm^Yc$1C zS8z6(}w@za-)AuKW=5mtWBUf*2WA>L&cJ8n}e48hKp$u4>6^QhUx2fZ2KTFV2E4Z7I2p zz}}0h*Z>fGe+H{ND0Z+aU}{oGGKn&GRwQPPYhVWYXy$R|d6Xx{51j{a?F)c;1U*R$ zlT?(*xJ17TKDnpsXE86FQwvPj#T|TNFQykC{VIJ0qD`=z(azopcyLY9dt*2B2D1?Nj?xj`M5hZ66}u$CREcd5(uqJTHSxstd?>P*!ec$=vR4Kv3ctB z!hXOHt6;I&xqk$qkqZkHoLez5^h1R(sA6f)s@IaMVFTIH6L92~jl`M`2cZ4%zx3i6 zPKeBn2H1*FVU!Ush)-|tgW-R*RsuPLI7Wu;VNdC1_mOMoisTI7OzF>-5~_U)!3m~a zolFY1X2gjbMB@mnqu_x{bKI)MujG5UDJCY=kC-me`geMRLpf%FK7kMBg!54c#B*7PbU`aGi zaPWtL1Rdau1wo|n2b-uP9@u05g(boGi*H#sCf~9dly#!-plDZS>LohGkFWou+w+Sb z;Om1=A#_~SNzRgfQ4;Y9a&8&J6d1R=nwe5<;Ny?`!Go_A?Jh~U@+L5Rq)36kUOoVw zeSqMvG#oO${U?>7d0$fQq^?zKEm@Xh*$yVR=5!-G&fp?#)@+ z@xG(6)C7eOaC{qbs%I#E5pyd+!$^8PPWqNs@|b1;_*7E$TL~7g8E~o&sQG=@=il*~ z=l>j%v?s!ka_s^&i3e$`V$*hT@{DQ?>|?gZXn;WTq=1A2f71_EyF38FCk)lp+Yo4u ztW|bX`ySES<8`%Cd6Rv+(s5n|mm8V{RxJkt*TPYS$oOW(IId&OwnZ?z8uGIfGI-m< zoqsaDD1;g}F;G!QB0n?+P+|#ip%=jGkW;O*SO(WYThS#P9{G=MU+TsKjP2<{-FKOX0@_Q>%+H9Vs#n z^;B&j)gSA6$D`x)+Gd(YwhLuoK!LJ{3gwSn4cQRhEosBs$>+0}DyHkK2et0Pxz&?` zLg}I(a!u%B<^Q>ZZ@Vi$jPdMEjKS(Sy2AmB_L*yUsMEAfwtEW3899noZ}}h)wL0j~ zV6YahhKf5U6A;YRtoOr0Kp1)CVg+yxcCaQG9tQQ{cs22ZNW|7p%sGx|ojH>s1d%aa9jiE`XvU_EaM(CHczIdqS2@v=YHqghWMqk7LgWYulPqQ(Pweebnj` z2Et3y4t}TSF1e`e`E+$&56*Y<=X-b5(V(^gcGE}Kk?*qNt1SJ-bnA+Zl2nEf7DWxN zCWJi99`4cl{jLI)7xI+AgDzNJo05JR>+-d}mXG#P6GNN`L%fm|u1TKRbm|UC!7Z!I z9&<%!n38r+Mc)@Y7jec)HwX>ysZ_j4$}r{(Ql&l(C=8S_x}hc@bH?7Zn?)L^Z$WZ2gcW6;O*~f0Uy${%4$`;vQK#7cpkP=xzIwNugd58VP@ETncD_1)ArFjt6E0=_m z5r|VzwXXwz3n`19r6vS5pjg)0)o^G>1H!OU5X=)ffT}J3?^^fazjjQt!g{sdY*ACg zuC_MQXW(BO3hfc#L1N}W@9+i5*Fr_*nDd99iy zEH=iWPWkvx1G6=;^FbA8We`*+%1T<>BR~VPyCxV|ga(A!`Ez7QqDo>f`hZCZ;$tuT z;1(eOUJMx)Nm zXs7xpxS0|Ybw&hpnPrsEIkOI*;0}a!v|MgClVHVq#Hk5frP`gU)Oe@TFHxd2{6=$j zjkLj(A8x@i=;^EgtQ`8MVT`?(Te)T$>D?9Wl`*nw~NIW%l(eFw)|1@1;%5``99 zq^`>Ts!V+^kmZ?G9 zWN`~c-Vtp+m1r+O_0OqZq7^uMaN+ueG|4xAUZahBATfm=fV@ATj&XqFb(B!)EbX5P zQpl1`z~H4m2BVM!RaDjLyF9}8M?{+`u)q1zq1L{_SxOe@)FEdEb^-1xJ%#V*6y*Xv zpPURy--}2CJhY?TBH5lELIihQA@#=sq`+>sl(}LOm@$zX3vbiGyQl^?km~4Qv<7xS z>7Q^;#(^b0D9cIr3n^tp+LE?q88TMfvNCdD1y)e#ywsppp7%N<2wIUZ>uDtlunX5w zWkh8D1REi@=!2qPzH|rlVotuEX}OF*qyID;aX2|=(ggy*GRdIiqv;0-J7MCrqIU)u zR6AwIkLB~B!F^k-fx&MQGoMrgNW*Ph)B;KA_GqEG0AQ?NtL8AM{Zyz|1Tp-#Rq?BW zb?{5{wCNv>TYDY=?Erx_?+-3vVA^(1V;?tf_RZsedl2hTt}=4FkpqMM#&wa+qnY9J ziJP4q6WmifbCm6V!E9k-Z(NyBntk)O;>O{tCHp4(DZqIWi37Fz740Ae3Ldn1)|byX z3HR`XvbgJcu>6(USGO%5=V9@gChFk9@$yftz>EWI850dpKj{Oz@yX*0Ap?Q{#cUo! z-&kHu5TAa#S)?4zi#3KlmEqu8lgkX~`BM5<*&l|ph*EHX(H;Er2Eeb7iw5i;vVxlM{FSlN^dX$t``KP^?=Tf zl$qw3mNGA7y}<_Fee6f2Z^xf%7O~CBid};n$a!KuHrwa~Y{!A60MSocQKZqg>hNt8ZGssp>Liqp{E>w+tzme$9oIS`@vM}6^V z?e`r97XC`K8{mEU%W%e(xi@mOB~7@4`YUrfSudaMK0rGTJ_;{56>ht1q28&=`*Ah5 za&>z@-o%(N^W$&MjBifyfkG~A&ZgR=2rS$mjlxc5*8PpF}iChxkhf4#s z;RatJFF7*_VUX`A)mcRO7S+X$7KiCh!bhr1)esXbcSj~R68sE|qSB-^i+~#lxYvRb zB7-8Mb}q3h7qbfC4gyPSUT0!-my^G0PShd}IM%E5g-s$KtaT?9F!O^qp)a0&WgKV^{BC|^lDj05 zilY^r*)*BY-0A&zHC^J~t3#LLElD?VMBmWU!Kwk2tCSb^L5G)nxI1uq8*4DQ3;6E% z%ThQSj8b1v9jwlM7=0!U^Mgk^ym6pMTwpLb)WwsHKpGK*MJphw}`-fC}PIw*6698d{ z=?-LXJS<5e4y%AjrH(?M7WfN){g{TEe%ACrZB>3OUzgvm&IK7T#(Xdy^6p@$B%5Qy z8@Uljuw{mB833Z66NYjLmlpchws=>?k9XINO>f|K>Kt)7N;xrFXJKQFbwd{pcC69= zy62vF6KUlXHFG5|cCqp>iHn9oXo^sZfRsDYn88D&Aq_8MJ)4TJ!F&{e3m&_Nqzbk7 zxX?%6tj|Z#<7UHy#{OVHj#$md9q9`)p=FRnMk_=I6UjdnCDn zj*9o$X$m+YK<2JT4m60rM+t{iEb$Rc>6BT6n# zxaIy|jlBbOBwP3Y9dnY2ZQHi(Oq_`(ww+9D+qP|+6Wf~D6J!2;*YjS@y}##q&swz_ ztLwARIkmgGtM)$My&c@M3?+2r8C~O=wacrq(h}6{Rcoy}N)7fYMp<{dNgsS>-6!Ou zsfKb>VK3sa0CJ3I31 z)=`*HH(HU#PQ^>?Hv0>2$&7VK1;bd~wIUALHP?r_=~a|U+NtDoZ<5`Kac6fTE8GQ1 ztBUw;KZJq8jFt#xwLQ38SmDom-J_T;7n~hq7A|njL={oAIHWr;%S=1U4;a5c&XhnE)=D?b`d&9XZpJmF~IoQ{qc6l@DuMgUv>A=0BpRbUo-%0qv z&rKF?l^-bhC$obNIll>%PoXTxzn+&fg)$@F_KqczY(jYXB)wV_W2mst%G_&d=!S{K zj0_!MFR5ihye@v^YZsAFT=|ht5wQk|?~??dq?l8RtAOy60`+Msn=P}?T6C=zJpc#{ zcDAnrcZjF;b4o3GsI}%i2y}_tXT;|E$vjQI!gPJ0D<{xn(F0mK89d{3A+#ctnxd8i z9Knd*J6Mr(ypV_-a3ja)^LTw$6Su~EH+Spu_E?J(YvaWA^};es!4-_ebM4^vJrzzK z4_2M~tlhM-%EN^W*Eu)veHm-tj7iFo)(pq*`x_Z&lb6V@I@=V{cAhRc=I9SIMbM_Y zy2&L4>1yfHbufWKYhj?x&QAT&&6}5mFlIXmVG`~dn#%xX@eYA7ts4{;m-0fG#5**x zY1vmm=`MN)539Ml)$n_?Kn*0KXaO)fkbM6p%zL;6$59kfbI-wpD& z!~$S&xkr&1YI#BpfYmJf3)Y1VVLwY!_bhk_YRe=R!p+19EOilI5*1A;=RP67hNg{$Io~nBQUqrxEtAiV41Ic`sRF` zzCWid{Q0i>kSn6-fU8CS1{`)-)c(}H>Ri^u*#yEJgJI0F%BcsHCf=*;tC_0bP$q22aYj%svTr zuZCBKps`D;8&aGvva%X-%BHc|s4O%;1O$48LO&@KDTkr57BVg16Ooz5fNfXLk@|Q7 zZ}S5$A>mHQ`Bkp+V=mi}`G8Z1O@lac37FvsNpg`1XspzWDXij}fRVZ;$-!+K2HVDU zmJatk!S`rPQDvHen;6+ZY-iy1hTb{E+m5Lr47mj+d!h#VT_R9CVTCzR=!o0l4HD8J zEfB}z-++X%E4)GtMk{KITqC0-!#lkQ^fu0-lZ-iTil9y=x*HcJnH)hEHFW$4NfOW0 zbQD@qUmP0KQL^@q;~iT#85^vd9b|(u>bN=EshWy#nJXyV;OQ_}0kgvmURnsaBbN3U zNo9O;IJ<#bu?rn?O~ucb1&YeBHaiAMph((>ML<-qssKB^XEkC>8Nq7{ zoH+)iM$OE{Hvfe_sD-T&uuy@<<%H)dE%5xoeFOL7tL1TJ<_#vU6ZAFc(I&>;(!7*c z3GHXM4@qc2`Zypi35yBtXh%40q5`-2#T!xpEpN@wj7%`zv%EoO$GAtd<)I`3Wgg%a^Pc;a=>QkK2Ttf1MKE{VCiQ_@ znTVU7s;!UIIpVl^99jg@hT&s2zRC!5<7u%4;XN%zbtZyI#h3jd3G1sVXU!r(i!C4V zo58!_tx`snm~rvBYRItdKaRvCk2|e2zvmg(k0UOmvG!nSI+tB_rKthy=Cc@p-vDDR z_j*0DTEMglfj$&Z2SCvXh0G!B23T18+chs`i; zhMaOORp~rPO~AO%v0CS0wl-GzbmSd6z++~2M8K6FEsC>lOC}B%WoZq*OY$q{;1e#X zAaskyWGm|kn80V|7yB48Z(Ep@$jNW9-(kArWTExYPt5ezN|tjqnCoq}M;m2Hl673$ z#JDvGdrt;^@TA`N8)1OIzXJEymJ`;6K^qWD%Exa4s=#0C6WB+Ww5IqehnU+dO+`Ai zWu*OiZ;YN_$7Vf9rV;r2*0%R|d^T60?`@tRovM>jK1FDYUxO^Bk$zl&;1?0K;0Yk% zKmugZJ~z?hE1s?qTx>_FNL6tE(0E)r+J4L*Q!Xed+$H0sR+`46hL^M5){s^MADix) ze^(fWD>qE@64D{Z_ltf_%rnE5EHvW!PB6CK4bED-4n!o-<4<6iH{$>r<)r(CmmPgH z)&6OVvjRHK*PAw6792zlp7(^XltR>s5+fMO>IL?y#ahda1E+ ziX)0*typ%6-KWURWWm;gym}o4PKIZV8hhYxu}L9Xp?M%wcH7z=6Hjn5o%EERkaOtO z(7HoQZ!+4FBDTjs8yh`cuDEjP-fG(Bs*IIE0R=KejMRO4$w>_h1S-BH#Y=GYLc=KL zeP+ZC8%%^;U*VU-o0O)3;3b#0oDx!9mq#0CR6o^{R7A>lFd)ZjL9Rx9gHb;=h>L<% zmcN0Jb9Atb5wARVL&(lg?Whpy!3hAuE#c0$H%@>>SiKu%@@K=4#3#Dl0nRh49;d`< zi88<%{1CRns3pZ1y`D1Mz&qq0sm76oPTH|GXg_&tefM^8Bs+NPI_u{Lp_RN;^T14@ zI&A4c&8SvdAd;S(k}}dfLh(HAqEN^hajj2{v|`+sxTnOT+@3u>0@_I6ZwpBo1(D81 zpyl@X(*}?fd1G>>+lcO{)U8{dJUc7T9g(H3Z95w-E-I67Bw%UFkFkT>R2C-sU$16UDfV#)gv4Eo5bx$H zY6-Et*A-4|NCZLiQcPiO&8doCQQU4`F3yPVCCE!}buwok+0M^P} z_QxtNP#)Zp=p5-*PLV_qP*uOVm6+Y9VBT6D4dA*fEl31n#&PO$+_n+SDer${NkjzU z^KHHv3l24e_Y;-^#Y=T01hTV&@-+|H89SPV0>Kz`6z>Ov&Itg6f&cJ5`9ohebl0bk zJ;L8ZoPW!^`+@?zT5Q)teWO}_{W84C89m6JohW(5|@W6X_~Q*AUXLB2I|NGw5Y zb3B)y3)YZb{6I>rwBwgroXtY0lpTR8h=vp-RBN*}N5{+R{lc$>X<5~ZRq;3on@Q5^LFq~Ii^}~TO~e3IL|f`W`D5T0 zZPGWL5%Q##vM$dArJ*n|)3~wL1is76+P(g0W?ufa1YV~131!-)@~+Lq&dEA4+-6d| z1Z|5B?Ijx2AN#}s@GK+m0YmEwJku%#)2Qg*OC(pkh9-Wdxo0l!P|&?BL*jVv2iG35 z%A=ra<5BsS7W$y~AVk!vOi3qKx%_G5CZLQ*-Y;maroBarWXRuAJWX zVm)c=fG`ik8de@xV0;RXf!7)@A7)^rpq&;5xq*7OqQ)@J5bt9l_mb@DcK>{G&h%vv zD5DYslctrhyL~`8RFjLUBrUZ<-3rs7aZ=89ODh+2sIrD&ybrl1_$7V8&$G&kKay}FGH%e%N=@x&c*%h) zpFQdeL`Lg8s7f=V*>5*H9`-u6_`K9BpV9c_`Mp$lU%QOFtTG>$U9ERR@$(gv400DJ z+!ytZ+(usV2%qZ+U!41mRktawLnuqnrVRVx&IBNKK?<~9{R(j^TE_arT*jm%pVTQ! z>*r80CFTq&#-antN={6i`_;m4Ps{E@%9+i9sf`BtYEJ0SW4BJ7V#Bp&-n6^cBIDQJ z8BXBy$Y~DG+pXBp@gu>rnkk1)QHZ5!p%tT#Ne4Y>bV51m6F3rdl#tFBB$%j$67YLm zGES{SKF3a)y*@7(ilOJ>A7=eRLxm1H(ykNafqZt`d5}wc)F&bj94E+V zP}?Iki0OAI*Nn$5?B3{lW0;mXvqwb=knD>Oj~`NiO@CzB1IIUaO(i*4=vGw@vEL7B zdJ4xoxUiitxi~R=&^jk@(d^epDAf)SGz4)oIE86jFE&jh)C+f5oD}t_Xmt3XCELtd z?YO6<&mmeQ=G$c%6Fc!Gad(_r#y-tWW^VVE@pwgm)K64_@wKPyV_hp(eowv<^2Y{GizTgy3aSELoEMj$S|)TEL?< zMYbibJ!0E`Ky8;>b<9JxR*Ly5(8~E`EVU9f$<%vYlaSL`QO@vWZc3`0zjiyCmmE=S zH+})PXp533?{?xW^==Ol#|$Ua)bM^_6I^yOJAo4s|1+MK3J-IDE$7=8lb}+_Or&Fx zaPhqcLmtZWni;NK-r%b{%)aS&=0$u1GLLWY!L&7AAGw~#6AJM%{TzDMH89b2<#=73 zE0t@upV`EkYU^!RFFsJ`ugyefg=U|gPBp{-eCNCiJ1ut5+1ZFuQ9^e=Xqeq0d)LKV zVPrF&;o%XHNm&_P)-h$RG&J&`oe1YaEo~o+?E|?4AMZfFhreQUYZagD;+1Qv>M)CQ ziLT4PKsz=&RhS)D)4{GIwAU|IupO&$PuW7Ccx2*oBj5u<8JhF#UiKF}7p#FS%&xE|{Z zr)Uwwd{yg*$9vR&*YXdzRh!|4%(w;}^*pKdy9Pdd<$v{oP2>FWn$FF~;<7IV_g`3d=H&Ji7+n^X^Mu zI=;SZL{^B%1}1MO>4#HnbD=m4b0!-fc4kNi^wIEv6|yp0CU&9O?bTf+L+cZwOk6LI z1{Te}b`bQw4#=dQeF^5b;eo88(?Q+qiBTSH8Q*=!s|&sM@0yW(baV;t!edMx!QzL= z$&R*y*kcf+-)LT~%ySF6)68?^ajA;w6rNGPB-1fHRdmb?BQc$)-|}#Md>@+^URFwtUKXABHsDa;m#;Te~_*ix;*egkv=d7GmB`(Zrnint7#-M5RQEOxv2%B=6 z2uIlJ>I+Y-lpx(wZ zJ5isN9k5X)s&YgB&>HWaJYl7hYeNhRheFK@LC)l+VvsST!YSLiQ367Ob%d2Fy3f54 z`8oZGt=lwkZ`(xkV@@;c^qZ=|2tji82LV^MGpFD?W`{G)W0yo2e?JCTZ`#zG`gAa_ zH>`7Rc{tRfA}H2^Hq<7rHn}`$5~_5+@>a~IJe|D=cuoD=k^n1LI|S04mu-?Ix=FM& z2bG_fp-JxmftLw2ga;(<=aNy9YP~L>)rl`zK->n}pR5{+4LFcGMq#TGf|^4Wy*R&I z43-A5aEHB#K`!X%>$5$qOh^$Da>MI_4KGF*oz#YRaAh1@eCLmIdM@9G2gY-gFzx0HlFnP+AD;jDAK+KXh`;{>aQZZEvo3;>jmf`w&sVonabsFm zQsz#Om{tgd3Y}*~j|&>k}>BNsZ8ly+7da;W!nBi+uZuyPM?|V{nbrAQ=qJmMbDdJMh78)h~$d#dm#KO@Va^- z$zlo8DM%!Pio9J0a_bb73c7+N6pc%n6SM_TWXUrsFyBSaa)6mA;7$%_nM-ffceEIV z?+O`^AD_P9poUF^| zbf@%BNdCwJtbHYS;cgbh;)I0vj|Wa@J|+TD=2+nSaEhKK=!nSkXU5= zcGc$d?eyl|l#xz6sK!USXE;(`k_T5!BsEE_CV&J54|M$;0c$A8$SU z{x&w8*%Z9=H4YPdFV;t-noY3;G7aP}c(-x0T!(n!uY%KtmOV%taKiLCp+oCAd!%Ez z_+;)LJmKIB=kG{xHUQ=`;{v+p{{?6$h zvAp$jnp@VKlaoPntIO6GmX;ef%ZZSu(HV=2EY+4U^EVvvjR(rF7 zs2^VB(p$Zv*GNOZ5jW!-^M(6945UKko-}qGb$w9O}@MA(7NTy)bzk; z$>8)SaMi>X^NwE98BWep>obv5xyi7bwsIy{N&d<9F0qNjR>Osd76C$skdvsfttRO^ zVG#bw4!Yalh63CZ9FRKUH_l+}O9jZ9DSyPu_@oV7s!i*OTwm_fDKqP=f5YEVL4uV) z;yDBpM>=Shg*;9~6I#29KdnDiK5o4?yRPv5%jrS^592@@83>4t90-W%f1WP>qdf8V zlSHTHhQpewb2mDWAQBO!RFcuS9nL_gfrEwgHQCBVizd6ZCHW>-gT=~Ik2PALD?O$IQd!}r>y_4SdX$XJH!M~YaJy5*{(klpI=+6_Y-V~rE$`2}S zm%5ru*RzCql_gsoCvA>tJ*oCmHH4|w8mWs}^re|5Rw$KQCYsfW4W=bbm8k?4f0u16 zsytp~JXNhN&>g}YTs1%w(VT4l0~>H7wDnopV!Z-=Ez|t?E6seizk8spkeAY8cCdS*te_W+ z@zgETS6>TRqo^yM^mifs;pGY9Nv`41%_pxuxjGH!UBrz-)yNqmrSumpPPOV8_4Yjx zi$9m2A)T#iydZ)m|^nQ=}JjwA-x$Ayu*x}gm$w}s<0{E)A!P{e?` zoCg}uQgV<2-v|;xS%vNCcX$|LNI2nh?WRYp6Jwm1Lr=b@55}D1<(HV#IWg8VA*J1k zZfX1Td*c$w*Lc|N%`aHkh)pYawl%pQ6iW-fRudK$=Y)*V5K>9pU)0BdDuu zlzx2!^ENLl=@YQ~5Vm~ui0$06p$XlQhGaqbkXE1l8PAnr)!}CMsSO6B3-KXAE3{OW=cg&=Zd-F)rdSAv}!-GK5lw_pOnK$rC#qa zWNJ2goC&rmBywO4nEMefE4;Zl1=@w;UIy1ZiJCk z6-_6il=>Uaerher0j+6cbw@z>P8nCl)@L03gz>&hX+${mBp$sabR7+h35ZIz_369* z5#YN>)Nb6vIpKk{kK$p`_&V2Z2)-qhqo32f+P-}V!CqxThRK^y3dE9UDSVm#DL{my zF26aXo0arg2Nwpy{Jn(U+LbYTyZwf7Wh5M(II&dy1y%i)UZrCw+UOGJ35@Os@#?1n z4Ge9wF7NHSdB^sHvpQ`z4Sp5pqo`TXr$^7Dx*DK!FZiu&alGltegkV7MuxVyWBw6G zIsTJbBs@=i3}#2Ox2KI0udn?dn>?9oKYnk$s6UiJyPD=lPkklSt~!1;ZWx**+;db% zx|2bYH;z*mm0Smvd`-Ea7OsOz(g62DHYt?Q1aGB4FgZ5O*Jk4=ObIqP<|eT}+krqT z9TbI6gR6puAoktmK--E>Cs%T*om!O!eTPr-0*5Y<7-tlQi)o)lmSsJ{7{#6_^>>5J zKk|odIA5+gSMA}#>q@qK5gfjb!}INb)rpIq^ z1J#ig^c{wlRzNvAlMyXPdXuE2_Yhmx$ttkElppCiuRR{Pw!_6$QwBdwjF*#7oXrF+ zRx=zH8^Z7^P6n-D0Dj}tiKDBf`Mle6rxK^ZGdwmOG$I08K!ONOKsukE+VZ-ZMoePU z3ljd!{#00V&hnUu&B{tgN@Q-sr4uX+$F0Eb6XY2l?C*ouzW_JG+MEu40EJ%AVXZkH5p4*wLfEa&vw5Xf=qMQAFz3 zlrWiC8BA|W*B%QJS8^IAC*8p9;e+k<0G&li4v-LT>Mt4^fq&pb;q6Jn23PDAri+Jy zN25es$8IS-oY~Cn3}f-U07_UMKaal>E4VzsoWDnvigVlUgMkW2xi7s*ZOs=E83SX0 zqur$Q!k)iHy@A}Pdt!|kI^-7Vi}mc}X2r*2p@3yX-OwgC*LfvO@dxqg^Z~vUs|aiO zM#7pJ>H-Qg%I5}Kb?VF1f+2#OBWR7S4ZS+4?%%#0Txo1)miY3jucckfRGycs(&`#B z+K2G`PlVp#OG zkZR<1V~|IWRk06MXC*xbg~b=3J$`Jgo{hz4WTFpur0wpKq=odGf#1+5I6y3BADiN zQ`r;Q3T`VsQh!0E@_oCRpkYVqqM04;?2yNP3UDy~!TS6Tgfd`yxW|QKno1D0*mkUW zY9P&z_AaJfEw(A;kPHM>G&8tDp#V7;phaQwmgphba{S2cZ!z;W-|`}egZ!m|ou3H~ zLLVv=6_e}etR$$8cIig9X9c^DI%)1I4iqS-*Uuu;R7#x>ovn08EmOk+wMo zC_wC<;N)*dOrbkvcpdTS+$n|+NR0(t14asuXMx^nwE1HHN&m~w{Q8a7`aN1Yz-6(; z(Sud{4yQhpHQHxa16N(%kxJhbpJucU+8*wQj|CatOy=J4f~<{ng?D0besQISUIY5O z&9w|n4lx)(84cppg4?9Vxkz8LsgYb2Mb$ON0pHCIOp|0?!Hl0XAi{+TP?ktbLQmk+Z&m49!l0i;H3Kf!bTI7NM&=LB#7Mfb z<*s=(r9$5L_<{T!WNX$G!*0K{5uy#P-w>hT$NK|)_3kMHCr@aI@V%?K@@VOss;RLX z2Z7=YsfIKsX=mVwBRa&vKtEg$Y3>PJ8QuM|AS(atEXhwaN0Byt{PRpMGKRg3sHSDo z528dFe{U$h>HN_fVe}J%`T5jy+rGqI%m6{4(CI1QZkO)_dbv0y4AvVz5hVj#NfCzL zjVxFBNEYHkPO5oA&?jLbWl=G>yWtakFZ^a|_&4Ra?>I1@7Jt%v3@)tQJ#)Kod+#^3 ze26WMSPNq#7(?B z#Sn+qKXV(|XRmbNnzMXB?gm=NSS5vVuqd~%kK9NMu>sccox>PmZ*+|)4!Vn_m>gl> zGqM3!;0OX!Y4+3MCAcb!PV`o!nB1b)kE* z>0nR{x;woko!CVF!R4Hcs(aK4lGB6EF}@<$t}wEPvQC`GYAO1U2m8sIEP)D|AiL(D za~!(#alf9wH^*baXm@sdUVhhF`a+9@ z8?{L*=^)gBgYN?NfQ$Xz$Ei*+o4BjZYLbm>;^7=ZooVe_6_k5F%CthmfyQU(o=pUa zZBMp%Pn08+o7UocBlm;&5#kYI<;f6JK!sX=`>SAZFrvxlgI!W-ueWb+X$FZxEi&8^ zv-PUA*)ivAG7N_~b7aOAj2aefp!#fFAQV9gJzE_Fw=mX8*Q{$mceYV;p?+xYlEa`L z*j25v%Sdq7Fu_nDlmX_$!Lkz<-Lx1kJQo4ed%?K9bELieu6PlB0fiDjj&{x4zxaI? z?`771f>5V);65UdoM6PwPK`5LNr7E9W5d}>C^ghxMlIUbG02}Ng3`2a&RG#rnxySA z2-4%DPq$E(I=W!`)H7@98Lj)F7Dc0gVxxw>S%B}W(sb?z`YGr zZPbNC!04iGDl6Oq`~Yj%KGSrrkGGKqoZIumHGiX4tHVf|5)s)oHF3@5a%JgM?d#>D z)5}Sv*5as&%-}feg#Gm>kpU9>(0*`>e|xgkdR}Yf(}yEkNo72;G|NQBa4v}<^w|;e z2v*N<>QY7zZnG*(q%N%>?Xe*ri3U4j-^xIlyb2zj$jP-uyQjFytzl4bCu{AwG z7Ni5V4~oTx;)MlTp^gxW+VXEe?z+I(_4!@ZUBa8BU2#F%Qk@>|Bbf!y5GRX{KIfLX zN$8Jv#H#Jq+*tz_G9yDYDilO3;6ga))%=#WqZ-Esbsa;(ZqcJCj(K_k?+g4IDJT;Dc<>Rj{}^upQ1Y7f4Y*s zV}$GC^yYr$=*dkUR>XD8`-vJ(2H`(^Ls5mNb6!pul)BaEq-`l+d&ojW$3WiM+Z_GA zA4FD@?5!pUZ|p`+N+*PKNMc|UU^NxVaE)TzdY~JX>y8C|Y(qOa#}7Z_<&3id84T*b z-zV-J7stY%Jsa_(ikc}?5?Q$}T5nh+XB34J^1*uEKJG9SF_{c=$_UrOQRr%bO_c$y zPD@V9GoX0dM>qG;aN-HZ;u~3W?n}00(qsYZ>QSz13 zNj>)`B6phG6#LECE?PS(=Z`sR&F)8j=mw${dP0w?+dwEKs1V82eEqu*%uxea zvv?LM{HR|A_r=*i0D1cE9YwbbY0a~`Q zv-2TMA(>`MdW<9W=n3+g=!};D?#wvX=hJBoN)-S%!d~Y?Eq4C|Jg83gQBKtG^Ou`! zBp+I;iK+APfYYofNiCVWi-UMa>Ny4FB&hMblFvO(7HzkLF%HOAp{K6a$ALD`2iBcYxYgT>Jn9upL#)V?#k5R=*QV#1)Wn4fkhVS(wmWpAaL5K zUGT6njo;DZVv}W2l-_IL9Pg)MCeC!8M_=8@EeS}<120o=5A7B-q3Dr3$s5NNz)euO z$`y{~CcE1?wlp(LzQ1B8A8vK{|`<08)|EquWWiJG7HzN3>uWXi@O?xC9!!#fn zDXTrsJ0i4c%@$eGTjG0!y_ZL5wou%<1tW8|9lTK4y|aNR|8P)f?vbpR9Mf*BuR0Qn ztdy8RnSJ9PRFWs|Sq#|Km<^9}20G7&4q12+11PjXa6~Yuguq8FoTzLQqa@Yu`6AH; zE5y*Xg^98g%`9}H@{T|_!W>kbOy3-b0|6=2x2R8wyB!T%qr>^Pv(TBUQc{AY6lt2J zDoJI^DwSm_I{X^*$6rAeq=7+DKv4j<;h=zi6Ke{o7=;T00|JT!$iX54-u=B5`2O}x zR9u`^Qcg^sUP1alPk!BjqIGv?2Z1l7vzLTNr2=CAV8iu z5E@|ny%qQ-{uSY9W2qrROJynkvqjSUoJL&>9K(6{9t-yB)!1!z7a$Q|B zYcoe(T|nW4zmDa%@(Y{p=J!d^KtSMlzs{^OAc5nr5Lo5NT`E5`fP?iuQ0Ly-WiT(S;X#D~Klu`c|#&7O=5RRAG1(=R= zz()VawET96)iD3wk-ut;{V_>@zpeb&3HuH1SlwQ|4>)u+08QlImIwUa3VaO!@BDR? ze}(@tdH>yn+2swCJ%F14*#6%CRLcKi;`d4Y?{J^0@tk^q+pz!+5#8^d2Rs(|s#E>n z@ZTr<|7QQ@Ouykk{y*m5zou&Je`o(LSNE@e$=>9tib_Uv5rzZ=bZQL*1dxgQqZRlj z{445rg}8ru_^$=cen$ZYEcI`l`eQ}MKT*HdD)|#NqV#X5e`YrQlliNh{+~=Yz&Ylx z{`8+Y`!!PfPvWn_-G37Gwf~LypJ2-0Ilp=OuX2Nba&Arjjq_K3|Nr@A`c(z+PZQmy z|I5VhNBsYG$=?R?>uR1qEocP(FAINo!hbjMYqr~;CNkpwmx+IR$bYx>YgGMDTR8w# z$bWjvUysSJp7Y-={Th(_)6!Jd|FZON;W-6qaKNw9Z&LDDK%W5h1? z_1PquaYrq(-gvZmcA=82#|*S$n`E%@uNWM*-X*tIK#rqz5V#_kPb*hrG6xl@k&h^q zB^@>FGT++b2OD5QRXl?6im4DMi<$Lt);E%kme!EXaG6_qI;!imjg`}St8vAVoFr+= zxoGGr_0-iC73yZu8mvcXv-Vx0PN*uM=j(MOeje?EuO?Do@77_LxKAP-yAnw61>zc> zd$~I8poWKo9rARXwh%pPAvd=@cJ$;Cia{*Q(Akp{KOMG-Nv(JXQ*gQDwoZ%t{CEY9 z#gK9UPDHp=wN(ap+-c|`T6ho$FXj!BY0n-rR|JO+w!kB*2zyD1oI`9uEype-Xcn`A z_A+?VY1EwFt`Z3KB5rrCll6}_FZ&A1o>q^H_Vg>`6>P5~P@p8rpg&$kHQ@SCrwo{Y z08rxP5miQDMUProX+|o6M6YK6r8ZoqUIe%rcP$BOC^j;IDz7<~)cO61$*LUsg&5Jt zM1d$=?m>&Y>XZ>}LIl`Kq>SH^Dr|TRl*NHH8eHofi=0SBd?K#S*aWi5>XDULZZa~k zS}!lRYb8e&zv-j$wzgp4WG%>tZaN4iSwVG+f4Y z#$#HJt@wBcOjDM`3>+&#{33x>vD@skj4^3gWUy6_NNTA7Iis0F%gyK5Uehs%qo@zF z*{`P|sB1?0VN$}AMf~-Jbzhig1fuSfvbL6;IHq|&rn9hk8f*{4OluNT9cl08rCfCO z0aL%}>NvEq=1KC0q?c-M?pHL4Ot;{f2lRK4Vhykf#dgf7K_#U)~lQ< z7tT~YC9Lxn)T&DFZ{FUMfGSLnWHv^M&79cQ-&4e01Eo>6M)p}s(P9vAIQ>;3w7T~S z6Q)CI@RNLA`ju>Yydxsr1G)1n(U2xv#PLHwoau1e%pjhK62cvC*FRPil@~(UAPK(H z2)-|7^<+wctuCQ)Y@;!4=+P+?aR)7T#3l~x-)QF+it`Uoi8ee5&J{ZM4@0xN?& zMj>%wodYtqdDf({9yV_JAh&>M$Ljigx!*;l(sm|}zZUovd6qQPyVT5L;Pdzjx`Tv! z+8ChNLuJBE2ru;&{-bCSD|Y17{Vi5UfB*phl(PRWTFUaOO2VSbbS}SKeBuS+0vQlQ zU$#Icx&sthMk&A!Je3RyVqCN~DFZg8S?ucTeQs#KWfE*((r6T25950zr!ZJ^gPZar zLQw<~P5e382P4I_L(uRNzpKlrhP@QQ9Rh8!rL9~%x$*xjKE;RGX6#1ks*&+HaFovz zEK4jyJH@7R*v&|XCZSzR-PR`Ks9a}pFG2mjS#w-sn&egMSG&FPyo$Lt@ zVoHa^hA>&r%l82OXMjmyt4zRu1N$2v0PepBXkzMMYG-0yy1mM#@%L`6zxE$qPNt5omQJSs*RxQ^8sG#N5CG=B zzDGnO!V6?CN!0!n90o3`P`|$%#N6zS>$4A#-oH}>$XhoLVmY^)yFrrV(tVX+(GZfD zw@LU0=O`}Y(Oc6qCvt;FSv|Mk@WYv_N?%+=1&*-78t&h8hzzP_cM zrHj5kor8z6+T8Mjs>HPPjQseNbej?dt>j`0K!jrn>d^_+DJlsXY3ULAQ7PI8x#=;b z$uZg?dI|+dzi7e41d?+6(RQTs!$E+N&cCI<*v?yR$J6RL6%qgd6z{)Y-oHQF!O+Rj z%-+e?)JfmS((dmHQZ)7BHro;XUdjyI@kvS=jc&-o2Fu}ljRc1W5!z%TK+WI zM4k(klbE(xC>Dv`>9p?hw*Fj;T!7z=Nquuy3ExkaAe|4Bq!GTo?hpNZFDFN$U;f>U z&m9D7efd_-SDB2T`ZAJha>L&)W^Y8^hOI*8#7hr!aN;gS^E2mJdgqAXaZ0EORaK)v z_KeMe6H}mIW{jc+_k+!gCA>m^7Rp|>6NSSzlOxemqT(hUyK>E+;2S6=AB+M%1mJCJ zbl(Py8YG;kNPjh!W|TwgY@JS*(hxgNN!K2%rP0g^DTOB=IZj;2UTo&57?8pQ^6vQ} zkY+V322>uRNIR^A-YNm^dxlb(%HzlkDuZL4BEFe*^k(c=I29ZWKuFm@I1ql(xK1y; z_E&gI#>~QooC>PK3mpnP<{mrQsRJl3YN#-R*jCG70E4|w!|-A()7(cV7s}%8rn7{v zVK$0AAG6Pkc9_nByxSD)qyo}uXl2#Zf1zE+9L4#S+s)7OJYo$%i+Xtov zUB>@<=Nwc3)RR^;Mi8&K>5pa|#Nb3U9gW)+u^1Z% zhzVoNEA>tV0P|;X?(T{wg#G}B=K2aaP5%iL>NO|uC=Nol(f*3(1=l$X)_yt(+o(?O ziR#ILk?xn5Fl!=|E07ZR;&}{I0Y3*j&H__*%_avyV`czh$zr;RF?xvexP&(z!CU`pX9b&HJm&MG?%{z4(`r1c1Mh z235-+!S)q0bOaSj)ulF)klH9>Mat@Hl=CLvz?N_1 zp;XVR2IgS-LL={eJLgqGd#*b#bcN-#>Ef|SG1Ztv-u75~KN_y|`gx+%2mtBO!-!*^ zn673sQX$v;E>sZFpY9!bfqa$1Dy~R>n|+LQtMEMStD?RKcOGmw-9l!Bu0YLPu~C*f zITV%P-;wj;+AZADEguzDFmm-Cs>g;;oV9)19314;N=azek=~mvScoJFkbXXojNEgF zjDyoksKCkiGPsP#UHhb$kiuyWk|Zc3_3(gyQh;WLvnT;l0dwGUPcjX6KzKW1Q?JmKv3dfIOWAMZ+(-XDMc{J;iS1dP3-e2u1MSsrD1x2c$~lxd7Igs>qBb5*4C6f8f;yp@c4e; z?Sy#%A^r8`)=(j(llA@v>xT}%c${5{6p9rHrpkorx8T&7?L$k-6Q*8#x$tUQ7d^Gf zzh%K{`eAXzoA=&KFTOx79tFnBj?+T<9q0vgNtJ_xasw3tAM+Vi4~a4}$D&lK2^uRD zusw9U$~*UvjSKU&eI>)FEshj?ZC z;w(M0Hz-$a_IyC2hCG+}ZYJH)-(UNjUu?nKsl6dHQgG~BhVJC($x#=0OQq3PLBO`+ zSQVq1SVyetua6F;u5eqMPOGbnVJxiV*>Mr?t3zHi3cwhy-xKdM4=cSVm|@5vAqVqP z{rsi$#S@icaN6wpJ*!f6bM?f`-0=sJZ}ASh!T<74dj&lf%fA2FdpG>^v%wi%=VNj{ zAc60>Et?f(=Z5+HaPCmYA*lv39JzZx#$ntItX6%FXwQ#uczcrvg)vYR->Y%Hr_&o4 z)ds1B7yPfm;>^{a9z>6J6mOK?H%%r{R~t|m{ zUR}ppH3V#hKepKhvYGFj7_WlS=D9*_wh?iw0Ig68``iQ^S9eEL={4d2a2&qaAYctW zIYTjg8d?v0k$_{d5tpBhz*-I~@xzwv z=)pFw8=O;3w-raRPBuk$llqFZA)OzNZ$|f9@+fxh;40mft*W{*RJTtp$kn>@FoA|~ z+_M_Ob6CFyf3eE+qs_UH2*^((JS!Knio$O^ijUO8)81IwUJ2^m$9}H`aY1-GuA-0g z8KZ9+3Nzp~c+N4WZ60lOb`H0(?MJEP=i5N_F^3U0ag${}7ln9}Z0o?_@_kwY5Fvu% zcz*Uf*V54zpxm<}Hx{+zJFr>RT?Cjkk*xsD*##?xC)feq??>XG>gL|(2}}U9b?7;)cs?Fee>cy2J?4i(MA&Uf{`TiHT<~+x zQgoo-?z-tIL2|pl-=GZzOJ73o>D2hd{}&uhu=ZB_iyr}i{yF_U{wJ7paJ4i3zo8uP zzu~}tFenFv4cHhs06^ehH2n{X{dXMbW@z%i0icF$+$I~+x6Kcf(Zj%kn%e4S;TWWY zqX1nL*l9@^TMxAX1`w?xn?0!#;n}up(C5ocJdvnJk8GEOO+=>0@t3HtXzHr;pa`io z-7_d>rr8_T?6LFClM*J8>yEN>I=hq`NS#(M@lexGF!=Eja3B9qf^|}YK4=;dF@CJi zB#;j2M^Yac2`TGS+riosxezPj*H*!?_>lG$AFtc}vA{C8$1KQCPDS*m!+8lO!?dfI zujkKqTVCAYZETOKRHcoa-sgv8hI^smIeKZ!A~NId*F_8pybNaW4zqHrQ=hpXR53=) zB=XgUOQ8#W%UMb_KyNlieg?yygPIr$vRu1ZYZpt9nS(w;1*#yQ?;PCdOM$kg;jdSxvK!JIsXVODmOE?U%E7#ZO?Disg(pP*G zTi)r5tSo1$#|fD4yHJR;31Ub(N#|z}*MgcGGiK2av%}c7%N`c@dkU+pM#-jE6b-&4 zEhgIr8v9;}902io0A>m%6^n1b@ev9-VrZ`%<4;kbvh3hLR-iw;;#c8jKh18b~Zm61rhkFtc@axGQG+8PYr&Hv5L> zjs-jfeFc&Lx0-ztVmGG7p=Rh!Rz7D85SY{e#nJ5FjK7MPQdHC;%GHqV1py+RP$Y`F z-lkBk>onw1sj0Pa*AyCb+BbnZ8%CRVY;1jn8k=QEz8$4{$=T`bGdQUbUv*HhDj>p` z3(MD<4z<$`QUM4=^M*WVxj#~>nHoENVDW$ae=Ox#Vc=8$qGlpST7W ztttA+m!9%MTWrsHVxRtaA&co^M}GoE<&sm%rXVQ!w#{fA{at>gmQ_kkUwiy~Sb)L) z_F9~7V_aCP6;oMQSt3l#jAA1?ub`T2A?DUJENrKuwcjkEA;EJ1QMI|@jZc+?Js=|n zT+>dcEV*$`fPPJl_( zlP4guW(#8Dm=}WqpMJ(IRtLcO5IiXHL>FX*r)+vg*W3bMIk6>zMXN81d^%nSI+F1g z4Q(L+kAr@dW8kw9ulm3_Qc?!`*&ABtyh26)n$J(n#{;Zr|3r{bBs2ulD>+f4L;J;4 z+)VHHv!&EW1jE$lkj4z5gxjsmyk?>aWcM`VkxyGE9W;r1yynkvn=AR&TyGE+h%q*R zl*XJcX*fIhQ19FkrXI=1<=~?le|PNca5Yy4Ch5FdISOawwY-jX*QTwGjE1h$Kn6=A z0Xzj_ZYStJo)~Q*s?&){*+AT}z4gpB3H~KigMHFFKsJk*idWkKa3kPdmgAV5Jj!f> zjA+{j5ojgf9;At=`EH7{sYef7kV84}9HG9(oo9Xe+zQXoxYcWhy?y}90*hlT!gi=U z{D|p{_rl)qi6^XL8r2m$87aBjm5xEjKH93)&Vs?%hR$JAf2F-fhK9#jz)r{V2ew!Z zgQB3x*|8MZQ{W|96+De=k#p^`ygkp&F?pY4M3>*8Ut2n2QtLnfSein)8&e*U*f;QH z$jKrv%nVMj6}%Ta2sj^|3~>0lZq1jV{yRdV&=wSG=88%fD~B7Q=v`hyV5llKBd9uz z)TVOu0irYYinbxdr$EP;B1%Pusfw$T|4%OA1@T{?w@NawO^_;?n#(^8!PhPMrxBdO zKwbw07=cA<%!7slXW$!24{eip@gj?Y$X7L^{01lO0{CkE*zBSbA&z0Ux~x@EfILC4 z8+~PrgQa(3OT>-)03mS9+3_n_VHCx$Cet8%Xb!pqU5_Mj+9T73m=~O`?u=n7yV9^& z7WfJbyVlIOddPx8>{ONoPZ|TsP(%!&T(TZ_%P#<==#Km54}uYkzNR~nnM2Ej%--Ufoqzwn3jJ@+*5wWqo@ zp+kcCWdYBfKwoA)?79F9Ifd-iLdZA4a%M;_wN5`o@;Y;DAbTR66LX7{tua=#KuAiEIP&SSXecPZ|lRJhc2)gAs-^Xjnx09gMIsrVn#*ncJ@L{l~%`?qUu zt{+Uj1`XPW_`;)DDh?rHTH&Qrb}$)fLg`VS);OVV>S!r>(CF?u& zb>x(G&+>O>7ukcGeNW%M9Rq1OKJ|4-9cA}$jyRNitm+GqxqFJ)8)S_~*SOOcF8{Wz zRmXP?gQW8xZbc074#yjVIp|!5W=pNN!5Urtt*Dnk3LG#Z_Z=>*;WNAVU`ahMFRzbn ze(vyvdVFuyaE-$#`>4kYMQ#N5Ad-RxtHzU6``ySQ@Bj%*f4O+ip7uq-UJ{@@FLB?# zuojj?0;!;K)L%gylHf#khA?B3yHEIAh>XjQR$@&alIJhrdUdqDha-wEYo5;nQj5Tv zBWyq(7_}3A^m9PG3>)X6Na;-W6Fk1a)C?U@2_v?8o+jerJL@##I%oLqt&2O&1_}f zkEl<3CRXvzeoxiNd+{#%;o7pjfc3uTJOVe+%9LLqKTWjgbUcXA&-5xZ~Ug2xNyD2qRl4${l^ zbRijuD0bAt3itQoF33B4{tke@5J(flC>eY?2tdbu>~TqhESZ&0~7HzqdL;Z+oOGmGTUkYq`mlH!so(Hs{Tmz=J4 znG!o%nwZ$yQi#)IijTgdV6f6E z<*b^DpY5KFIpbYLbz0uGC$xJ!bh2!1Ei-mr<>6lV2e1NPO43;wpUB`PJZnq zi)EK^ja+%O{!r$uv34Vbb`Yv%X^cgqU`J6*%*shDb*Uw`Wpe`j@;i%|xrLBXvb7Ya ziWu5B*}%`t>=Ca~Ic+6GP zdTb*^k+rcS!ZH4%7OAJ4+R|ed@n-_%y*T9#s7($7Cvw}cXyhgNtH#}ue7QPXm0{CJ z?PwzR3lQ0=^_O;spP`C0dK1<&2v4+ick;53*w@RCwI8+a#N4-rD0eKTbx}z*TV>gk z4W%v~z4;TeTeVOxkCj2R=Zb(Nc1^UJxb<4Vf-Mg7)Pv7dZ=$7B+$0HafObhoV~P!H zRZtdd<_&PG{`&3&jFa(U>=%yv2L*IQL4 zeW`G~O`^3~R+(h%!MHu3aVqB;d@&DF4iMp1r8?YKhY`?;v3RGbZc_|br;7r5qKR}K z3qX-VD#SnVU;-AkutNdm&A&bx5wgFX2@pEFm&>iyle6M1DNm=}34b3N)s^8`7He_hr1-o_H^0 zy`Y7y1;K;2o`KrDcDBZ#{#|J(+KXamP5O}BcCjj-ly>9&rA2|Cd}@m6B87pP+!X~* z^IVd6B1PE2mx~5Pq<0=_bR5f$dmI5Sxjxp8rAkVDjWv&(@ zRtYSdFHJiNG21g|5`jZY*&Iz&wJ}8h$K-_EGh-)hh9#MsN}VE&n}&%<&zib87^d^u z0mNvB55|E(kk6V1kHY@tbz^8+=^P^vXwtB>C?%^PA`SJ#e#k#5Q4{6Kn%hobmZPI7 z$VY{PldvH`=)JER;(kDfUH+NECIPL`4xSisgJ_!ai4&L+Ll<6|udw>DDmxgo2B3)u zD=57S3XLJIm&DEx=^y+jRewfc;&D_P@~}>gV(KPy5{tGd^3Sa7UASs@z;fWwcTYpO zzhnE}W(Ksu_j?cua@U_f3~$OS_z2CHGln7S!s?iX>a7Rl6?hJ;ybj`pZ$ z__r=U-|ypoJ~sQl%d(O%zyBQ9Kcg2_F*|Q!iQukE49(a@0#0yq-w}3;LL)jkQ@X$V zN-I9(AyI;l1zjLrt%DkM8kkJshkTq=+agu@S=A^?d@iucs#Rn7Oa`ix_BBwW4+ z^)FXYt_|_KOoR6@KKPssU zgR4Kl*D2m(iHS%nK-mGNql1KD2KcOUJbLfA+G+`dWO{+w9groyjD6}8)Ksw5tE50Q zmAu&2PfA?8F0GN~+y3%^Q$9&ZRl7~ukQ=5RYi{OjSJcb8F1>Bd_9^}ru>^L| z@ltuCJQ$5m_|!n9H?I!D(2&|Q9C>=d6wPAJfLrh4H#~**=WAqk!%iSN2av&x(CV9` zKuQ_4v1eCpkn~E0Ha?+=_Lgj!v#AbW2iS!`nrdLNhZC-5&jT8J2B_ zrg(J?n0d1Jo7>sL=@Y2S4!C02`of!iOqj8(MYhueT&uHHtbUbdSeGN(OYd@4Oy)2^ z5^Eso@UTz=UwC1tfQ>2KPZ8!)p@mUB@TX@yT1$W_*HD~S>KT&BtPZr*R!(_v*<`oT z!aR;naUBbfBrc5(OBX9G;zG?J5F`$4|J;WU3YkXVDujBEjwPG2GMlUiiYxqXPnEFG zCp4douZmQVoLySN$Qtg4Sf#L6^tCAyYXebRaR`~;IM+QgZryeE&a6)KvmJR^5up&Lwjgsx+i4#7LJ+bs@uBfDN)Gh>1XT&ZQ* z)hXN8$x?y>%PooJ7rG~2kkmB_&vt;tcN!PL7;539ofVhm5YJ?-^GEDi-3lxMQ(BQDTpTHbNYbghOnQ@e^wE?YZWY!+d?!D zVm_tpmErJ=PNDM!#zRK_wN8G#$Ssw}jLji(zT%YgKK<%$A(;mY)PsO0;Ny3WHmw8* zwIQf&xANtD5veP*qRR@|H_rI(ryLZgxknkw0t@uec0y!9Shy}#Xfaw9Y*Pu3||%9Ky0d z^OX=T{X(4H-i6EXCNpV{RV2|Syg4`|Id}DRjm!>pePvxD6WI+6%E1r6TnZgB5{CaM zF?UK-F*GjcNNMX=_9FLhFevwzZ<8?c^K9aP2j@s?*xEZlwG+vvY(0FkS)}HfIL(5< zrclOQOfcp=IQ`5`JI3=4wSi%Ei*v}AB)(w-HWw!BANhn&-WYo+9slTDxU}3Xz@hk) zY4npDPK}!fE7FC$9_Iv6X%&H*xU*r+G}(L4O1Bo7G}06);~Z;TuuOJ26p$^?uo_Ae zQc4NFq}A|*0Zg6X`Ih$7j7KE_TX8k3GSW^39w7<8>cA12c0_X>vfqMfP_=VjYHf|R z3iyd*<;Ua7p8>`$KMXRF+K;FaVLJR$4soo9``MhH6Ud-jIfcYSE6>$LgBcEMDHVy1 z=a;t~+8a-~ySw|}G5Q6M{4B?}FL{e>n&Y1tXuO%wCaJ)JRUXe%jh>+$Z=xk)C-!w!yoi0t6oJ<@*#Ln4vx42DDSnzw`5L@E%AjR_itvGP9EvW9(TdAY_ zEwQJTM^zJdw3x4k=Fh~fWl3eDG-=M6)RX0i2D?is)p42A)^W*JWD>3CK1yB*saA3c zN9Qk0(QqWO93$;_8)kNjH#}AbfTe*16@P6$i6Om4Qsp-?tu^QZwcdaUcV)r2<$KI! zUFwb5ugO8`_l3=9ZCWeyy7EW5y;_vPMewLXVBYO;{@n9(9#SBjk@=muQa6*KNGvK| zrM2^@g;J`@im$&Dm;W)Cd2cRKdbbF3L|y+SD#W&(6r%*Dka=r!XtP-axTBCBo0j ztIJ=*Lf_C%!N*TuC)AhELeI%Z$t+)&e~%U>Yv(IvDOZU*mOo2C{XZs{{A(xkoBq!4 zu>5V9(ET4+|Nm};{O9U@G^XP=M^R5;_(d#9Yt*_fp45>>+@PvtS0a#$Ev+y_k?fO7 z*FhQ{6v^Nwmld>XMV0WJaJBOzi7r~{w8~CPc$X+wdO%M*e#CCNTha5NALjtpzjque zzUq-7ab{r5L+y7ydUm~rUa(>^LwEUW77v<4Wk1r~PRsthBi0_?Py~~$DL7+Xdyffk z&tpEF2@|v6GB@K;S974MMK|GUZ&%5wZd=bpGX!&1hryO1vgMd@Tug=1mZ8pYIvZ|U zhCR&i9Agb)(FI4%6jrK-iVu>SOD{+0a^TfPCd^4Y#_5TI&LL1 zYi)l>H%M!-`vM&A+c538i%qL_f7*Jrj4$qZ(?c0nw9{M1f&Bu9G44$1zB}(EuNz+d zq`>%Ks_XH!Mk%}SMI!OIxLRI@-`$R0Ab6yg#GLegw$128eDd*Fmq*g7DlzN5+o*DM z8A_w1Ww=##$AAYbS1-ANd`U%zBb*dE5U;?0@^(idKa&{ctq%CkfbYhpW-2ug44vQH zB6%82;J%zh-{y$4Td0>lq3^kgXfxV}_X@K$O>6bU>AFwJ!J(6T>$HD0%fnyvmR_b+ zt;;u!nBbLOEA-rSxeQn__f=iGW@IEjKdq>kXU2Bz+sFRh+#9P2JKz3WRC#KLExVNC zD;yDRwE6nXNUphIlqPqN^Ih8tht9Jr{Wle~5n|QU+tnFH)b#;`8GS@iTMX^-R9X|< zG=C`L?-9r;eT6SI!T^ql@883JPW5@_en8&24#U(_Qt9P`w;tm1IDf{vhW-NWEgC=5j`i#p_JkNka=5r=qPh(k-5Zb zhx8`PMdD#^=#uTlQL8}Pi+HqzEnI;M?^-^(-(5c+l<%6}bG#i?AGRN*dI~1a))<^x zW!Da8B~Wa|{g$+g-dpHsKm<>)G3d@Nk#y@QBvG2IYLMCX$5Ex5Pv<>9N5$GW4=eK*m}p2Y(^8AEYj|{651nh)kPxVEa~o? z$?P77SyA!J(vH4yu=BCSrZsEe!k$f$(4#XCqP!$<^3t>4zPa=1C-zpC+N+TXAYWq@ zZe`*SgUYvUSZ%H{C+9iBes3joDYW&6?H7u3&69nMwgu*R!Cb7xFVWPsRz*sh0Hqqs z!TU^%^iZnLxe&$=rtY!WJK@1PLaQW#!wveoTGF$cggo{4p*86|{Z zV%R1E_GP!^(5OObcr4wE8Gp>gg~1=u1j|hY%~IQX*mCorZaMjBnXo)e%-vRz{fc9OEDcN1(~N+I z=Tx|qZ{Qs*(Jmog=?%0`)E)KlOZ|X(GSzCG$;y}Y-`+}DQ{^quE@sk0R!`NjsTnJ( zRRIYx)quh*qx1$fW@;Br)pzRO9ibi5<2Gzr%OM_sjdoLJ)76<7rbA`&G zsWZA&<9^IB-U>tPLSS2_n{MiC)PmcEtszahH?nb{DCMhU5y~WA&~f2JEJ^M(a>vemE@6D zoQd&DB9t$TO?OqFtb_>b>Ax_Hs_%8qDm(DN6!l*01HHW}HeY&J*d>a$Fa}=^ zUo4|F75Xz4(n!~t>l9qz3I+8BYKc@1p_`*RfSl@E`AVhI71_RE-h=o&(o+y9DY8I;U+HeWJk=7*F<&9&E>?y1ObtjgErPTU^@AwMTF{H+A+4z`M z_fXm=(L)(sGIVt4%5V@;ZqQTK90BUOwesq@BY*vj)nTW#@FEK<^?p_(!3pW|QK(!2 z3D%>B6-5bBriQRb_dVDeUCCmt)>VJt9&E^M-9*bLXe-gc7tV+~q?U2dTW})fChydg zakE@dH4@;;A2v-GyX76V5I_D4P&(~VaIV-Yu(2GY2IAUfK1 z&%p;~Necn|P(j|ILPTA7k>fg&OhrFG-lQMjPttDfYj(q7KT*Uy2;txjD! z{D#cTyFW$!JYTg;VaT%ciY#dn^#Ll7_C9IoPE}9n?xF%Q!6Pn9d0mBUe))llDUZnP zq$PRgTa@kg3Nbkrb%JpZtL;h!vTGy?Vr^L+KhhyiKoD{B`ttgbZx@}Q@4^E8q=%oTf27M>JGgOu4 zj@vV!1P55BnK_sD5vp2Pkv1;`9c(Zs@}piY;<~Ktgg@bIoHl>C>)fJo!0iGI*MLd5 zg`?$^f!ToI*F$e>F{E*SDhm+w%s9);FoiM$2)J_)lY7*?6IN6=uDPjU#DpzvmCe3C zO94)AE3Ysi@W2onR!$E4xj1XeWRbWj1H9WU@CXK;T+>KLICa@rjCO<{uAX!&r-4W7 zBw(5}`gnbdbqZ0|dP#lz5|d279bYnm9){O{=I%ABglC01;A}ZCZHV4=ud7V|==ljtTP(rpkme&Pce** zi7BQWdPnTw{HlEn0=*+r@r@Cw#sEu4MP*oy3+@Ph^Y*3u;zjZk7RSZeSilq8hAm&N z-*D+!zLZ(~p$7GcqpKpNQ%jdfg{ufTJbPUS9o%P@aOJA*dnqGLu7nff8z1UFd=_+6 z3kUwlT{@-Q_&1)h)L^Z*^royAiJ-dZNq2005?Ifi{Hk$5xMD*bPOA1++f`2w^7?%1 z^4m7ujKj!J71y8Sm&Ko|_XxL|hvL>DO>F-I?kA5d*1lSHPHs)s^a57*^JB$D1D&Q` zHETC~aB{NAM^jNg!2{KDDG%p6QLqSN+@vMDXeZ*1`6%(WLv9t;{hIQ{(VUEF77Vh< z%H+q90)N7!WN8# zqHMd|7<;R?8Y&shbtDo1z=a(E0M7rbvni9Q{$J+w zw^e+FtNWKJ-F^9j!WM;}E{-QAp-eBE_m6Xm`g1^9vC}&6BJfzc8p$S5NsjdPf}RQo z6kiGV`4{?}Do6D;Q_PhS{E)f5Gj}%WXK&|_R&aCU`-85fwm1DfnE&}bHd3u3Qny}E zXxmRzMqJN$Ln%W2Q)R_R(NySBXA`SAbPTI4 zB0fI_diZ05kyVFR|G-L(1?Myg+M!MZLKb|y%qu&ysoH4N+uJOJ`RVy2rq~OG9 zOWW+#5tgvy-X z3QMhIB)i4`s+eQx%QneY$e_usxa&8+hu@-!sp2FenhhqYq?(CrV)QWbSA}LT&ZhhV zD?U-hD-B$c48+1^*W%eys8)DF5-Yxzu;xRSOA`0~Mtp+D6c$#I;5 zeFo#5E6rO@lKUX!u;k(pxdhynYN*V(W_4Y-C?*-WQonBYrqFD1a~itw zyR=w1!vj~O*>b#B!SotUlub>KfM(VLxS31vvPO`?;b<5k&;(4NV`1ESkm1}W_T@^; zd`xN+yS=v(9T;-j$0F#eF1s$&`UzAWBW z|A*wRu%wTB?NPwbq7m^GE!X`CpUc}CmY1RCOEdt}*j`{7`H#eImn*!tv*2CU=22Xe znX67|(fy_QmdQx4J?v}Gv=BuJhPhRnf34!|Z0jT`Z~_l!3-+<;e6hEf&O8n`L2C~K zgW4}37sb;1_)_vmVYQ}XZGLQP)RBZoU^*6Vx;zw>c((lr8A(BSw%N%@MOY~gbiS_C zWv+xUBc3q)sBL+?{E13)DC6^LMzU6+bpsEfL zH>p$?FS+cX>qPz>40*?68LP6`!ZHR3t-qvL%14fRon3-(7(4NJzn1m$zQ2;n^1kmT z{l3Qu?R#sGPU|TlO*d6Mp%;gm3(_NSQ|}t|DvNq_bUaV?0Lx>bImBt4*k3MNiW%OK z_6(a9BQ=xnQI#CL?M7(HEOmG3)6=sXi=~<8g@*xbAwhW;;0D|6)*49< zn6{=;EgnxP)S8fapz2&t;My0$86L{mSyBLy3MLt7n$TW~S9C1up% zyc8CSF{u429 z@{8KPeT&5aPGQH?`QY?%GNTesDq@y++zm=Iw@RUs_ML~S@4 z?jLP3y;boUPJN=i-1krbgO=AybxqIIF2KK@eK%v`nHa%*h zv3kX%h~okG!ekU8hx+=sV^LxxLvN!vjqR*N*3&Yn7b}aUpWv!|K|dtIBNprXlQ#Zh zQTn>;cLS%W-$|;hT9}kF0laKNiQ7 zMs)yGfc|-(P&fRhkeXerF1f#pyurkUySEKq$%HY-K^N2pH-@OtTTjOIyvHW9#}7M_ zbeOo#4o2KTcuc}_X_3tFlzy%E)_)E1rflh@6j~uurzzI@0a6TF3C4G+CQ_u>x`;-s zKuI`_uo4yRE2kv^?pE+2p;9eiO!guUQ9$0!)Oj``cJr`pI?NETb=NZONV; z@3x!6sSmHh?qM}j$o`>$+c{@@PJg+>7=?D)FwBruWUS|J+)(*dI7@-Ct8r%Ma%eqf z(3*-)HiydMW*)h!K*#qu`|CHpUn3`ejBCR*uML>P2(upPq3b=Ctfvm-m)E*vPf+HNJY%>*ev+epzh2hP z$Nm>%?+_(g6l86tZQHhO+qP{RFKyelZTqEd+jeHZs()2?Rd)?~aK~re!8t2p$KGG` zR5o3GWoyTorJ%j|@c_Qsq3CzHO>2?+|M^)P`inY0n<6W3wqKop_m>@=h$L_pCY&J* zu=T6{LS$<5hmT}5W{NQ|@=)>1x2h>{L{1i+GCbBX01)`RXyUT$hST~g$Z0wg72{=0 z71H0?nQF@>zMO@hp8ChY7E-Frz8qyx+urz5t4=h#4#b6Lyi^EuJbzm_4zqR{r9mXv zJpfynaD+9-)&mJ=<~0hOiK%#qj;tuloT^t?jp#)@9RciCLUcIH4ITlJoAKC(h^rc< z40%Pi-yUJ)2f`w0h4o}POu=_~tRld83w^NYErdK|76if##u40;e(Zz%6B^KFI z+b5Inpfq=Rv%8`06X9`SilUjoV`z58WUn+T2#!ZDD4oa>JhCx@${{BIEP$Vey?TcY3c#dkYqTe)$9aRy3YHp2on90?ndy1=Cemgx-w!dG&MWV%}RwCgXe5u zH0UK0@PM|^*vyBD;)F{5aE!L0U9nBr9kG?o>)S{aR(xXm6^^oKP{>5V8@z(?&`6t5 zbWF?M{1m42e3;FC_{z`Aaa2pw+x(qQJUmFA>oqdOsu~J#RYU>GdF&h;1slOBcs`TV zF?fTKe_!vq?ZS9Bgkk&3$RH!-k6)KoUz_f(t=1yL#S?A3Rv6@tHu*Ua&HC@UNp;DonZ1f3>n)PRDGk+{vIK^;= z`7PyWDLgk}hwxpBa}%ln*-{+mqmoO7wm2rG!;Kx}p>N6pLAj)78+okh3Y{nmuBo(+ zoi7XQyPz$$!&=eqnM|{RI+Ryq?(R|;vXE{{SCG{K+d|E|5VvNBG?=};l+b#VB8BUrpfJjFk`|B%nqz3?0?^{UVpt_&d1Cxhury7^3#3aPxbje zZ^?f@SMhDYDhp)xenIwwdE}kF780GQ;C)WmZTjg(8CO;Wtv%sH(jUKpYLTM0L!_kMs=PZW~7c6+Sn#K8Zo>7UTRt zxtpU->IVUrFfj}Y>#{6XxG2{78q>)u-PIE3w)Kh-2?3Or>Ym}sIm8j? z{AtVeTC&Pp_Ou4`D+2LqiCx`}X?|LADL9cvvo9bbe^fOY6Vh z6AF~VLiaY#)JA)W`*%rx(2XHESOCO@0g%HQ4Pzz%1G@$ibYdvPe&%m3^7l4nyUgiX zXHBhk_68(Qxd`bLB}m9o=)pYx9*Ss@?w+Oa#p-R&SqNqop;uKECpXTzjUZUUUb%z{)gGunE00wCKDa5~br&r~J!?(loa?M{NG z^3~2@iwz|%Yr9SH4s1$%+uoH;ru9^(RSz7&A6J5(;jT~Jn}(fRa)`SisM7H0xY=J5 z5s8XR9GQap!xl8>>}8>@wW*fQ3#~vmq4Bge^}5z*4CS!69%w(@-m4UZQ?(wr;FX^d zFZK&5aq|cbo2DHSw4DLyGP%I=L%LV9=XIBi+~uNm6DYm}Waei5vC{@$j^iNt2=4sm znMm&X!Jc-X4#E=F^r)!8fxS9n;T?(8r9GqPmP(hE+hD9@Oni?K-QgRA=h!&;os2ol zxVsT3CS~S3ywG!|H=}qhzz4C6gVCr-@(*EQOTBu6K`kTS=qm};;bzAseq{7|J}N9!VhA%`c?H`@}5oO46r)1NpXeiCQ9ApB^6nz&9#Npq*`T z;%8zt?z6f^crAW$}Y*gjFi7fPkU8prY%Fhcwy5As! z3meft4~{%f&qIInOY0~ScbA_)@7T+1TY>MKld4^Vpen&$2(QrhX{&nk3)8A z)CrdT}yd3%%4+6rI(a8YnUqyQ$?P2fwQ^U875963sE zpEc*fFLG!<*}&-6(ci(oH1$T*Dho8qZg_@kv&!t-ZvE%NLN3VPX|wvC>8-)Pb_ zB=@G@p0PfE-)aSyotql}!v9YJjM!<&nH2&6AOQRS3&Hq50+^YTy`9THn8|;LSeo1a z2ZEtLV24L@J+bkuhXf+#V-fEb2xM6+(b@w(L|_ro7?DDeQpn~Q`E#3tc@fljr8B?v z{Gq^qwUC{$3v(KU?>CCJ3Qgv!R4R3=Ufr;MfI2hHq>NUL^b%i%t1PODHT46ek2`=z zy?1-EduBZRlaGYmz^`dJOqYh}U1s(6qld!eYeAL$PGMMx`>R@|G;2m1U>A8YE;8-1 zd^qJxEVEG|0mtNdGb`(6@ zoGL9ltRcZQgMWe`n%WS+iEV1?TMw(Yxl!uqB1%T^0{=IA+Dww}BJ-Oh_eQp&X2}wv z)(nPEU0vN?`Wxh+9xI`#x}<3%ldK^ofrc8^`|)0Q>weZEelproX~a{2dAlst6D`Ip z@jxF`66ScL>^(r%S&?|D}d1az*{CCpE{bN@eQXzn;6zwQH6*3;g%qE8GSwby#Ejk z{aq)1oWa0tmyXH1_5^f8HWsq`e9G;!ARDUd9ocdmh8)^-(p{=3ekzSIZ)xEw4(Cov z=llGk?EtJzXdQjO2r#^gOkAthjeB zU@H#XBqgW5hAc#yCEhJlD#0oHt}z^9snDZzQ;AP}Yl2t({4@bQ0>)NaOHbP{i!ZPt zBVL2S7Ej^lKvD?#Pv@d)wX$I}hf;_F`3_7{!dN}-%F2YXmfMgu8qNCUs?Gbs!$N(k z-%`iILvLG*keeV0nj{hR6gO#5|9WGTm$i}hN&PCwK7b66D|)yd@JSu9BeAI0)jPBm zb-jZw?P`i!jTQ^GF+M>~5g9YrCa_cv&KW_s=5G znHzGO>Ngx5E6m`vts)0&to5bM0itROKBoiN3@dOd`^tV_kGE`tWe&%6U^X<#jIS|? ze;lBSB;j>|jJG+wv`VLy&9yhz!%Sj=yIU~?jLi_%?zaCP9GK0nO&BR}zq|MuI^&!I z_dG2fMFJH(-dfD!X-kOCpkOr;eWz8VKDCbSupIG;CLt0RzBOYTy|% z2eBszAUul|Cl6xtXQ1zxTp+zGyw;Rtkpf*2^b1x4TEE}+$3yYv=g-hnKlNNNDF1dQ zc(ETyYhokqcF)}M0Fx2)F6>*Na!Gcj|0tC*8Iql$18S2+jI}Sam`#x|3D+)2(Q*hf z<=_SKK$G9*CRC{0NdJhBLqi#NHr>D~upwWwVQB?jZ`Oztw%4=U`X>om>_V5#6-gA> zz@0U0XN@epshd)$-fEWG;Yj7sryaXr5XaSZ-tP-uE&EIVU2!%P^$b&+o}{XyQtr?S zRY+}*(Ph@=i5sYA3y@-U7uxu2Be~%WG@ZJ;KI&Cf^R&~rMI%wLi{PBHI~Se(>lr=B z(Rh>|@oO@|iNRI_F&q11xqpr3?0JhB-hs6^n;gg1@tGa^VPJ6Y`>HNe*5Uc6tMD$X zd8xb>CT^n(nCPz3t4yW310{(NA>QjBD^q_wLe3yi;;w&pw7N0@+R+nz6~-@{jyZntXfv+AV1SVZrY z*N&6ts#4~r44SQr^lq&5Ab5ATYpgoFs@=+Si;R*HdS;kgv<{yU&>>}MYWS?AExbQzUgif(rLhUO#q`{H5oYLC~ zcUrfN1eZ}qY9IY57C0%P1lwK*J~0{L0c0>ZtxZHxQ^-24p=%N)20dLqKN(fsq2tBC z%Vf7I!?D(}oZ2Qilm2(R{VW0sFBCVBgpqlS5UFLVl`DDuTy$MANZSJt+n8C ze+#ZJLhgs%jAmfY`aF7GS%#entM)l`(qgepDm`f|u<9K$e&m`W2d{(8ixFg>HW#-O z8Zlwa>FVBtMT9E1vdWvbY)}3(v2rhUFWvDk)Zwj9VSEdSGK=vuYc&iQ)Dv`P6|Xyo z`W<*PIpp>Yl@NqI<4-NL-|5^>SG(F1>gon8PBlDPRct3BRUzp?vWQ2OWKIoWM^Y%$ zUmvXJ2EpVwD5Y@_7lDu(CzDu;1Mt<1f2SSn!&C^2cd`z+%C4D#K%Ms!XdaN( z&a((e78wx%$+$|?q^6Ylttfd|rPN3mv}VDAe~mTL-TltxLHnx7l=nk1M|E!$=D(Q5 z?vUBb9|^)KiOEd-ZO-b+(tAG%UzYF+lOV&WTlHxvD6DLjL~-_xxrJ03$yd@>%9Sk0Y?)qwl}2P8~_PIy6*s_7$B80K=0*Wgw*ny+5-17~J$ zvnoZ@m1Y1&`T;S;en2!U4O5XOu16cQsXA1hDb<@SL0DpMs>}QHWxfg_>qk!b(`Ulfq>!^i|-l zb<%Lp4LFG+VcIQ|I=kpeobRcpc%S8S{N#Us7ywFt}lB*X|mktW7NI?sqD@vl$ zDyQvCG%GLj=rbZ*F}v=CkF4TdI1-$y6w5Isd^=HSe1MJ#5OVEEr{m4J(>qY`hiCRl z{?HJ0H`XTV%wlk*EDZE7P=G8fqCzaF>qIEhPDhxt663}uN z`%@40CZXnlXlJEoDiR)bJ9N#&-MOAz?9Cga>$i+!8@^>EiPW)LSwZMET4tFxCl4(r+^8xJZ3{`Y`tgP4G6W zvprkcGJ8qekzGa=w@a{cbm9EcdkgH)GosDl^X~XBf0Qm*k?n+#^N6L?EHxubIW^%> zi}A$gYXl4lwGeo7ViraGOP|)=L)8muHr4~7+ea>TM_li3O@Bz>7`)o-W_5TUJX`$2 z8RRxH7$3ec#rTlwoySo(0`AxbsCX>Sz(1;4=Og2mCEbS=bL|SLnY^Hu_W|S7)>@PV zb}IAz?nb=c`*h^^2wboZNs+#9$JE)0KSPE3vlj&gg=_n#@=_teM8IHXCxG1Bd|L~ddKpA?yV ztQ!Yn)bepr#u+Sq%I5~gVgB{H%; z)#Mm1t8^S^g$Z%oGO**TmtN2giRL?DOtz>C4ioJQasKy;=&X) z)b?`>xffn4?EhyO4B?(VNIG-P*6YaDyt z#KBM9J8sAKlS9*+gMV1*YC;PwpHDWjPWGWv^l?9DcaZuUYvkRmb%1jJZm9;SYDK)^ zj1u;`(@;O)q5@M(3F)I*gZ;JwO`@dLK@uHVGvAbEw^fE4%rp@RWVUXP(yBS{_RK)P z%S;;WaK&N$Q9LRVhXY`y?_x`wsiM8op;{euE4iEtv?W6>2O4Y_ktwHzWQPIeRmYBy z+;OO&6e}!D9dy36?mW*g9Huy;L7yp5IW07;3{YvR7VBK40E-yw8EueQ9(n02Owdy= zKv5zSui2F)X~juM08!O9bce8ju8-m1K}K}(-}6kv_d*!PHWR(77grpyru6BZwM9=9BwDW&AuXaqU*FH^iF&x3?~$9mjZ{c0Ri=1n zVWuSMwi`QH75|z=4*Tke!?wdU;^NyXOIfMGa3^qvgDY*!`3blm{5W5%dpnMB3)G-9 zL3j^}$QbkPNExR$qp4A-P2c$QBewjk{|N{z3{$+tw8O{_ZEHcQ>oFk43YOgQn5ylTkVn^ zr)^;1+9jAXFAtN?Xbe+j^#QrCL6)dZ3{ET}Vc ziZ41>Fluc6z}RA1*D%t3_+?Z1A|MfVunRd)38_hzY}?RMD2T5I@dRJ5vwT3%b4I&j zG@IzGkH7!tixcuvdzKqOJS6mYq^h>|$M%kYX^F>cFRr*JbqjXHN7vlBpvTbnO^TW6 z6R`cCUC2drQ`~5XpF#iXL;io>^Xa5&BdvdMacv?1fPVyt|HTRQU&UAnrnUVh$Ah;Y zsJf`Z`5wiz5kF4!qmi?1=QUv#n51xEETkKxcd0)-F$((?Y zi`P?{iKfu9>Mbsn_4w;qW^pMr#sx=4_kwjxGp!GqSp=$DC8L}5hl(H#n>Q@K;ode^dXarIS#yid;Pt9oPtFV zm?WBo$hlX@S_rbVJA&J%VmD6!^q!l0tbj|Njl?!HeY4Rx{%ZjU>mjI>*h0g zLo69~O)kRC9b6e%u6d03DPr*uY(e!*6j{Nx7~Bh3iJ@76=6}Qh@wmM(cU)8pjr5V2 zY1(b|j6D0?LpW5fJXMJqCz4CS&^VEr(rtS<^&wtW6g>Pb)S)1fKl+MLoT838;hV=! zMepZuuwY~`MefM-@Q^>?gU7Yaq?S>0I3)B%ptq$7qQLt!l#`aS@P6+iAr-Wok|@;x zQRjv10NZ6{vk7J$4yH3T%j&&1=ypOt{-cZcpqk<234yW&=w#8Tk$Ao$OD!Z85Nqy9 zJ3*qvzYxl_YDB{TpbFR~w``Y)>WAh?`Cb5P0}Y_nYS$z{ZlwvhOhz@)Pf7^Z!P-M< zO&R#!Sk4KlmPBjyYqVl`mR<;+JvL)cy5>u|1}f|XJ6omH#Okr3eA;e4c3EXNzqm=H z*~pPb11N^AmsZ0RL*x&Vv%v`Vv|>@=+$LdO6957;jxq1X%n=-$*@&A<)_)@)92H2L zXG{B91guw1VlDBjzwwpLYx2%lm7OY^>lMZ;e}-h9bOEZv8GVr^Yv$3kpC}Rn&76zIMcCQ{5gKGLwx|RHblhycK|1b` zWIi}Z=RrY5X%-HMFtcm%Y;JB9(K78$H14FL(n=?&DXCEq_*?B1`E=`1Nwu+?5G#?! zhMyi=JxjO80i6S*R@2dH=B0U6&LePj)@$gS-k;Wqn68>4^S3B`dF)c@=XBAixO%|-V5IUY^|ViRFD zotZAEHE28DMVyNlWgP9qh;1AY%s<|DKBqU+Z+|Qu*phe={xKmx;zLkz!nrxe$jgT_ z_BW5G0!q@9)9W9vYrGQh9xOzQ{d6>TqE1lYof=|K9Zc64fu~DXcN95m)-%Msj+Z+L zZ?v6!rTwrAm@0Oa3CL(A>uNs>dH~G!kY&oR4 zd{BoCkT4V_|9Sxt`0El4?ka}24HAdGfItpJq4u6jhE-SOY@0THG+1X|O4;F1-C82P zkgAAk-svT-D~{c~L8gCX5jZfZns*UfI>^1)pt%-5U+APct}p*?NaM_FTPO!4FDbp0 zuQ-H}fYskCoe{W$nen}VicwqSm|V+-aGs|Pg@?a9_*L(!RrcKD83QU4Plb&hpjq(U z!};-(oVn!<&7#86MG=-X*5z@N2Z<6EuJo@AX+BPNo~}%TTpG_f^hNd)*nwX03BTrQ z9pD%dIM>w4X07EO*a_%p(~ZwC?% z2qf*xv-rL2Cg?D)8@E_+RxFdpiMqBt@~b#OHnFvh4+L7hbm*5TSG1_!&V=5=wMh08oOpBJ9)U7bLJRVdOy+-9ADhdqeYm@N@l@9JuBY(G(u|EE> z)0vTcFivfO$;0xN$UZo>y5&?lZ$h-WHr@>WY*`E0h2;_n0Z-qGbP67 zDcRANqdNjqcw!aRTG6RX3Djp|mA5T)%h%sKM%reR9x33LJIaGbG>vPWqpQt>C=TqO z*m7_lE2tWNrqbJ#Jqt}QOgM|isNJV!e2vNq~W?}Kwz#Sf!>h&If^UaV=a5a%& zP5oo>vVb>Fw=#qeUCj?@)ux95h$-#|4(FA0>CP##@sV*{F>>A4Q2w!P_*E>T2<7u$ zPEEw*rH~_L5gt62ot>{&e*$|4x;Wy`B`^PPh>s^@;<@>x`ckAEwCwM~HHyK@!kTUj zFDp{`*}0U<;-gDMZS2inwz_t~YdKlF?@xHzZ03jktfm)4ksqd?8D_?xLON_J!RjD; zF=c_rq1!$A2$_7M2xKDx*XFgk!+%D=&qAN?iU_{JHg4l;5Ppk%6_(DV*3z;s_L73| z_GiAk(8vh92$LU7x45Du3M5=jI!~MfELa4lLxHJhHjv4s{i!@?yJxG2QD((1&x+cI zME)tCD;x=;7Aq30h!mp!%9ipRu}7@&#R=T4V z%o+uAz8;T+eMqFLK;EShR2Nr~}c(YE@KJGIW74O;@J z)3n-fv&rTlNP5Y_%J_DzQ=6cPDyVHvp3v?MICnDKH3ekmjpoX(kux zrzB}v8)#>wSep^yXXVxDr)8;U7U& z>C>$zcuWhIwe^*}kgdWU@0cfu_TQrt|M4%0&!|Nb|1uIc{Qsin|7&;g-)VdQ6`S~< zVe=Yo>v&>y*PApgrAtPv7_rB-jBQQt?|4J^o~Q;`%@sXw-r-?9`-UjMk=VJvw$l>C zuCqIK>s?8Bc)6fiZG>=;hQOI9qK!wQ5G13FdkQL%B*lpWe$VigKq}**Qjz#7AquGW zGn{Q*%L^7bV=0Q$io52!-Z#9bKBrq%ZAVU_ApZD1_gpW=(usZqu-}$<{_J1eyC8!k zWNjs}Iw__;Y8z7GHI=wEm99w3BI0!y$#tzs%Ot*KmFMhIGR6_oNYd#Kty-)EHBBVN zNy#$;Tq6tda>`uLmO-WjIR!3o3!14xE_T`#Cxj{H#_?FpWBJVh=RnwDPaGvY6Q*L+o*6&AfO0*j8{!yPwbn@=r^)2OTJ@|x&r zYN4KlVcqFqMpCL7lByvSswtGJF%`=oi}}C7{4WkUIj)zr?*_i6HI^Cb4g0E~5l|)- zg`X8#)0OUjwWisvT4JX=ZGC1|tWLY0-4q(@;l8)*73Brmy+`e|ZW6C^*>s&pSTp#s zs{r+}OFJ6fR&=h@Q!j6$Qnt+*t+3g;iS8XSAQJNObZ*!-ug;i&>2S`M)NM`&_l?J5 zx4hbCxZ8H!mUMJ^H_o(N!&YskIvtuX**0esAKNv(t7l?ubdDs?Kz?y?C8SpNggxuh zeMgDF!RGqw^n& z03>2(jt@)A9nIw;S6t8lGUcl)(?BLky@x34t~B53R%urmp|B7#wy+?f3{_Fssw1;3 zKHbTfuNc>iG{=HBQO8kPwSGqgewk%4O?5F<2)gD!B>Kj6Y&3&;L%eY(dbXF69!atp z<-RFQ?9kA}m=M;>JUZ70j>P)TnKAm&sv-?_XN*nzwOZD+PPeu9Z+5mAy<~lSHoOV0 zbzl=uiOX34j`PGSK$D4Zc9A&|WRKg!$qcJ#Yf%rNloq}O!JqNG1(XJn&)JtR3M65c zPYVQ1)Z!f*+HEYElX#VOR{MD7$~K%!M^p}&1!Oe& zo$M=@?Y_)~ZZ^}Fz277S3h3$rge>`>wR_%Nv@v``67VB2e@^HQUsu6MJmUU>$v117 zf3WWS3|?5D-GaN<;PtxSjP3UMzSJ)7evth59$yO1yWeVcOKD?o>Snw$%k=RCP>vlOR4i5KmuH|=aZa~FQmZ;=;V#k}*9LhxF!*P#3%-C|v}c|npPZ)^ zfOKKb%<}Ivr+;&}G+C~2#qz^`FYBCzfCPHgbXaceoKU!}1 zVeNVhgrt2i+ivRV*Va`>sWl*e{M-DY&wMG9ENc8zW)a&QXC13>WRxg6RhdEH+j86r zt9vm1&7*C2?}CL%A;wM)wk(jN4ot0n$wgMhYNHp#P^kx0esV5=tcO&7 zQxZVd!!nm_4j}7|nID=4ntAUe(4>cFF@p`9`3fa4*6o-NS_k2>l?9;nBqc!JEtyYR z2jjAk1*G*-8Yt( z8BasxNt~!J;>4fGeytV`p_!?+ShHL!UV4aK0;=n0nm4T^gNqBvtygIhio$)a^UuA; z2iVnT{XrEO(Q314%Rg_i>6%7dd;kvz*Iqa3SiiQ@5n!dGx_3M|^<(jUdP_YyWgnKe z=G`k_v}VecJm>NkHXW{1C$S&y&whYC173cwYoE{V{pZ|p0>}2EZFkFgZBFYYwNo`e zCmjkquptSG$gwJ9$e)6MauL=^`8hR}v~esNJwO906jX@(SS!fBw62)YiSG*3O{YtO z?EG}FrFE0;{JJR>v(>usuRz482ZMe%AZR5YeH;wlS&OKzUzzm05yk9(t8|Fnybh*I zjXN)i!8zK-04gTzOo-S{h`q08*nu@uiQ*Ha6lZq0O%Z5AxI^X^(lIawwwUwfUQd=> zg|msc#wqs{MRh+E*RtKfiJS|@6t#l@5h`<#Ak9Zk3 zj$r(AJw45l7*z^lPpm7-_@abIA@Y)t!7gnm-ccC>Nu$U`JeZMP(qM(?BlK!|<+X9(+z#VOa zQn3rCc!?sV49!>sT4YQzgF+}nC2~-Q(C3IO5iBvx;RK?8*vG=kFL5D&YT;=QiB0GH z0gH^?CTxp=FGO~X%_H1eq=&H@eKsA}gxY2pRYVcQ*bID~TS3k5uN_doO7)uV zoUv)cx%+Wkovz8e1_g9+Uh{UjXBDqub4>a;TiB2q9{)(EDSSPX>)S2p0rF(cXR%%{<#L%GZ zzI&xiBaHlc{;4_7d|E^9-t69{qvbv?rVsq`;5`re(q!$_7pD#8(xhlheVx6qoL~O- zD}nh{HOVL}S`mI_1&Wb&5e+CJkRQQ@P_y?+5o_VLF`;0@9vjlID2}f~FTD3sunTYK z(Y-|zY>l-Z&9-o=q*!qnw1ag07|cDql4|I7>OXEF%T0TLSpmu~`)Sxqzd>A`1vcBu zMkpp?>YwlD{73YEep7CiP>q&tf(*TRaI9?!x`<;4pZAdg_g7Fv*h4jnaEucLl4C75 zCi|o113s#4Q?8_eHfJ$BhMdd{3kv3ZbGj*cDcB4ci`L4hJ9@AohAVB^f_o6;2thrT zbTMjR&h;rN1C}N;xCP_*(f(Ss=`7B-=zZ{qs5DoY1OY$53t1t4Q#Q~FM=Ms?8nZQ; z;;1V)8&FSYgY47wAL)pT;4|Q^SA|P2HwY&(!kELZz-@>Mr})l@ zZI}wmGs8^PJ=(TxESk7o??*u%!Ch5Jg8~XAs z@QO3ABK9O+N#xIVqnb!(b7^Rz0SJgp;(`3?6#z8XcInynVh@3S>3Kq-8`!nNo8D^` zV!x~O;SdSPATc&KWtnm{^UW8weZv=aJz#Q*dmx|){}|%|kp8`}CxB?s*@a?B?lLRj zL>jjM2jK+|1kFTLe~Z|co3Cn_H4xIVQ227k7-J7vARd;u4zdGz8tNC<*2q5A&vGOF zT2^gpvT+_OphSk0L0JOEBw@LKsxBv}e_8$b?gi9zrqC%PVzQK zLw{|L&%P1}&W6|%cLX2xNEF7*?1mjllE@Q!@+ZO*2m;U=Y2b@I5R%}FJs^_ci$1U< z8YejT!$5)#@Wp~4Quu>S)DaKtvH!x7VEo0mtQ(VW*$m1$(RWa^D>L;Ho#MyW|IzLF z#SifH!KV;9F6ty_NxvwG_yjq(jA06l+g;5}sW$NO$Nk{J*NS$RBwTqD7(P;@z+W#P zfX+TZ@K+iRnO=X}0w?bs#fRU zFh}5F?x(^7wBG;JSfs0`1Q=A=P;eX9b(3_^;vzsr-56g)QU*8By$K!|%}niP^Ss2hikEcfNW11LhR7+0zq4>_)E{ z!Ty(&FcwBn^w%Ql{}o8)9!}O_sB1E5IXPnZ0@d?}^hi`(GR&Mbg}aP+`+fql^@1<_9n()bsXK{fJOVvwL8>l+9umQ1>=ky#j3Y_5QthG zbZ9VG3s*zMos$U&=4#gaVId%lJaVxDI0rje6ATZ7`f$9O_(3FM>nG+MN3_l#TLAzs z3$u<_cYyV(2cx*E2P+pqQ4xEp5tWjBW$8U3OD9^1U@JnRqP@qlXN0wFO{pm^6aGGG z^$7#vC20r0({q2T_PJ|&|$qLsb&ult%hos<^ zRc4R5qBBfMyQiY>i=B%&W2GB}hWAt|-Xvuha|WqWp9T~L${5{H6OcJ$Z`#cw4b-@T21#zRPntRS5cxq`gIeqwlyu8EZ^o%_-}2OFR}Bew0do}1I@x4XPn z%@P(H<4~u3{HKB0n%McE3bZl^suN`;t?dz@0oh#>3@kze!tDGxG9*zYu@`;7qy+J? z7k+Sy5CE?T+>LuW$0v9fCh-qWEw|BIBd8-{5gW9vFdG=zdnRbg9427kz*#2o063#j zXJ)iheH7eGiHSNRg1O8x%IBO}hfi<^!a7WpuRsdEG{nM~T^PNVKs|~}r`-5mj{o>^C!R)dq_Qu=A;)`;xA72>G zpU#6`Y^uBOobS$3;0|G{j6+3;fLqYUmp(Moi$rliAFD8Y2(M`#;*7q7W7 z1}<*Ru`%r%WQ<$hp`rLKvt>TRa&c(}OJm>0OVnF6Iu<7sh}J~v3{wFRt)c#&4NZaL z8d|crg(B~WHlIqg7ohs*R4>sAoISX3{X&}Ln?JA7#yyajLJvURA5h0Q!0|dtsC1V0 zPX#Gt$tGa%QXhj+NP;S=YV}EK;dgBwV7bTC>2 zJD~JWI49%4k{*=hr2B=GG9qnB+p-K9D{fgCIj{mND0E(GP%F=Soe>1B$d~oB5(U_W z>!>m!GJk@NkX!UY(Jx=R19~whU(d8$MxfDunvFP|oHOYH0brSAQ1a3A1B9J0@mkS4 z0}QI2vg60{`Ox6LE!M!`H;I`~ssW_oHZE#`q;z|<&|Cm8)~{7_7}S0$)GLA*{@bef zRlz#=C3@QQkH)P%4}f-nz?%05moPAGyQi^_n>YLB@xMKYbtqREx!uTt!G7bq$mY?^ z@cG2ePL2ugshv5>cE4b@FtInTOeoF1d0TPg@YRxill>InJc-1CTK$T4kOBn{+C1ya zXPkt4ctTm+^*mVqO6{xL7LW6=_)HUZ@Zfm)r&eIb0k({ZhNqwOf!+Az@r959L4aa5 zkD+fYFD8gjzuhcSj^@Q0!=B1;aIMK@26Xa)X`sFJP_q}!AV|^jdjbFeypSy})u2@H z7mS$hxnrx~p$Z$&*4-K~C^fdC9pVb+AWY zNXtAi42oJzGFhoo3&|u(pCr|R-BrbDXzz8wl~GIUO=5BCVZWR$=W!@30cx2`Y#OE;)z_ zTZCx={3F8&bxVqp{&taa!Nsj1iq%nK=#!@44CezISlwG~oV!_})Rm$Gk-&0L8?7xz zTX*Fzi^Af_E}w-BgZA}dGV-$b&&Cn$&c?@I26H#LWY$u8e0R#UH^B_*2dhLb3GlU^bSL2>Ri%-F^+Fs@1wnhchyChK>X5EQBAB|tED*Ov(UEwJYA z!dO7=lTgtW2HK=cv+lNqcw*9n+S$y!&%Mh^9OvjIv!6VP;QbmUtJed>+m(E}zQGrr z-qQvRUV08I{xv9 zM^IpZn$3k~aF7fas(2@Gs2)HYOS(YfG&WsphqAE?$#u9yM~Sg2Yyz_^=k^&T>cpE%twf31 z$^TM>ktx-N10~CgIWUwNxTEivhxys*B|YSX4o!u0D{Vjl9_B`|yTL@64tL534ZZy$ zysAsoh!pj83_-6mF}lmiUo|Ic5eFRWRreCx=;>h90LoR$3;UqM%RSs3IK7QE7~BPX zcl>23oDD{)FQ^Vy=RS-+lZN@hBOTs2&?7D|7#!;2$wnZJ2*RQjkkoSdK5eghVG$5? z`G(sPjM@D|s=e{l!rvcnqn+<{p&d|bPls%J#@JImC!Afs z3CAtCLyde#im!w~$1nR^)wdiIe7N4>P1I9a`g^x^FSH#$+#cNBU^EJJjeHJv6~FKu zpC0dWSK&)Btjkl!F%K`v=yNihxJRCD#bb`0EJlgw+2+oawF@*0S9`>&&t@mQ4(17f zu)}l*vNs-afR7!P@OFjSJw zvEhx}h$GlCL$?e7(a#A(xr9p#{cBsitK!GI>&B)xa65I5xE!UN7_GCgvBtWg3kN&a z=zraFPrQk=a*CR{k{7#Jd6>jS!yq(8C`CZZooLM9A<~eBm$9Br#n)gy3cv-A-9u7^ z+Iw8+qi@#dBj|Cn;Xz}6Fd#>)=Hrg^1)0z?NFt*ZqJxR#pNf*|#MCZ9NnAYPZ?7Yi z-8x6b`|LCYoDd*$*CX=U=Q8+2Df&o4+CzEt9(>QkXg#cR;F5)pzU3u|NOf@Cxv^US zVRfe?HyNp)phqt-Q0MkZzX!{?7%%)_K_?KF)F0>J+eL7!$1@4o{XYWAwNRvo1V`;;S-3rm_mF49H! zF=?B2Cr=w_psJ&UvJ{=QQN$@1Et8t)c}g7dB#*-b?wNkOgt4r0pY;gx8G|Pnx8qSg z7M*%559l2-sG1Ihfa^}H&$Wo=ng`geTRAJ!hH)eAML2krs{Td z_O!*aN69#}8_}A80A^qncDk^=T|<_{V+oog?=)-rW}nMlhCuE4MBC~7D%`ZPXGk4|96!2kHxTm4^1|z zPdWf}u(v*YA^=gI%q1Ce!^e}^rGyr6*l8C#ZM4Z{M7K#9a7#kYp#-+T!k_?8!zwg! zY_lM9Hj_>6&xVr|<1)|j=MUp?^KP#Si#OwO@%~VF=me*ntJ}~T>FqbD&Axo_zU&+n zS5}7i-OA?oNc!p=7tW3`XPLfAgsBZij&8!OTxC#%m3Nf_`Q)OMs zuvOKt-Fd$y%DmXA+#2q}-G!>G{Z4VCft$D_Wuwr;;iZ82k}hC?_nvJBg8dT7`hw|! zquTbA&JMkG61Rmu)JSK;;D)-kL0S*x9Kvi7-ZoKI^9cWO#dwaYVfsWJCPOc zf}~YNe77IMKw(BpgtFQmTrRBe=e_PxOqUDJjxh@txMre?C|VrS9hhaN9pwj%>f7V? zHZ8BdHB3HO8w<~rcD9ne^8O?ay@JUeV82$kXp6 z{NU#%3%AM-6#SFfL5G~*1j?sS7UW;g%b7x%5pR3P5=k~8ynK>gt%)&I*k@(#wKR0Y z#9~H<4zQQhG9g|UKk~JUNGPuS$ft-{gT(hqf=^P+DaBPl_(_5KG?mSk*=H@fR*N10 z1O_|X*MU34)A~837CqEja~=e`#O*U;bNytVCSPH?KG2mD=&|SlEu9RWak>y%5lT%_ z%K?sHMDHD}$T?m}L=L!-WAu5vKC6jaW4@cab$NTN#fi0X;`(}FnWf+gM&h}4aQmJL zCyxiK&VANyT3O}cLWb*{8~DDAwQt5GO zn%K1LE1+~2#kX)J7k8yf1x59ob9)Zz3-L`cjACoq0ih-r9t@(`s`w%ODdY}XrKRr% z`CDQEFu2^K$PBeSp$5Qemi-0m!iKP)C8>KBJOs665)0vG?uERIOC)2%CZw79Pm3ot zbBjT)1_bvRQHB+1^3;WVomm($FPa&etwFyiKjH9ritmrxzD&upKkFw~TiFu4o9{Qr z{rEPvQeR$O@yNvW)mpY;%|ej>Uip~R&eNnaJ8;9%%5tT=Gqdleg8`m-DqT3;4YAU6 z%n*(CC+nlu{3oOMR0W~q?M!$yvX*)-yrNN<)241(AF5bS7%_`lXwb?Yqdq4i%N))F zN8b--V)s1)T$cEgBK~nr#d{9JM=6{eRxp&YvODI;f}#DW?ALbgGv+cRd#ZiyEzfL z{-KB0v$!&NpLv>3)679IOAMjxa(IIQZA=0c>E?r~{7Yrhk<3@@c;mvT*u8V1pc5%-OTdtLI34 zJb}0QftQeQC*=GpSNSoQZODAUDa58h9JvI{aD*he$OJT2>ctdRaZSKTU6bVCwhe=A z<2p-+d!FEXG^VIB&A?5JY#_EXaC<}V9O7-q)DVW;0+T&a1N|-$D4wvw94K_e?eGQ( z>5vwPWASf5!q^pFp$4NBwMDLxQIg@EUIcm@XVFQ<95+Q!CllR`3zJNapo!jp-;^d&lvPEu4%E*3AyG!5MYjob6OiMYzlr6mIZz7_5NV;RY`)1l$oz zdyJ$qJ~^D-K&{w?4!Ne{=gR^`WmuaXgCtNS?ZYA<&P5SN6-1b4I}oHkK`Tm51VK$ISqIl|*D7$jsinJu?^HAGxWm+Y)a zHb}Fy-iT?QIjxqr=4VDG81Gr$AhToKBiiy%5`i)g@QQiQ{mOKJi{1k#bc`aHJ8qNu zK&?!~O;6RZqe{%U_*^w)*!CYsVv@(5R+``QjO)h{7t&aJFf^UZuDa6HfOYd(48U)I zv6g$i9$775T7^I#il+mhXoN!M5OxDBto`kpnoulyeed0Yg0+C_MU#)y(ht^zk^1#6 z<3>YHxt6MQ9;7B<+~-)W^DtW*t9&~04jtezvpXW-%8wSsS+^wb3 zmsAkCMPss+bp%Y{GxLjm44JnrOiJYBx7hD6-Ep$edgv!+dTS-ixf;y%Hru0(vLwkm zu5Du68ic(kgFbjt@B57~z~5hi`)kVy>%yQ7h$ZFYHvv`Puk{J+qf1&-{FFn??Ukk? z9ojO|{=7Fv&#z;%9wgHU{C#WN`#V0HE712g&yP;k$ta&9w8gJM7Sl*SERYvS^>1=I<59!Q*=}n{tAUSA zcg?>m48xThrg;hJkmLJBzb59HVM`VoaeXHkTki&Etz8Eq66oiWP=p~V1Gu(z(QB!Z>tBZQjM?tI!;37g-<*3fzNb*S|!#jYYQOc-}qQ6Ih3 zSUANIMX^>ayTtBOWM;BpYe8PUjshpcvqp_Q@VD5c5UtQW5GuQE?T(2jIGIj*N>9i+ z^lE6`p`|w&ZAlT^W1x+Vo-S8hxpZ$eZF5z|%AkM(nIcB&KE33mh6Mr@Uy|Y_xO$;s z6!ShaVuuYTLawjy%i&E*(?IZ&%UezfsjkbTjWepBYDp?0Wjh#PjWYSOVMyW=-R=PAnN^Qd z;O?{ zXCu&Zd;DnwNQ%5MIn!-KcU0=uEl-}EmFJGgQrEVf5BV62@}+=}?05-Ga@Ds2m}KvV zbP+tVW-7PO)*Ls4i56;WTB`P1VfLQZ=RvZh{%a75($RSUOD)c3AymqaKovwo3KFWd*_xx{W%YjH*TS@{YS>MzKWtXj zltdxD9&?_(VW-RRI9QjmA(u9^Kn}Xi*HNp`PgZn8g-Gh!WB>6?5{M|5^es)BA)n?NWKyW@6`LofvL2 zDPDrMMThnh4eF15;sAJ-k@tY1bp@Vjm4azh^zS8-D_%nrKhxYZ7k4P=UX~$oy!V4^ z4_W0=P_^->{7Vac(0dRfYE|-d6Gl)&pRz^NC}~l~`o9=_!icH4cOFYv5&JlM@e)@~ zZ+o$xv~@t3hhYsXk1H@fg~z~ajh7EIFjCM?3xnK1y<1UZ7-xw0v5D0pFVf# zj4%O7h!TnjvfplnY0x++XS=193p!L;LonWlToe3~zToFsWyK#!I1w2)=x3#-_A|WX zz?IJ)^#vlMbskiunbGXGn;j2(9b0@}>XpxEeDeHWD!i{6xSh?rDs!y{cvXj5W64+TCaYEI2A2p{b4R+Qj$;V zl%@4^sF)IS1{Gt`0c9m8CeHn8;kTz{_aWuX=D^fOgM2k7^yjf#r%tiqS~G9jU2Bo? z>+cLF@Ok7khv)59Z0Pur;91R-L#HUj(zMWu(Z{5N9yB_k9Q6qt2|7wh=L-@{)ItgP zJuVq1Rnemp>^~ppe|Thn&8lB=xSxyUmZy;=IGL?j`T6v{1-ooP7JN$ex$>T%%jF=F zT~()hcJ}RbN!4G@#K{7u(O?5dWTtb$ZWeBDVTV92$bA}qx5bQvy8p$k6uaB285`Vg zJN9Q(sA-^N(KyC}2HjWY=@a#y`yY(J!KPfQK`~pkd~Iph3GzTbyX`#4r9J8skq3?w zWHhMl5gNqwJCtk2V;6RBbiFZ5%beMxA_YkH#fQfaDZr*bvh0E5o4clx94vIJDu>wb z2Q@u~;~iYsPMBPr7(Qs76S!#h>m!tE2M8L1xEY+nw5=DLrV;9eJ1kC$dQ>zze9)3@ z=B#$y)6(Y#CgPAy}f<|Z?@d&~Gd_Y*X~QKrbA$eGq(F+u9!J|GIF zqKFzVH(A2Jz~zJFM~S3Gr$msVs~?!=XeErM7xhbQCry0Pexl|&M4#5J*I|NPv>=o^ z9bv4yde70Uo`D13I}dZ325|0~Zd=~zHke}9ezXs?W;?zJGRtYc>0cZXV4jvt9_m$o z>wl)NAW=29s*eEG@fF{#MIFcaL5C-QU6D|eU;|o*8x(#}?0rJ;vM81;$$m#KpbIVF z(V8OL64xHF?LVNl%dI-*p;{}&d=+Tr{4$nWiJD~Uy{<{fX{;z`crrI7RnA|#9nDLQ zD7G8FfLpXh$&+_Gah7_whlpc_lWA&rKd=cdJDHupiHQFh&r5}eIlz|l?TblJDP$(n zu}HZ1UV|YI<$28vS1xbx)g5Nv^gHt+z5$uXH~3)M8n2IB&*KS&c$t0n?vTCf z;;k^U8PD+Wh{&X@j4tb#vQ`=z`Oi*-^PrZt561R^T!N2xpx?t^F}k&iPj>OjHC1(( z#koY+{2I=>=V7lOgJ!c>` zu(3NiFdsSQ2bm1%%>SG^h3tVE#u|iyr z^@UTkh+)2}^~2*m>c4CG2i&U7a6@KXgN}Nh)cRcmAHMRx`oN}f{&-F2=40{PLCp90 z0z(^Ifx`ibWS`BcZ4bGw3<~XFCXZn8 zL*!&fTS4qG2-0sfuU6){1>I@pIr6wv#dHeKs9%!ln4T&+=7o`%&eLysxIVs*&5Lr4 z_9rYW6jrcQbEoM=o7~4#&uV)`Bw!+P&CQYTZ*-7JNE6(R&pA?>yqnhb=W%o73V9@E zKe>+8hX@EJ8#DIC4+rz{dMj!REHWQ?ruUgo^&e5b-hQyca*WL8xMfhp$CS^$rh6Vo zA+D;<3Yvk&Cbtda>%A1ovPBsEDe&Pi`^Wdd7SYCnf-&T5gB9a$;Ra%_lAl%}NOI>^ z@Mj{%dsV8+g6tnS+$q$+?>%ExO(c9QyA|w}B+GMFPO1_YW;M-=7ieS9H2SDDG7N-G zIZcEkY<2a8Css<3?l4!&E6@W+sfz?Cq>#s|A@5f?PBs|Mn)M|BOT1III^4vSC$G1) z*`J-L&&m$is1jAVp?_$NcTb+MQpvR;hJ{0+=7k_0X88o0M@qWLkWnRWV2)nJ4mIs1cvE8Ce<@ExP>w* znAaQDIk!9cQdUIe_R{%uKsm8%^BY0k?w$r9Zp zTAG8(&&$xHcYwglgc`yF68Ce-C`q+mm(S|Nmn%4aEi=NFAfFRS7}Op^9Fd zUoHkqgIKu3-ozjmboBMv9#$r#2no62b-{)gBaBXJ!#lV#jxE0P$2mQh@52M*IoY%0 zIC8Pr#=QvtNQ2Ut`>LR+4DJpTbNCGAp@UMW)-x%+;YQLx$JKk*_9OMW zU>Len`X?lR3u19X!u!VqCo~@ufhcn<@O?PNSNq;nVV3TYT}sBp-4wIN z$b}Ewdze21F%y{I1Hd^6&ng=XzB9^|s|1^;GTk(E1)!nzZh@QJ`7yndf;J#~+CoSy zGJd;i^Z9mq^KQyWCmvMeBi%C`DKE)`t0t0~BvzDymN>5_nwLd&ncLzi1b-o1JR!5( z!QYfxq^jmy55nVBnB&V`dH5YGZH&zE(K&*(yYz5M(yUDYZlRK5Hcg$1$esygV zytU5z6?`hUO>^(=+ibH5Hg&bmrjdHtXrSO_zOYr|OXqWI0r&Ak?fspmAN=Iub>ok> z9)5ot8_sMBUiun`iM_@)x|@xLK}4yzp1SX+z5%qzyP>`kc_A^_)G@ zv0QvIcaNTD>jVmK1X{K8*BEXO?=@)NW{Y%>7di)bzGzE}rPD`W)lYtG5Zpt;p$>kCWE4V&de$kXVI#YOT-+2U(kk4`5Wi#ax}2Rs&B4i>Au z*+A3}FLLRvUeRl$q2Gv`@s0VyeIEu=p>ofer=UG9vP`gNWk5)l?XQ}m>NUGdq*iBnGldB~EWP6v`#9^!9!b6Jyp+m??)Yw*& z^qnvW|6~W-kV)lc>m>ep@4^RAdL(J#6}JT#PmN;7ynV7 z`1?tsQ**;%P1U&@9Y_#~h*B!aXxt8GAk@IYLi(C)<)THC-P)3TldHjE<*COSEzp&o zRDwW2*C%dJ>DYSTGER=!EZuv;F&@+Ge)Ov_etWzex7P{r^K|e%=}#fDN{?^g0!l;+ z4fKg(=q+1^Ssr}ehXN1dM*clhHs0PzZ~HWaKjq-xOs^iOUIXbBgfR5yiB)e3&=Tbb zm9|S=&86#E!o13oEsm2m$F!bQd#M`2RBMgY#Vq>LOcN`V$}JPk>cj@q5~j*j0*k-P zHWpPLFEXC0TMVU{;WkG#8&_&@vK?lcW;Vzb-F~m&th!cicf2h1s`FlTUnN~kU4ov| zct_NK`aI|kVGgbupowTsHvfSQxDnd=tZcDffxeb$e*BeYKHJ|tP*%uGX)!z4JyBNB z3&wcrmg%dng{)E36;JxRkpA%U1o0%-@aX1~SD##+hVw4sMxkotjFD3M3l^tZ^^AJ^ zo`^-GP=(H2^Oy6qwUFJu$q-nR!M$4Nak%{D5^maW7f!3wd33iF-$*Z8U%Id#8?|{} z+H6-Xn%832?AztQBAXw{9(GS;yGOwvusu_Q30Z~{e}E4|cw=O%AH<$pa#aPttXLsh zwnc<9u+{iq%QpA=j-(D@*;p<|sh>Br;IGU$rw2z8gX`nM$X(sg2A12xaTzR+ddHOYHM3Qdz7O%>v_PZb0%7XjknM3#6`DUyW1FPva3``20Ja2 zYj+5aZi$Hus6X)&$K7E+n&v#d;)HYihNl6Es7)4ahAW}QdqJL{%osk*Jwq?ID_Cgx zn3>dx$&!&6br@pB?b!a>Y%4gbxXQ~fw160|*X?&I?ui;8%>(Wrejy_Jvk#xFVK$_S zx;@M_SA0oDjQ!KKPyr*jMK_TyRr?RnXGAllBBpc2T=8l|9t~QxpI9F^y}nP%VWd*8 zcNj7?8$C}T_N{d3RXGwlu!c;4Rwya$h#CisgsJE)cqHOJGipEzeILoD{@8Gkz&ox@663WrfXNW1 z!cmvs9Ma87`mBQs17ZGNLT~NLn7!S8!?-dMj!v9ds{VqieoL>?F%)fdiSqJx?sA}Q#ix@ixztXrN`t<`CwYNGmq?5=3d6;;&mzmR9$}1PPn7z* zLFOO%!#12R*PN^NaN%_&TfPVmU&rD3_R+*R)nbVWO^mYFaO`59aRd*#(b%i!KqJ%R zH@Jc7$O`%nLrW{59G%ICmLt7MQqp^ft?OhJSYOJI^qkio4_w>f;;JcwA121j$tTWc zf)=Y84vP(8coipuRxkj+aq7g;)zW<4?YUEl)8H8%n+_Tg0WBawgeD-JPfu-mT}>k< zvFQZ~e`bFwtT|_SOvGkor6VOWx8c$W7KY31&J#;4U>~@;P&vr_IiNMq9g}M2siZ?4UND*@S*VbBw>Rq_6pO* zL&2j_BCcb%lpfA(=5~g$cwPV{ERUbZUx^i59$?Pjqe{iOZTG=I1*F`U-lVqXi-?SY zF~HGo(s^Oe-=f|??$bT7MhqQti}b~Mc5<`gjomBU4-wv)cwlhn7dDYj_u4O9E%T;N0 zjT!An=QjLQ*V;&AN{vy2{+t&2O_!jvmM$87$c@T!eoPMX+JS6 z`dUaea=S6eqsOY)hpMxZ9)rT-3(y`vHdfEZ;xjWJ#u-k((-#C3k8GlQnUe^;arg$F zX}k$Cbw}}2;FA@As-yA_?xDpfPLa_bzCil`ABfExwt3i=U>~5A@0ppWxKjw605TCw z^SY_*iEIV8l^&_Tpi=q1-AvH1BX!Zt4tI9QV?PBr82?~>eg{GsFg@Jk!ZA%Hh+1qr z);u+k=0|%M)2}c298fwJuh33_gdR)OTEp^M$joX z2b`{+ma&`KCaXxUFP}m9y1=@;O6bq03g1vy_*0usq@l%f%=EO4Ahb_JEIVp9;DOYF zte1KDT-j5RImyIMVQZUkZy;v{23l(nq3}*`5c`l@MudHb`88W~WCCF$*Rty5iA1hf zt@HAh)7Iu5psA>o`BffnUXE%TE*vOJq$Qyz@aeazbS+`f&!UmFU)l)KhSqP0Q1IjZfxde8l!22cG(`B`)m(YB^i9>& z*o}igafVbwnv=9MaKsTE;$Wa3u7@=D1g?zkepwKee|DDSCz_*3n?C+|CKnmQUPe^Y zGU*3VqKv;c6yJ3I=#4P?3Bmk)>bY%S;x1-@AW-P^6mYl8cLKd!oDv4>jh~2;fvuzn zL+?hGt9&F2aUmzwJR#_lu#mE-7~I|PiM|(pGd291a@=%=8gVZ(p=Jd>XrnW5w#*dNk!r& zUY=rzL+hWpjqI~mI&jTdJ|K4ktz)c`!Z=uzTi8c#B!<`k>-f%LjIcMlMid9##ZpX; zuX_ zJ=t_HCIBK@LFX7>5o}i&*+W?;&SSL{{l|m-WKEVp1x=7$ z^UpaBUHZ6R&)*x%m+)`IAVuHh(uNwy*iW_@F2S_4*R40#U}*N9xoxyNyFD+zYb|}D zMZ%5Rq?L3K>cGKw0eis3{_f*cr!fSeHK04&sJT!-GTh@fa$$p+}=6TUVc}+h`xYAi62M1=Ivkn zK8yD<>pwxLQ#x=T5lBuj;%2ADnXRP2E}OC8Y$cQ$YA>S}ZR;51PZU9E+BfH{h$v0c zb{Pce@zJMSD9*#v)6>D9cWt+X>;32x&8mGg$W{)gfI$~`?XE#<>{xX)I2|(uH(}u3 z2C6pdLLy*vQ8$$pZUKIPHEf@0I@ibBNCVF8`Qe(sQLEKqBu$Bk?3$Xm=5o2RbgK6C za?$DKq*7~fR7GZR9CpI~dX&fjiG654xW&Ic*=jwnHS+1h5v`;$9$A`YB4aq0#1Q)I z2zdmnXE=2!qX)NH6(&-bR*?4CkdH)z9kFj^piEu`k51&|+M?Z4T;*0W8}1b32J<7% z@uY4y@vX|9-+8ytn!V_%en{=?mR}wvOk(%e30DX5L1>W~5Fi$XVvCFVI9Kz~4YNv< zxtL(nPW~~qVTYje?D1A+Ak<;&Qb zo*)a-0ow<~Vngx50U2V4Zxb zwfn7DK7$kpHq{*$9MJe);+CrH2Epo?VOzs!G7R{rvZx?+4#{068NYFa@g4*VFAl+X zj9pxw^F5X_tl&Y0vS&EVs4^WcnMcCK$Vmw*XyM5BWm}v+q|X%Ze7DB|&e=~XE3jmmY$fbxohB;+~YvCw#HNd9I zfL5m^r{x(?Jnf^K`)D}v1Y_}ytU323TQX^~0QGTU$kd(fhv~5_e}z63xIRV(DGze5 z8G}ZQr-HvKs;2G`4p|X=oT}O5yt|8(F&-(Dq`d>|#wZ^ZA8VvKN@UG;9iO$3C9lY}QS%+YTSf`B6H8?i*eQ8HBDCa2y} zDD17X%w#3NfH&r-E|`7JwFevf*)%6L#m|QNd4ld{_$@pk*_BJ^HA_3?H3zL!5-F{; zeaxhu`xB8n&25VPW^5O&Bz3V{f0gsc9JOZmBR_Nl(F#4GN7Zd0loC{kWNN>u(6((8~9yh!{bT*rsZey(oTf!Izh`pC|!&^JIR=fa^32+Q#!GM@oYG47(TTSup z)wfWjNgba(+Hs7-NW=U9iQzT-CSi4nD#%a0AUAhq>TvYq?5~2(DTlx!i*)JDNJJ1g zZPYG!SeeG}=y9>hvM5ULHE@piQ!x`~I?to8Zse8(B;|pZskeuA3z|^$NS@@4;|ky= zs9fa=$8wY1?HpU087ALfv6BzCI(WK2#AQKB9!#o&sdi9UOECsw%ic#9$J3q@%7v}j zPr${VLMu7DJ7Z>on`^PPOlV(Qi*7 z$TocZnuC^7f=&vGe5FSMXHEg#v&gCuzpees#pwUlKl-v40=F9x{Lxo7OU)%%IG^aStlV6Zb3z>}t%0$2kL?=R=1qyodo5+8{V07*s;wqZUq7wuw=a>i2w+ z=zCa~6zW^lC&k^4hON=z{M%XROjRiIZ&d=vkQaI~@2wKQ@zvix-0UID!{f}1Qy98kTwQ#wv zu9>x&qpmKXaKc~5@>}_ZO?UJABxoQYaJ*k_o^)IMc zi@%pn|9gK(08aBal#}(Be}ViqC=Mt~h!KEgKfuKPePXnJfdI;={|nE47qqzt}0s=a<1_A=e#Qo6< zd=vf^^}9mcKRx`{f@Z&?00Wl#w@&@BqT`>aUu%{8i5gM*H`G5foBqlCRZjm;rW@cK z^H+cR&z$`lDg7t$SK;nIiTc|AM*L4O!9O{-CjZ9ytH1yM{4)Kj0{Ewi zZqxr|;`bx|f4k&wgZOnd&z}}F0{@qVKRn^ToA@=`?N1XKasSK2KRx8X+xj)C{->=R zfGXraz2&dRwb-%s(f9VHd AssI20 From cf00b4a13f5fdedda19c3cae214943fc28df52ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 00:42:32 +0800 Subject: [PATCH 08/55] export --- .../python/onnxruntime/funasr_onnx/vad_bin.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index 533b4b7fd..9568ac981 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -23,7 +23,7 @@ class Fsmn_vad(): device_id: Union[str, int] = "-1", quantize: bool = False, intra_op_num_threads: int = 4, - max_end_sil: int = 800, + max_end_sil: int = None, ): if not Path(model_dir).exists(): @@ -43,14 +43,17 @@ class Fsmn_vad(): self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) self.batch_size = batch_size self.vad_scorer = E2EVadModel(config["vad_post_conf"]) - self.max_end_sil = max_end_sil + self.max_end_sil = max_end_sil if max_end_sil is not None else config["vad_post_conf"]["max_end_silence_time"] + self.encoder_conf = config["encoder_conf"] def prepare_cache(self, in_cache: list = []): if len(in_cache) > 0: return in_cache - - for i in range(4): - cache = np.random.rand(1, 128, 19, 1).astype(np.float32) + fsmn_layers = self.encoder_conf["fsmn_layers"] + proj_dim = self.encoder_conf["proj_dim"] + lorder = self.encoder_conf["lorder"] + for i in range(fsmn_layers): + cache = np.random.rand(1, proj_dim, lorder-1, 1).astype(np.float32) in_cache.append(cache) return in_cache From ee3f4e5236a1a4cbd49f0defe4a8ee720995074f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 10:45:58 +0800 Subject: [PATCH 09/55] export --- funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index 9568ac981..0b7ecffc0 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -53,7 +53,7 @@ class Fsmn_vad(): proj_dim = self.encoder_conf["proj_dim"] lorder = self.encoder_conf["lorder"] for i in range(fsmn_layers): - cache = np.random.rand(1, proj_dim, lorder-1, 1).astype(np.float32) + cache = np.zeros(1, proj_dim, lorder-1, 1).astype(np.float32) in_cache.append(cache) return in_cache From c039cbc3bf3311c370d891c1bf67b275e95f0cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 13:20:27 +0800 Subject: [PATCH 10/55] export --- funasr/runtime/python/onnxruntime/demo_vad.py | 28 +++++++++++++++---- .../python/onnxruntime/funasr_onnx/vad_bin.py | 10 +++---- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/demo_vad.py b/funasr/runtime/python/onnxruntime/demo_vad.py index ae033cc5b..2e171978c 100644 --- a/funasr/runtime/python/onnxruntime/demo_vad.py +++ b/funasr/runtime/python/onnxruntime/demo_vad.py @@ -1,12 +1,30 @@ - +import soundfile from funasr_onnx import Fsmn_vad model_dir = "/Users/zhifu/Downloads/speech_fsmn_vad_zh-cn-16k-common-pytorch" - +wav_path = "/Users/zhifu/Downloads/speech_fsmn_vad_zh-cn-16k-common-pytorch/example/vad_example.wav" model = Fsmn_vad(model_dir) -wav_path = "/Users/zhifu/Downloads/speech_fsmn_vad_zh-cn-16k-common-pytorch/example/vad_example.wav" +#offline vad +# result = model(wav_path) +# print(result) + +#online vad +speech, sample_rate = soundfile.read(wav_path) +speech_length = speech.shape[0] + +sample_offset = 0 +step = 160 * 10 +param_dict = {'in_cache': []} +for sample_offset in range(0, speech_length, min(step, speech_length - sample_offset)): + if sample_offset + step >= speech_length - 1: + step = speech_length - sample_offset + is_final = True + else: + is_final = False + param_dict['is_final'] = is_final + segments_result = model(audio_in=speech[sample_offset: sample_offset + step], + param_dict=param_dict) + print(segments_result) -result = model(wav_path) -print(result) \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index 0b7ecffc0..cdd4578d1 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -53,13 +53,13 @@ class Fsmn_vad(): proj_dim = self.encoder_conf["proj_dim"] lorder = self.encoder_conf["lorder"] for i in range(fsmn_layers): - cache = np.zeros(1, proj_dim, lorder-1, 1).astype(np.float32) + cache = np.zeros((1, proj_dim, lorder-1, 1)).astype(np.float32) in_cache.append(cache) return in_cache - def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs) -> List: - waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) + def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> List: + waveform_list = self.load_data(audio_in, self.frontend.opts.frame_opts.samp_freq) waveform_nums = len(waveform_list) is_final = kwargs.get('kwargs', False) @@ -70,13 +70,13 @@ class Fsmn_vad(): waveform = waveform_list[beg_idx:end_idx] feats, feats_len = self.extract_feat(waveform) param_dict = kwargs.get('param_dict', dict()) - in_cache = param_dict.get('cache', list()) + in_cache = param_dict.get('in_cache', list()) in_cache = self.prepare_cache(in_cache) try: inputs = [feats] inputs.extend(in_cache) scores, out_caches = self.infer(inputs) - param_dict['cache'] = out_caches + param_dict['in_cache'] = out_caches segments = self.vad_scorer(scores, waveform[0][None, :], is_final=is_final, max_end_sil=self.max_end_sil) except ONNXRuntimeError: From 5016f29452ce6705e038d1484044404a45b7b54c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 15:54:04 +0800 Subject: [PATCH 11/55] export --- .../onnxruntime/funasr_onnx.egg-info/PKG-INFO | 80 +++++++++++++++++++ .../funasr_onnx.egg-info/SOURCES.txt | 17 ++++ .../funasr_onnx.egg-info/dependency_links.txt | 1 + .../funasr_onnx.egg-info/requires.txt | 7 ++ .../funasr_onnx.egg-info/top_level.txt | 1 + 5 files changed, 106 insertions(+) create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO new file mode 100644 index 000000000..94d2cb8fd --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO @@ -0,0 +1,80 @@ +Metadata-Version: 2.1 +Name: funasr-onnx +Version: 0.0.3 +Summary: FunASR: A Fundamental End-to-End Speech Recognition Toolkit +Home-page: https://github.com/alibaba-damo-academy/FunASR.git +Author: Speech Lab, Alibaba Group, China +Author-email: funasr@list.alibaba-inc.com +License: MIT +Keywords: funasr,asr +Platform: Any +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Description-Content-Type: text/markdown + +## Using funasr with ONNXRuntime + + +### Introduction +- Model comes from [speech_paraformer](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary). + + +### Steps: +1. Export the model. + - Command: (`Tips`: torch >= 1.11.0 is required.) + + More details ref to ([export docs](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export)) + + - `e.g.`, Export model from modelscope + ```shell + python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False + ``` + - `e.g.`, Export model from local path, the model'name must be `model.pb`. + ```shell + python -m funasr.export.export_model --model-name ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False + ``` + + +2. Install the `funasr_onnx` + +install from pip +```shell +pip install --upgrade funasr_onnx -i https://pypi.Python.org/simple +``` + +or install from source code + +```shell +git clone https://github.com/alibaba/FunASR.git && cd FunASR +cd funasr/runtime/python/funasr_onnx +python setup.py build +python setup.py install +``` + +3. Run the demo. + - Model_dir: the model path, which contains `model.onnx`, `config.yaml`, `am.mvn`. + - Input: wav formt file, support formats: `str, np.ndarray, List[str]` + - Output: `List[str]`: recognition result. + - Example: + ```python + from funasr_onnx import Paraformer + + model_dir = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" + model = Paraformer(model_dir, batch_size=1) + + wav_path = ['/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav'] + + result = model(wav_path) + print(result) + ``` + +## Performance benchmark + +Please ref to [benchmark](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/python/benchmark_onnx.md) + +## Acknowledge +1. This project is maintained by [FunASR community](https://github.com/alibaba-damo-academy/FunASR). +2. We acknowledge [SWHL](https://github.com/RapidAI/RapidASR) for contributing the onnxruntime (for paraformer model). diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt new file mode 100644 index 000000000..e759e2778 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt @@ -0,0 +1,17 @@ +README.md +setup.py +funasr_onnx/__init__.py +funasr_onnx/paraformer_bin.py +funasr_onnx/punc_bin.py +funasr_onnx/vad_bin.py +funasr_onnx.egg-info/PKG-INFO +funasr_onnx.egg-info/SOURCES.txt +funasr_onnx.egg-info/dependency_links.txt +funasr_onnx.egg-info/requires.txt +funasr_onnx.egg-info/top_level.txt +funasr_onnx/utils/__init__.py +funasr_onnx/utils/e2e_vad.py +funasr_onnx/utils/frontend.py +funasr_onnx/utils/postprocess_utils.py +funasr_onnx/utils/timestamp_utils.py +funasr_onnx/utils/utils.py \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt new file mode 100644 index 000000000..6fcb63279 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt @@ -0,0 +1,7 @@ +librosa +onnxruntime>=1.7.0 +scipy +numpy>=1.19.3 +typeguard +kaldi-native-fbank +PyYAML>=5.1.2 diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt new file mode 100644 index 000000000..de41eb90e --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt @@ -0,0 +1 @@ +funasr_onnx From 02652ef9891234494005a92ac937b9d1e9964ec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 15:54:59 +0800 Subject: [PATCH 12/55] export --- .../onnxruntime/funasr_onnx.egg-info/PKG-INFO | 80 ------------------- .../funasr_onnx.egg-info/SOURCES.txt | 17 ---- .../funasr_onnx.egg-info/dependency_links.txt | 1 - .../funasr_onnx.egg-info/requires.txt | 7 -- .../funasr_onnx.egg-info/top_level.txt | 1 - 5 files changed, 106 deletions(-) delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO deleted file mode 100644 index 94d2cb8fd..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/PKG-INFO +++ /dev/null @@ -1,80 +0,0 @@ -Metadata-Version: 2.1 -Name: funasr-onnx -Version: 0.0.3 -Summary: FunASR: A Fundamental End-to-End Speech Recognition Toolkit -Home-page: https://github.com/alibaba-damo-academy/FunASR.git -Author: Speech Lab, Alibaba Group, China -Author-email: funasr@list.alibaba-inc.com -License: MIT -Keywords: funasr,asr -Platform: Any -Classifier: Programming Language :: Python :: 3.6 -Classifier: Programming Language :: Python :: 3.7 -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Description-Content-Type: text/markdown - -## Using funasr with ONNXRuntime - - -### Introduction -- Model comes from [speech_paraformer](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary). - - -### Steps: -1. Export the model. - - Command: (`Tips`: torch >= 1.11.0 is required.) - - More details ref to ([export docs](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/export)) - - - `e.g.`, Export model from modelscope - ```shell - python -m funasr.export.export_model --model-name damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False - ``` - - `e.g.`, Export model from local path, the model'name must be `model.pb`. - ```shell - python -m funasr.export.export_model --model-name ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch --export-dir ./export --type onnx --quantize False - ``` - - -2. Install the `funasr_onnx` - -install from pip -```shell -pip install --upgrade funasr_onnx -i https://pypi.Python.org/simple -``` - -or install from source code - -```shell -git clone https://github.com/alibaba/FunASR.git && cd FunASR -cd funasr/runtime/python/funasr_onnx -python setup.py build -python setup.py install -``` - -3. Run the demo. - - Model_dir: the model path, which contains `model.onnx`, `config.yaml`, `am.mvn`. - - Input: wav formt file, support formats: `str, np.ndarray, List[str]` - - Output: `List[str]`: recognition result. - - Example: - ```python - from funasr_onnx import Paraformer - - model_dir = "/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" - model = Paraformer(model_dir, batch_size=1) - - wav_path = ['/nfs/zhifu.gzf/export/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/example/asr_example.wav'] - - result = model(wav_path) - print(result) - ``` - -## Performance benchmark - -Please ref to [benchmark](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/python/benchmark_onnx.md) - -## Acknowledge -1. This project is maintained by [FunASR community](https://github.com/alibaba-damo-academy/FunASR). -2. We acknowledge [SWHL](https://github.com/RapidAI/RapidASR) for contributing the onnxruntime (for paraformer model). diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt deleted file mode 100644 index e759e2778..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/SOURCES.txt +++ /dev/null @@ -1,17 +0,0 @@ -README.md -setup.py -funasr_onnx/__init__.py -funasr_onnx/paraformer_bin.py -funasr_onnx/punc_bin.py -funasr_onnx/vad_bin.py -funasr_onnx.egg-info/PKG-INFO -funasr_onnx.egg-info/SOURCES.txt -funasr_onnx.egg-info/dependency_links.txt -funasr_onnx.egg-info/requires.txt -funasr_onnx.egg-info/top_level.txt -funasr_onnx/utils/__init__.py -funasr_onnx/utils/e2e_vad.py -funasr_onnx/utils/frontend.py -funasr_onnx/utils/postprocess_utils.py -funasr_onnx/utils/timestamp_utils.py -funasr_onnx/utils/utils.py \ No newline at end of file diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt deleted file mode 100644 index 8b1378917..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt deleted file mode 100644 index 6fcb63279..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/requires.txt +++ /dev/null @@ -1,7 +0,0 @@ -librosa -onnxruntime>=1.7.0 -scipy -numpy>=1.19.3 -typeguard -kaldi-native-fbank -PyYAML>=5.1.2 diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt b/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt deleted file mode 100644 index de41eb90e..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -funasr_onnx From 1f8b46402c2852b68e2f68134364842e351d353d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 15:57:22 +0800 Subject: [PATCH 13/55] export --- funasr/export/export_model.py | 3 ++- funasr/models/e2e_vad.py | 6 ++++-- funasr/tasks/vad.py | 25 +++++++++++++++++++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/funasr/export/export_model.py b/funasr/export/export_model.py index cad3367a3..4aed49f54 100644 --- a/funasr/export/export_model.py +++ b/funasr/export/export_model.py @@ -191,9 +191,10 @@ class ModelExport: cmvn_file = os.path.join(model_dir, 'vad.mvn') model, vad_infer_args = VADTask.build_model_from_file( - config, model_file, 'cpu' + config, model_file, cmvn_file=cmvn_file, device='cpu' ) self.export_config["feats_dim"] = 400 + self.frontend = model.frontend self._export(model, tag_name) diff --git a/funasr/models/e2e_vad.py b/funasr/models/e2e_vad.py index e6cd7c0d7..ff3742949 100644 --- a/funasr/models/e2e_vad.py +++ b/funasr/models/e2e_vad.py @@ -192,7 +192,7 @@ class WindowDetector(object): class E2EVadModel(nn.Module): - def __init__(self, encoder: FSMN, vad_post_args: Dict[str, Any]): + def __init__(self, encoder: FSMN, vad_post_args: Dict[str, Any], frontend=None): super(E2EVadModel, self).__init__() self.vad_opts = VADXOptions(**vad_post_args) self.windows_detector = WindowDetector(self.vad_opts.window_size_ms, @@ -229,6 +229,7 @@ class E2EVadModel(nn.Module): self.data_buf_all = None self.waveform = None self.ResetDetection() + self.frontend = frontend def AllResetDetection(self): self.is_final = False @@ -477,8 +478,9 @@ class E2EVadModel(nn.Module): ) -> Tuple[List[List[List[int]]], Dict[str, torch.Tensor]]: self.max_end_sil_frame_cnt_thresh = max_end_sil - self.vad_opts.speech_to_sil_time_thres self.waveform = waveform # compute decibel for each frame - self.ComputeDecibel() + self.ComputeScores(feats, in_cache) + self.ComputeDecibel() if not is_final: self.DetectCommonFrames() else: diff --git a/funasr/tasks/vad.py b/funasr/tasks/vad.py index 22a5cb3d3..1ac77d39a 100644 --- a/funasr/tasks/vad.py +++ b/funasr/tasks/vad.py @@ -40,7 +40,7 @@ from funasr.models.encoder.transformer_encoder import TransformerEncoder from funasr.models.frontend.abs_frontend import AbsFrontend from funasr.models.frontend.default import DefaultFrontend from funasr.models.frontend.fused import FusedFrontends -from funasr.models.frontend.wav_frontend import WavFrontend +from funasr.models.frontend.wav_frontend import WavFrontend, WavFrontendOnline from funasr.models.frontend.s3prl import S3prlFrontend from funasr.models.frontend.windowing import SlidingWindow from funasr.models.postencoder.abs_postencoder import AbsPostEncoder @@ -81,6 +81,7 @@ frontend_choices = ClassChoices( s3prl=S3prlFrontend, fused=FusedFrontends, wav_frontend=WavFrontend, + wav_frontend_online=WavFrontendOnline, ), type_check=AbsFrontend, default="default", @@ -291,7 +292,24 @@ class VADTask(AbsTask): model_class = model_choices.get_class(args.model) except AttributeError: model_class = model_choices.get_class("e2evad") - model = model_class(encoder=encoder, vad_post_args=args.vad_post_conf) + + # 1. frontend + if args.input_size is None: + # Extract features in the model + frontend_class = frontend_choices.get_class(args.frontend) + if args.frontend == 'wav_frontend': + frontend = frontend_class(cmvn_file=args.cmvn_file, **args.frontend_conf) + else: + frontend = frontend_class(**args.frontend_conf) + input_size = frontend.output_size() + else: + # Give features from data-loader + args.frontend = None + args.frontend_conf = {} + frontend = None + input_size = args.input_size + + model = model_class(encoder=encoder, vad_post_args=args.vad_post_conf, frontend=frontend) return model @@ -301,6 +319,7 @@ class VADTask(AbsTask): cls, config_file: Union[Path, str] = None, model_file: Union[Path, str] = None, + cmvn_file: Union[Path, str] = None, device: str = "cpu", ): """Build model from the files. @@ -325,6 +344,8 @@ class VADTask(AbsTask): with config_file.open("r", encoding="utf-8") as f: args = yaml.safe_load(f) + if cmvn_file is not None: + args["cmvn_file"] = cmvn_file args = argparse.Namespace(**args) model = cls.build_model(args) model.to(device) From 6ebd2676480068c1cb27ffc1b9318b09e1662173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Wed, 29 Mar 2023 16:48:57 +0800 Subject: [PATCH 14/55] general punc model conversion onnx --- funasr/export/export_model.py | 12 +- .../export/models/target_delay_transformer.py | 160 ++++++++++++++++++ 2 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 funasr/export/models/target_delay_transformer.py diff --git a/funasr/export/export_model.py b/funasr/export/export_model.py index 4aed49f54..9afa7b1af 100644 --- a/funasr/export/export_model.py +++ b/funasr/export/export_model.py @@ -174,7 +174,10 @@ class ModelExport: json_file = os.path.join(model_dir, 'configuration.json') with open(json_file, 'r') as f: config_data = json.load(f) - mode = config_data['model']['model_config']['mode'] + if config_data['task'] == "punctuation": + mode = config_data['model']['punc_model_config']['mode'] + else: + mode = config_data['model']['model_config']['mode'] if mode.startswith('paraformer'): from funasr.tasks.asr import ASRTaskParaformer as ASRTask config = os.path.join(model_dir, 'config.yaml') @@ -195,6 +198,13 @@ class ModelExport: ) self.export_config["feats_dim"] = 400 self.frontend = model.frontend + elif mode.startswith('punc'): + from funasr.tasks.punctuation import PunctuationTask as PUNCTask + punc_train_config = os.path.join(model_dir, 'config.yaml') + punc_model_file = os.path.join(model_dir, 'punc.pb') + model, punc_train_args = PUNCTask.build_model_from_file( + punc_train_config, punc_model_file, 'cpu' + ) self._export(model, tag_name) diff --git a/funasr/export/models/target_delay_transformer.py b/funasr/export/models/target_delay_transformer.py new file mode 100644 index 000000000..0a2586c93 --- /dev/null +++ b/funasr/export/models/target_delay_transformer.py @@ -0,0 +1,160 @@ +from typing import Any +from typing import List +from typing import Tuple + +import torch +import torch.nn as nn + +from funasr.export.utils.torch_function import MakePadMask +from funasr.export.utils.torch_function import sequence_mask +#from funasr.models.encoder.sanm_encoder import SANMEncoder as Encoder +from funasr.punctuation.sanm_encoder import SANMEncoder +from funasr.export.models.encoder.sanm_encoder import SANMEncoder as SANMEncoder_export +from funasr.punctuation.abs_model import AbsPunctuation + + +class TargetDelayTransformer(nn.Module): + + def __init__( + self, + model, + max_seq_len=512, + model_name='punc_model', + **kwargs, + ): + super().__init__() + onnx = False + if "onnx" in kwargs: + onnx = kwargs["onnx"] + self.embed = model.embed + self.decoder = model.decoder + self.model = model + self.feats_dim = self.embed.embedding_dim + self.num_embeddings = self.embed.num_embeddings + self.model_name = model_name + from typing import Any + from typing import List + from typing import Tuple + + import torch + import torch.nn as nn + + from funasr.export.utils.torch_function import MakePadMask + from funasr.export.utils.torch_function import sequence_mask + # from funasr.models.encoder.sanm_encoder import SANMEncoder as Encoder + from funasr.punctuation.sanm_encoder import SANMEncoder + from funasr.export.models.encoder.sanm_encoder import SANMEncoder as SANMEncoder_export + from funasr.punctuation.abs_model import AbsPunctuation + + class TargetDelayTransformer(nn.Module): + + def __init__( + self, + model, + max_seq_len=512, + model_name='punc_model', + **kwargs, + ): + super().__init__() + onnx = False + if "onnx" in kwargs: + onnx = kwargs["onnx"] + self.embed = model.embed + self.decoder = model.decoder + self.model = model + self.feats_dim = self.embed.embedding_dim + self.num_embeddings = self.embed.num_embeddings + self.model_name = model_name + + if isinstance(model.encoder, SANMEncoder): + self.encoder = SANMEncoder_export(model.encoder, onnx=onnx) + else: + assert False, "Only support samn encode." + + def forward(self, input: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]: + """Compute loss value from buffer sequences. + + Args: + input (torch.Tensor): Input ids. (batch, len) + hidden (torch.Tensor): Target ids. (batch, len) + + """ + x = self.embed(input) + # mask = self._target_mask(input) + h, _ = self.encoder(x, text_lengths) + y = self.decoder(h) + return y + + def get_dummy_inputs(self): + length = 120 + text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)) + text_lengths = torch.tensor([length - 20, length], dtype=torch.int32) + return (text_indexes, text_lengths) + + def get_input_names(self): + return ['input', 'text_lengths'] + + def get_output_names(self): + return ['logits'] + + def get_dynamic_axes(self): + return { + 'input': { + 0: 'batch_size', + 1: 'feats_length' + }, + 'text_lengths': { + 0: 'batch_size', + }, + 'logits': { + 0: 'batch_size', + 1: 'logits_length' + }, + } + + if isinstance(model.encoder, SANMEncoder): + self.encoder = SANMEncoder_export(model.encoder, onnx=onnx) + else: + assert False, "Only support samn encode." + + def forward(self, input: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]: + """Compute loss value from buffer sequences. + + Args: + input (torch.Tensor): Input ids. (batch, len) + hidden (torch.Tensor): Target ids. (batch, len) + + """ + x = self.embed(input) + # mask = self._target_mask(input) + h, _ = self.encoder(x, text_lengths) + y = self.decoder(h) + return y + + def get_dummy_inputs(self): + length = 120 + text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)) + text_lengths = torch.tensor([length-20, length], dtype=torch.int32) + return (text_indexes, text_lengths) + + def get_input_names(self): + return ['input', 'text_lengths'] + + def get_output_names(self): + return ['logits'] + + def get_dynamic_axes(self): + return { + 'input': { + 0: 'batch_size', + 1: 'feats_length' + }, + 'text_lengths': { + 0: 'batch_size', + }, + 'logits': { + 0: 'batch_size', + 1: 'logits_length' + }, + } + From a62b45743934cc44a62c6feea3594a4918ca64a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Wed, 29 Mar 2023 16:53:30 +0800 Subject: [PATCH 15/55] general punc model conversion onnx, add test_onnx_punc --- funasr/export/test/test_onnx_punc.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 funasr/export/test/test_onnx_punc.py diff --git a/funasr/export/test/test_onnx_punc.py b/funasr/export/test/test_onnx_punc.py new file mode 100644 index 000000000..dc63176fb --- /dev/null +++ b/funasr/export/test/test_onnx_punc.py @@ -0,0 +1,18 @@ +import onnxruntime +import numpy as np + + +if __name__ == '__main__': + onnx_path = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/model.onnx" + sess = onnxruntime.InferenceSession(onnx_path) + input_name = [nd.name for nd in sess.get_inputs()] + output_name = [nd.name for nd in sess.get_outputs()] + + def _get_feed_dict(text_length): + return {'input': np.ones((1, text_length), dtype=np.int64), 'text_lengths': np.array([text_length,], dtype=np.int32)} + + def _run(feed_dict): + output = sess.run(output_name, input_feed=feed_dict) + for name, value in zip(output_name, output): + print('{}: {}'.format(name, value)) + _run(_get_feed_dict(10)) From 9232f066040c909f79eb7eeb70d5a3a0e1b97b4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Wed, 29 Mar 2023 17:01:26 +0800 Subject: [PATCH 16/55] general punc model conversion onnx, fix bug --- funasr/export/models/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/funasr/export/models/__init__.py b/funasr/export/models/__init__.py index 71e8f3b57..8a2172dde 100644 --- a/funasr/export/models/__init__.py +++ b/funasr/export/models/__init__.py @@ -3,6 +3,7 @@ from funasr.export.models.e2e_asr_paraformer import Paraformer as Paraformer_exp from funasr.export.models.e2e_asr_paraformer import BiCifParaformer as BiCifParaformer_export from funasr.models.e2e_vad import E2EVadModel from funasr.export.models.e2e_vad import E2EVadModel as E2EVadModel_export +from funasr.export.models.target_delay_transformer import TargetDelayTransformer as TargetDelayTransformer_export def get_model(model, export_config=None): if isinstance(model, BiCifParaformer): @@ -11,5 +12,8 @@ def get_model(model, export_config=None): return Paraformer_export(model, **export_config) elif isinstance(model, E2EVadModel): return E2EVadModel_export(model, **export_config) + elif isinstance(model, ESPnetPunctuationModel): + if isinstance(model.punc_model, TargetDelayTransformer): + return TargetDelayTransformer_export(model.punc_model, **export_config) else: - raise "Funasr does not support the given model type currently." \ No newline at end of file + raise "Funasr does not support the given model type currently." From 3964f678a77c5fa8fb548ed7511a23c35539354d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Wed, 29 Mar 2023 17:11:14 +0800 Subject: [PATCH 17/55] general punc model conversion onnx, fix bug --- funasr/export/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/funasr/export/models/__init__.py b/funasr/export/models/__init__.py index 8a2172dde..f125f5b98 100644 --- a/funasr/export/models/__init__.py +++ b/funasr/export/models/__init__.py @@ -4,6 +4,7 @@ from funasr.export.models.e2e_asr_paraformer import BiCifParaformer as BiCifPara from funasr.models.e2e_vad import E2EVadModel from funasr.export.models.e2e_vad import E2EVadModel as E2EVadModel_export from funasr.export.models.target_delay_transformer import TargetDelayTransformer as TargetDelayTransformer_export +from funasr.punctuation.espnet_model import ESPnetPunctuationModel def get_model(model, export_config=None): if isinstance(model, BiCifParaformer): From 7a6753aa52ef5e2024470ca8d83c5d8bd9d3437a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Wed, 29 Mar 2023 17:14:09 +0800 Subject: [PATCH 18/55] general punc model conversion onnx, fix bug --- funasr/export/models/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/funasr/export/models/__init__.py b/funasr/export/models/__init__.py index f125f5b98..a34133841 100644 --- a/funasr/export/models/__init__.py +++ b/funasr/export/models/__init__.py @@ -3,6 +3,7 @@ from funasr.export.models.e2e_asr_paraformer import Paraformer as Paraformer_exp from funasr.export.models.e2e_asr_paraformer import BiCifParaformer as BiCifParaformer_export from funasr.models.e2e_vad import E2EVadModel from funasr.export.models.e2e_vad import E2EVadModel as E2EVadModel_export +from funasr.punctuation.target_delay_transformer import TargetDelayTransformer from funasr.export.models.target_delay_transformer import TargetDelayTransformer as TargetDelayTransformer_export from funasr.punctuation.espnet_model import ESPnetPunctuationModel From bf918fe311aa1271abcdabf0157a53b71b45906e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Wed, 29 Mar 2023 17:17:12 +0800 Subject: [PATCH 19/55] general punc model conversion onnx, fix bug --- funasr/export/test/test_onnx_punc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funasr/export/test/test_onnx_punc.py b/funasr/export/test/test_onnx_punc.py index dc63176fb..62689a904 100644 --- a/funasr/export/test/test_onnx_punc.py +++ b/funasr/export/test/test_onnx_punc.py @@ -3,7 +3,7 @@ import numpy as np if __name__ == '__main__': - onnx_path = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/model.onnx" + onnx_path = "../damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/model.onnx" sess = onnxruntime.InferenceSession(onnx_path) input_name = [nd.name for nd in sess.get_inputs()] output_name = [nd.name for nd in sess.get_outputs()] From a030ff0f85fd6b1cc2a1d443d2fcfb11ccb1aa8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 29 Mar 2023 21:15:55 +0800 Subject: [PATCH 20/55] export --- funasr/export/models/__init__.py | 4 + funasr/export/models/encoder/sanm_encoder.py | 99 +++++++++++++ .../export/models/target_delay_transformer.py | 132 +++++++++--------- .../export/models/vad_realtime_transformer.py | 79 +++++++++++ 4 files changed, 248 insertions(+), 66 deletions(-) create mode 100644 funasr/export/models/vad_realtime_transformer.py diff --git a/funasr/export/models/__init__.py b/funasr/export/models/__init__.py index a34133841..62ee72354 100644 --- a/funasr/export/models/__init__.py +++ b/funasr/export/models/__init__.py @@ -6,6 +6,8 @@ from funasr.export.models.e2e_vad import E2EVadModel as E2EVadModel_export from funasr.punctuation.target_delay_transformer import TargetDelayTransformer from funasr.export.models.target_delay_transformer import TargetDelayTransformer as TargetDelayTransformer_export from funasr.punctuation.espnet_model import ESPnetPunctuationModel +from funasr.punctuation.vad_realtime_transformer import VadRealtimeTransformer +from funasr.export.models.vad_realtime_transformer import VadRealtimeTransformer as VadRealtimeTransformer_export def get_model(model, export_config=None): if isinstance(model, BiCifParaformer): @@ -17,5 +19,7 @@ def get_model(model, export_config=None): elif isinstance(model, ESPnetPunctuationModel): if isinstance(model.punc_model, TargetDelayTransformer): return TargetDelayTransformer_export(model.punc_model, **export_config) + elif isinstance(model.punc_model, VadRealtimeTransformer): + return VadRealtimeTransformer_export(model.punc_model, **export_config) else: raise "Funasr does not support the given model type currently." diff --git a/funasr/export/models/encoder/sanm_encoder.py b/funasr/export/models/encoder/sanm_encoder.py index 8a5053870..3b7b4143f 100644 --- a/funasr/export/models/encoder/sanm_encoder.py +++ b/funasr/export/models/encoder/sanm_encoder.py @@ -107,3 +107,102 @@ class SANMEncoder(nn.Module): } } + + +class SANMVadEncoder(nn.Module): + def __init__( + self, + model, + max_seq_len=512, + feats_dim=560, + model_name='encoder', + onnx: bool = True, + ): + super().__init__() + self.embed = model.embed + self.model = model + self.feats_dim = feats_dim + self._output_size = model._output_size + + if onnx: + self.make_pad_mask = MakePadMask(max_seq_len, flip=False) + else: + self.make_pad_mask = sequence_mask(max_seq_len, flip=False) + + if hasattr(model, 'encoders0'): + for i, d in enumerate(self.model.encoders0): + if isinstance(d.self_attn, MultiHeadedAttentionSANM): + d.self_attn = MultiHeadedAttentionSANM_export(d.self_attn) + if isinstance(d.feed_forward, PositionwiseFeedForward): + d.feed_forward = PositionwiseFeedForward_export(d.feed_forward) + self.model.encoders0[i] = EncoderLayerSANM_export(d) + + for i, d in enumerate(self.model.encoders): + if isinstance(d.self_attn, MultiHeadedAttentionSANM): + d.self_attn = MultiHeadedAttentionSANM_export(d.self_attn) + if isinstance(d.feed_forward, PositionwiseFeedForward): + d.feed_forward = PositionwiseFeedForward_export(d.feed_forward) + self.model.encoders[i] = EncoderLayerSANM_export(d) + + self.model_name = model_name + self.num_heads = model.encoders[0].self_attn.h + self.hidden_size = model.encoders[0].self_attn.linear_out.out_features + + def prepare_mask(self, mask): + mask_3d_btd = mask[:, :, None] + if len(mask.shape) == 2: + mask_4d_bhlt = 1 - mask[:, None, None, :] + elif len(mask.shape) == 3: + mask_4d_bhlt = 1 - mask[:, None, :] + mask_4d_bhlt = mask_4d_bhlt * -10000.0 + + return mask_3d_btd, mask_4d_bhlt + + def forward(self, + speech: torch.Tensor, + speech_lengths: torch.Tensor, + ): + speech = speech * self._output_size ** 0.5 + mask = self.make_pad_mask(speech_lengths) + mask = self.prepare_mask(mask) + if self.embed is None: + xs_pad = speech + else: + xs_pad = self.embed(speech) + + encoder_outs = self.model.encoders0(xs_pad, mask) + xs_pad, masks = encoder_outs[0], encoder_outs[1] + + encoder_outs = self.model.encoders(xs_pad, mask) + xs_pad, masks = encoder_outs[0], encoder_outs[1] + + xs_pad = self.model.after_norm(xs_pad) + + return xs_pad, speech_lengths + + def get_output_size(self): + return self.model.encoders[0].size + + def get_dummy_inputs(self): + feats = torch.randn(1, 100, self.feats_dim) + return (feats) + + def get_input_names(self): + return ['feats'] + + def get_output_names(self): + return ['encoder_out', 'encoder_out_lens', 'predictor_weight'] + + def get_dynamic_axes(self): + return { + 'feats': { + 1: 'feats_length' + }, + 'encoder_out': { + 1: 'enc_out_length' + }, + 'predictor_weight': { + 1: 'pre_out_length' + } + + } diff --git a/funasr/export/models/target_delay_transformer.py b/funasr/export/models/target_delay_transformer.py index 0a2586c93..fd90835c9 100644 --- a/funasr/export/models/target_delay_transformer.py +++ b/funasr/export/models/target_delay_transformer.py @@ -28,7 +28,7 @@ class TargetDelayTransformer(nn.Module): onnx = kwargs["onnx"] self.embed = model.embed self.decoder = model.decoder - self.model = model + # self.model = model self.feats_dim = self.embed.embedding_dim self.num_embeddings = self.embed.num_embeddings self.model_name = model_name @@ -46,71 +46,71 @@ class TargetDelayTransformer(nn.Module): from funasr.export.models.encoder.sanm_encoder import SANMEncoder as SANMEncoder_export from funasr.punctuation.abs_model import AbsPunctuation - class TargetDelayTransformer(nn.Module): - - def __init__( - self, - model, - max_seq_len=512, - model_name='punc_model', - **kwargs, - ): - super().__init__() - onnx = False - if "onnx" in kwargs: - onnx = kwargs["onnx"] - self.embed = model.embed - self.decoder = model.decoder - self.model = model - self.feats_dim = self.embed.embedding_dim - self.num_embeddings = self.embed.num_embeddings - self.model_name = model_name - - if isinstance(model.encoder, SANMEncoder): - self.encoder = SANMEncoder_export(model.encoder, onnx=onnx) - else: - assert False, "Only support samn encode." - - def forward(self, input: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]: - """Compute loss value from buffer sequences. - - Args: - input (torch.Tensor): Input ids. (batch, len) - hidden (torch.Tensor): Target ids. (batch, len) - - """ - x = self.embed(input) - # mask = self._target_mask(input) - h, _ = self.encoder(x, text_lengths) - y = self.decoder(h) - return y - - def get_dummy_inputs(self): - length = 120 - text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)) - text_lengths = torch.tensor([length - 20, length], dtype=torch.int32) - return (text_indexes, text_lengths) - - def get_input_names(self): - return ['input', 'text_lengths'] - - def get_output_names(self): - return ['logits'] - - def get_dynamic_axes(self): - return { - 'input': { - 0: 'batch_size', - 1: 'feats_length' - }, - 'text_lengths': { - 0: 'batch_size', - }, - 'logits': { - 0: 'batch_size', - 1: 'logits_length' - }, - } + # class TargetDelayTransformer(nn.Module): + # + # def __init__( + # self, + # model, + # max_seq_len=512, + # model_name='punc_model', + # **kwargs, + # ): + # super().__init__() + # onnx = False + # if "onnx" in kwargs: + # onnx = kwargs["onnx"] + # self.embed = model.embed + # self.decoder = model.decoder + # self.model = model + # self.feats_dim = self.embed.embedding_dim + # self.num_embeddings = self.embed.num_embeddings + # self.model_name = model_name + # + # if isinstance(model.encoder, SANMEncoder): + # self.encoder = SANMEncoder_export(model.encoder, onnx=onnx) + # else: + # assert False, "Only support samn encode." + # + # def forward(self, input: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]: + # """Compute loss value from buffer sequences. + # + # Args: + # input (torch.Tensor): Input ids. (batch, len) + # hidden (torch.Tensor): Target ids. (batch, len) + # + # """ + # x = self.embed(input) + # # mask = self._target_mask(input) + # h, _ = self.encoder(x, text_lengths) + # y = self.decoder(h) + # return y + # + # def get_dummy_inputs(self): + # length = 120 + # text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)) + # text_lengths = torch.tensor([length - 20, length], dtype=torch.int32) + # return (text_indexes, text_lengths) + # + # def get_input_names(self): + # return ['input', 'text_lengths'] + # + # def get_output_names(self): + # return ['logits'] + # + # def get_dynamic_axes(self): + # return { + # 'input': { + # 0: 'batch_size', + # 1: 'feats_length' + # }, + # 'text_lengths': { + # 0: 'batch_size', + # }, + # 'logits': { + # 0: 'batch_size', + # 1: 'logits_length' + # }, + # } if isinstance(model.encoder, SANMEncoder): self.encoder = SANMEncoder_export(model.encoder, onnx=onnx) diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py new file mode 100644 index 000000000..44583d853 --- /dev/null +++ b/funasr/export/models/vad_realtime_transformer.py @@ -0,0 +1,79 @@ +from typing import Any +from typing import List +from typing import Tuple + +import torch +import torch.nn as nn + +from funasr.modules.embedding import SinusoidalPositionEncoder +from funasr.punctuation.sanm_encoder import SANMVadEncoder as Encoder +from funasr.punctuation.abs_model import AbsPunctuation +from funasr.punctuation.sanm_encoder import SANMVadEncoder +from funasr.export.models.encoder.sanm_encoder import SANMVadEncoder as SANMVadEncoder_export + +class VadRealtimeTransformer(AbsPunctuation): + + def __init__( + self, + model, + max_seq_len=512, + model_name='punc_model', + **kwargs, + ): + super().__init__() + + + self.embed = model.embed + if isinstance(model.encoder, SANMVadEncoder): + self.encoder = SANMVadEncoder_export(model.encoder, onnx=onnx) + else: + assert False, "Only support samn encode." + # self.encoder = model.encoder + self.decoder = model.decoder + + + + def forward(self, input: torch.Tensor, text_lengths: torch.Tensor, + vad_indexes: torch.Tensor) -> Tuple[torch.Tensor, None]: + """Compute loss value from buffer sequences. + + Args: + input (torch.Tensor): Input ids. (batch, len) + hidden (torch.Tensor): Target ids. (batch, len) + + """ + x = self.embed(input) + # mask = self._target_mask(input) + h, _, _ = self.encoder(x, text_lengths, vad_indexes) + y = self.decoder(h) + return y + + def with_vad(self): + return True + + def get_dummy_inputs(self): + length = 120 + text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)) + text_lengths = torch.tensor([length-20, length], dtype=torch.int32) + return (text_indexes, text_lengths) + + def get_input_names(self): + return ['input', 'text_lengths'] + + def get_output_names(self): + return ['logits'] + + def get_dynamic_axes(self): + return { + 'input': { + 0: 'batch_size', + 1: 'feats_length' + }, + 'text_lengths': { + 0: 'batch_size', + }, + 'logits': { + 0: 'batch_size', + 1: 'logits_length' + }, + } From 00bec8f243d5067f0b6719aacab52a73c2a530e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 00:09:47 +0800 Subject: [PATCH 21/55] export --- .../onnxruntime/funasr_onnx/utils/utils.py | 2 +- .../python/onnxruntime/funasr_onnx/vad_bin.py | 53 ++++++++++--------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py index fccd5a095..fb414ad33 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py @@ -226,7 +226,7 @@ def read_yaml(yaml_path: Union[str, Path]) -> Dict: @functools.lru_cache() -def get_logger(name='rapdi_paraformer'): +def get_logger(name='funasr_onnx'): """Initialize and get a logger by name. If the logger has not been initialized, this method will initialize the logger by adding one or two handlers, otherwise the initialized logger will diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index cdd4578d1..47d429349 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -59,33 +59,38 @@ class Fsmn_vad(): def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> List: - waveform_list = self.load_data(audio_in, self.frontend.opts.frame_opts.samp_freq) - waveform_nums = len(waveform_list) + # waveform_list = self.load_data(audio_in, self.frontend.opts.frame_opts.samp_freq) is_final = kwargs.get('kwargs', False) - - asr_res = [] - for beg_idx in range(0, waveform_nums, self.batch_size): + param_dict = kwargs.get('param_dict', dict()) + audio_in_cache = param_dict.get('audio_in_cache', None) + audio_in_cum = audio_in + if audio_in_cache is not None: + audio_in_cum = np.concatenate((audio_in_cache, audio_in_cum)) + param_dict['audio_in_cache'] = audio_in_cum + feats, feats_len = self.extract_feat([audio_in_cum]) + + in_cache = param_dict.get('in_cache', list()) + in_cache = self.prepare_cache(in_cache) + beg_idx = param_dict.get('beg_idx',0) + feats = feats[:, beg_idx:beg_idx+8, :] + param_dict['beg_idx'] = beg_idx + feats.shape[1] + try: + inputs = [feats] + inputs.extend(in_cache) + scores, out_caches = self.infer(inputs) + param_dict['in_cache'] = out_caches + segments = self.vad_scorer(scores, audio_in[None, :], is_final=is_final, max_end_sil=self.max_end_sil) + # print(segments) + if len(segments) == 1 and segments[0][0][1] != -1: + self.frontend.reset_status() - end_idx = min(waveform_nums, beg_idx + self.batch_size) - waveform = waveform_list[beg_idx:end_idx] - feats, feats_len = self.extract_feat(waveform) - param_dict = kwargs.get('param_dict', dict()) - in_cache = param_dict.get('in_cache', list()) - in_cache = self.prepare_cache(in_cache) - try: - inputs = [feats] - inputs.extend(in_cache) - scores, out_caches = self.infer(inputs) - param_dict['in_cache'] = out_caches - segments = self.vad_scorer(scores, waveform[0][None, :], is_final=is_final, max_end_sil=self.max_end_sil) - - except ONNXRuntimeError: - # logging.warning(traceback.format_exc()) - logging.warning("input wav is silence or noise") - segments = '' - asr_res.append(segments) + + except ONNXRuntimeError: + logging.warning(traceback.format_exc()) + logging.warning("input wav is silence or noise") + segments = [] - return asr_res + return segments def load_data(self, wav_content: Union[str, np.ndarray, List[str]], fs: int = None) -> List: From 97ec4cd7b590e948ca129fc6d9e4ef66cb3d761e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 00:13:03 +0800 Subject: [PATCH 22/55] export --- funasr/runtime/python/libtorch/funasr_torch/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funasr/runtime/python/libtorch/funasr_torch/utils/utils.py b/funasr/runtime/python/libtorch/funasr_torch/utils/utils.py index 2f09de8c9..cafc43bf9 100644 --- a/funasr/runtime/python/libtorch/funasr_torch/utils/utils.py +++ b/funasr/runtime/python/libtorch/funasr_torch/utils/utils.py @@ -134,7 +134,7 @@ def read_yaml(yaml_path: Union[str, Path]) -> Dict: @functools.lru_cache() -def get_logger(name='torch_paraformer'): +def get_logger(name='funasr_torch'): """Initialize and get a logger by name. If the logger has not been initialized, this method will initialize the logger by adding one or two handlers, otherwise the initialized logger will From e55178abc21a3a692b7b18cc12922b4004c15f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 14:11:02 +0800 Subject: [PATCH 23/55] general punc model runtime --- .../python/onnxruntime/demo_punc_offline.py | 9 + .../onnxruntime/funasr_onnx/__init__.py | 2 + .../onnxruntime/funasr_onnx/punc_bin.py | 133 +++++ .../funasr_onnx/utils/preprocessor.py | 470 ++++++++++++++++++ .../onnxruntime/funasr_onnx/utils/utils.py | 13 + 5 files changed, 627 insertions(+) create mode 100644 funasr/runtime/python/onnxruntime/demo_punc_offline.py create mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx/utils/preprocessor.py diff --git a/funasr/runtime/python/onnxruntime/demo_punc_offline.py b/funasr/runtime/python/onnxruntime/demo_punc_offline.py new file mode 100644 index 000000000..056f73751 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/demo_punc_offline.py @@ -0,0 +1,9 @@ +from funasr_onnx import TargetDelayTransformer + +model_dir = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch" +model = TargetDelayTransformer(model_dir) + +text_in = "我们都是木头人不会讲话不会动" + +result = model(text_in) +print(result) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py index 475047903..1620a0b25 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py @@ -1,3 +1,5 @@ # -*- encoding: utf-8 -*- from .paraformer_bin import Paraformer from .vad_bin import Fsmn_vad +from .punc_bin import TargetDelayTransformer +#from .punc_bin import VadRealtimeTransformer diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index e69de29bb..64ced69be 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -0,0 +1,133 @@ +# -*- encoding: utf-8 -*- + +import os.path +from pathlib import Path +from typing import List, Union, Tuple +import numpy as np + +from .utils.utils import (ONNXRuntimeError, + OrtInferSession, get_logger, + read_yaml) +from .utils.preprocessor import CodeMixTokenizerCommonPreprocessor +from .utils.utils import split_to_mini_sentence +logging = get_logger() + + +class TargetDelayTransformer(): + def __init__(self, model_dir: Union[str, Path] = None, + batch_size: int = 1, + device_id: Union[str, int] = "-1", + quantize: bool = False, + intra_op_num_threads: int = 4 + ): + + if not Path(model_dir).exists(): + raise FileNotFoundError(f'{model_dir} does not exist.') + + model_file = os.path.join(model_dir, 'model.onnx') + if quantize: + model_file = os.path.join(model_dir, 'model_quant.onnx') + config_file = os.path.join(model_dir, 'punc.yaml') + config = read_yaml(config_file) + + self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) + self.batch_size = 1 + self.encoder_conf = config["encoder_conf"] + self.punc_list = config.punc_list + self.period = 0 + for i in range(len(self.punc_list)): + if self.punc_list[i] == ",": + self.punc_list[i] = "，" + elif self.punc_list[i] == "?": + self.punc_list[i] = "？" + elif self.punc_list[i] == "。": + self.period = i + self.preprocessor = CodeMixTokenizerCommonPreprocessor( + train=False, + token_type=config.token_type, + token_list=config.token_list, + bpemodel=config.bpemodel, + text_cleaner=config.cleaner, + g2p_type=config.g2p, + text_name="text", + non_linguistic_symbols=config.non_linguistic_symbols, + ) + + def __call__(self, text: Union[list, str], split_size=20): + data = {"text": text} + result = self.preprocessor(data=data, uid="12938712838719") + split_text = self.preprocessor.pop_split_text_data(result) + mini_sentences = split_to_mini_sentence(split_text, split_size) + mini_sentences_id = split_to_mini_sentence(data["text"], split_size) + assert len(mini_sentences) == len(mini_sentences_id) + cache_sent = [] + cache_sent_id = [] + new_mini_sentence = "" + new_mini_sentence_punc = [] + cache_pop_trigger_limit = 200 + for mini_sentence_i in range(len(mini_sentences)): + mini_sentence = mini_sentences[mini_sentence_i] + mini_sentence_id = mini_sentences_id[mini_sentence_i] + mini_sentence = cache_sent + mini_sentence + mini_sentence_id = np.concatenate((cache_sent_id, mini_sentence_id), axis=0) + data = { + "text": mini_sentence_id, + "text_lengths": len(mini_sentence_id), + } + try: + outputs = self.infer(data['text'], data['text_lengths']) + y = outputs[0] + _, indices = y.view(-1, y.shape[-1]).topk(1, dim=1) + punctuations = indices + assert punctuations.size()[0] == len(mini_sentence) + except ONNXRuntimeError: + logging.warning("error") + + # Search for the last Period/QuestionMark as cache + if mini_sentence_i < len(mini_sentences) - 1: + sentenceEnd = -1 + last_comma_index = -1 + for i in range(len(punctuations) - 2, 1, -1): + if self.punc_list[punctuations[i]] == "。" or self.punc_list[punctuations[i]] == "？": + sentenceEnd = i + break + if last_comma_index < 0 and self.punc_list[punctuations[i]] == "，": + last_comma_index = i + + if sentenceEnd < 0 and len(mini_sentence) > cache_pop_trigger_limit and last_comma_index >= 0: + # The sentence it too long, cut off at a comma. + sentenceEnd = last_comma_index + punctuations[sentenceEnd] = self.period + cache_sent = mini_sentence[sentenceEnd + 1:] + cache_sent_id = mini_sentence_id[sentenceEnd + 1:] + mini_sentence = mini_sentence[0:sentenceEnd + 1] + punctuations = punctuations[0:sentenceEnd + 1] + + punctuations_np = punctuations.cpu().numpy() + new_mini_sentence_punc += [int(x) for x in punctuations_np] + words_with_punc = [] + for i in range(len(mini_sentence)): + if i > 0: + if len(mini_sentence[i][0].encode()) == 1 and len(mini_sentence[i - 1][0].encode()) == 1: + mini_sentence[i] = " " + mini_sentence[i] + words_with_punc.append(mini_sentence[i]) + if self.punc_list[punctuations[i]] != "_": + words_with_punc.append(self.punc_list[punctuations[i]]) + new_mini_sentence += "".join(words_with_punc) + # Add Period for the end of the sentence + new_mini_sentence_out = new_mini_sentence + new_mini_sentence_punc_out = new_mini_sentence_punc + if mini_sentence_i == len(mini_sentences) - 1: + if new_mini_sentence[-1] == "，" or new_mini_sentence[-1] == "、": + new_mini_sentence_out = new_mini_sentence[:-1] + "。" + new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + [self.period] + elif new_mini_sentence[-1] != "。" and new_mini_sentence[-1] != "？": + new_mini_sentence_out = new_mini_sentence + "。" + new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + [self.period] + return new_mini_sentence_out, new_mini_sentence_punc_out + + def infer(self, feats: List) -> Tuple[np.ndarray, np.ndarray]: + + outputs = self.ort_infer(feats) + return outputs + diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/preprocessor.py b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/preprocessor.py new file mode 100644 index 000000000..4c9710371 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/preprocessor.py @@ -0,0 +1,470 @@ +import re +from abc import ABC +from abc import abstractmethod +from pathlib import Path +from typing import Collection +from typing import Dict +from typing import Iterable +from typing import List +from typing import Union + +import numpy as np +import scipy.signal +import soundfile +from typeguard import check_argument_types +from typeguard import check_return_type + +from funasr.text.build_tokenizer import build_tokenizer +from funasr.text.cleaner import TextCleaner +from funasr.text.token_id_converter import TokenIDConverter + + +class AbsPreprocessor(ABC): + def __init__(self, train: bool): + self.train = train + + @abstractmethod + def __call__( + self, uid: str, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + raise NotImplementedError + + +def forward_segment(text, dic): + word_list = [] + i = 0 + while i < len(text): + longest_word = text[i] + for j in range(i + 1, len(text) + 1): + word = text[i:j] + if word in dic: + if len(word) > len(longest_word): + longest_word = word + word_list.append(longest_word) + i += len(longest_word) + return word_list + + +def seg_tokenize(txt, seg_dict): + out_txt = "" + for word in txt: + if word in seg_dict: + out_txt += seg_dict[word] + " " + else: + out_txt += "" + " " + return out_txt.strip().split() + +def seg_tokenize_wo_pattern(txt, seg_dict): + out_txt = "" + for word in txt: + if word in seg_dict: + out_txt += seg_dict[word] + " " + else: + out_txt += "" + " " + return out_txt.strip().split() + + +def framing( + x, + frame_length: int = 512, + frame_shift: int = 256, + centered: bool = True, + padded: bool = True, +): + if x.size == 0: + raise ValueError("Input array size is zero") + if frame_length < 1: + raise ValueError("frame_length must be a positive integer") + if frame_length > x.shape[-1]: + raise ValueError("frame_length is greater than input length") + if 0 >= frame_shift: + raise ValueError("frame_shift must be greater than 0") + + if centered: + pad_shape = [(0, 0) for _ in range(x.ndim - 1)] + [ + (frame_length // 2, frame_length // 2) + ] + x = np.pad(x, pad_shape, mode="constant", constant_values=0) + + if padded: + # Pad to integer number of windowed segments + # I.e make x.shape[-1] = frame_length + (nseg-1)*nstep, + # with integer nseg + nadd = (-(x.shape[-1] - frame_length) % frame_shift) % frame_length + pad_shape = [(0, 0) for _ in range(x.ndim - 1)] + [(0, nadd)] + x = np.pad(x, pad_shape, mode="constant", constant_values=0) + + # Created strided array of data segments + if frame_length == 1 and frame_length == frame_shift: + result = x[..., None] + else: + shape = x.shape[:-1] + ( + (x.shape[-1] - frame_length) // frame_shift + 1, + frame_length, + ) + strides = x.strides[:-1] + (frame_shift * x.strides[-1], x.strides[-1]) + result = np.lib.stride_tricks.as_strided(x, shape=shape, strides=strides) + return result + + +def detect_non_silence( + x: np.ndarray, + threshold: float = 0.01, + frame_length: int = 1024, + frame_shift: int = 512, + window: str = "boxcar", +) -> np.ndarray: + """Power based voice activity detection. + + Args: + x: (Channel, Time) + >>> x = np.random.randn(1000) + >>> detect = detect_non_silence(x) + >>> assert x.shape == detect.shape + >>> assert detect.dtype == np.bool + """ + if x.shape[-1] < frame_length: + return np.full(x.shape, fill_value=True, dtype=np.bool) + + if x.dtype.kind == "i": + x = x.astype(np.float64) + # framed_w: (C, T, F) + framed_w = framing( + x, + frame_length=frame_length, + frame_shift=frame_shift, + centered=False, + padded=True, + ) + framed_w *= scipy.signal.get_window(window, frame_length).astype(framed_w.dtype) + # power: (C, T) + power = (framed_w ** 2).mean(axis=-1) + # mean_power: (C, 1) + mean_power = np.mean(power, axis=-1, keepdims=True) + if np.all(mean_power == 0): + return np.full(x.shape, fill_value=True, dtype=np.bool) + # detect_frames: (C, T) + detect_frames = power / mean_power > threshold + # detects: (C, T, F) + detects = np.broadcast_to( + detect_frames[..., None], detect_frames.shape + (frame_shift,) + ) + # detects: (C, TF) + detects = detects.reshape(*detect_frames.shape[:-1], -1) + # detects: (C, TF) + return np.pad( + detects, + [(0, 0)] * (x.ndim - 1) + [(0, x.shape[-1] - detects.shape[-1])], + mode="edge", + ) + + +class CommonPreprocessor(AbsPreprocessor): + def __init__( + self, + train: bool, + token_type: str = None, + token_list: Union[Path, str, Iterable[str]] = None, + bpemodel: Union[Path, str, Iterable[str]] = None, + text_cleaner: Collection[str] = None, + g2p_type: str = None, + unk_symbol: str = "", + space_symbol: str = "", + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + delimiter: str = None, + rir_scp: str = None, + rir_apply_prob: float = 1.0, + noise_scp: str = None, + noise_apply_prob: float = 1.0, + noise_db_range: str = "3_10", + speech_volume_normalize: float = None, + speech_name: str = "speech", + text_name: str = "text", + split_with_space: bool = False, + seg_dict_file: str = None, + ): + super().__init__(train) + self.train = train + self.speech_name = speech_name + self.text_name = text_name + self.speech_volume_normalize = speech_volume_normalize + self.rir_apply_prob = rir_apply_prob + self.noise_apply_prob = noise_apply_prob + self.split_with_space = split_with_space + self.seg_dict = None + if seg_dict_file is not None: + self.seg_dict = {} + with open(seg_dict_file) as f: + lines = f.readlines() + for line in lines: + s = line.strip().split() + key = s[0] + value = s[1:] + self.seg_dict[key] = " ".join(value) + + if token_type is not None: + if token_list is None: + raise ValueError("token_list is required if token_type is not None") + self.text_cleaner = TextCleaner(text_cleaner) + + self.tokenizer = build_tokenizer( + token_type=token_type, + bpemodel=bpemodel, + delimiter=delimiter, + space_symbol=space_symbol, + non_linguistic_symbols=non_linguistic_symbols, + g2p_type=g2p_type, + ) + self.token_id_converter = TokenIDConverter( + token_list=token_list, + unk_symbol=unk_symbol, + ) + else: + self.text_cleaner = None + self.tokenizer = None + self.token_id_converter = None + + if train and rir_scp is not None: + self.rirs = [] + with open(rir_scp, "r", encoding="utf-8") as f: + for line in f: + sps = line.strip().split(None, 1) + if len(sps) == 1: + self.rirs.append(sps[0]) + else: + self.rirs.append(sps[1]) + else: + self.rirs = None + + if train and noise_scp is not None: + self.noises = [] + with open(noise_scp, "r", encoding="utf-8") as f: + for line in f: + sps = line.strip().split(None, 1) + if len(sps) == 1: + self.noises.append(sps[0]) + else: + self.noises.append(sps[1]) + sps = noise_db_range.split("_") + if len(sps) == 1: + self.noise_db_low, self.noise_db_high = float(sps[0]) + elif len(sps) == 2: + self.noise_db_low, self.noise_db_high = float(sps[0]), float(sps[1]) + else: + raise ValueError( + "Format error: '{noise_db_range}' e.g. -3_4 -> [-3db,4db]" + ) + else: + self.noises = None + + def _speech_process( + self, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, Union[str, np.ndarray]]: + assert check_argument_types() + if self.speech_name in data: + if self.train and (self.rirs is not None or self.noises is not None): + speech = data[self.speech_name] + nsamples = len(speech) + + # speech: (Nmic, Time) + if speech.ndim == 1: + speech = speech[None, :] + else: + speech = speech.T + # Calc power on non shlence region + power = (speech[detect_non_silence(speech)] ** 2).mean() + + # 1. Convolve RIR + if self.rirs is not None and self.rir_apply_prob >= np.random.random(): + rir_path = np.random.choice(self.rirs) + if rir_path is not None: + rir, _ = soundfile.read( + rir_path, dtype=np.float64, always_2d=True + ) + + # rir: (Nmic, Time) + rir = rir.T + + # speech: (Nmic, Time) + # Note that this operation doesn't change the signal length + speech = scipy.signal.convolve(speech, rir, mode="full")[ + :, : speech.shape[1] + ] + # Reverse mean power to the original power + power2 = (speech[detect_non_silence(speech)] ** 2).mean() + speech = np.sqrt(power / max(power2, 1e-10)) * speech + + # 2. Add Noise + if ( + self.noises is not None + and self.noise_apply_prob >= np.random.random() + ): + noise_path = np.random.choice(self.noises) + if noise_path is not None: + noise_db = np.random.uniform( + self.noise_db_low, self.noise_db_high + ) + with soundfile.SoundFile(noise_path) as f: + if f.frames == nsamples: + noise = f.read(dtype=np.float64, always_2d=True) + elif f.frames < nsamples: + offset = np.random.randint(0, nsamples - f.frames) + # noise: (Time, Nmic) + noise = f.read(dtype=np.float64, always_2d=True) + # Repeat noise + noise = np.pad( + noise, + [(offset, nsamples - f.frames - offset), (0, 0)], + mode="wrap", + ) + else: + offset = np.random.randint(0, f.frames - nsamples) + f.seek(offset) + # noise: (Time, Nmic) + noise = f.read( + nsamples, dtype=np.float64, always_2d=True + ) + if len(noise) != nsamples: + raise RuntimeError(f"Something wrong: {noise_path}") + # noise: (Nmic, Time) + noise = noise.T + + noise_power = (noise ** 2).mean() + scale = ( + 10 ** (-noise_db / 20) + * np.sqrt(power) + / np.sqrt(max(noise_power, 1e-10)) + ) + speech = speech + scale * noise + + speech = speech.T + ma = np.max(np.abs(speech)) + if ma > 1.0: + speech /= ma + data[self.speech_name] = speech + + if self.speech_volume_normalize is not None: + speech = data[self.speech_name] + ma = np.max(np.abs(speech)) + data[self.speech_name] = speech * self.speech_volume_normalize / ma + assert check_return_type(data) + return data + + def _text_process( + self, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + if self.text_name in data and self.tokenizer is not None: + text = data[self.text_name] + text = self.text_cleaner(text) + if self.split_with_space: + tokens = text.strip().split(" ") + if self.seg_dict is not None: + tokens = forward_segment("".join(tokens), self.seg_dict) + tokens = seg_tokenize(tokens, self.seg_dict) + else: + tokens = self.tokenizer.text2tokens(text) + text_ints = self.token_id_converter.tokens2ids(tokens) + data[self.text_name] = np.array(text_ints, dtype=np.int64) + assert check_return_type(data) + return data + + def __call__( + self, uid: str, data: Dict[str, Union[str, np.ndarray]] + ) -> Dict[str, np.ndarray]: + assert check_argument_types() + + data = self._speech_process(data) + data = self._text_process(data) + return data + +class CodeMixTokenizerCommonPreprocessor(CommonPreprocessor): + def __init__( + self, + train: bool, + token_type: str = None, + token_list: Union[Path, str, Iterable[str]] = None, + bpemodel: Union[Path, str, Iterable[str]] = None, + text_cleaner: Collection[str] = None, + g2p_type: str = None, + unk_symbol: str = "", + space_symbol: str = "", + non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, + delimiter: str = None, + rir_scp: str = None, + rir_apply_prob: float = 1.0, + noise_scp: str = None, + noise_apply_prob: float = 1.0, + noise_db_range: str = "3_10", + speech_volume_normalize: float = None, + speech_name: str = "speech", + text_name: str = "text", + split_text_name: str = "split_text", + split_with_space: bool = False, + seg_dict_file: str = None, + ): + super().__init__( + train=train, + # Force to use word. + token_type="word", + token_list=token_list, + bpemodel=bpemodel, + text_cleaner=text_cleaner, + g2p_type=g2p_type, + unk_symbol=unk_symbol, + space_symbol=space_symbol, + non_linguistic_symbols=non_linguistic_symbols, + delimiter=delimiter, + speech_name=speech_name, + text_name=text_name, + rir_scp=rir_scp, + rir_apply_prob=rir_apply_prob, + noise_scp=noise_scp, + noise_apply_prob=noise_apply_prob, + noise_db_range=noise_db_range, + speech_volume_normalize=speech_volume_normalize, + split_with_space=split_with_space, + seg_dict_file=seg_dict_file, + ) + # The data field name for split text. + self.split_text_name = split_text_name + + @classmethod + def split_words(cls, text: str): + words = [] + segs = text.split() + for seg in segs: + # There is no space in seg. + current_word = "" + for c in seg: + if len(c.encode()) == 1: + # This is an ASCII char. + current_word += c + else: + # This is a Chinese char. + if len(current_word) > 0: + words.append(current_word) + current_word = "" + words.append(c) + if len(current_word) > 0: + words.append(current_word) + return words + + def __call__( + self, uid: str, data: Dict[str, Union[list, str, np.ndarray]] + ) -> Dict[str, Union[list, np.ndarray]]: + assert check_argument_types() + # Split words. + if isinstance(data[self.text_name], str): + split_text = self.split_words(data[self.text_name]) + else: + split_text = data[self.text_name] + data[self.text_name] = " ".join(split_text) + data = self._speech_process(data) + data = self._text_process(data) + data[self.split_text_name] = split_text + return data + + def pop_split_text_data(self, data: Dict[str, Union[str, np.ndarray]]): + result = data[self.split_text_name] + del data[self.split_text_name] + return result diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py index fccd5a095..c7e607691 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py @@ -215,6 +215,19 @@ class OrtInferSession(): if not model_path.is_file(): raise FileExistsError(f'{model_path} is not a file.') +def split_to_mini_sentence(words: list, word_limit: int = 20): + assert word_limit > 1 + if len(words) <= word_limit: + return [words] + sentences = [] + length = len(words) + sentence_len = length // word_limit + for i in range(sentence_len): + sentences.append(words[i * word_limit:(i + 1) * word_limit]) + if length % word_limit > 0: + sentences.append(words[sentence_len * word_limit:]) + return sentences + def read_yaml(yaml_path: Union[str, Path]) -> Dict: if not Path(yaml_path).exists(): From 5eed382e118926b830174fe666c4aec48daf550d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 14:13:48 +0800 Subject: [PATCH 24/55] export --- funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py index 47d429349..221867dbf 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/vad_bin.py @@ -60,8 +60,9 @@ class Fsmn_vad(): def __call__(self, audio_in: Union[str, np.ndarray, List[str]], **kwargs) -> List: # waveform_list = self.load_data(audio_in, self.frontend.opts.frame_opts.samp_freq) - is_final = kwargs.get('kwargs', False) + param_dict = kwargs.get('param_dict', dict()) + is_final = param_dict.get('is_final', False) audio_in_cache = param_dict.get('audio_in_cache', None) audio_in_cum = audio_in if audio_in_cache is not None: From 30374838287efa8337c68c283d5663c672950f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 14:15:23 +0800 Subject: [PATCH 25/55] export --- funasr/bin/asr_inference_uniasr.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/funasr/bin/asr_inference_uniasr.py b/funasr/bin/asr_inference_uniasr.py index 7961d5af3..1286bc25a 100644 --- a/funasr/bin/asr_inference_uniasr.py +++ b/funasr/bin/asr_inference_uniasr.py @@ -37,9 +37,6 @@ from funasr.utils import asr_utils, wav_utils, postprocess_utils from funasr.models.frontend.wav_frontend import WavFrontend -header_colors = '\033[95m' -end_colors = '\033[0m' - class Speech2Text: """Speech2Text class From 19bda23f5e87e91bb7363e988fba842a69be36ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 14:20:35 +0800 Subject: [PATCH 26/55] fix --- .../python/onnxruntime/funasr_onnx/punc_bin.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index 64ced69be..8ea4517ec 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -32,8 +32,7 @@ class TargetDelayTransformer(): self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) self.batch_size = 1 - self.encoder_conf = config["encoder_conf"] - self.punc_list = config.punc_list + self.punc_list = config['punc_list'] self.period = 0 for i in range(len(self.punc_list)): if self.punc_list[i] == ",": @@ -44,13 +43,13 @@ class TargetDelayTransformer(): self.period = i self.preprocessor = CodeMixTokenizerCommonPreprocessor( train=False, - token_type=config.token_type, - token_list=config.token_list, - bpemodel=config.bpemodel, - text_cleaner=config.cleaner, - g2p_type=config.g2p, + token_type=config['token_type'], + token_list=config['token_list'], + bpemodel=config['bpemodel'], + text_cleaner=config['cleaner'], + g2p_type=config['g2p'], text_name="text", - non_linguistic_symbols=config.non_linguistic_symbols, + non_linguistic_symbols=config['non_linguistic_symbols'], ) def __call__(self, text: Union[list, str], split_size=20): From 1d4dda939cd8998119d2a9dd4a170e8db657f21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 15:44:37 +0800 Subject: [PATCH 27/55] fix --- .../runtime/python/onnxruntime/funasr_onnx/punc_bin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index 8ea4517ec..034475c6c 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -70,8 +70,8 @@ class TargetDelayTransformer(): mini_sentence = cache_sent + mini_sentence mini_sentence_id = np.concatenate((cache_sent_id, mini_sentence_id), axis=0) data = { - "text": mini_sentence_id, - "text_lengths": len(mini_sentence_id), + "text": mini_sentence_id[None,:].astype(np.int64), + "text_lengths": np.array([len(mini_sentence_id)], dtype='int32'), } try: outputs = self.infer(data['text'], data['text_lengths']) @@ -125,8 +125,8 @@ class TargetDelayTransformer(): new_mini_sentence_punc_out = new_mini_sentence_punc[:-1] + [self.period] return new_mini_sentence_out, new_mini_sentence_punc_out - def infer(self, feats: List) -> Tuple[np.ndarray, np.ndarray]: - - outputs = self.ort_infer(feats) + def infer(self, feats: np.ndarray, + feats_len: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + outputs = self.ort_infer([feats, feats_len]) return outputs From c4882b43fce3c32cb0ce3c9fc2c164f0ce0e8213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 16:04:13 +0800 Subject: [PATCH 28/55] export --- funasr/export/export_model.py | 7 +++++ funasr/export/models/encoder/sanm_encoder.py | 29 ++++++++++++++++--- .../export/models/vad_realtime_transformer.py | 18 +++++------- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/funasr/export/export_model.py b/funasr/export/export_model.py index 9afa7b1af..444ccf436 100644 --- a/funasr/export/export_model.py +++ b/funasr/export/export_model.py @@ -205,6 +205,13 @@ class ModelExport: model, punc_train_args = PUNCTask.build_model_from_file( punc_train_config, punc_model_file, 'cpu' ) + elif mode.startswith('punc_VadRealtime'): + from funasr.tasks.punctuation import PunctuationTask as PUNCTask + punc_train_config = os.path.join(model_dir, 'config.yaml') + punc_model_file = os.path.join(model_dir, 'punc.pb') + model, punc_train_args = PUNCTask.build_model_from_file( + punc_train_config, punc_model_file, 'cpu' + ) self._export(model, tag_name) diff --git a/funasr/export/models/encoder/sanm_encoder.py b/funasr/export/models/encoder/sanm_encoder.py index 3b7b4143f..118e24008 100644 --- a/funasr/export/models/encoder/sanm_encoder.py +++ b/funasr/export/models/encoder/sanm_encoder.py @@ -9,6 +9,21 @@ from funasr.export.models.modules.encoder_layer import EncoderLayerSANM as Encod from funasr.modules.positionwise_feed_forward import PositionwiseFeedForward from funasr.export.models.modules.feedforward import PositionwiseFeedForward as PositionwiseFeedForward_export +def subsequent_mask(size, device="cpu", dtype=torch.bool): + """Create mask for subsequent steps (size, size). + + :param int size: size of mask + :param str device: "cpu" or "cuda" or torch.Tensor.device + :param torch.dtype dtype: result dtype + :rtype: torch.Tensor + >>> subsequent_mask(3) + [[1, 0, 0], + [1, 1, 0], + [1, 1, 1]] + """ + ret = torch.ones(size, size, device=device, dtype=dtype) + return torch.tril(ret, out=ret) + class SANMEncoder(nn.Module): def __init__( self, @@ -150,10 +165,11 @@ class SANMVadEncoder(nn.Module): def prepare_mask(self, mask): mask_3d_btd = mask[:, :, None] + sub_masks = subsequent_mask(mask.size(-1)) if len(mask.shape) == 2: - mask_4d_bhlt = 1 - mask[:, None, None, :] + mask_4d_bhlt = 1 - sub_masks[:, None, None, :] elif len(mask.shape) == 3: - mask_4d_bhlt = 1 - mask[:, None, :] + mask_4d_bhlt = 1 - sub_masks[:, None, :] mask_4d_bhlt = mask_4d_bhlt * -10000.0 return mask_3d_btd, mask_4d_bhlt @@ -161,6 +177,7 @@ class SANMVadEncoder(nn.Module): def forward(self, speech: torch.Tensor, speech_lengths: torch.Tensor, + vad_mask: torch.Tensor, ): speech = speech * self._output_size ** 0.5 mask = self.make_pad_mask(speech_lengths) @@ -173,8 +190,12 @@ class SANMVadEncoder(nn.Module): encoder_outs = self.model.encoders0(xs_pad, mask) xs_pad, masks = encoder_outs[0], encoder_outs[1] - encoder_outs = self.model.encoders(xs_pad, mask) - xs_pad, masks = encoder_outs[0], encoder_outs[1] + # encoder_outs = self.model.encoders(xs_pad, mask) + for layer_idx, encoder_layer in enumerate(self.model.encoders): + if layer_idx == len(self.model.encoders) - 1: + mask = (mask[0], vad_mask) + encoder_outs = encoder_layer(xs_pad, mask) + xs_pad, masks = encoder_outs[0], encoder_outs[1] xs_pad = self.model.after_norm(xs_pad) diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index 44583d853..7c573fcbb 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -21,7 +21,9 @@ class VadRealtimeTransformer(AbsPunctuation): **kwargs, ): super().__init__() - + onnx = False + if "onnx" in kwargs: + onnx = kwargs["onnx"] self.embed = model.embed if isinstance(model.encoder, SANMVadEncoder): @@ -53,12 +55,13 @@ class VadRealtimeTransformer(AbsPunctuation): def get_dummy_inputs(self): length = 120 - text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)) - text_lengths = torch.tensor([length-20, length], dtype=torch.int32) - return (text_indexes, text_lengths) + text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)) + text_lengths = torch.tensor([length], dtype=torch.int32) + vad_mask = torch.ones(length, length)[None, None, :, :] + return (text_indexes, text_lengths, vad_mask) def get_input_names(self): - return ['input', 'text_lengths'] + return ['input', 'text_lengths', 'vad_mask'] def get_output_names(self): return ['logits'] @@ -66,14 +69,9 @@ class VadRealtimeTransformer(AbsPunctuation): def get_dynamic_axes(self): return { 'input': { - 0: 'batch_size', 1: 'feats_length' }, - 'text_lengths': { - 0: 'batch_size', - }, 'logits': { - 0: 'batch_size', 1: 'logits_length' }, } From 8c09872873c48bf23e22c62da0896e72c6e0186e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 16:05:13 +0800 Subject: [PATCH 29/55] export --- funasr/export/models/vad_realtime_transformer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index 7c573fcbb..381d02d35 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -71,6 +71,10 @@ class VadRealtimeTransformer(AbsPunctuation): 'input': { 1: 'feats_length' }, + 'vad_mask': { + 2: 'feats_length1', + 3: 'feats_length2' + }, 'logits': { 1: 'logits_length' }, From 7df8452a8520cf3e4609114b47510371b2c621a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 16:15:49 +0800 Subject: [PATCH 30/55] fix --- funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index 034475c6c..c00a3d7f2 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -76,9 +76,8 @@ class TargetDelayTransformer(): try: outputs = self.infer(data['text'], data['text_lengths']) y = outputs[0] - _, indices = y.view(-1, y.shape[-1]).topk(1, dim=1) - punctuations = indices - assert punctuations.size()[0] == len(mini_sentence) + punctuations = np.argmax(y,axis=-1)[0] + assert punctuations.size == len(mini_sentence) except ONNXRuntimeError: logging.warning("error") From 3266665f3d984c6a4b8faaa0f53570507ea25efc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 16:21:44 +0800 Subject: [PATCH 31/55] fix --- funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index c00a3d7f2..3f649bcdb 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -101,8 +101,7 @@ class TargetDelayTransformer(): mini_sentence = mini_sentence[0:sentenceEnd + 1] punctuations = punctuations[0:sentenceEnd + 1] - punctuations_np = punctuations.cpu().numpy() - new_mini_sentence_punc += [int(x) for x in punctuations_np] + new_mini_sentence_punc += [int(x) for x in punctuations] words_with_punc = [] for i in range(len(mini_sentence)): if i > 0: From 0b15e6ea5cccbea3c590958d60e623800bbe3dfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 16:27:07 +0800 Subject: [PATCH 32/55] fix --- funasr/runtime/python/onnxruntime/demo_punc_offline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/demo_punc_offline.py b/funasr/runtime/python/onnxruntime/demo_punc_offline.py index 056f73751..3ca8b4a4f 100644 --- a/funasr/runtime/python/onnxruntime/demo_punc_offline.py +++ b/funasr/runtime/python/onnxruntime/demo_punc_offline.py @@ -4,6 +4,6 @@ model_dir = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-tran model = TargetDelayTransformer(model_dir) text_in = "我们都是木头人不会讲话不会动" - +text_in="跨境河流是养育沿岸人民的生命之源长期以来为帮助下游地区防灾减灾中方技术人员在上游地区极为恶劣的自然条件下克服巨大困难甚至冒着生命危险向印方提供汛期水文资料处理紧急事件中方重视印方在跨境河流问题上的关切愿意进一步完善双方联合工作机制凡是中方能做的我们都会去做而且会做得更好我请印度朋友们放心中国在上游的任何开发利用都会经过科学规划和论证兼顾上下游的利益" result = model(text_in) -print(result) +print(result[0]) From 953952f2480e13bb62b8b786756cff248fcd4af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 16:34:33 +0800 Subject: [PATCH 33/55] export --- funasr/export/models/encoder/sanm_encoder.py | 2 +- .../export/models/vad_realtime_transformer.py | 5 +++-- .../export/test/test_onnx_punc_vadrealtime.py | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 funasr/export/test/test_onnx_punc_vadrealtime.py diff --git a/funasr/export/models/encoder/sanm_encoder.py b/funasr/export/models/encoder/sanm_encoder.py index 118e24008..a4b112fb8 100644 --- a/funasr/export/models/encoder/sanm_encoder.py +++ b/funasr/export/models/encoder/sanm_encoder.py @@ -165,7 +165,7 @@ class SANMVadEncoder(nn.Module): def prepare_mask(self, mask): mask_3d_btd = mask[:, :, None] - sub_masks = subsequent_mask(mask.size(-1)) + sub_masks = subsequent_mask(mask.size(-1)).type(torch.float32) if len(mask.shape) == 2: mask_4d_bhlt = 1 - sub_masks[:, None, None, :] elif len(mask.shape) == 3: diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index 381d02d35..de7c721eb 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -32,6 +32,7 @@ class VadRealtimeTransformer(AbsPunctuation): assert False, "Only support samn encode." # self.encoder = model.encoder self.decoder = model.decoder + self.model_name = model_name @@ -46,7 +47,7 @@ class VadRealtimeTransformer(AbsPunctuation): """ x = self.embed(input) # mask = self._target_mask(input) - h, _, _ = self.encoder(x, text_lengths, vad_indexes) + h, _ = self.encoder(x, text_lengths, vad_indexes) y = self.decoder(h) return y @@ -57,7 +58,7 @@ class VadRealtimeTransformer(AbsPunctuation): length = 120 text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)) text_lengths = torch.tensor([length], dtype=torch.int32) - vad_mask = torch.ones(length, length)[None, None, :, :] + vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :] return (text_indexes, text_lengths, vad_mask) def get_input_names(self): diff --git a/funasr/export/test/test_onnx_punc_vadrealtime.py b/funasr/export/test/test_onnx_punc_vadrealtime.py new file mode 100644 index 000000000..c5cc17ea1 --- /dev/null +++ b/funasr/export/test/test_onnx_punc_vadrealtime.py @@ -0,0 +1,18 @@ +import onnxruntime +import numpy as np + + +if __name__ == '__main__': + onnx_path = "./export/damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727/model.onnx" + sess = onnxruntime.InferenceSession(onnx_path) + input_name = [nd.name for nd in sess.get_inputs()] + output_name = [nd.name for nd in sess.get_outputs()] + + def _get_feed_dict(text_length): + return {'input': np.ones((1, text_length), dtype=np.int64), 'text_lengths': np.array([text_length,], dtype=np.int32), 'vad_mask': np.ones((1, 1, text_length, text_length), dtype=np.float32)} + + def _run(feed_dict): + output = sess.run(output_name, input_feed=feed_dict) + for name, value in zip(output_name, output): + print('{}: {}'.format(name, value)) + _run(_get_feed_dict(10)) From 0fd9640ced9c8ae9af43e5300068a8837d8ce26e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 16:48:55 +0800 Subject: [PATCH 34/55] fix --- .../onnxruntime/funasr_onnx/punc_bin.py | 25 ++++++------------- .../onnxruntime/funasr_onnx/utils/utils.py | 19 ++++++++++++++ 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index 3f649bcdb..e1f35f207 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -8,8 +8,7 @@ import numpy as np from .utils.utils import (ONNXRuntimeError, OrtInferSession, get_logger, read_yaml) -from .utils.preprocessor import CodeMixTokenizerCommonPreprocessor -from .utils.utils import split_to_mini_sentence +from .utils.utils import (TokenIDConverter, split_to_mini_sentence,code_mix_split_words) logging = get_logger() @@ -30,6 +29,7 @@ class TargetDelayTransformer(): config_file = os.path.join(model_dir, 'punc.yaml') config = read_yaml(config_file) + self.converter = TokenIDConverter(config['token_list']) self.ort_infer = OrtInferSession(model_file, device_id, intra_op_num_threads=intra_op_num_threads) self.batch_size = 1 self.punc_list = config['punc_list'] @@ -41,23 +41,12 @@ class TargetDelayTransformer(): self.punc_list[i] = "？" elif self.punc_list[i] == "。": self.period = i - self.preprocessor = CodeMixTokenizerCommonPreprocessor( - train=False, - token_type=config['token_type'], - token_list=config['token_list'], - bpemodel=config['bpemodel'], - text_cleaner=config['cleaner'], - g2p_type=config['g2p'], - text_name="text", - non_linguistic_symbols=config['non_linguistic_symbols'], - ) def __call__(self, text: Union[list, str], split_size=20): - data = {"text": text} - result = self.preprocessor(data=data, uid="12938712838719") - split_text = self.preprocessor.pop_split_text_data(result) + split_text = code_mix_split_words(text) + split_text_id = self.converter.tokens2ids(split_text) mini_sentences = split_to_mini_sentence(split_text, split_size) - mini_sentences_id = split_to_mini_sentence(data["text"], split_size) + mini_sentences_id = split_to_mini_sentence(split_text_id, split_size) assert len(mini_sentences) == len(mini_sentences_id) cache_sent = [] cache_sent_id = [] @@ -68,9 +57,9 @@ class TargetDelayTransformer(): mini_sentence = mini_sentences[mini_sentence_i] mini_sentence_id = mini_sentences_id[mini_sentence_i] mini_sentence = cache_sent + mini_sentence - mini_sentence_id = np.concatenate((cache_sent_id, mini_sentence_id), axis=0) + mini_sentence_id = np.array(cache_sent_id + mini_sentence_id, dtype='int64') data = { - "text": mini_sentence_id[None,:].astype(np.int64), + "text": mini_sentence_id[None,:], "text_lengths": np.array([len(mini_sentence_id)], dtype='int32'), } try: diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py index 63bc0e46f..0df954ed7 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/utils.py @@ -228,6 +228,25 @@ def split_to_mini_sentence(words: list, word_limit: int = 20): sentences.append(words[sentence_len * word_limit:]) return sentences +def code_mix_split_words(text: str): + words = [] + segs = text.split() + for seg in segs: + # There is no space in seg. + current_word = "" + for c in seg: + if len(c.encode()) == 1: + # This is an ASCII char. + current_word += c + else: + # This is a Chinese char. + if len(current_word) > 0: + words.append(current_word) + current_word = "" + words.append(c) + if len(current_word) > 0: + words.append(current_word) + return words def read_yaml(yaml_path: Union[str, Path]) -> Dict: if not Path(yaml_path).exists(): From 85b8628dbf3020e73580b73240804d587ead4eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 17:03:50 +0800 Subject: [PATCH 35/55] export --- funasr/export/models/encoder/sanm_encoder.py | 5 +++-- funasr/export/models/vad_realtime_transformer.py | 15 ++++++++++----- funasr/export/test/test_onnx_punc_vadrealtime.py | 6 +++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/funasr/export/models/encoder/sanm_encoder.py b/funasr/export/models/encoder/sanm_encoder.py index a4b112fb8..5437440a1 100644 --- a/funasr/export/models/encoder/sanm_encoder.py +++ b/funasr/export/models/encoder/sanm_encoder.py @@ -163,9 +163,9 @@ class SANMVadEncoder(nn.Module): self.num_heads = model.encoders[0].self_attn.h self.hidden_size = model.encoders[0].self_attn.linear_out.out_features - def prepare_mask(self, mask): + def prepare_mask(self, mask, sub_masks): mask_3d_btd = mask[:, :, None] - sub_masks = subsequent_mask(mask.size(-1)).type(torch.float32) + # sub_masks = subsequent_mask(mask.size(-1)).type(torch.float32) if len(mask.shape) == 2: mask_4d_bhlt = 1 - sub_masks[:, None, None, :] elif len(mask.shape) == 3: @@ -178,6 +178,7 @@ class SANMVadEncoder(nn.Module): speech: torch.Tensor, speech_lengths: torch.Tensor, vad_mask: torch.Tensor, + sub_masks: torch.Tensor, ): speech = speech * self._output_size ** 0.5 mask = self.make_pad_mask(speech_lengths) diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index de7c721eb..a3d486432 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -11,7 +11,7 @@ from funasr.punctuation.abs_model import AbsPunctuation from funasr.punctuation.sanm_encoder import SANMVadEncoder from funasr.export.models.encoder.sanm_encoder import SANMVadEncoder as SANMVadEncoder_export -class VadRealtimeTransformer(AbsPunctuation): +class VadRealtimeTransformer(nn.Module): def __init__( self, @@ -36,8 +36,11 @@ class VadRealtimeTransformer(AbsPunctuation): - def forward(self, input: torch.Tensor, text_lengths: torch.Tensor, - vad_indexes: torch.Tensor) -> Tuple[torch.Tensor, None]: + def forward(self, input: torch.Tensor, + text_lengths: torch.Tensor, + vad_indexes: torch.Tensor, + sub_masks: torch.Tensor, + ) -> Tuple[torch.Tensor, None]: """Compute loss value from buffer sequences. Args: @@ -47,7 +50,7 @@ class VadRealtimeTransformer(AbsPunctuation): """ x = self.embed(input) # mask = self._target_mask(input) - h, _ = self.encoder(x, text_lengths, vad_indexes) + h, _ = self.encoder(x, text_lengths, vad_indexes, sub_masks) y = self.decoder(h) return y @@ -59,7 +62,9 @@ class VadRealtimeTransformer(AbsPunctuation): text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)) text_lengths = torch.tensor([length], dtype=torch.int32) vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :] - return (text_indexes, text_lengths, vad_mask) + sub_masks = torch.ones(length, length, dtype=torch.float32) + sub_masks = torch.tril(sub_masks) + return (text_indexes, text_lengths, vad_mask, sub_masks) def get_input_names(self): return ['input', 'text_lengths', 'vad_mask'] diff --git a/funasr/export/test/test_onnx_punc_vadrealtime.py b/funasr/export/test/test_onnx_punc_vadrealtime.py index c5cc17ea1..6544a898f 100644 --- a/funasr/export/test/test_onnx_punc_vadrealtime.py +++ b/funasr/export/test/test_onnx_punc_vadrealtime.py @@ -9,7 +9,11 @@ if __name__ == '__main__': output_name = [nd.name for nd in sess.get_outputs()] def _get_feed_dict(text_length): - return {'input': np.ones((1, text_length), dtype=np.int64), 'text_lengths': np.array([text_length,], dtype=np.int32), 'vad_mask': np.ones((1, 1, text_length, text_length), dtype=np.float32)} + return {'input': np.ones((1, text_length), dtype=np.int64), + 'text_lengths': np.array([text_length,], dtype=np.int32), + 'vad_mask': np.ones((1, 1, text_length, text_length), dtype=np.float32), + 'sub_masks': np.tril(np.ones((text_length, text_length), dtype=np.float32)) + } def _run(feed_dict): output = sess.run(output_name, input_feed=feed_dict) From e3822504654b1fb2f51030a1e990566eb67bfc95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 17:07:47 +0800 Subject: [PATCH 36/55] export --- funasr/export/models/encoder/sanm_encoder.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/funasr/export/models/encoder/sanm_encoder.py b/funasr/export/models/encoder/sanm_encoder.py index 5437440a1..8198d18a3 100644 --- a/funasr/export/models/encoder/sanm_encoder.py +++ b/funasr/export/models/encoder/sanm_encoder.py @@ -9,20 +9,6 @@ from funasr.export.models.modules.encoder_layer import EncoderLayerSANM as Encod from funasr.modules.positionwise_feed_forward import PositionwiseFeedForward from funasr.export.models.modules.feedforward import PositionwiseFeedForward as PositionwiseFeedForward_export -def subsequent_mask(size, device="cpu", dtype=torch.bool): - """Create mask for subsequent steps (size, size). - - :param int size: size of mask - :param str device: "cpu" or "cuda" or torch.Tensor.device - :param torch.dtype dtype: result dtype - :rtype: torch.Tensor - >>> subsequent_mask(3) - [[1, 0, 0], - [1, 1, 0], - [1, 1, 1]] - """ - ret = torch.ones(size, size, device=device, dtype=dtype) - return torch.tril(ret, out=ret) class SANMEncoder(nn.Module): def __init__( @@ -182,7 +168,7 @@ class SANMVadEncoder(nn.Module): ): speech = speech * self._output_size ** 0.5 mask = self.make_pad_mask(speech_lengths) - mask = self.prepare_mask(mask) + mask = self.prepare_mask(mask, sub_masks) if self.embed is None: xs_pad = speech else: From d66d4b7d8377708b4efdf74d54af40008f32b813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 17:13:22 +0800 Subject: [PATCH 37/55] fix --- .../onnxruntime/funasr_onnx/punc_bin.py | 2 +- .../funasr_onnx/utils/preprocessor.py | 470 ------------------ 2 files changed, 1 insertion(+), 471 deletions(-) delete mode 100644 funasr/runtime/python/onnxruntime/funasr_onnx/utils/preprocessor.py diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index e1f35f207..d72b0ce2f 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -86,7 +86,7 @@ class TargetDelayTransformer(): sentenceEnd = last_comma_index punctuations[sentenceEnd] = self.period cache_sent = mini_sentence[sentenceEnd + 1:] - cache_sent_id = mini_sentence_id[sentenceEnd + 1:] + cache_sent_id = mini_sentence_id[sentenceEnd + 1:].tolist() mini_sentence = mini_sentence[0:sentenceEnd + 1] punctuations = punctuations[0:sentenceEnd + 1] diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/preprocessor.py b/funasr/runtime/python/onnxruntime/funasr_onnx/utils/preprocessor.py deleted file mode 100644 index 4c9710371..000000000 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/utils/preprocessor.py +++ /dev/null @@ -1,470 +0,0 @@ -import re -from abc import ABC -from abc import abstractmethod -from pathlib import Path -from typing import Collection -from typing import Dict -from typing import Iterable -from typing import List -from typing import Union - -import numpy as np -import scipy.signal -import soundfile -from typeguard import check_argument_types -from typeguard import check_return_type - -from funasr.text.build_tokenizer import build_tokenizer -from funasr.text.cleaner import TextCleaner -from funasr.text.token_id_converter import TokenIDConverter - - -class AbsPreprocessor(ABC): - def __init__(self, train: bool): - self.train = train - - @abstractmethod - def __call__( - self, uid: str, data: Dict[str, Union[str, np.ndarray]] - ) -> Dict[str, np.ndarray]: - raise NotImplementedError - - -def forward_segment(text, dic): - word_list = [] - i = 0 - while i < len(text): - longest_word = text[i] - for j in range(i + 1, len(text) + 1): - word = text[i:j] - if word in dic: - if len(word) > len(longest_word): - longest_word = word - word_list.append(longest_word) - i += len(longest_word) - return word_list - - -def seg_tokenize(txt, seg_dict): - out_txt = "" - for word in txt: - if word in seg_dict: - out_txt += seg_dict[word] + " " - else: - out_txt += "" + " " - return out_txt.strip().split() - -def seg_tokenize_wo_pattern(txt, seg_dict): - out_txt = "" - for word in txt: - if word in seg_dict: - out_txt += seg_dict[word] + " " - else: - out_txt += "" + " " - return out_txt.strip().split() - - -def framing( - x, - frame_length: int = 512, - frame_shift: int = 256, - centered: bool = True, - padded: bool = True, -): - if x.size == 0: - raise ValueError("Input array size is zero") - if frame_length < 1: - raise ValueError("frame_length must be a positive integer") - if frame_length > x.shape[-1]: - raise ValueError("frame_length is greater than input length") - if 0 >= frame_shift: - raise ValueError("frame_shift must be greater than 0") - - if centered: - pad_shape = [(0, 0) for _ in range(x.ndim - 1)] + [ - (frame_length // 2, frame_length // 2) - ] - x = np.pad(x, pad_shape, mode="constant", constant_values=0) - - if padded: - # Pad to integer number of windowed segments - # I.e make x.shape[-1] = frame_length + (nseg-1)*nstep, - # with integer nseg - nadd = (-(x.shape[-1] - frame_length) % frame_shift) % frame_length - pad_shape = [(0, 0) for _ in range(x.ndim - 1)] + [(0, nadd)] - x = np.pad(x, pad_shape, mode="constant", constant_values=0) - - # Created strided array of data segments - if frame_length == 1 and frame_length == frame_shift: - result = x[..., None] - else: - shape = x.shape[:-1] + ( - (x.shape[-1] - frame_length) // frame_shift + 1, - frame_length, - ) - strides = x.strides[:-1] + (frame_shift * x.strides[-1], x.strides[-1]) - result = np.lib.stride_tricks.as_strided(x, shape=shape, strides=strides) - return result - - -def detect_non_silence( - x: np.ndarray, - threshold: float = 0.01, - frame_length: int = 1024, - frame_shift: int = 512, - window: str = "boxcar", -) -> np.ndarray: - """Power based voice activity detection. - - Args: - x: (Channel, Time) - >>> x = np.random.randn(1000) - >>> detect = detect_non_silence(x) - >>> assert x.shape == detect.shape - >>> assert detect.dtype == np.bool - """ - if x.shape[-1] < frame_length: - return np.full(x.shape, fill_value=True, dtype=np.bool) - - if x.dtype.kind == "i": - x = x.astype(np.float64) - # framed_w: (C, T, F) - framed_w = framing( - x, - frame_length=frame_length, - frame_shift=frame_shift, - centered=False, - padded=True, - ) - framed_w *= scipy.signal.get_window(window, frame_length).astype(framed_w.dtype) - # power: (C, T) - power = (framed_w ** 2).mean(axis=-1) - # mean_power: (C, 1) - mean_power = np.mean(power, axis=-1, keepdims=True) - if np.all(mean_power == 0): - return np.full(x.shape, fill_value=True, dtype=np.bool) - # detect_frames: (C, T) - detect_frames = power / mean_power > threshold - # detects: (C, T, F) - detects = np.broadcast_to( - detect_frames[..., None], detect_frames.shape + (frame_shift,) - ) - # detects: (C, TF) - detects = detects.reshape(*detect_frames.shape[:-1], -1) - # detects: (C, TF) - return np.pad( - detects, - [(0, 0)] * (x.ndim - 1) + [(0, x.shape[-1] - detects.shape[-1])], - mode="edge", - ) - - -class CommonPreprocessor(AbsPreprocessor): - def __init__( - self, - train: bool, - token_type: str = None, - token_list: Union[Path, str, Iterable[str]] = None, - bpemodel: Union[Path, str, Iterable[str]] = None, - text_cleaner: Collection[str] = None, - g2p_type: str = None, - unk_symbol: str = "", - space_symbol: str = "", - non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, - delimiter: str = None, - rir_scp: str = None, - rir_apply_prob: float = 1.0, - noise_scp: str = None, - noise_apply_prob: float = 1.0, - noise_db_range: str = "3_10", - speech_volume_normalize: float = None, - speech_name: str = "speech", - text_name: str = "text", - split_with_space: bool = False, - seg_dict_file: str = None, - ): - super().__init__(train) - self.train = train - self.speech_name = speech_name - self.text_name = text_name - self.speech_volume_normalize = speech_volume_normalize - self.rir_apply_prob = rir_apply_prob - self.noise_apply_prob = noise_apply_prob - self.split_with_space = split_with_space - self.seg_dict = None - if seg_dict_file is not None: - self.seg_dict = {} - with open(seg_dict_file) as f: - lines = f.readlines() - for line in lines: - s = line.strip().split() - key = s[0] - value = s[1:] - self.seg_dict[key] = " ".join(value) - - if token_type is not None: - if token_list is None: - raise ValueError("token_list is required if token_type is not None") - self.text_cleaner = TextCleaner(text_cleaner) - - self.tokenizer = build_tokenizer( - token_type=token_type, - bpemodel=bpemodel, - delimiter=delimiter, - space_symbol=space_symbol, - non_linguistic_symbols=non_linguistic_symbols, - g2p_type=g2p_type, - ) - self.token_id_converter = TokenIDConverter( - token_list=token_list, - unk_symbol=unk_symbol, - ) - else: - self.text_cleaner = None - self.tokenizer = None - self.token_id_converter = None - - if train and rir_scp is not None: - self.rirs = [] - with open(rir_scp, "r", encoding="utf-8") as f: - for line in f: - sps = line.strip().split(None, 1) - if len(sps) == 1: - self.rirs.append(sps[0]) - else: - self.rirs.append(sps[1]) - else: - self.rirs = None - - if train and noise_scp is not None: - self.noises = [] - with open(noise_scp, "r", encoding="utf-8") as f: - for line in f: - sps = line.strip().split(None, 1) - if len(sps) == 1: - self.noises.append(sps[0]) - else: - self.noises.append(sps[1]) - sps = noise_db_range.split("_") - if len(sps) == 1: - self.noise_db_low, self.noise_db_high = float(sps[0]) - elif len(sps) == 2: - self.noise_db_low, self.noise_db_high = float(sps[0]), float(sps[1]) - else: - raise ValueError( - "Format error: '{noise_db_range}' e.g. -3_4 -> [-3db,4db]" - ) - else: - self.noises = None - - def _speech_process( - self, data: Dict[str, Union[str, np.ndarray]] - ) -> Dict[str, Union[str, np.ndarray]]: - assert check_argument_types() - if self.speech_name in data: - if self.train and (self.rirs is not None or self.noises is not None): - speech = data[self.speech_name] - nsamples = len(speech) - - # speech: (Nmic, Time) - if speech.ndim == 1: - speech = speech[None, :] - else: - speech = speech.T - # Calc power on non shlence region - power = (speech[detect_non_silence(speech)] ** 2).mean() - - # 1. Convolve RIR - if self.rirs is not None and self.rir_apply_prob >= np.random.random(): - rir_path = np.random.choice(self.rirs) - if rir_path is not None: - rir, _ = soundfile.read( - rir_path, dtype=np.float64, always_2d=True - ) - - # rir: (Nmic, Time) - rir = rir.T - - # speech: (Nmic, Time) - # Note that this operation doesn't change the signal length - speech = scipy.signal.convolve(speech, rir, mode="full")[ - :, : speech.shape[1] - ] - # Reverse mean power to the original power - power2 = (speech[detect_non_silence(speech)] ** 2).mean() - speech = np.sqrt(power / max(power2, 1e-10)) * speech - - # 2. Add Noise - if ( - self.noises is not None - and self.noise_apply_prob >= np.random.random() - ): - noise_path = np.random.choice(self.noises) - if noise_path is not None: - noise_db = np.random.uniform( - self.noise_db_low, self.noise_db_high - ) - with soundfile.SoundFile(noise_path) as f: - if f.frames == nsamples: - noise = f.read(dtype=np.float64, always_2d=True) - elif f.frames < nsamples: - offset = np.random.randint(0, nsamples - f.frames) - # noise: (Time, Nmic) - noise = f.read(dtype=np.float64, always_2d=True) - # Repeat noise - noise = np.pad( - noise, - [(offset, nsamples - f.frames - offset), (0, 0)], - mode="wrap", - ) - else: - offset = np.random.randint(0, f.frames - nsamples) - f.seek(offset) - # noise: (Time, Nmic) - noise = f.read( - nsamples, dtype=np.float64, always_2d=True - ) - if len(noise) != nsamples: - raise RuntimeError(f"Something wrong: {noise_path}") - # noise: (Nmic, Time) - noise = noise.T - - noise_power = (noise ** 2).mean() - scale = ( - 10 ** (-noise_db / 20) - * np.sqrt(power) - / np.sqrt(max(noise_power, 1e-10)) - ) - speech = speech + scale * noise - - speech = speech.T - ma = np.max(np.abs(speech)) - if ma > 1.0: - speech /= ma - data[self.speech_name] = speech - - if self.speech_volume_normalize is not None: - speech = data[self.speech_name] - ma = np.max(np.abs(speech)) - data[self.speech_name] = speech * self.speech_volume_normalize / ma - assert check_return_type(data) - return data - - def _text_process( - self, data: Dict[str, Union[str, np.ndarray]] - ) -> Dict[str, np.ndarray]: - if self.text_name in data and self.tokenizer is not None: - text = data[self.text_name] - text = self.text_cleaner(text) - if self.split_with_space: - tokens = text.strip().split(" ") - if self.seg_dict is not None: - tokens = forward_segment("".join(tokens), self.seg_dict) - tokens = seg_tokenize(tokens, self.seg_dict) - else: - tokens = self.tokenizer.text2tokens(text) - text_ints = self.token_id_converter.tokens2ids(tokens) - data[self.text_name] = np.array(text_ints, dtype=np.int64) - assert check_return_type(data) - return data - - def __call__( - self, uid: str, data: Dict[str, Union[str, np.ndarray]] - ) -> Dict[str, np.ndarray]: - assert check_argument_types() - - data = self._speech_process(data) - data = self._text_process(data) - return data - -class CodeMixTokenizerCommonPreprocessor(CommonPreprocessor): - def __init__( - self, - train: bool, - token_type: str = None, - token_list: Union[Path, str, Iterable[str]] = None, - bpemodel: Union[Path, str, Iterable[str]] = None, - text_cleaner: Collection[str] = None, - g2p_type: str = None, - unk_symbol: str = "", - space_symbol: str = "", - non_linguistic_symbols: Union[Path, str, Iterable[str]] = None, - delimiter: str = None, - rir_scp: str = None, - rir_apply_prob: float = 1.0, - noise_scp: str = None, - noise_apply_prob: float = 1.0, - noise_db_range: str = "3_10", - speech_volume_normalize: float = None, - speech_name: str = "speech", - text_name: str = "text", - split_text_name: str = "split_text", - split_with_space: bool = False, - seg_dict_file: str = None, - ): - super().__init__( - train=train, - # Force to use word. - token_type="word", - token_list=token_list, - bpemodel=bpemodel, - text_cleaner=text_cleaner, - g2p_type=g2p_type, - unk_symbol=unk_symbol, - space_symbol=space_symbol, - non_linguistic_symbols=non_linguistic_symbols, - delimiter=delimiter, - speech_name=speech_name, - text_name=text_name, - rir_scp=rir_scp, - rir_apply_prob=rir_apply_prob, - noise_scp=noise_scp, - noise_apply_prob=noise_apply_prob, - noise_db_range=noise_db_range, - speech_volume_normalize=speech_volume_normalize, - split_with_space=split_with_space, - seg_dict_file=seg_dict_file, - ) - # The data field name for split text. - self.split_text_name = split_text_name - - @classmethod - def split_words(cls, text: str): - words = [] - segs = text.split() - for seg in segs: - # There is no space in seg. - current_word = "" - for c in seg: - if len(c.encode()) == 1: - # This is an ASCII char. - current_word += c - else: - # This is a Chinese char. - if len(current_word) > 0: - words.append(current_word) - current_word = "" - words.append(c) - if len(current_word) > 0: - words.append(current_word) - return words - - def __call__( - self, uid: str, data: Dict[str, Union[list, str, np.ndarray]] - ) -> Dict[str, Union[list, np.ndarray]]: - assert check_argument_types() - # Split words. - if isinstance(data[self.text_name], str): - split_text = self.split_words(data[self.text_name]) - else: - split_text = data[self.text_name] - data[self.text_name] = " ".join(split_text) - data = self._speech_process(data) - data = self._text_process(data) - data[self.split_text_name] = split_text - return data - - def pop_split_text_data(self, data: Dict[str, Union[str, np.ndarray]]): - result = data[self.split_text_name] - del data[self.split_text_name] - return result From 795b6e04864d7a8ea1cb8e41a412152651c47eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 17:14:29 +0800 Subject: [PATCH 38/55] export --- funasr/export/models/encoder/sanm_encoder.py | 7 +------ funasr/export/models/vad_realtime_transformer.py | 10 +++++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/funasr/export/models/encoder/sanm_encoder.py b/funasr/export/models/encoder/sanm_encoder.py index 8198d18a3..8390f6822 100644 --- a/funasr/export/models/encoder/sanm_encoder.py +++ b/funasr/export/models/encoder/sanm_encoder.py @@ -151,12 +151,7 @@ class SANMVadEncoder(nn.Module): def prepare_mask(self, mask, sub_masks): mask_3d_btd = mask[:, :, None] - # sub_masks = subsequent_mask(mask.size(-1)).type(torch.float32) - if len(mask.shape) == 2: - mask_4d_bhlt = 1 - sub_masks[:, None, None, :] - elif len(mask.shape) == 3: - mask_4d_bhlt = 1 - sub_masks[:, None, :] - mask_4d_bhlt = mask_4d_bhlt * -10000.0 + mask_4d_bhlt = (1 - sub_masks) * -10000.0 return mask_3d_btd, mask_4d_bhlt diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index a3d486432..093e71de1 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -63,11 +63,11 @@ class VadRealtimeTransformer(nn.Module): text_lengths = torch.tensor([length], dtype=torch.int32) vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :] sub_masks = torch.ones(length, length, dtype=torch.float32) - sub_masks = torch.tril(sub_masks) - return (text_indexes, text_lengths, vad_mask, sub_masks) + sub_masks = torch.tril(sub_masks).type(torch.float32) + return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :]) def get_input_names(self): - return ['input', 'text_lengths', 'vad_mask'] + return ['input', 'text_lengths', 'vad_mask', 'sub_masks'] def get_output_names(self): return ['logits'] @@ -81,6 +81,10 @@ class VadRealtimeTransformer(nn.Module): 2: 'feats_length1', 3: 'feats_length2' }, + 'sub_masks': { + 2: 'feats_length1', + 3: 'feats_length2' + }, 'logits': { 1: 'logits_length' }, From e98c18110341ff0de12e5497205478722739653f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 17:17:44 +0800 Subject: [PATCH 39/55] export --- funasr/export/test/test_onnx_punc_vadrealtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funasr/export/test/test_onnx_punc_vadrealtime.py b/funasr/export/test/test_onnx_punc_vadrealtime.py index 6544a898f..05f770438 100644 --- a/funasr/export/test/test_onnx_punc_vadrealtime.py +++ b/funasr/export/test/test_onnx_punc_vadrealtime.py @@ -12,7 +12,7 @@ if __name__ == '__main__': return {'input': np.ones((1, text_length), dtype=np.int64), 'text_lengths': np.array([text_length,], dtype=np.int32), 'vad_mask': np.ones((1, 1, text_length, text_length), dtype=np.float32), - 'sub_masks': np.tril(np.ones((text_length, text_length), dtype=np.float32)) + 'sub_masks': np.tril(np.ones((text_length, text_length), dtype=np.float32))[None, None, :, :] } def _run(feed_dict): From 496cd9b6b319988371c1bc0a6db018a23b8549c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Thu, 30 Mar 2023 17:18:18 +0800 Subject: [PATCH 40/55] export --- funasr/export/test/test_onnx_punc_vadrealtime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funasr/export/test/test_onnx_punc_vadrealtime.py b/funasr/export/test/test_onnx_punc_vadrealtime.py index 05f770438..54f85f194 100644 --- a/funasr/export/test/test_onnx_punc_vadrealtime.py +++ b/funasr/export/test/test_onnx_punc_vadrealtime.py @@ -12,7 +12,7 @@ if __name__ == '__main__': return {'input': np.ones((1, text_length), dtype=np.int64), 'text_lengths': np.array([text_length,], dtype=np.int32), 'vad_mask': np.ones((1, 1, text_length, text_length), dtype=np.float32), - 'sub_masks': np.tril(np.ones((text_length, text_length), dtype=np.float32))[None, None, :, :] + 'sub_masks': np.tril(np.ones((text_length, text_length), dtype=np.float32))[None, None, :, :].astype(np.float32) } def _run(feed_dict): From 5afbcdc39545ee1b417165d48cef3eddd4fc13ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 17:26:44 +0800 Subject: [PATCH 41/55] change name --- funasr/runtime/python/onnxruntime/demo_punc_offline.py | 4 ++-- funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py | 4 ++-- funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/demo_punc_offline.py b/funasr/runtime/python/onnxruntime/demo_punc_offline.py index 3ca8b4a4f..277bc1d72 100644 --- a/funasr/runtime/python/onnxruntime/demo_punc_offline.py +++ b/funasr/runtime/python/onnxruntime/demo_punc_offline.py @@ -1,7 +1,7 @@ -from funasr_onnx import TargetDelayTransformer +from funasr_onnx import CT-Transformer model_dir = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch" -model = TargetDelayTransformer(model_dir) +model = CT-Transformer(model_dir) text_in = "我们都是木头人不会讲话不会动" text_in="跨境河流是养育沿岸人民的生命之源长期以来为帮助下游地区防灾减灾中方技术人员在上游地区极为恶劣的自然条件下克服巨大困难甚至冒着生命危险向印方提供汛期水文资料处理紧急事件中方重视印方在跨境河流问题上的关切愿意进一步完善双方联合工作机制凡是中方能做的我们都会去做而且会做得更好我请印度朋友们放心中国在上游的任何开发利用都会经过科学规划和论证兼顾上下游的利益" diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py index 1620a0b25..6741d3be9 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- from .paraformer_bin import Paraformer from .vad_bin import Fsmn_vad -from .punc_bin import TargetDelayTransformer -#from .punc_bin import VadRealtimeTransformer +from .punc_bin import CT-Transformer +#from .punc_bin import VadRealtimeCT-Transformer diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index d72b0ce2f..d5969e22d 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -12,7 +12,7 @@ from .utils.utils import (TokenIDConverter, split_to_mini_sentence,code_mix_spli logging = get_logger() -class TargetDelayTransformer(): +class CT-Transformer(): def __init__(self, model_dir: Union[str, Path] = None, batch_size: int = 1, device_id: Union[str, int] = "-1", From 3cd71a385a31f987f2db99df902ca36ee02b1813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 17:29:12 +0800 Subject: [PATCH 42/55] change name --- funasr/runtime/python/onnxruntime/demo_punc_offline.py | 4 ++-- funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py | 4 ++-- funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/demo_punc_offline.py b/funasr/runtime/python/onnxruntime/demo_punc_offline.py index 277bc1d72..d13c06bdb 100644 --- a/funasr/runtime/python/onnxruntime/demo_punc_offline.py +++ b/funasr/runtime/python/onnxruntime/demo_punc_offline.py @@ -1,7 +1,7 @@ -from funasr_onnx import CT-Transformer +from funasr_onnx import CT_Transformer model_dir = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch" -model = CT-Transformer(model_dir) +model = CT_Transformer(model_dir) text_in = "我们都是木头人不会讲话不会动" text_in="跨境河流是养育沿岸人民的生命之源长期以来为帮助下游地区防灾减灾中方技术人员在上游地区极为恶劣的自然条件下克服巨大困难甚至冒着生命危险向印方提供汛期水文资料处理紧急事件中方重视印方在跨境河流问题上的关切愿意进一步完善双方联合工作机制凡是中方能做的我们都会去做而且会做得更好我请印度朋友们放心中国在上游的任何开发利用都会经过科学规划和论证兼顾上下游的利益" diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py index 6741d3be9..825f2ca38 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- from .paraformer_bin import Paraformer from .vad_bin import Fsmn_vad -from .punc_bin import CT-Transformer -#from .punc_bin import VadRealtimeCT-Transformer +from .punc_bin import CT_Transformer +#from .punc_bin import VadRealtimeCT_Transformer diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index d5969e22d..949172eb1 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -12,7 +12,7 @@ from .utils.utils import (TokenIDConverter, split_to_mini_sentence,code_mix_spli logging = get_logger() -class CT-Transformer(): +class CT_Transformer(): def __init__(self, model_dir: Union[str, Path] = None, batch_size: int = 1, device_id: Union[str, int] = "-1", From 141cf8e8bf40f6316b8cf86fca964e73eb54cadd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Thu, 30 Mar 2023 20:30:59 +0800 Subject: [PATCH 43/55] fix --- funasr/export/models/encoder/sanm_encoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/funasr/export/models/encoder/sanm_encoder.py b/funasr/export/models/encoder/sanm_encoder.py index 8390f6822..44a48ff6a 100644 --- a/funasr/export/models/encoder/sanm_encoder.py +++ b/funasr/export/models/encoder/sanm_encoder.py @@ -164,6 +164,7 @@ class SANMVadEncoder(nn.Module): speech = speech * self._output_size ** 0.5 mask = self.make_pad_mask(speech_lengths) mask = self.prepare_mask(mask, sub_masks) + vad_mask = self.prepare_mask(mask, vad_mask) if self.embed is None: xs_pad = speech else: @@ -175,7 +176,7 @@ class SANMVadEncoder(nn.Module): # encoder_outs = self.model.encoders(xs_pad, mask) for layer_idx, encoder_layer in enumerate(self.model.encoders): if layer_idx == len(self.model.encoders) - 1: - mask = (mask[0], vad_mask) + mask = vad_mask encoder_outs = encoder_layer(xs_pad, mask) xs_pad, masks = encoder_outs[0], encoder_outs[1] From d0cd484fdc21c06b8bc892bb2ab1c2a25fb1da8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 31 Mar 2023 15:05:37 +0800 Subject: [PATCH 44/55] export --- funasr/bin/punctuation_infer.py | 2 +- funasr/bin/punctuation_infer_vadrealtime.py | 2 +- funasr/datasets/preprocessor.py | 14 + funasr/export/models/__init__.py | 8 +- .../export/models/target_delay_transformer.py | 87 +-- .../export/models/vad_realtime_transformer.py | 7 +- funasr/lm/espnet_model.py | 2 +- funasr/models/encoder/sanm_encoder.py | 232 ++++++- .../target_delay_transformer.py | 3 +- .../vad_realtime_transformer.py | 2 +- funasr/punctuation/abs_model.py | 31 - funasr/punctuation/sanm_encoder.py | 590 ------------------ funasr/punctuation/text_preprocessor.py | 13 +- funasr/tasks/lm.py | 8 +- funasr/tasks/punctuation.py | 14 +- .../espnet_model.py => train/abs_model.py} | 56 +- 16 files changed, 309 insertions(+), 762 deletions(-) rename funasr/{punctuation => models}/target_delay_transformer.py (97%) rename funasr/{punctuation => models}/vad_realtime_transformer.py (98%) delete mode 100644 funasr/punctuation/abs_model.py delete mode 100644 funasr/punctuation/sanm_encoder.py rename funasr/{punctuation/espnet_model.py => train/abs_model.py} (86%) diff --git a/funasr/bin/punctuation_infer.py b/funasr/bin/punctuation_infer.py index a801ee8c6..dd28ef8da 100644 --- a/funasr/bin/punctuation_infer.py +++ b/funasr/bin/punctuation_infer.py @@ -23,7 +23,7 @@ from funasr.torch_utils.set_all_random_seed import set_all_random_seed from funasr.utils import config_argparse from funasr.utils.types import str2triple_str from funasr.utils.types import str_or_none -from funasr.punctuation.text_preprocessor import split_to_mini_sentence +from funasr.datasets.preprocessor import split_to_mini_sentence class Text2Punc: diff --git a/funasr/bin/punctuation_infer_vadrealtime.py b/funasr/bin/punctuation_infer_vadrealtime.py index ce1cee8b0..81f9d7ae8 100644 --- a/funasr/bin/punctuation_infer_vadrealtime.py +++ b/funasr/bin/punctuation_infer_vadrealtime.py @@ -23,7 +23,7 @@ from funasr.torch_utils.set_all_random_seed import set_all_random_seed from funasr.utils import config_argparse from funasr.utils.types import str2triple_str from funasr.utils.types import str_or_none -from funasr.punctuation.text_preprocessor import split_to_mini_sentence +from funasr.datasets.preprocessor import split_to_mini_sentence class Text2Punc: diff --git a/funasr/datasets/preprocessor.py b/funasr/datasets/preprocessor.py index 98cca1dcd..afeff4ee6 100644 --- a/funasr/datasets/preprocessor.py +++ b/funasr/datasets/preprocessor.py @@ -800,3 +800,17 @@ class PuncTrainTokenizerCommonPreprocessor(CommonPreprocessor): data[self.vad_name] = np.array([vad], dtype=np.int64) text_ints = self.token_id_converter[i].tokens2ids(tokens) data[text_name] = np.array(text_ints, dtype=np.int64) + + +def split_to_mini_sentence(words: list, word_limit: int = 20): + assert word_limit > 1 + if len(words) <= word_limit: + return [words] + sentences = [] + length = len(words) + sentence_len = length // word_limit + for i in range(sentence_len): + sentences.append(words[i * word_limit:(i + 1) * word_limit]) + if length % word_limit > 0: + sentences.append(words[sentence_len * word_limit:]) + return sentences \ No newline at end of file diff --git a/funasr/export/models/__init__.py b/funasr/export/models/__init__.py index 62ee72354..4ac0456b9 100644 --- a/funasr/export/models/__init__.py +++ b/funasr/export/models/__init__.py @@ -3,10 +3,10 @@ from funasr.export.models.e2e_asr_paraformer import Paraformer as Paraformer_exp from funasr.export.models.e2e_asr_paraformer import BiCifParaformer as BiCifParaformer_export from funasr.models.e2e_vad import E2EVadModel from funasr.export.models.e2e_vad import E2EVadModel as E2EVadModel_export -from funasr.punctuation.target_delay_transformer import TargetDelayTransformer +from funasr.models.target_delay_transformer import TargetDelayTransformer from funasr.export.models.target_delay_transformer import TargetDelayTransformer as TargetDelayTransformer_export -from funasr.punctuation.espnet_model import ESPnetPunctuationModel -from funasr.punctuation.vad_realtime_transformer import VadRealtimeTransformer +from funasr.train.abs_model import PunctuationModel +from funasr.models.vad_realtime_transformer import VadRealtimeTransformer from funasr.export.models.vad_realtime_transformer import VadRealtimeTransformer as VadRealtimeTransformer_export def get_model(model, export_config=None): @@ -16,7 +16,7 @@ def get_model(model, export_config=None): return Paraformer_export(model, **export_config) elif isinstance(model, E2EVadModel): return E2EVadModel_export(model, **export_config) - elif isinstance(model, ESPnetPunctuationModel): + elif isinstance(model, PunctuationModel): if isinstance(model.punc_model, TargetDelayTransformer): return TargetDelayTransformer_export(model.punc_model, **export_config) elif isinstance(model.punc_model, VadRealtimeTransformer): diff --git a/funasr/export/models/target_delay_transformer.py b/funasr/export/models/target_delay_transformer.py index fd90835c9..bfe3ec423 100644 --- a/funasr/export/models/target_delay_transformer.py +++ b/funasr/export/models/target_delay_transformer.py @@ -1,18 +1,8 @@ -from typing import Any -from typing import List from typing import Tuple import torch import torch.nn as nn -from funasr.export.utils.torch_function import MakePadMask -from funasr.export.utils.torch_function import sequence_mask -#from funasr.models.encoder.sanm_encoder import SANMEncoder as Encoder -from funasr.punctuation.sanm_encoder import SANMEncoder -from funasr.export.models.encoder.sanm_encoder import SANMEncoder as SANMEncoder_export -from funasr.punctuation.abs_model import AbsPunctuation - - class TargetDelayTransformer(nn.Module): def __init__( @@ -32,85 +22,10 @@ class TargetDelayTransformer(nn.Module): self.feats_dim = self.embed.embedding_dim self.num_embeddings = self.embed.num_embeddings self.model_name = model_name - from typing import Any - from typing import List - from typing import Tuple - import torch - import torch.nn as nn - - from funasr.export.utils.torch_function import MakePadMask - from funasr.export.utils.torch_function import sequence_mask # from funasr.models.encoder.sanm_encoder import SANMEncoder as Encoder - from funasr.punctuation.sanm_encoder import SANMEncoder + from funasr.models.encoder.sanm_encoder import SANMEncoder from funasr.export.models.encoder.sanm_encoder import SANMEncoder as SANMEncoder_export - from funasr.punctuation.abs_model import AbsPunctuation - - # class TargetDelayTransformer(nn.Module): - # - # def __init__( - # self, - # model, - # max_seq_len=512, - # model_name='punc_model', - # **kwargs, - # ): - # super().__init__() - # onnx = False - # if "onnx" in kwargs: - # onnx = kwargs["onnx"] - # self.embed = model.embed - # self.decoder = model.decoder - # self.model = model - # self.feats_dim = self.embed.embedding_dim - # self.num_embeddings = self.embed.num_embeddings - # self.model_name = model_name - # - # if isinstance(model.encoder, SANMEncoder): - # self.encoder = SANMEncoder_export(model.encoder, onnx=onnx) - # else: - # assert False, "Only support samn encode." - # - # def forward(self, input: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]: - # """Compute loss value from buffer sequences. - # - # Args: - # input (torch.Tensor): Input ids. (batch, len) - # hidden (torch.Tensor): Target ids. (batch, len) - # - # """ - # x = self.embed(input) - # # mask = self._target_mask(input) - # h, _ = self.encoder(x, text_lengths) - # y = self.decoder(h) - # return y - # - # def get_dummy_inputs(self): - # length = 120 - # text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)) - # text_lengths = torch.tensor([length - 20, length], dtype=torch.int32) - # return (text_indexes, text_lengths) - # - # def get_input_names(self): - # return ['input', 'text_lengths'] - # - # def get_output_names(self): - # return ['logits'] - # - # def get_dynamic_axes(self): - # return { - # 'input': { - # 0: 'batch_size', - # 1: 'feats_length' - # }, - # 'text_lengths': { - # 0: 'batch_size', - # }, - # 'logits': { - # 0: 'batch_size', - # 1: 'logits_length' - # }, - # } if isinstance(model.encoder, SANMEncoder): self.encoder = SANMEncoder_export(model.encoder, onnx=onnx) diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index 093e71de1..693b9c844 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -1,14 +1,9 @@ -from typing import Any -from typing import List from typing import Tuple import torch import torch.nn as nn -from funasr.modules.embedding import SinusoidalPositionEncoder -from funasr.punctuation.sanm_encoder import SANMVadEncoder as Encoder -from funasr.punctuation.abs_model import AbsPunctuation -from funasr.punctuation.sanm_encoder import SANMVadEncoder +from funasr.models.encoder.sanm_encoder import SANMVadEncoder from funasr.export.models.encoder.sanm_encoder import SANMVadEncoder as SANMVadEncoder_export class VadRealtimeTransformer(nn.Module): diff --git a/funasr/lm/espnet_model.py b/funasr/lm/espnet_model.py index db11b6741..a9b8130c6 100644 --- a/funasr/lm/espnet_model.py +++ b/funasr/lm/espnet_model.py @@ -12,7 +12,7 @@ from funasr.torch_utils.device_funcs import force_gatherable from funasr.train.abs_espnet_model import AbsESPnetModel -class ESPnetLanguageModel(AbsESPnetModel): +class LanguageModel(AbsESPnetModel): def __init__(self, lm: AbsLM, vocab_size: int, ignore_id: int = 0): assert check_argument_types() super().__init__() diff --git a/funasr/models/encoder/sanm_encoder.py b/funasr/models/encoder/sanm_encoder.py index 57890efe6..2a3a35353 100644 --- a/funasr/models/encoder/sanm_encoder.py +++ b/funasr/models/encoder/sanm_encoder.py @@ -10,7 +10,7 @@ from funasr.modules.streaming_utils.chunk_utilis import overlap_chunk from typeguard import check_argument_types import numpy as np from funasr.modules.nets_utils import make_pad_mask -from funasr.modules.attention import MultiHeadedAttention, MultiHeadedAttentionSANM +from funasr.modules.attention import MultiHeadedAttention, MultiHeadedAttentionSANM, MultiHeadedAttentionSANMwithMask from funasr.modules.embedding import SinusoidalPositionEncoder from funasr.modules.layer_norm import LayerNorm from funasr.modules.multi_layer_conv import Conv1dLinear @@ -27,7 +27,7 @@ from funasr.modules.subsampling import TooShortUttError from funasr.modules.subsampling import check_short_utt from funasr.models.ctc import CTC from funasr.models.encoder.abs_encoder import AbsEncoder - +from funasr.modules.mask import subsequent_mask, vad_mask class EncoderLayerSANM(nn.Module): def __init__( @@ -958,3 +958,231 @@ class SANMEncoderChunkOpt(AbsEncoder): var_dict_tf[name_tf].shape)) return var_dict_torch_update + + +class SANMVadEncoder(AbsEncoder): + """ + author: Speech Lab, Alibaba Group, China + + """ + + def __init__( + self, + input_size: int, + output_size: int = 256, + attention_heads: int = 4, + linear_units: int = 2048, + num_blocks: int = 6, + dropout_rate: float = 0.1, + positional_dropout_rate: float = 0.1, + attention_dropout_rate: float = 0.0, + input_layer: Optional[str] = "conv2d", + pos_enc_class=SinusoidalPositionEncoder, + normalize_before: bool = True, + concat_after: bool = False, + positionwise_layer_type: str = "linear", + positionwise_conv_kernel_size: int = 1, + padding_idx: int = -1, + interctc_layer_idx: List[int] = [], + interctc_use_conditioning: bool = False, + kernel_size : int = 11, + sanm_shfit : int = 0, + selfattention_layer_type: str = "sanm", + ): + assert check_argument_types() + super().__init__() + self._output_size = output_size + + if input_layer == "linear": + self.embed = torch.nn.Sequential( + torch.nn.Linear(input_size, output_size), + torch.nn.LayerNorm(output_size), + torch.nn.Dropout(dropout_rate), + torch.nn.ReLU(), + pos_enc_class(output_size, positional_dropout_rate), + ) + elif input_layer == "conv2d": + self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) + elif input_layer == "conv2d2": + self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) + elif input_layer == "conv2d6": + self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) + elif input_layer == "conv2d8": + self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) + elif input_layer == "embed": + self.embed = torch.nn.Sequential( + torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), + SinusoidalPositionEncoder(), + ) + elif input_layer is None: + if input_size == output_size: + self.embed = None + else: + self.embed = torch.nn.Linear(input_size, output_size) + elif input_layer == "pe": + self.embed = SinusoidalPositionEncoder() + else: + raise ValueError("unknown input_layer: " + input_layer) + self.normalize_before = normalize_before + if positionwise_layer_type == "linear": + positionwise_layer = PositionwiseFeedForward + positionwise_layer_args = ( + output_size, + linear_units, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d": + positionwise_layer = MultiLayeredConv1d + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + elif positionwise_layer_type == "conv1d-linear": + positionwise_layer = Conv1dLinear + positionwise_layer_args = ( + output_size, + linear_units, + positionwise_conv_kernel_size, + dropout_rate, + ) + else: + raise NotImplementedError("Support only linear or conv1d.") + + if selfattention_layer_type == "selfattn": + encoder_selfattn_layer = MultiHeadedAttention + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + attention_dropout_rate, + ) + + elif selfattention_layer_type == "sanm": + self.encoder_selfattn_layer = MultiHeadedAttentionSANMwithMask + encoder_selfattn_layer_args0 = ( + attention_heads, + input_size, + output_size, + attention_dropout_rate, + kernel_size, + sanm_shfit, + ) + + encoder_selfattn_layer_args = ( + attention_heads, + output_size, + output_size, + attention_dropout_rate, + kernel_size, + sanm_shfit, + ) + + self.encoders0 = repeat( + 1, + lambda lnum: EncoderLayerSANM( + input_size, + output_size, + self.encoder_selfattn_layer(*encoder_selfattn_layer_args0), + positionwise_layer(*positionwise_layer_args), + dropout_rate, + normalize_before, + concat_after, + ), + ) + + self.encoders = repeat( + num_blocks-1, + lambda lnum: EncoderLayerSANM( + output_size, + output_size, + self.encoder_selfattn_layer(*encoder_selfattn_layer_args), + positionwise_layer(*positionwise_layer_args), + dropout_rate, + normalize_before, + concat_after, + ), + ) + if self.normalize_before: + self.after_norm = LayerNorm(output_size) + + self.interctc_layer_idx = interctc_layer_idx + if len(interctc_layer_idx) > 0: + assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks + self.interctc_use_conditioning = interctc_use_conditioning + self.conditioning_layer = None + self.dropout = nn.Dropout(dropout_rate) + + def output_size(self) -> int: + return self._output_size + + def forward( + self, + xs_pad: torch.Tensor, + ilens: torch.Tensor, + vad_indexes: torch.Tensor, + prev_states: torch.Tensor = None, + ctc: CTC = None, + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """Embed positions in tensor. + + Args: + xs_pad: input tensor (B, L, D) + ilens: input length (B) + prev_states: Not to be used now. + Returns: + position embedded tensor and mask + """ + masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) + sub_masks = subsequent_mask(masks.size(-1), device=xs_pad.device).unsqueeze(0) + no_future_masks = masks & sub_masks + xs_pad *= self.output_size()**0.5 + if self.embed is None: + xs_pad = xs_pad + elif (isinstance(self.embed, Conv2dSubsampling) or isinstance(self.embed, Conv2dSubsampling2) + or isinstance(self.embed, Conv2dSubsampling6) or isinstance(self.embed, Conv2dSubsampling8)): + short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) + if short_status: + raise TooShortUttError( + f"has {xs_pad.size(1)} frames and is too short for subsampling " + + f"(it needs more than {limit_size} frames), return empty results", + xs_pad.size(1), + limit_size, + ) + xs_pad, masks = self.embed(xs_pad, masks) + else: + xs_pad = self.embed(xs_pad) + + # xs_pad = self.dropout(xs_pad) + mask_tup0 = [masks, no_future_masks] + encoder_outs = self.encoders0(xs_pad, mask_tup0) + xs_pad, _ = encoder_outs[0], encoder_outs[1] + intermediate_outs = [] + + + for layer_idx, encoder_layer in enumerate(self.encoders): + if layer_idx + 1 == len(self.encoders): + # This is last layer. + coner_mask = torch.ones(masks.size(0), + masks.size(-1), + masks.size(-1), + device=xs_pad.device, + dtype=torch.bool) + for word_index, length in enumerate(ilens): + coner_mask[word_index, :, :] = vad_mask(masks.size(-1), + vad_indexes[word_index], + device=xs_pad.device) + layer_mask = masks & coner_mask + else: + layer_mask = no_future_masks + mask_tup1 = [masks, layer_mask] + encoder_outs = encoder_layer(xs_pad, mask_tup1) + xs_pad, layer_mask = encoder_outs[0], encoder_outs[1] + + if self.normalize_before: + xs_pad = self.after_norm(xs_pad) + + olens = masks.squeeze(1).sum(1) + if len(intermediate_outs) > 0: + return (xs_pad, intermediate_outs), olens, None + return xs_pad, olens, None diff --git a/funasr/punctuation/target_delay_transformer.py b/funasr/models/target_delay_transformer.py similarity index 97% rename from funasr/punctuation/target_delay_transformer.py rename to funasr/models/target_delay_transformer.py index 219af263f..a71952b15 100644 --- a/funasr/punctuation/target_delay_transformer.py +++ b/funasr/models/target_delay_transformer.py @@ -5,12 +5,11 @@ from typing import Tuple import torch import torch.nn as nn -from funasr.modules.embedding import PositionalEncoding from funasr.modules.embedding import SinusoidalPositionEncoder #from funasr.models.encoder.transformer_encoder import TransformerEncoder as Encoder from funasr.punctuation.sanm_encoder import SANMEncoder as Encoder #from funasr.modules.mask import subsequent_n_mask -from funasr.punctuation.abs_model import AbsPunctuation +from funasr.train.abs_model import AbsPunctuation class TargetDelayTransformer(AbsPunctuation): diff --git a/funasr/punctuation/vad_realtime_transformer.py b/funasr/models/vad_realtime_transformer.py similarity index 98% rename from funasr/punctuation/vad_realtime_transformer.py rename to funasr/models/vad_realtime_transformer.py index 35224f9bd..2945572f5 100644 --- a/funasr/punctuation/vad_realtime_transformer.py +++ b/funasr/models/vad_realtime_transformer.py @@ -7,7 +7,7 @@ import torch.nn as nn from funasr.modules.embedding import SinusoidalPositionEncoder from funasr.punctuation.sanm_encoder import SANMVadEncoder as Encoder -from funasr.punctuation.abs_model import AbsPunctuation +from funasr.train.abs_model import AbsPunctuation class VadRealtimeTransformer(AbsPunctuation): diff --git a/funasr/punctuation/abs_model.py b/funasr/punctuation/abs_model.py deleted file mode 100644 index 404d5e893..000000000 --- a/funasr/punctuation/abs_model.py +++ /dev/null @@ -1,31 +0,0 @@ -from abc import ABC -from abc import abstractmethod -from typing import Tuple - -import torch - -from funasr.modules.scorers.scorer_interface import BatchScorerInterface - - -class AbsPunctuation(torch.nn.Module, BatchScorerInterface, ABC): - """The abstract class - - To share the loss calculation way among different models, - We uses delegate pattern here: - The instance of this class should be passed to "LanguageModel" - - >>> from funasr.punctuation.abs_model import AbsPunctuation - >>> punc = AbsPunctuation() - >>> model = ESPnetPunctuationModel(punc=punc) - - This "model" is one of mediator objects for "Task" class. - - """ - - @abstractmethod - def forward(self, input: torch.Tensor, hidden: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - raise NotImplementedError - - @abstractmethod - def with_vad(self) -> bool: - raise NotImplementedError diff --git a/funasr/punctuation/sanm_encoder.py b/funasr/punctuation/sanm_encoder.py deleted file mode 100644 index 896209323..000000000 --- a/funasr/punctuation/sanm_encoder.py +++ /dev/null @@ -1,590 +0,0 @@ -from typing import List -from typing import Optional -from typing import Sequence -from typing import Tuple -from typing import Union -import logging -import torch -import torch.nn as nn -from funasr.modules.streaming_utils.chunk_utilis import overlap_chunk -from typeguard import check_argument_types -import numpy as np -from funasr.modules.nets_utils import make_pad_mask -from funasr.modules.attention import MultiHeadedAttention, MultiHeadedAttentionSANM, MultiHeadedAttentionSANMwithMask -from funasr.modules.embedding import SinusoidalPositionEncoder -from funasr.modules.layer_norm import LayerNorm -from funasr.modules.multi_layer_conv import Conv1dLinear -from funasr.modules.multi_layer_conv import MultiLayeredConv1d -from funasr.modules.positionwise_feed_forward import ( - PositionwiseFeedForward, # noqa: H301 -) -from funasr.modules.repeat import repeat -from funasr.modules.subsampling import Conv2dSubsampling -from funasr.modules.subsampling import Conv2dSubsampling2 -from funasr.modules.subsampling import Conv2dSubsampling6 -from funasr.modules.subsampling import Conv2dSubsampling8 -from funasr.modules.subsampling import TooShortUttError -from funasr.modules.subsampling import check_short_utt -from funasr.models.ctc import CTC -from funasr.models.encoder.abs_encoder import AbsEncoder - -from funasr.modules.nets_utils import make_pad_mask -from funasr.modules.mask import subsequent_mask, vad_mask - -class EncoderLayerSANM(nn.Module): - def __init__( - self, - in_size, - size, - self_attn, - feed_forward, - dropout_rate, - normalize_before=True, - concat_after=False, - stochastic_depth_rate=0.0, - ): - """Construct an EncoderLayer object.""" - super(EncoderLayerSANM, self).__init__() - self.self_attn = self_attn - self.feed_forward = feed_forward - self.norm1 = LayerNorm(in_size) - self.norm2 = LayerNorm(size) - self.dropout = nn.Dropout(dropout_rate) - self.in_size = in_size - self.size = size - self.normalize_before = normalize_before - self.concat_after = concat_after - if self.concat_after: - self.concat_linear = nn.Linear(size + size, size) - self.stochastic_depth_rate = stochastic_depth_rate - self.dropout_rate = dropout_rate - - def forward(self, x, mask, cache=None, mask_shfit_chunk=None, mask_att_chunk_encoder=None): - """Compute encoded features. - - Args: - x_input (torch.Tensor): Input tensor (#batch, time, size). - mask (torch.Tensor): Mask tensor for the input (#batch, time). - cache (torch.Tensor): Cache tensor of the input (#batch, time - 1, size). - - Returns: - torch.Tensor: Output tensor (#batch, time, size). - torch.Tensor: Mask tensor (#batch, time). - - """ - skip_layer = False - # with stochastic depth, residual connection `x + f(x)` becomes - # `x <- x + 1 / (1 - p) * f(x)` at training time. - stoch_layer_coeff = 1.0 - if self.training and self.stochastic_depth_rate > 0: - skip_layer = torch.rand(1).item() < self.stochastic_depth_rate - stoch_layer_coeff = 1.0 / (1 - self.stochastic_depth_rate) - - if skip_layer: - if cache is not None: - x = torch.cat([cache, x], dim=1) - return x, mask - - residual = x - if self.normalize_before: - x = self.norm1(x) - - if self.concat_after: - x_concat = torch.cat((x, self.self_attn(x, mask, mask_shfit_chunk=mask_shfit_chunk, mask_att_chunk_encoder=mask_att_chunk_encoder)), dim=-1) - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.concat_linear(x_concat) - else: - x = stoch_layer_coeff * self.concat_linear(x_concat) - else: - if self.in_size == self.size: - x = residual + stoch_layer_coeff * self.dropout( - self.self_attn(x, mask, mask_shfit_chunk=mask_shfit_chunk, mask_att_chunk_encoder=mask_att_chunk_encoder) - ) - else: - x = stoch_layer_coeff * self.dropout( - self.self_attn(x, mask, mask_shfit_chunk=mask_shfit_chunk, mask_att_chunk_encoder=mask_att_chunk_encoder) - ) - if not self.normalize_before: - x = self.norm1(x) - - residual = x - if self.normalize_before: - x = self.norm2(x) - x = residual + stoch_layer_coeff * self.dropout(self.feed_forward(x)) - if not self.normalize_before: - x = self.norm2(x) - - - return x, mask, cache, mask_shfit_chunk, mask_att_chunk_encoder - -class SANMEncoder(AbsEncoder): - """ - author: Speech Lab, Alibaba Group, China - - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - pos_enc_class=SinusoidalPositionEncoder, - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 1, - padding_idx: int = -1, - interctc_layer_idx: List[int] = [], - interctc_use_conditioning: bool = False, - kernel_size : int = 11, - sanm_shfit : int = 0, - selfattention_layer_type: str = "sanm", - ): - assert check_argument_types() - super().__init__() - self._output_size = output_size - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - SinusoidalPositionEncoder(), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - elif input_layer == "pe": - self.embed = SinusoidalPositionEncoder() - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - - if selfattention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - - elif selfattention_layer_type == "sanm": - self.encoder_selfattn_layer = MultiHeadedAttentionSANM - encoder_selfattn_layer_args0 = ( - attention_heads, - input_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - ) - - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - ) - - self.encoders0 = repeat( - 1, - lambda lnum: EncoderLayerSANM( - input_size, - output_size, - self.encoder_selfattn_layer(*encoder_selfattn_layer_args0), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.encoders = repeat( - num_blocks-1, - lambda lnum: EncoderLayerSANM( - output_size, - output_size, - self.encoder_selfattn_layer(*encoder_selfattn_layer_args), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - self.dropout = nn.Dropout(dropout_rate) - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - xs_pad *= self.output_size()**0.5 - if self.embed is None: - xs_pad = xs_pad - elif ( - isinstance(self.embed, Conv2dSubsampling) - or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) - or isinstance(self.embed, Conv2dSubsampling8) - ): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " - + f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - - # xs_pad = self.dropout(xs_pad) - encoder_outs = self.encoders0(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - intermediate_outs = [] - if len(self.interctc_layer_idx) == 0: - encoder_outs = self.encoders(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - encoder_outs = encoder_layer(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - - # intermediate outputs are also normalized - if self.normalize_before: - encoder_out = self.after_norm(encoder_out) - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None - -class SANMVadEncoder(AbsEncoder): - """ - author: Speech Lab, Alibaba Group, China - - """ - - def __init__( - self, - input_size: int, - output_size: int = 256, - attention_heads: int = 4, - linear_units: int = 2048, - num_blocks: int = 6, - dropout_rate: float = 0.1, - positional_dropout_rate: float = 0.1, - attention_dropout_rate: float = 0.0, - input_layer: Optional[str] = "conv2d", - pos_enc_class=SinusoidalPositionEncoder, - normalize_before: bool = True, - concat_after: bool = False, - positionwise_layer_type: str = "linear", - positionwise_conv_kernel_size: int = 1, - padding_idx: int = -1, - interctc_layer_idx: List[int] = [], - interctc_use_conditioning: bool = False, - kernel_size : int = 11, - sanm_shfit : int = 0, - selfattention_layer_type: str = "sanm", - ): - assert check_argument_types() - super().__init__() - self._output_size = output_size - - if input_layer == "linear": - self.embed = torch.nn.Sequential( - torch.nn.Linear(input_size, output_size), - torch.nn.LayerNorm(output_size), - torch.nn.Dropout(dropout_rate), - torch.nn.ReLU(), - pos_enc_class(output_size, positional_dropout_rate), - ) - elif input_layer == "conv2d": - self.embed = Conv2dSubsampling(input_size, output_size, dropout_rate) - elif input_layer == "conv2d2": - self.embed = Conv2dSubsampling2(input_size, output_size, dropout_rate) - elif input_layer == "conv2d6": - self.embed = Conv2dSubsampling6(input_size, output_size, dropout_rate) - elif input_layer == "conv2d8": - self.embed = Conv2dSubsampling8(input_size, output_size, dropout_rate) - elif input_layer == "embed": - self.embed = torch.nn.Sequential( - torch.nn.Embedding(input_size, output_size, padding_idx=padding_idx), - SinusoidalPositionEncoder(), - ) - elif input_layer is None: - if input_size == output_size: - self.embed = None - else: - self.embed = torch.nn.Linear(input_size, output_size) - elif input_layer == "pe": - self.embed = SinusoidalPositionEncoder() - else: - raise ValueError("unknown input_layer: " + input_layer) - self.normalize_before = normalize_before - if positionwise_layer_type == "linear": - positionwise_layer = PositionwiseFeedForward - positionwise_layer_args = ( - output_size, - linear_units, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d": - positionwise_layer = MultiLayeredConv1d - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - elif positionwise_layer_type == "conv1d-linear": - positionwise_layer = Conv1dLinear - positionwise_layer_args = ( - output_size, - linear_units, - positionwise_conv_kernel_size, - dropout_rate, - ) - else: - raise NotImplementedError("Support only linear or conv1d.") - - if selfattention_layer_type == "selfattn": - encoder_selfattn_layer = MultiHeadedAttention - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - attention_dropout_rate, - ) - - elif selfattention_layer_type == "sanm": - self.encoder_selfattn_layer = MultiHeadedAttentionSANMwithMask - encoder_selfattn_layer_args0 = ( - attention_heads, - input_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - ) - - encoder_selfattn_layer_args = ( - attention_heads, - output_size, - output_size, - attention_dropout_rate, - kernel_size, - sanm_shfit, - ) - - self.encoders0 = repeat( - 1, - lambda lnum: EncoderLayerSANM( - input_size, - output_size, - self.encoder_selfattn_layer(*encoder_selfattn_layer_args0), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - - self.encoders = repeat( - num_blocks-1, - lambda lnum: EncoderLayerSANM( - output_size, - output_size, - self.encoder_selfattn_layer(*encoder_selfattn_layer_args), - positionwise_layer(*positionwise_layer_args), - dropout_rate, - normalize_before, - concat_after, - ), - ) - if self.normalize_before: - self.after_norm = LayerNorm(output_size) - - self.interctc_layer_idx = interctc_layer_idx - if len(interctc_layer_idx) > 0: - assert 0 < min(interctc_layer_idx) and max(interctc_layer_idx) < num_blocks - self.interctc_use_conditioning = interctc_use_conditioning - self.conditioning_layer = None - self.dropout = nn.Dropout(dropout_rate) - - def output_size(self) -> int: - return self._output_size - - def forward( - self, - xs_pad: torch.Tensor, - ilens: torch.Tensor, - vad_indexes: torch.Tensor, - prev_states: torch.Tensor = None, - ctc: CTC = None, - ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: - """Embed positions in tensor. - - Args: - xs_pad: input tensor (B, L, D) - ilens: input length (B) - prev_states: Not to be used now. - Returns: - position embedded tensor and mask - """ - masks = (~make_pad_mask(ilens)[:, None, :]).to(xs_pad.device) - sub_masks = subsequent_mask(masks.size(-1), device=xs_pad.device).unsqueeze(0) - no_future_masks = masks & sub_masks - xs_pad *= self.output_size()**0.5 - if self.embed is None: - xs_pad = xs_pad - elif (isinstance(self.embed, Conv2dSubsampling) or isinstance(self.embed, Conv2dSubsampling2) - or isinstance(self.embed, Conv2dSubsampling6) or isinstance(self.embed, Conv2dSubsampling8)): - short_status, limit_size = check_short_utt(self.embed, xs_pad.size(1)) - if short_status: - raise TooShortUttError( - f"has {xs_pad.size(1)} frames and is too short for subsampling " + - f"(it needs more than {limit_size} frames), return empty results", - xs_pad.size(1), - limit_size, - ) - xs_pad, masks = self.embed(xs_pad, masks) - else: - xs_pad = self.embed(xs_pad) - - # xs_pad = self.dropout(xs_pad) - mask_tup0 = [masks, no_future_masks] - encoder_outs = self.encoders0(xs_pad, mask_tup0) - xs_pad, _ = encoder_outs[0], encoder_outs[1] - intermediate_outs = [] - #if len(self.interctc_layer_idx) == 0: - if False: - # Here, we should not use the repeat operation to do it for all layers. - encoder_outs = self.encoders(xs_pad, masks) - xs_pad, masks = encoder_outs[0], encoder_outs[1] - else: - for layer_idx, encoder_layer in enumerate(self.encoders): - if layer_idx + 1 == len(self.encoders): - # This is last layer. - coner_mask = torch.ones(masks.size(0), - masks.size(-1), - masks.size(-1), - device=xs_pad.device, - dtype=torch.bool) - for word_index, length in enumerate(ilens): - coner_mask[word_index, :, :] = vad_mask(masks.size(-1), - vad_indexes[word_index], - device=xs_pad.device) - layer_mask = masks & coner_mask - else: - layer_mask = no_future_masks - mask_tup1 = [masks, layer_mask] - encoder_outs = encoder_layer(xs_pad, mask_tup1) - xs_pad, layer_mask = encoder_outs[0], encoder_outs[1] - - if layer_idx + 1 in self.interctc_layer_idx: - encoder_out = xs_pad - - # intermediate outputs are also normalized - if self.normalize_before: - encoder_out = self.after_norm(encoder_out) - - intermediate_outs.append((layer_idx + 1, encoder_out)) - - if self.interctc_use_conditioning: - ctc_out = ctc.softmax(encoder_out) - xs_pad = xs_pad + self.conditioning_layer(ctc_out) - - if self.normalize_before: - xs_pad = self.after_norm(xs_pad) - - olens = masks.squeeze(1).sum(1) - if len(intermediate_outs) > 0: - return (xs_pad, intermediate_outs), olens, None - return xs_pad, olens, None - diff --git a/funasr/punctuation/text_preprocessor.py b/funasr/punctuation/text_preprocessor.py index c9c4bac57..8b1378917 100644 --- a/funasr/punctuation/text_preprocessor.py +++ b/funasr/punctuation/text_preprocessor.py @@ -1,12 +1 @@ -def split_to_mini_sentence(words: list, word_limit: int = 20): - assert word_limit > 1 - if len(words) <= word_limit: - return [words] - sentences = [] - length = len(words) - sentence_len = length // word_limit - for i in range(sentence_len): - sentences.append(words[i * word_limit:(i + 1) * word_limit]) - if length % word_limit > 0: - sentences.append(words[sentence_len * word_limit:]) - return sentences + diff --git a/funasr/tasks/lm.py b/funasr/tasks/lm.py index 608c1d3eb..dc8fd3e25 100644 --- a/funasr/tasks/lm.py +++ b/funasr/tasks/lm.py @@ -15,7 +15,7 @@ from typeguard import check_return_type from funasr.datasets.collate_fn import CommonCollateFn from funasr.datasets.preprocessor import CommonPreprocessor from funasr.lm.abs_model import AbsLM -from funasr.lm.espnet_model import ESPnetLanguageModel +from funasr.lm.espnet_model import LanguageModel from funasr.lm.seq_rnn_lm import SequentialRNNLM from funasr.lm.transformer_lm import TransformerLM from funasr.tasks.abs_task import AbsTask @@ -83,7 +83,7 @@ class LMTask(AbsTask): group.add_argument( "--model_conf", action=NestedDictAction, - default=get_default_kwargs(ESPnetLanguageModel), + default=get_default_kwargs(LanguageModel), help="The keyword arguments for model class.", ) @@ -178,7 +178,7 @@ class LMTask(AbsTask): return retval @classmethod - def build_model(cls, args: argparse.Namespace) -> ESPnetLanguageModel: + def build_model(cls, args: argparse.Namespace) -> LanguageModel: assert check_argument_types() if isinstance(args.token_list, str): with open(args.token_list, encoding="utf-8") as f: @@ -201,7 +201,7 @@ class LMTask(AbsTask): # 2. Build ESPnetModel # Assume the last-id is sos_and_eos - model = ESPnetLanguageModel(lm=lm, vocab_size=vocab_size, **args.model_conf) + model = LanguageModel(lm=lm, vocab_size=vocab_size, **args.model_conf) # 3. Initialize if args.init is not None: diff --git a/funasr/tasks/punctuation.py b/funasr/tasks/punctuation.py index ea1e10284..0170f28a8 100644 --- a/funasr/tasks/punctuation.py +++ b/funasr/tasks/punctuation.py @@ -14,10 +14,10 @@ from typeguard import check_return_type from funasr.datasets.collate_fn import CommonCollateFn from funasr.datasets.preprocessor import PuncTrainTokenizerCommonPreprocessor -from funasr.punctuation.abs_model import AbsPunctuation -from funasr.punctuation.espnet_model import ESPnetPunctuationModel -from funasr.punctuation.target_delay_transformer import TargetDelayTransformer -from funasr.punctuation.vad_realtime_transformer import VadRealtimeTransformer +from funasr.train.abs_model import AbsPunctuation +from funasr.train.abs_model import PunctuationModel +from funasr.models.target_delay_transformer import TargetDelayTransformer +from funasr.models.vad_realtime_transformer import VadRealtimeTransformer from funasr.tasks.abs_task import AbsTask from funasr.text.phoneme_tokenizer import g2p_choices from funasr.torch_utils.initialize import initialize @@ -79,7 +79,7 @@ class PunctuationTask(AbsTask): group.add_argument( "--model_conf", action=NestedDictAction, - default=get_default_kwargs(ESPnetPunctuationModel), + default=get_default_kwargs(PunctuationModel), help="The keyword arguments for model class.", ) @@ -183,7 +183,7 @@ class PunctuationTask(AbsTask): return retval @classmethod - def build_model(cls, args: argparse.Namespace) -> ESPnetPunctuationModel: + def build_model(cls, args: argparse.Namespace) -> PunctuationModel: assert check_argument_types() if isinstance(args.token_list, str): with open(args.token_list, encoding="utf-8") as f: @@ -218,7 +218,7 @@ class PunctuationTask(AbsTask): # Assume the last-id is sos_and_eos if "punc_weight" in args.model_conf: args.model_conf.pop("punc_weight") - model = ESPnetPunctuationModel(punc_model=punc, vocab_size=vocab_size, punc_weight=punc_weight_list, **args.model_conf) + model = PunctuationModel(punc_model=punc, vocab_size=vocab_size, punc_weight=punc_weight_list, **args.model_conf) # FIXME(kamo): Should be done in model? # 3. Initialize diff --git a/funasr/punctuation/espnet_model.py b/funasr/train/abs_model.py similarity index 86% rename from funasr/punctuation/espnet_model.py rename to funasr/train/abs_model.py index 7266b387d..8bfba4513 100644 --- a/funasr/punctuation/espnet_model.py +++ b/funasr/train/abs_model.py @@ -1,3 +1,9 @@ +from abc import ABC +from abc import abstractmethod +from typing import Tuple + +import torch + from typing import Dict from typing import Optional from typing import Tuple @@ -7,13 +13,34 @@ import torch.nn.functional as F from typeguard import check_argument_types from funasr.modules.nets_utils import make_pad_mask -from funasr.punctuation.abs_model import AbsPunctuation from funasr.torch_utils.device_funcs import force_gatherable from funasr.train.abs_espnet_model import AbsESPnetModel +from funasr.modules.scorers.scorer_interface import BatchScorerInterface -class ESPnetPunctuationModel(AbsESPnetModel): +class AbsPunctuation(torch.nn.Module, BatchScorerInterface, ABC): + """The abstract class + + To share the loss calculation way among different models, + We uses delegate pattern here: + The instance of this class should be passed to "LanguageModel" + + This "model" is one of mediator objects for "Task" class. + + """ + + @abstractmethod + def forward(self, input: torch.Tensor, hidden: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError + + @abstractmethod + def with_vad(self) -> bool: + raise NotImplementedError + + +class PunctuationModel(AbsESPnetModel): + def __init__(self, punc_model: AbsPunctuation, vocab_size: int, ignore_id: int = 0, punc_weight: list = None): assert check_argument_types() super().__init__() @@ -21,12 +48,12 @@ class ESPnetPunctuationModel(AbsESPnetModel): self.punc_weight = torch.Tensor(punc_weight) self.sos = 1 self.eos = 2 - + # ignore_id may be assumed as 0, shared with CTC-blank symbol for ASR. self.ignore_id = ignore_id - #if self.punc_model.with_vad(): + # if self.punc_model.with_vad(): # print("This is a vad puncuation model.") - + def nll( self, text: torch.Tensor, @@ -54,7 +81,7 @@ class ESPnetPunctuationModel(AbsESPnetModel): else: text = text[:, :max_length] punc = punc[:, :max_length] - + if self.punc_model.with_vad(): # Should be VadRealtimeTransformer assert vad_indexes is not None @@ -62,7 +89,7 @@ class ESPnetPunctuationModel(AbsESPnetModel): else: # Should be TargetDelayTransformer, y, _ = self.punc_model(text, text_lengths) - + # Calc negative log likelihood # nll: (BxL,) if self.training == False: @@ -75,7 +102,8 @@ class ESPnetPunctuationModel(AbsESPnetModel): return nll, text_lengths else: self.punc_weight = self.punc_weight.to(punc.device) - nll = F.cross_entropy(y.view(-1, y.shape[-1]), punc.view(-1), self.punc_weight, reduction="none", ignore_index=self.ignore_id) + nll = F.cross_entropy(y.view(-1, y.shape[-1]), punc.view(-1), self.punc_weight, reduction="none", + ignore_index=self.ignore_id) # nll: (BxL,) -> (BxL,) if max_length is None: nll.masked_fill_(make_pad_mask(text_lengths).to(nll.device).view(-1), 0.0) @@ -87,7 +115,7 @@ class ESPnetPunctuationModel(AbsESPnetModel): # nll: (BxL,) -> (B, L) nll = nll.view(batch_size, -1) return nll, text_lengths - + def batchify_nll(self, text: torch.Tensor, punc: torch.Tensor, @@ -113,7 +141,7 @@ class ESPnetPunctuationModel(AbsESPnetModel): nlls = [] x_lengths = [] max_length = text_lengths.max() - + start_idx = 0 while True: end_idx = min(start_idx + batch_size, total_num) @@ -132,7 +160,7 @@ class ESPnetPunctuationModel(AbsESPnetModel): assert nll.size(0) == total_num assert x_lengths.size(0) == total_num return nll, x_lengths - + def forward( self, text: torch.Tensor, @@ -146,15 +174,15 @@ class ESPnetPunctuationModel(AbsESPnetModel): ntokens = y_lengths.sum() loss = nll.sum() / ntokens stats = dict(loss=loss.detach()) - + # force_gatherable: to-device and to-tensor if scalar for DataParallel loss, stats, weight = force_gatherable((loss, stats, ntokens), loss.device) return loss, stats, weight - + def collect_feats(self, text: torch.Tensor, punc: torch.Tensor, text_lengths: torch.Tensor) -> Dict[str, torch.Tensor]: return {} - + def inference(self, text: torch.Tensor, text_lengths: torch.Tensor, From 8a73c61e255f55182804834bdb243e70732705a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 31 Mar 2023 15:06:03 +0800 Subject: [PATCH 45/55] export --- funasr/punctuation/__init__.py | 0 funasr/punctuation/text_preprocessor.py | 1 - 2 files changed, 1 deletion(-) delete mode 100644 funasr/punctuation/__init__.py delete mode 100644 funasr/punctuation/text_preprocessor.py diff --git a/funasr/punctuation/__init__.py b/funasr/punctuation/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/funasr/punctuation/text_preprocessor.py b/funasr/punctuation/text_preprocessor.py deleted file mode 100644 index 8b1378917..000000000 --- a/funasr/punctuation/text_preprocessor.py +++ /dev/null @@ -1 +0,0 @@ - From 5788a4ca1786a393244ef92afa7e16e34089c1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 31 Mar 2023 15:23:35 +0800 Subject: [PATCH 46/55] export --- funasr/models/target_delay_transformer.py | 2 +- funasr/models/vad_realtime_transformer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/funasr/models/target_delay_transformer.py b/funasr/models/target_delay_transformer.py index a71952b15..84a2e6c99 100644 --- a/funasr/models/target_delay_transformer.py +++ b/funasr/models/target_delay_transformer.py @@ -7,7 +7,7 @@ import torch.nn as nn from funasr.modules.embedding import SinusoidalPositionEncoder #from funasr.models.encoder.transformer_encoder import TransformerEncoder as Encoder -from funasr.punctuation.sanm_encoder import SANMEncoder as Encoder +from funasr.models.encoder.sanm_encoder import SANMEncoder as Encoder #from funasr.modules.mask import subsequent_n_mask from funasr.train.abs_model import AbsPunctuation diff --git a/funasr/models/vad_realtime_transformer.py b/funasr/models/vad_realtime_transformer.py index 2945572f5..66f7fadeb 100644 --- a/funasr/models/vad_realtime_transformer.py +++ b/funasr/models/vad_realtime_transformer.py @@ -6,7 +6,7 @@ import torch import torch.nn as nn from funasr.modules.embedding import SinusoidalPositionEncoder -from funasr.punctuation.sanm_encoder import SANMVadEncoder as Encoder +from funasr.models.encoder.sanm_encoder import SANMVadEncoder as Encoder from funasr.train.abs_model import AbsPunctuation From 4ba1011b42e041ee1d71448eefd7ef2e7bd61bb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 31 Mar 2023 15:31:26 +0800 Subject: [PATCH 47/55] export --- funasr/lm/abs_model.py | 130 +++++++++++++++++++++++++++++++++++++ funasr/lm/espnet_model.py | 131 -------------------------------------- funasr/tasks/lm.py | 2 +- 3 files changed, 131 insertions(+), 132 deletions(-) delete mode 100644 funasr/lm/espnet_model.py diff --git a/funasr/lm/abs_model.py b/funasr/lm/abs_model.py index 0ad1e71bc..997aad9eb 100644 --- a/funasr/lm/abs_model.py +++ b/funasr/lm/abs_model.py @@ -5,7 +5,18 @@ from typing import Tuple import torch from funasr.modules.scorers.scorer_interface import BatchScorerInterface +from typing import Dict +from typing import Optional +from typing import Tuple +import torch +import torch.nn.functional as F +from typeguard import check_argument_types + +from funasr.modules.nets_utils import make_pad_mask +from funasr.lm.abs_model import AbsLM +from funasr.torch_utils.device_funcs import force_gatherable +from funasr.train.abs_espnet_model import AbsESPnetModel class AbsLM(torch.nn.Module, BatchScorerInterface, ABC): """The abstract LM class @@ -27,3 +38,122 @@ class AbsLM(torch.nn.Module, BatchScorerInterface, ABC): self, input: torch.Tensor, hidden: torch.Tensor ) -> Tuple[torch.Tensor, torch.Tensor]: raise NotImplementedError + + +class LanguageModel(AbsESPnetModel): + def __init__(self, lm: AbsLM, vocab_size: int, ignore_id: int = 0): + assert check_argument_types() + super().__init__() + self.lm = lm + self.sos = 1 + self.eos = 2 + + # ignore_id may be assumed as 0, shared with CTC-blank symbol for ASR. + self.ignore_id = ignore_id + + def nll( + self, + text: torch.Tensor, + text_lengths: torch.Tensor, + max_length: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute negative log likelihood(nll) + + Normally, this function is called in batchify_nll. + Args: + text: (Batch, Length) + text_lengths: (Batch,) + max_lengths: int + """ + batch_size = text.size(0) + # For data parallel + if max_length is None: + text = text[:, : text_lengths.max()] + else: + text = text[:, :max_length] + + # 1. Create a sentence pair like ' w1 w2 w3' and 'w1 w2 w3 ' + # text: (Batch, Length) -> x, y: (Batch, Length + 1) + x = F.pad(text, [1, 0], "constant", self.sos) + t = F.pad(text, [0, 1], "constant", self.ignore_id) + for i, l in enumerate(text_lengths): + t[i, l] = self.eos + x_lengths = text_lengths + 1 + + # 2. Forward Language model + # x: (Batch, Length) -> y: (Batch, Length, NVocab) + y, _ = self.lm(x, None) + + # 3. Calc negative log likelihood + # nll: (BxL,) + nll = F.cross_entropy(y.view(-1, y.shape[-1]), t.view(-1), reduction="none") + # nll: (BxL,) -> (BxL,) + if max_length is None: + nll.masked_fill_(make_pad_mask(x_lengths).to(nll.device).view(-1), 0.0) + else: + nll.masked_fill_( + make_pad_mask(x_lengths, maxlen=max_length + 1).to(nll.device).view(-1), + 0.0, + ) + # nll: (BxL,) -> (B, L) + nll = nll.view(batch_size, -1) + return nll, x_lengths + + def batchify_nll( + self, text: torch.Tensor, text_lengths: torch.Tensor, batch_size: int = 100 + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Compute negative log likelihood(nll) from transformer language model + + To avoid OOM, this fuction seperate the input into batches. + Then call nll for each batch and combine and return results. + Args: + text: (Batch, Length) + text_lengths: (Batch,) + batch_size: int, samples each batch contain when computing nll, + you may change this to avoid OOM or increase + + """ + total_num = text.size(0) + if total_num <= batch_size: + nll, x_lengths = self.nll(text, text_lengths) + else: + nlls = [] + x_lengths = [] + max_length = text_lengths.max() + + start_idx = 0 + while True: + end_idx = min(start_idx + batch_size, total_num) + batch_text = text[start_idx:end_idx, :] + batch_text_lengths = text_lengths[start_idx:end_idx] + # batch_nll: [B * T] + batch_nll, batch_x_lengths = self.nll( + batch_text, batch_text_lengths, max_length=max_length + ) + nlls.append(batch_nll) + x_lengths.append(batch_x_lengths) + start_idx = end_idx + if start_idx == total_num: + break + nll = torch.cat(nlls) + x_lengths = torch.cat(x_lengths) + assert nll.size(0) == total_num + assert x_lengths.size(0) == total_num + return nll, x_lengths + + def forward( + self, text: torch.Tensor, text_lengths: torch.Tensor + ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: + nll, y_lengths = self.nll(text, text_lengths) + ntokens = y_lengths.sum() + loss = nll.sum() / ntokens + stats = dict(loss=loss.detach()) + + # force_gatherable: to-device and to-tensor if scalar for DataParallel + loss, stats, weight = force_gatherable((loss, stats, ntokens), loss.device) + return loss, stats, weight + + def collect_feats( + self, text: torch.Tensor, text_lengths: torch.Tensor + ) -> Dict[str, torch.Tensor]: + return {} diff --git a/funasr/lm/espnet_model.py b/funasr/lm/espnet_model.py deleted file mode 100644 index a9b8130c6..000000000 --- a/funasr/lm/espnet_model.py +++ /dev/null @@ -1,131 +0,0 @@ -from typing import Dict -from typing import Optional -from typing import Tuple - -import torch -import torch.nn.functional as F -from typeguard import check_argument_types - -from funasr.modules.nets_utils import make_pad_mask -from funasr.lm.abs_model import AbsLM -from funasr.torch_utils.device_funcs import force_gatherable -from funasr.train.abs_espnet_model import AbsESPnetModel - - -class LanguageModel(AbsESPnetModel): - def __init__(self, lm: AbsLM, vocab_size: int, ignore_id: int = 0): - assert check_argument_types() - super().__init__() - self.lm = lm - self.sos = 1 - self.eos = 2 - - # ignore_id may be assumed as 0, shared with CTC-blank symbol for ASR. - self.ignore_id = ignore_id - - def nll( - self, - text: torch.Tensor, - text_lengths: torch.Tensor, - max_length: Optional[int] = None, - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute negative log likelihood(nll) - - Normally, this function is called in batchify_nll. - Args: - text: (Batch, Length) - text_lengths: (Batch,) - max_lengths: int - """ - batch_size = text.size(0) - # For data parallel - if max_length is None: - text = text[:, : text_lengths.max()] - else: - text = text[:, :max_length] - - # 1. Create a sentence pair like ' w1 w2 w3' and 'w1 w2 w3 ' - # text: (Batch, Length) -> x, y: (Batch, Length + 1) - x = F.pad(text, [1, 0], "constant", self.sos) - t = F.pad(text, [0, 1], "constant", self.ignore_id) - for i, l in enumerate(text_lengths): - t[i, l] = self.eos - x_lengths = text_lengths + 1 - - # 2. Forward Language model - # x: (Batch, Length) -> y: (Batch, Length, NVocab) - y, _ = self.lm(x, None) - - # 3. Calc negative log likelihood - # nll: (BxL,) - nll = F.cross_entropy(y.view(-1, y.shape[-1]), t.view(-1), reduction="none") - # nll: (BxL,) -> (BxL,) - if max_length is None: - nll.masked_fill_(make_pad_mask(x_lengths).to(nll.device).view(-1), 0.0) - else: - nll.masked_fill_( - make_pad_mask(x_lengths, maxlen=max_length + 1).to(nll.device).view(-1), - 0.0, - ) - # nll: (BxL,) -> (B, L) - nll = nll.view(batch_size, -1) - return nll, x_lengths - - def batchify_nll( - self, text: torch.Tensor, text_lengths: torch.Tensor, batch_size: int = 100 - ) -> Tuple[torch.Tensor, torch.Tensor]: - """Compute negative log likelihood(nll) from transformer language model - - To avoid OOM, this fuction seperate the input into batches. - Then call nll for each batch and combine and return results. - Args: - text: (Batch, Length) - text_lengths: (Batch,) - batch_size: int, samples each batch contain when computing nll, - you may change this to avoid OOM or increase - - """ - total_num = text.size(0) - if total_num <= batch_size: - nll, x_lengths = self.nll(text, text_lengths) - else: - nlls = [] - x_lengths = [] - max_length = text_lengths.max() - - start_idx = 0 - while True: - end_idx = min(start_idx + batch_size, total_num) - batch_text = text[start_idx:end_idx, :] - batch_text_lengths = text_lengths[start_idx:end_idx] - # batch_nll: [B * T] - batch_nll, batch_x_lengths = self.nll( - batch_text, batch_text_lengths, max_length=max_length - ) - nlls.append(batch_nll) - x_lengths.append(batch_x_lengths) - start_idx = end_idx - if start_idx == total_num: - break - nll = torch.cat(nlls) - x_lengths = torch.cat(x_lengths) - assert nll.size(0) == total_num - assert x_lengths.size(0) == total_num - return nll, x_lengths - - def forward( - self, text: torch.Tensor, text_lengths: torch.Tensor - ) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]: - nll, y_lengths = self.nll(text, text_lengths) - ntokens = y_lengths.sum() - loss = nll.sum() / ntokens - stats = dict(loss=loss.detach()) - - # force_gatherable: to-device and to-tensor if scalar for DataParallel - loss, stats, weight = force_gatherable((loss, stats, ntokens), loss.device) - return loss, stats, weight - - def collect_feats( - self, text: torch.Tensor, text_lengths: torch.Tensor - ) -> Dict[str, torch.Tensor]: - return {} diff --git a/funasr/tasks/lm.py b/funasr/tasks/lm.py index dc8fd3e25..80d66d52f 100644 --- a/funasr/tasks/lm.py +++ b/funasr/tasks/lm.py @@ -15,7 +15,7 @@ from typeguard import check_return_type from funasr.datasets.collate_fn import CommonCollateFn from funasr.datasets.preprocessor import CommonPreprocessor from funasr.lm.abs_model import AbsLM -from funasr.lm.espnet_model import LanguageModel +from funasr.lm.abs_model import LanguageModel from funasr.lm.seq_rnn_lm import SequentialRNNLM from funasr.lm.transformer_lm import TransformerLM from funasr.tasks.abs_task import AbsTask From cbe0c2e915070e71ac74d3a924a88ad3e44029a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 31 Mar 2023 15:34:37 +0800 Subject: [PATCH 48/55] export --- funasr/lm/abs_model.py | 1 - funasr/train/abs_model.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/funasr/lm/abs_model.py b/funasr/lm/abs_model.py index 997aad9eb..1f3c8a7b1 100644 --- a/funasr/lm/abs_model.py +++ b/funasr/lm/abs_model.py @@ -14,7 +14,6 @@ import torch.nn.functional as F from typeguard import check_argument_types from funasr.modules.nets_utils import make_pad_mask -from funasr.lm.abs_model import AbsLM from funasr.torch_utils.device_funcs import force_gatherable from funasr.train.abs_espnet_model import AbsESPnetModel diff --git a/funasr/train/abs_model.py b/funasr/train/abs_model.py index 8bfba4513..1c7ff3d33 100644 --- a/funasr/train/abs_model.py +++ b/funasr/train/abs_model.py @@ -1,8 +1,6 @@ from abc import ABC from abc import abstractmethod -from typing import Tuple -import torch from typing import Dict from typing import Optional From 54409a2485f6b16300414277879148432146a758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 7 Apr 2023 11:40:35 +0800 Subject: [PATCH 49/55] onnx --- .../export/models/vad_realtime_transformer.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index 693b9c844..a8948ebcd 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -52,11 +52,21 @@ class VadRealtimeTransformer(nn.Module): def with_vad(self): return True - def get_dummy_inputs(self): - length = 120 - text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)) + # def get_dummy_inputs(self): + # length = 120 + # text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)) + # text_lengths = torch.tensor([length], dtype=torch.int32) + # vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :] + # sub_masks = torch.ones(length, length, dtype=torch.float32) + # sub_masks = torch.tril(sub_masks).type(torch.float32) + # return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :]) + + def get_dummy_inputs(self, txt_dir): + from funasr.modules.mask import vad_mask + length = 10 + text_indexes = torch.tensor([[266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757]], dtype=torch.int32) text_lengths = torch.tensor([length], dtype=torch.int32) - vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :] + vad_mask = vad_mask(10, 3, dtype=torch.float32)[None, None, :, :] sub_masks = torch.ones(length, length, dtype=torch.float32) sub_masks = torch.tril(sub_masks).type(torch.float32) return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :]) From 8b802ea8a0876192cba09823d061e520a3d3bb39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 7 Apr 2023 11:47:25 +0800 Subject: [PATCH 50/55] onnx --- funasr/export/models/vad_realtime_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index a8948ebcd..c8f5364db 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -61,7 +61,7 @@ class VadRealtimeTransformer(nn.Module): # sub_masks = torch.tril(sub_masks).type(torch.float32) # return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :]) - def get_dummy_inputs(self, txt_dir): + def get_dummy_inputs(self, txt_dir=None): from funasr.modules.mask import vad_mask length = 10 text_indexes = torch.tensor([[266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757]], dtype=torch.int32) From eb82674d880b1bad0319339b2036644a538e99e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 7 Apr 2023 14:08:34 +0800 Subject: [PATCH 51/55] onnx --- funasr/export/models/encoder/sanm_encoder.py | 53 ++++++++++--------- .../export/models/vad_realtime_transformer.py | 8 +-- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/funasr/export/models/encoder/sanm_encoder.py b/funasr/export/models/encoder/sanm_encoder.py index 44a48ff6a..f583f56ad 100644 --- a/funasr/export/models/encoder/sanm_encoder.py +++ b/funasr/export/models/encoder/sanm_encoder.py @@ -158,13 +158,14 @@ class SANMVadEncoder(nn.Module): def forward(self, speech: torch.Tensor, speech_lengths: torch.Tensor, - vad_mask: torch.Tensor, + vad_masks: torch.Tensor, sub_masks: torch.Tensor, ): speech = speech * self._output_size ** 0.5 mask = self.make_pad_mask(speech_lengths) + vad_masks = self.prepare_mask(mask, vad_masks) mask = self.prepare_mask(mask, sub_masks) - vad_mask = self.prepare_mask(mask, vad_mask) + if self.embed is None: xs_pad = speech else: @@ -176,7 +177,7 @@ class SANMVadEncoder(nn.Module): # encoder_outs = self.model.encoders(xs_pad, mask) for layer_idx, encoder_layer in enumerate(self.model.encoders): if layer_idx == len(self.model.encoders) - 1: - mask = vad_mask + mask = vad_masks encoder_outs = encoder_layer(xs_pad, mask) xs_pad, masks = encoder_outs[0], encoder_outs[1] @@ -187,26 +188,26 @@ class SANMVadEncoder(nn.Module): def get_output_size(self): return self.model.encoders[0].size - def get_dummy_inputs(self): - feats = torch.randn(1, 100, self.feats_dim) - return (feats) - - def get_input_names(self): - return ['feats'] - - def get_output_names(self): - return ['encoder_out', 'encoder_out_lens', 'predictor_weight'] - - def get_dynamic_axes(self): - return { - 'feats': { - 1: 'feats_length' - }, - 'encoder_out': { - 1: 'enc_out_length' - }, - 'predictor_weight': { - 1: 'pre_out_length' - } - - } + # def get_dummy_inputs(self): + # feats = torch.randn(1, 100, self.feats_dim) + # return (feats) + # + # def get_input_names(self): + # return ['feats'] + # + # def get_output_names(self): + # return ['encoder_out', 'encoder_out_lens', 'predictor_weight'] + # + # def get_dynamic_axes(self): + # return { + # 'feats': { + # 1: 'feats_length' + # }, + # 'encoder_out': { + # 1: 'enc_out_length' + # }, + # 'predictor_weight': { + # 1: 'pre_out_length' + # } + # + # } diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index c8f5364db..c34525b90 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -66,13 +66,13 @@ class VadRealtimeTransformer(nn.Module): length = 10 text_indexes = torch.tensor([[266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757]], dtype=torch.int32) text_lengths = torch.tensor([length], dtype=torch.int32) - vad_mask = vad_mask(10, 3, dtype=torch.float32)[None, None, :, :] + vad_masks = vad_mask(10, 3, dtype=torch.float32)[None, None, :, :] sub_masks = torch.ones(length, length, dtype=torch.float32) sub_masks = torch.tril(sub_masks).type(torch.float32) - return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :]) + return (text_indexes, text_lengths, vad_masks, sub_masks[None, None, :, :]) def get_input_names(self): - return ['input', 'text_lengths', 'vad_mask', 'sub_masks'] + return ['input', 'text_lengths', 'vad_masks', 'sub_masks'] def get_output_names(self): return ['logits'] @@ -82,7 +82,7 @@ class VadRealtimeTransformer(nn.Module): 'input': { 1: 'feats_length' }, - 'vad_mask': { + 'vad_masks': { 2: 'feats_length1', 3: 'feats_length2' }, From 9f6445d39b14fa17f2c32a383c1054a8e073a9c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 7 Apr 2023 14:11:55 +0800 Subject: [PATCH 52/55] onnx --- funasr/export/models/vad_realtime_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py index c34525b90..24a8e7247 100644 --- a/funasr/export/models/vad_realtime_transformer.py +++ b/funasr/export/models/vad_realtime_transformer.py @@ -66,7 +66,7 @@ class VadRealtimeTransformer(nn.Module): length = 10 text_indexes = torch.tensor([[266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757]], dtype=torch.int32) text_lengths = torch.tensor([length], dtype=torch.int32) - vad_masks = vad_mask(10, 3, dtype=torch.float32)[None, None, :, :] + vad_masks = vad_mask(10, 2, dtype=torch.float32)[None, None, :, :] sub_masks = torch.ones(length, length, dtype=torch.float32) sub_masks = torch.tril(sub_masks).type(torch.float32) return (text_indexes, text_lengths, vad_masks, sub_masks[None, None, :, :]) From b9837bfc73c14f74cbb4c351bb51b35f1d354ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Fri, 7 Apr 2023 14:37:30 +0800 Subject: [PATCH 53/55] onnx --- funasr/export/models/__init__.py | 8 +- .../export/models/target_delay_transformer.py | 97 +++++++++++++++++-- .../export/models/vad_realtime_transformer.py | 96 ------------------ funasr/export/test/test_onnx_punc.py | 2 +- .../export/test/test_onnx_punc_vadrealtime.py | 4 +- 5 files changed, 95 insertions(+), 112 deletions(-) delete mode 100644 funasr/export/models/vad_realtime_transformer.py diff --git a/funasr/export/models/__init__.py b/funasr/export/models/__init__.py index 4ac0456b9..f81ff6454 100644 --- a/funasr/export/models/__init__.py +++ b/funasr/export/models/__init__.py @@ -4,10 +4,10 @@ from funasr.export.models.e2e_asr_paraformer import BiCifParaformer as BiCifPara from funasr.models.e2e_vad import E2EVadModel from funasr.export.models.e2e_vad import E2EVadModel as E2EVadModel_export from funasr.models.target_delay_transformer import TargetDelayTransformer -from funasr.export.models.target_delay_transformer import TargetDelayTransformer as TargetDelayTransformer_export +from funasr.export.models.target_delay_transformer import CT_Transformer as CT_Transformer_export from funasr.train.abs_model import PunctuationModel from funasr.models.vad_realtime_transformer import VadRealtimeTransformer -from funasr.export.models.vad_realtime_transformer import VadRealtimeTransformer as VadRealtimeTransformer_export +from funasr.export.models.target_delay_transformer import CT_Transformer_VadRealtime as CT_Transformer_VadRealtime_export def get_model(model, export_config=None): if isinstance(model, BiCifParaformer): @@ -18,8 +18,8 @@ def get_model(model, export_config=None): return E2EVadModel_export(model, **export_config) elif isinstance(model, PunctuationModel): if isinstance(model.punc_model, TargetDelayTransformer): - return TargetDelayTransformer_export(model.punc_model, **export_config) + return CT_Transformer_export(model.punc_model, **export_config) elif isinstance(model.punc_model, VadRealtimeTransformer): - return VadRealtimeTransformer_export(model.punc_model, **export_config) + return CT_Transformer_VadRealtime_export(model.punc_model, **export_config) else: raise "Funasr does not support the given model type currently." diff --git a/funasr/export/models/target_delay_transformer.py b/funasr/export/models/target_delay_transformer.py index bfe3ec423..2780d8275 100644 --- a/funasr/export/models/target_delay_transformer.py +++ b/funasr/export/models/target_delay_transformer.py @@ -3,7 +3,12 @@ from typing import Tuple import torch import torch.nn as nn -class TargetDelayTransformer(nn.Module): +from funasr.models.encoder.sanm_encoder import SANMEncoder +from funasr.export.models.encoder.sanm_encoder import SANMEncoder as SANMEncoder_export +from funasr.models.encoder.sanm_encoder import SANMVadEncoder +from funasr.export.models.encoder.sanm_encoder import SANMVadEncoder as SANMVadEncoder_export + +class CT_Transformer(nn.Module): def __init__( self, @@ -23,16 +28,12 @@ class TargetDelayTransformer(nn.Module): self.num_embeddings = self.embed.num_embeddings self.model_name = model_name - # from funasr.models.encoder.sanm_encoder import SANMEncoder as Encoder - from funasr.models.encoder.sanm_encoder import SANMEncoder - from funasr.export.models.encoder.sanm_encoder import SANMEncoder as SANMEncoder_export - if isinstance(model.encoder, SANMEncoder): self.encoder = SANMEncoder_export(model.encoder, onnx=onnx) else: assert False, "Only support samn encode." - def forward(self, input: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]: + def forward(self, inputs: torch.Tensor, text_lengths: torch.Tensor) -> Tuple[torch.Tensor, None]: """Compute loss value from buffer sequences. Args: @@ -40,7 +41,7 @@ class TargetDelayTransformer(nn.Module): hidden (torch.Tensor): Target ids. (batch, len) """ - x = self.embed(input) + x = self.embed(inputs) # mask = self._target_mask(input) h, _ = self.encoder(x, text_lengths) y = self.decoder(h) @@ -53,14 +54,14 @@ class TargetDelayTransformer(nn.Module): return (text_indexes, text_lengths) def get_input_names(self): - return ['input', 'text_lengths'] + return ['inputs', 'text_lengths'] def get_output_names(self): return ['logits'] def get_dynamic_axes(self): return { - 'input': { + 'inputs': { 0: 'batch_size', 1: 'feats_length' }, @@ -73,3 +74,81 @@ class TargetDelayTransformer(nn.Module): }, } + +class CT_Transformer_VadRealtime(nn.Module): + + def __init__( + self, + model, + max_seq_len=512, + model_name='punc_model', + **kwargs, + ): + super().__init__() + onnx = False + if "onnx" in kwargs: + onnx = kwargs["onnx"] + + self.embed = model.embed + if isinstance(model.encoder, SANMVadEncoder): + self.encoder = SANMVadEncoder_export(model.encoder, onnx=onnx) + else: + assert False, "Only support samn encode." + self.decoder = model.decoder + self.model_name = model_name + + + + def forward(self, inputs: torch.Tensor, + text_lengths: torch.Tensor, + vad_indexes: torch.Tensor, + sub_masks: torch.Tensor, + ) -> Tuple[torch.Tensor, None]: + """Compute loss value from buffer sequences. + + Args: + input (torch.Tensor): Input ids. (batch, len) + hidden (torch.Tensor): Target ids. (batch, len) + + """ + x = self.embed(inputs) + # mask = self._target_mask(input) + h, _ = self.encoder(x, text_lengths, vad_indexes, sub_masks) + y = self.decoder(h) + return y + + def with_vad(self): + return True + + def get_dummy_inputs(self): + length = 120 + text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)) + text_lengths = torch.tensor([length], dtype=torch.int32) + vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :] + sub_masks = torch.ones(length, length, dtype=torch.float32) + sub_masks = torch.tril(sub_masks).type(torch.float32) + return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :]) + + def get_input_names(self): + return ['inputs', 'text_lengths', 'vad_masks', 'sub_masks'] + + def get_output_names(self): + return ['logits'] + + def get_dynamic_axes(self): + return { + 'inputs': { + 1: 'feats_length' + }, + 'vad_masks': { + 2: 'feats_length1', + 3: 'feats_length2' + }, + 'sub_masks': { + 2: 'feats_length1', + 3: 'feats_length2' + }, + 'logits': { + 1: 'logits_length' + }, + } diff --git a/funasr/export/models/vad_realtime_transformer.py b/funasr/export/models/vad_realtime_transformer.py deleted file mode 100644 index 24a8e7247..000000000 --- a/funasr/export/models/vad_realtime_transformer.py +++ /dev/null @@ -1,96 +0,0 @@ -from typing import Tuple - -import torch -import torch.nn as nn - -from funasr.models.encoder.sanm_encoder import SANMVadEncoder -from funasr.export.models.encoder.sanm_encoder import SANMVadEncoder as SANMVadEncoder_export - -class VadRealtimeTransformer(nn.Module): - - def __init__( - self, - model, - max_seq_len=512, - model_name='punc_model', - **kwargs, - ): - super().__init__() - onnx = False - if "onnx" in kwargs: - onnx = kwargs["onnx"] - - self.embed = model.embed - if isinstance(model.encoder, SANMVadEncoder): - self.encoder = SANMVadEncoder_export(model.encoder, onnx=onnx) - else: - assert False, "Only support samn encode." - # self.encoder = model.encoder - self.decoder = model.decoder - self.model_name = model_name - - - - def forward(self, input: torch.Tensor, - text_lengths: torch.Tensor, - vad_indexes: torch.Tensor, - sub_masks: torch.Tensor, - ) -> Tuple[torch.Tensor, None]: - """Compute loss value from buffer sequences. - - Args: - input (torch.Tensor): Input ids. (batch, len) - hidden (torch.Tensor): Target ids. (batch, len) - - """ - x = self.embed(input) - # mask = self._target_mask(input) - h, _ = self.encoder(x, text_lengths, vad_indexes, sub_masks) - y = self.decoder(h) - return y - - def with_vad(self): - return True - - # def get_dummy_inputs(self): - # length = 120 - # text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)) - # text_lengths = torch.tensor([length], dtype=torch.int32) - # vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :] - # sub_masks = torch.ones(length, length, dtype=torch.float32) - # sub_masks = torch.tril(sub_masks).type(torch.float32) - # return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :]) - - def get_dummy_inputs(self, txt_dir=None): - from funasr.modules.mask import vad_mask - length = 10 - text_indexes = torch.tensor([[266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757, 266757]], dtype=torch.int32) - text_lengths = torch.tensor([length], dtype=torch.int32) - vad_masks = vad_mask(10, 2, dtype=torch.float32)[None, None, :, :] - sub_masks = torch.ones(length, length, dtype=torch.float32) - sub_masks = torch.tril(sub_masks).type(torch.float32) - return (text_indexes, text_lengths, vad_masks, sub_masks[None, None, :, :]) - - def get_input_names(self): - return ['input', 'text_lengths', 'vad_masks', 'sub_masks'] - - def get_output_names(self): - return ['logits'] - - def get_dynamic_axes(self): - return { - 'input': { - 1: 'feats_length' - }, - 'vad_masks': { - 2: 'feats_length1', - 3: 'feats_length2' - }, - 'sub_masks': { - 2: 'feats_length1', - 3: 'feats_length2' - }, - 'logits': { - 1: 'logits_length' - }, - } diff --git a/funasr/export/test/test_onnx_punc.py b/funasr/export/test/test_onnx_punc.py index 62689a904..39f85f457 100644 --- a/funasr/export/test/test_onnx_punc.py +++ b/funasr/export/test/test_onnx_punc.py @@ -9,7 +9,7 @@ if __name__ == '__main__': output_name = [nd.name for nd in sess.get_outputs()] def _get_feed_dict(text_length): - return {'input': np.ones((1, text_length), dtype=np.int64), 'text_lengths': np.array([text_length,], dtype=np.int32)} + return {'inputs': np.ones((1, text_length), dtype=np.int64), 'text_lengths': np.array([text_length,], dtype=np.int32)} def _run(feed_dict): output = sess.run(output_name, input_feed=feed_dict) diff --git a/funasr/export/test/test_onnx_punc_vadrealtime.py b/funasr/export/test/test_onnx_punc_vadrealtime.py index 54f85f194..86be026dc 100644 --- a/funasr/export/test/test_onnx_punc_vadrealtime.py +++ b/funasr/export/test/test_onnx_punc_vadrealtime.py @@ -9,9 +9,9 @@ if __name__ == '__main__': output_name = [nd.name for nd in sess.get_outputs()] def _get_feed_dict(text_length): - return {'input': np.ones((1, text_length), dtype=np.int64), + return {'inputs': np.ones((1, text_length), dtype=np.int64), 'text_lengths': np.array([text_length,], dtype=np.int32), - 'vad_mask': np.ones((1, 1, text_length, text_length), dtype=np.float32), + 'vad_masks': np.ones((1, 1, text_length, text_length), dtype=np.float32), 'sub_masks': np.tril(np.ones((text_length, text_length), dtype=np.float32))[None, None, :, :].astype(np.float32) } From 79007d36f1636eb51e0cead6bc0e6b18ff1f8253 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Fri, 7 Apr 2023 14:43:32 +0800 Subject: [PATCH 54/55] vad_realtime onnx runnable --- .../python/onnxruntime/demo_punc_online.py | 15 ++ .../onnxruntime/funasr_onnx/__init__.py | 2 +- .../onnxruntime/funasr_onnx/punc_bin.py | 130 ++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 funasr/runtime/python/onnxruntime/demo_punc_online.py diff --git a/funasr/runtime/python/onnxruntime/demo_punc_online.py b/funasr/runtime/python/onnxruntime/demo_punc_online.py new file mode 100644 index 000000000..853db50e2 --- /dev/null +++ b/funasr/runtime/python/onnxruntime/demo_punc_online.py @@ -0,0 +1,15 @@ +from funasr_onnx import CT_Transformer_VadRealtime + +model_dir = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727" +model = CT_Transformer_VadRealtime(model_dir) + +text_in = "跨境河流是养育沿岸|人民的生命之源长期以来为帮助下游地区防灾减灾中方技术人员|在上游地区极为恶劣的自然条件下克服巨大困难甚至冒着生命危险|向印方提供汛期水文资料处理紧急事件中方重视印方在跨境河流>问题上的关切|愿意进一步完善双方联合工作机制|凡是|中方能做的我们|都会去做而且会做得更好我请印度朋友们放心中国在上游的|任何开发利用都会经过科学|规划和论证兼顾上下游的利益" + +vads = text_in.split("|") +rec_result_all="outputs:" +param_dict = {"cache": []} +for vad in vads: + result = model(vad, param_dict=param_dict) + rec_result_all += result[0] + +print(rec_result_all) diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py index 825f2ca38..86f0e8e52 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/__init__.py @@ -2,4 +2,4 @@ from .paraformer_bin import Paraformer from .vad_bin import Fsmn_vad from .punc_bin import CT_Transformer -#from .punc_bin import VadRealtimeCT_Transformer +from .punc_bin import CT_Transformer_VadRealtime diff --git a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py index 949172eb1..0dc728ad3 100644 --- a/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py +++ b/funasr/runtime/python/onnxruntime/funasr_onnx/punc_bin.py @@ -117,3 +117,133 @@ class CT_Transformer(): outputs = self.ort_infer([feats, feats_len]) return outputs + +class CT_Transformer_VadRealtime(CT_Transformer): + def __init__(self, model_dir: Union[str, Path] = None, + batch_size: int = 1, + device_id: Union[str, int] = "-1", + quantize: bool = False, + intra_op_num_threads: int = 4 + ): + super(CT_Transformer_VadRealtime, self).__init__(model_dir, batch_size, device_id, quantize, intra_op_num_threads) + + def __call__(self, text: str, param_dict: map, split_size=20): + cache_key = "cache" + assert cache_key in param_dict + cache = param_dict[cache_key] + if cache is not None and len(cache) > 0: + precache = "".join(cache) + else: + precache = "" + cache = [] + full_text = precache + text + split_text = code_mix_split_words(full_text) + split_text_id = self.converter.tokens2ids(split_text) + mini_sentences = split_to_mini_sentence(split_text, split_size) + mini_sentences_id = split_to_mini_sentence(split_text_id, split_size) + new_mini_sentence_punc = [] + assert len(mini_sentences) == len(mini_sentences_id) + + cache_sent = [] + cache_sent_id = np.array([], dtype='int32') + sentence_punc_list = [] + sentence_words_list = [] + cache_pop_trigger_limit = 200 + skip_num = 0 + for mini_sentence_i in range(len(mini_sentences)): + mini_sentence = mini_sentences[mini_sentence_i] + mini_sentence_id = mini_sentences_id[mini_sentence_i] + mini_sentence = cache_sent + mini_sentence + mini_sentence_id = np.concatenate((cache_sent_id, mini_sentence_id), axis=0) + text_length = len(mini_sentence_id) + data = { + "input": mini_sentence_id[None,:], + "text_lengths": np.array([text_length], dtype='int32'), + "vad_mask": self.vad_mask(text_length, len(cache) - 1)[None, None, :, :].astype(np.float32), + "sub_masks": np.tril(np.ones((text_length, text_length), dtype=np.float32))[None, None, :, :].astype(np.float32) + } + try: + outputs = self.infer(data['input'], data['text_lengths'], data['vad_mask'], data["sub_masks"]) + y = outputs[0] + punctuations = np.argmax(y,axis=-1)[0] + assert punctuations.size == len(mini_sentence) + except ONNXRuntimeError: + logging.warning("error") + + # Search for the last Period/QuestionMark as cache + if mini_sentence_i < len(mini_sentences) - 1: + sentenceEnd = -1 + last_comma_index = -1 + for i in range(len(punctuations) - 2, 1, -1): + if self.punc_list[punctuations[i]] == "。" or self.punc_list[punctuations[i]] == "？": + sentenceEnd = i + break + if last_comma_index < 0 and self.punc_list[punctuations[i]] == "，": + last_comma_index = i + + if sentenceEnd < 0 and len(mini_sentence) > cache_pop_trigger_limit and last_comma_index >= 0: + # The sentence it too long, cut off at a comma. + sentenceEnd = last_comma_index + punctuations[sentenceEnd] = self.period + cache_sent = mini_sentence[sentenceEnd + 1:] + cache_sent_id = mini_sentence_id[sentenceEnd + 1:] + mini_sentence = mini_sentence[0:sentenceEnd + 1] + punctuations = punctuations[0:sentenceEnd + 1] + + punctuations_np = [int(x) for x in punctuations] + new_mini_sentence_punc += punctuations_np + sentence_punc_list += [self.punc_list[int(x)] for x in punctuations_np] + sentence_words_list += mini_sentence + + assert len(sentence_punc_list) == len(sentence_words_list) + words_with_punc = [] + sentence_punc_list_out = [] + for i in range(0, len(sentence_words_list)): + if i > 0: + if len(sentence_words_list[i][0].encode()) == 1 and len(sentence_words_list[i - 1][-1].encode()) == 1: + sentence_words_list[i] = " " + sentence_words_list[i] + if skip_num < len(cache): + skip_num += 1 + else: + words_with_punc.append(sentence_words_list[i]) + if skip_num >= len(cache): + sentence_punc_list_out.append(sentence_punc_list[i]) + if sentence_punc_list[i] != "_": + words_with_punc.append(sentence_punc_list[i]) + sentence_out = "".join(words_with_punc) + + sentenceEnd = -1 + for i in range(len(sentence_punc_list) - 2, 1, -1): + if sentence_punc_list[i] == "。" or sentence_punc_list[i] == "？": + sentenceEnd = i + break + cache_out = sentence_words_list[sentenceEnd + 1:] + if sentence_out[-1] in self.punc_list: + sentence_out = sentence_out[:-1] + sentence_punc_list_out[-1] = "_" + param_dict[cache_key] = cache_out + return sentence_out, sentence_punc_list_out, cache_out + + def vad_mask(self, size, vad_pos, dtype=np.bool): + """Create mask for decoder self-attention. + + :param int size: size of mask + :param int vad_pos: index of vad index + :param torch.dtype dtype: result dtype + :rtype: torch.Tensor (B, Lmax, Lmax) + """ + ret = np.ones((size, size), dtype=dtype) + if vad_pos <= 0 or vad_pos >= size: + return ret + sub_corner = np.zeros( + (vad_pos - 1, size - vad_pos), dtype=dtype) + ret[0:vad_pos - 1, vad_pos:] = sub_corner + return ret + + def infer(self, feats: np.ndarray, + feats_len: np.ndarray, + vad_mask: np.ndarray, + sub_masks: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + outputs = self.ort_infer([feats, feats_len, vad_mask, sub_masks]) + return outputs + From c4490d3575aa7accac36ae668b08b839d2ace478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B9=9D=E8=80=B3?= Date: Fri, 7 Apr 2023 15:41:53 +0800 Subject: [PATCH 55/55] fix --- funasr/runtime/python/onnxruntime/demo_punc_offline.py | 3 +-- funasr/runtime/python/onnxruntime/demo_punc_online.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/funasr/runtime/python/onnxruntime/demo_punc_offline.py b/funasr/runtime/python/onnxruntime/demo_punc_offline.py index d13c06bdb..469addad6 100644 --- a/funasr/runtime/python/onnxruntime/demo_punc_offline.py +++ b/funasr/runtime/python/onnxruntime/demo_punc_offline.py @@ -1,9 +1,8 @@ from funasr_onnx import CT_Transformer -model_dir = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch" +model_dir = "../../../export/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch" model = CT_Transformer(model_dir) -text_in = "我们都是木头人不会讲话不会动" text_in="跨境河流是养育沿岸人民的生命之源长期以来为帮助下游地区防灾减灾中方技术人员在上游地区极为恶劣的自然条件下克服巨大困难甚至冒着生命危险向印方提供汛期水文资料处理紧急事件中方重视印方在跨境河流问题上的关切愿意进一步完善双方联合工作机制凡是中方能做的我们都会去做而且会做得更好我请印度朋友们放心中国在上游的任何开发利用都会经过科学规划和论证兼顾上下游的利益" result = model(text_in) print(result[0]) diff --git a/funasr/runtime/python/onnxruntime/demo_punc_online.py b/funasr/runtime/python/onnxruntime/demo_punc_online.py index 853db50e2..63f2f5eab 100644 --- a/funasr/runtime/python/onnxruntime/demo_punc_online.py +++ b/funasr/runtime/python/onnxruntime/demo_punc_online.py @@ -1,12 +1,12 @@ from funasr_onnx import CT_Transformer_VadRealtime -model_dir = "/disk1/mengzhe.cmz/workspace/FunASR/funasr/export/damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727" +model_dir = "../../../export/damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727" model = CT_Transformer_VadRealtime(model_dir) text_in = "跨境河流是养育沿岸|人民的生命之源长期以来为帮助下游地区防灾减灾中方技术人员|在上游地区极为恶劣的自然条件下克服巨大困难甚至冒着生命危险|向印方提供汛期水文资料处理紧急事件中方重视印方在跨境河流>问题上的关切|愿意进一步完善双方联合工作机制|凡是|中方能做的我们|都会去做而且会做得更好我请印度朋友们放心中国在上游的|任何开发利用都会经过科学|规划和论证兼顾上下游的利益" vads = text_in.split("|") -rec_result_all="outputs:" +rec_result_all="" param_dict = {"cache": []} for vad in vads: result = model(vad, param_dict=param_dict)