speech_recognition/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR synced 2025-09-15 14:48:36 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #21 from alibaba-damo-academy/dev

Browse Source 
update inverse_text_normalization/zh into fun_text_processing, update…
This commit is contained in:
Zhang Chong 2022-12-28 18:19:33 +08:00 committed by GitHub
commit 0a4b242611
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 7 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
fun_text_processing/inverse_text_normalization/zh/taggers/tokenize_and_classify.py
View File

@ -28,6 +28,7 @@ from fun_text_processing.inverse_text_normalization.zh.graph_utils import (
GraphFst, GraphFst,
delete_extra_space, delete_extra_space,
delete_space, delete_space,
insert_space,
generator_main, generator_main,
) )
from pynini.lib import pynutil from pynini.lib import pynutil
@ -94,10 +95,10 @@ class ClassifyFst(GraphFst):
punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }") punct = pynutil.insert("tokens { ") + pynutil.add_weight(punct_graph, weight=1.1) + pynutil.insert(" }")
token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }") token = pynutil.insert("tokens { ") + classify + pynutil.insert(" }")
token_plus_punct = ( token_plus_punct = (
pynini.closure(punct + pynutil.insert(" ")) + token + pynini.closure(pynutil.insert(" ") + punct) pynini.closure(punct + insert_space) + token + pynini.closure(insert_space + punct)
) )
graph = token_plus_punct + pynini.closure(delete_extra_space + token_plus_punct) graph = token_plus_punct + pynini.closure(insert_space + token_plus_punct)
graph = delete_space + graph + delete_space graph = delete_space + graph + delete_space
self.fst = graph.optimize() self.fst = graph.optimize()

4
fun_text_processing/inverse_text_normalization/zh/taggers/word.py
View File

@ -11,7 +11,7 @@
# limitations under the License. # limitations under the License.
import pynini import pynini
from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_SPACE, GraphFst from fun_text_processing.inverse_text_normalization.zh.graph_utils import DAMO_NOT_SPACE, DAMO_CHAR, GraphFst
from pynini.lib import pynutil from pynini.lib import pynutil
@ -23,5 +23,5 @@ class WordFst(GraphFst):
def __init__(self): def __init__(self):
super().__init__(name="word", kind="classify") super().__init__(name="word", kind="classify")
word = pynutil.insert("name: \"") + pynini.closure(DAMO_NOT_SPACE, 1) + pynutil.insert("\"") word = pynutil.insert("name: \"") + DAMO_NOT_SPACE + pynutil.insert("\"")
self.fst = word.optimize() self.fst = word.optimize()

2
fun_text_processing/inverse_text_normalization/zh/verbalizers/verbalize_final.py
View File

@ -37,5 +37,5 @@ class VerbalizeFinalFst(GraphFst):
+ delete_space + delete_space
+ pynutil.delete("}") + pynutil.delete("}")
) )
graph = delete_space + pynini.closure(graph + delete_extra_space) + graph + delete_space graph = delete_space + pynini.closure(graph + delete_space) + graph + delete_space
self.fst = graph self.fst = graph

2
setup.py
View File

@ -124,7 +124,7 @@ setup(
long_description=open(os.path.join(dirname, "README.md"), encoding="utf-8").read(), long_description=open(os.path.join(dirname, "README.md"), encoding="utf-8").read(),
long_description_content_type="text/markdown", long_description_content_type="text/markdown",
license="The MIT License", license="The MIT License",
packages=find_packages(include=["funasr*"]), packages=find_packages(include=["funasr*", "fun_text_processing*"]),
package_data={"funasr": ["version.txt"]}, package_data={"funasr": ["version.txt"]},
install_requires=install_requires, install_requires=install_requires,
setup_requires=setup_requires, setup_requires=setup_requires,