/**
 * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
 * MIT License  (https://opensource.org/licenses/MIT)
*/

#pragma once
#include <yaml-cpp/yaml.h>
#include "cppjieba/DictTrie.hpp"
#include "cppjieba/HMMModel.hpp"
#include "cppjieba/Jieba.hpp"
#include "nlohmann/json.hpp"

namespace funasr {
class CTokenizer {
private:

	bool  m_ready = false;
	vector<string>   m_id2token,m_id2punc;
	map<string, int>  m_token2id,m_punc2id;

	cppjieba::DictTrie *jieba_dict_trie_=nullptr;
    cppjieba::HMMModel *jieba_model_=nullptr;
	cppjieba::Jieba jieba_processor_;

public:

	CTokenizer(const char* sz_yamlfile);
	CTokenizer();
	~CTokenizer();
	bool OpenYaml(const char* sz_yamlfile);
	bool OpenYaml(const char* sz_yamlfile, const char* token_file);
	void ReadYaml(const YAML::Node& node);
	vector<string> Id2String(vector<int> input);
	vector<int> String2Ids(vector<string> input);
	int String2Id(string input);
	vector<string> Id2Punc(vector<int> input);
	string Id2Punc(int n_punc_id);
	vector<int> Punc2Ids(vector<string> input);
	vector<string> SplitChineseString(const string& str_info);
	vector<string> SplitChineseJieba(const string& str_info);
	void StrSplit(const string& str, const char split, vector<string>& res);
	void Tokenize(const char* str_info, vector<string>& str_out, vector<int>& id_out);
	bool IsPunc(string& Punc);
	bool seg_jieba = false;
	void SetJiebaRes(cppjieba::DictTrie *dict, cppjieba::HMMModel *hmm);
	void JiebaInit(std::string punc_config);
};

} // namespace funasr