diff --git a/glc/cmn/cmn.go b/glc/cmn/cmn.go index b41210d..09ffa39 100644 --- a/glc/cmn/cmn.go +++ b/glc/cmn/cmn.go @@ -25,33 +25,38 @@ func StringToInt(s string, defaultVal int) int { return v } -// // 字符串(10进制无符号整数形式)转uint32,超过uint32最大值会丢失精度 -// // 转换失败时返回默认值 -// func StringToUint32(s string, defaultVal uint32) uint32 { -// v, err := strconv.ParseUint(s, 10, 32) -// if err != nil { -// return defaultVal -// } -// return uint32(v & 0xFFFFFFFF) -// } - -// 字符串(指定进制无符号整数形式)转uint64,进制base范围为2~36 -// 参数错误或转换失败都返回默认值 -func StringToUint64(s string, base int, defaultVal uint64) uint64 { - if s == "" || base < 2 || base > 36 { - return defaultVal - } - - v, err := strconv.ParseUint(s, base, 64) +// 字符串(10进制无符号整数形式)转uint32,超过uint32最大值会丢失精度 +// 转换失败时返回默认值 +func StringToUint32(s string, defaultVal uint32) uint32 { + v, err := strconv.ParseUint(s, 10, 32) if err != nil { return defaultVal } - return v + return uint32(v & 0xFFFFFFFF) } -// Uint64转指定进制形式字符串 -func Uint64ToString(val uint64, base int) string { - return strconv.FormatUint(val, base) +// // 字符串(指定进制无符号整数形式)转uint64,进制base范围为2~36 +// // 参数错误或转换失败都返回默认值 +// func StringToUint64(s string, base int, defaultVal uint64) uint64 { +// if s == "" || base < 2 || base > 36 { +// return defaultVal +// } + +// v, err := strconv.ParseUint(s, base, 64) +// if err != nil { +// return defaultVal +// } +// return v +// } + +// // Uint64转指定进制形式字符串 +// func Uint64ToString(val uint64, base int) string { +// return strconv.FormatUint(val, base) +// } + +// Uint32转字符串 +func Uint32ToString(val uint32) string { + return fmt.Sprintf("%d", val) } func StringToBytes(s string) []byte { @@ -77,26 +82,26 @@ func BytesToString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) } -// func Uint32ToBytes(num uint32) []byte { -// bkey := make([]byte, 4) -// binary.BigEndian.PutUint32(bkey, num) -// return bkey -// } - -// func BytesToUint32(bytes []byte) uint32 { -// return uint32(binary.BigEndian.Uint32(bytes)) -// } - -func Uint64ToBytes(num uint64) []byte { - bkey := make([]byte, 8) - binary.BigEndian.PutUint64(bkey, num) +func Uint32ToBytes(num uint32) []byte { + bkey := make([]byte, 4) + binary.BigEndian.PutUint32(bkey, num) return bkey } -func BytesToUint64(bytes []byte) uint64 { - return binary.BigEndian.Uint64(bytes) +func BytesToUint32(bytes []byte) uint32 { + return uint32(binary.BigEndian.Uint32(bytes)) } +// func Uint64ToBytes(num uint64) []byte { +// bkey := make([]byte, 8) +// binary.BigEndian.PutUint64(bkey, num) +// return bkey +// } + +// func BytesToUint64(bytes []byte) uint64 { +// return binary.BigEndian.Uint64(bytes) +// } + func LenRune(str string) int { return utf8.RuneCountInString(str) } diff --git a/glc/ldb/engine.go b/glc/ldb/engine.go index 46257d4..9cf9610 100644 --- a/glc/ldb/engine.go +++ b/glc/ldb/engine.go @@ -38,7 +38,7 @@ func (e *Engine) AddTextLog(date string, logText string, system string) { e.logStorage.AddTextLog(date, logText, system) } -func (e *Engine) Search(searchKey string, pageSize int, currentDocId uint64, forward bool) *search.SearchResult { +func (e *Engine) Search(searchKey string, pageSize int, currentDocId uint32, forward bool) *search.SearchResult { // 检查修正pageSize if pageSize < 1 { diff --git a/glc/ldb/engine_test.go b/glc/ldb/engine_test.go index 5eb5bd4..8f68ac3 100644 --- a/glc/ldb/engine_test.go +++ b/glc/ldb/engine_test.go @@ -1,7 +1,6 @@ package ldb import ( - "fmt" "log" "testing" "time" @@ -10,10 +9,10 @@ import ( func Test_all(t *testing.T) { engine := NewDefaultEngine() - for i := 1; i <= 100; i++ { - engine.AddTextLog("date", fmt.Sprintf(`java.sql.SQLException: ddduse them aalav_%d`, i), "sssss_ssss") - } - time.Sleep(time.Duration(5) * time.Second) + // for i := 1; i <= 10000; i++ { + // engine.AddTextLog("date", fmt.Sprintf(`java.sql.SQLException: ddduse them aalav_%d`, i), "sssss_ssss") + // } + time.Sleep(time.Duration(30) * time.Second) // for i := 1; i <= 10000; i++ { // engine.AddTextLog(` java.sql.SQLException: them aalav`) diff --git a/glc/ldb/search/search_keys.go b/glc/ldb/search/search_keys.go index 4d31eb5..73de47f 100644 --- a/glc/ldb/search/search_keys.go +++ b/glc/ldb/search/search_keys.go @@ -17,7 +17,7 @@ type SearchResult struct { } // 多关键词浏览日志 -func Search(storeName string, kws []string, pageSize int, currentDocId uint64, forward bool) *SearchResult { +func Search(storeName string, kws []string, pageSize int, currentDocId uint32, forward bool) *SearchResult { storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据 var widxs []*storage.WordIndexStorage for _, word := range kws { @@ -27,12 +27,12 @@ func Search(storeName string, kws []string, pageSize int, currentDocId uint64, f } // 无关键词时走全量检索 -func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward bool) *SearchResult { +func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward bool) *SearchResult { var rs = new(SearchResult) // 检索结果 storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据 totalCount := storeLogData.TotalCount() // 总件数 - rs.Total = cmn.Uint64ToString(totalCount, 10) // 返回的总件数用10进制字符串形式以避免出现科学计数法 + rs.Total = cmn.Uint32ToString(totalCount) // 返回的总件数用10进制字符串形式以避免出现科学计数法 if totalCount == 0 { return rs @@ -40,10 +40,10 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward if currentDocId == 0 { // 第一页 - var min, max uint64 + var min, max uint32 max = totalCount - if max > uint64(pageSize) { - min = max - uint64(pageSize) + 1 + if max > uint32(pageSize) { + min = max - uint32(pageSize) + 1 } else { min = 1 } @@ -54,14 +54,14 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward } else if forward { // 后一页 if currentDocId > 1 { - var min, max uint64 + var min, max uint32 if currentDocId > totalCount { max = totalCount } else { max = currentDocId - 1 } - if max > uint64(pageSize) { - min = max - uint64(pageSize) + 1 + if max > uint32(pageSize) { + min = max - uint32(pageSize) + 1 } else { min = 1 } @@ -73,9 +73,9 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward } else { // 前一页 if totalCount > currentDocId { - var min, max uint64 + var min, max uint32 min = currentDocId + 1 - max = min + uint64(pageSize) - 1 + max = min + uint32(pageSize) - 1 if max > totalCount { max = totalCount } @@ -90,13 +90,13 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward } // 有关键词时走索引检索 -func SearchWordIndex(storeName string, word string, pageSize int, currentDocId uint64, forward bool) *SearchResult { +func SearchWordIndex(storeName string, word string, pageSize int, currentDocId uint32, forward bool) *SearchResult { - var rs = new(SearchResult) // 检索结果 - storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据 - storeIndex := storage.NewWordIndexStorage(storeName, word) // 索引 - totalCount := storeIndex.TotalCount() // 总件数 - rs.Total = cmn.Uint64ToString(storeLogData.TotalCount(), 10) // 返回的总件数用10进制字符串形式以避免出现科学计数法 + var rs = new(SearchResult) // 检索结果 + storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据 + storeIndex := storage.NewWordIndexStorage(storeName, word) // 索引 + totalCount := storeIndex.TotalCount() // 总件数 + rs.Total = cmn.Uint32ToString(storeLogData.TotalCount()) // 返回的总件数用10进制字符串形式以避免出现科学计数法 if totalCount == 0 { return rs @@ -104,10 +104,10 @@ func SearchWordIndex(storeName string, word string, pageSize int, currentDocId u if currentDocId == 0 { // 第一页 - var min, max uint64 + var min, max uint32 max = totalCount - if max > uint64(pageSize) { - min = max - uint64(pageSize) + 1 + if max > uint32(pageSize) { + min = max - uint32(pageSize) + 1 } else { min = 1 } @@ -118,14 +118,14 @@ func SearchWordIndex(storeName string, word string, pageSize int, currentDocId u } else if forward { // 后一页 if currentDocId > 1 { - var min, max uint64 + var min, max uint32 if currentDocId > totalCount { max = totalCount } else { max = currentDocId - 1 } - if max > uint64(pageSize) { - min = max - uint64(pageSize) + 1 + if max > uint32(pageSize) { + min = max - uint32(pageSize) + 1 } else { min = 1 } @@ -137,9 +137,9 @@ func SearchWordIndex(storeName string, word string, pageSize int, currentDocId u } else { // 前一页 if totalCount > currentDocId { - var min, max uint64 + var min, max uint32 min = currentDocId + 1 - max = min + uint64(pageSize) - 1 + max = min + uint32(pageSize) - 1 if max > totalCount { max = totalCount } diff --git a/glc/ldb/search/search_same.go b/glc/ldb/search/search_same.go index 35e0594..8d29c6e 100644 --- a/glc/ldb/search/search_same.go +++ b/glc/ldb/search/search_same.go @@ -9,7 +9,7 @@ import ( ) // 参数widxs长度要求大于1,currentDocId不传就是查第一页 -func findSame(pageSize int, currentDocId uint64, forward bool, storeLogData *storage.LogDataStorageHandle, widxs ...*storage.WordIndexStorage) *SearchResult { +func findSame(pageSize int, currentDocId uint32, forward bool, storeLogData *storage.LogDataStorageHandle, widxs ...*storage.WordIndexStorage) *SearchResult { var rs = new(SearchResult) diff --git a/glc/ldb/storage/logdata_document.go b/glc/ldb/storage/logdata_document.go index a70ef0b..2c15b89 100644 --- a/glc/ldb/storage/logdata_document.go +++ b/glc/ldb/storage/logdata_document.go @@ -11,7 +11,7 @@ import ( // 日志的文档索引 type LogDataDocument struct { - Id uint64 `json:"id,omitempty"` // 文档ID,从1开始递增 + Id uint32 `json:"id,omitempty"` // 文档ID,从1开始递增 Content string `json:"content,omitempty"` // 文档内容,内容格式自行定义 } diff --git a/glc/ldb/storage/logdata_storage.go b/glc/ldb/storage/logdata_storage.go index 6117db6..a66d341 100644 --- a/glc/ldb/storage/logdata_storage.go +++ b/glc/ldb/storage/logdata_storage.go @@ -26,7 +26,7 @@ type LogDataStorage struct { subPath string // 存储目录下的相对路径(存放数据) storeChan chan *LogDataModel // 存储通道 leveldb *leveldb.DB // leveldb - currentCount uint64 // 当前件数 + currentCount uint32 // 当前件数 lastTime int64 // 最后一次访问时间 closing bool // 是否关闭中状态 mu sync.Mutex // 锁 @@ -131,15 +131,15 @@ func readyGo(store *LogDataStorage) { func saveLogData(store *LogDataStorage, model *LogDataModel) { //store.wg.Done() - store.currentCount++ // ID递增 - doc := new(LogDataDocument) // 文档 - doc.Id = store.currentCount // 已递增好的值 - model.Id = cmn.Uint64ToString(store.currentCount, 36) // 模型数据要转Json存,也得更新ID,ID用36进制字符串形式表示 - doc.Content = model.ToJson() // 转json作为内容(含Id) + store.currentCount++ // ID递增 + doc := new(LogDataDocument) // 文档 + doc.Id = store.currentCount // 已递增好的值 + model.Id = cmn.Uint32ToString(store.currentCount) // 模型数据要转Json存,也得更新ID,ID用36进制字符串形式表示 + doc.Content = model.ToJson() // 转json作为内容(含Id) // 保存 - store.put(cmn.Uint64ToBytes(doc.Id), doc.ToBytes()) // 日志数据 - store.leveldb.Put(cmn.Uint64ToBytes(0), cmn.Uint64ToBytes(store.currentCount), nil) // 保存日志总件数 + store.put(cmn.Uint32ToBytes(doc.Id), doc.ToBytes()) // 日志数据 + store.leveldb.Put(cmn.Uint32ToBytes(0), cmn.Uint32ToBytes(store.currentCount), nil) // 保存日志总件数 log.Println("保存日志数据 ", doc.Id) } @@ -171,9 +171,9 @@ func createInvertedIndex(s *LogDataStorage) int { // 每个关键词都创建反向索引 for _, word := range kws { idx := NewWordIndexStorage(s.StoreName(), word) - idx.Add(cmn.StringToUint64(m.Id, 36, 0)) // 日志ID加入索引 + idx.Add(cmn.StringToUint32(m.Id, 0)) // 日志ID加入索引 } - log.Println("创建日志索引:", cmn.StringToUint64(m.Id, 36, 0)) + log.Println("创建日志索引:", cmn.StringToUint32(m.Id, 0)) // 保存索引信息 mnt.AddKeyWords(kws) // 关键词信息 @@ -215,12 +215,12 @@ func (s *LogDataStorage) Get(key []byte) ([]byte, error) { } // 直接从leveldb取数据并转换为LogDataModel -func (s *LogDataStorage) GetLogDataModel(id uint64) (*LogDataModel, error) { +func (s *LogDataStorage) GetLogDataModel(id uint32) (*LogDataModel, error) { if s.closing { return nil, errors.New("current storage is closed") // 关闭中或已关闭时拒绝服务 } s.lastTime = time.Now().Unix() - bts, err := s.leveldb.Get(cmn.Uint64ToBytes(id), nil) + bts, err := s.leveldb.Get(cmn.Uint32ToBytes(id), nil) if err != nil { return nil, err } @@ -266,16 +266,16 @@ func (s *LogDataStorage) Close() { log.Println("关闭LogDataStorage:", s.storeName+cmn.PathSeparator()+s.subPath) } -func (s *LogDataStorage) loadTotalCount() uint64 { - bytes, err := s.leveldb.Get(cmn.Uint64ToBytes(0), nil) +func (s *LogDataStorage) loadTotalCount() uint32 { + bytes, err := s.leveldb.Get(cmn.Uint32ToBytes(0), nil) if err != nil || bytes == nil { return 0 } - return cmn.BytesToUint64(bytes) + return cmn.BytesToUint32(bytes) } // 总件数 -func (s *LogDataStorage) TotalCount() uint64 { +func (s *LogDataStorage) TotalCount() uint32 { return s.currentCount } diff --git a/glc/ldb/storage/logdata_storage_handle.go b/glc/ldb/storage/logdata_storage_handle.go index 5022276..ecb19d1 100644 --- a/glc/ldb/storage/logdata_storage_handle.go +++ b/glc/ldb/storage/logdata_storage_handle.go @@ -75,15 +75,15 @@ func (s *LogDataStorageHandle) AddTextLog(date string, logText string, system st // } // 取日志(文档) -func (s *LogDataStorageHandle) GetLogDataDocument(id uint64) *LogDataDocument { - bytes, _ := s.storage.Get(cmn.Uint64ToBytes(id)) +func (s *LogDataStorageHandle) GetLogDataDocument(id uint32) *LogDataDocument { + bytes, _ := s.storage.Get(cmn.Uint32ToBytes(id)) doc := new(LogDataDocument) doc.LoadBytes(bytes) return doc } // 取日志(模型) -func (s *LogDataStorageHandle) GetLogDataModel(id uint64) *LogDataModel { +func (s *LogDataStorageHandle) GetLogDataModel(id uint32) *LogDataModel { d := s.GetLogDataDocument(id) m := new(LogDataModel) m.LoadJson(d.Content) @@ -91,6 +91,6 @@ func (s *LogDataStorageHandle) GetLogDataModel(id uint64) *LogDataModel { } // 总件数 -func (s *LogDataStorageHandle) TotalCount() uint64 { +func (s *LogDataStorageHandle) TotalCount() uint32 { return s.storage.TotalCount() } diff --git a/glc/ldb/storage/word_index_storage.go b/glc/ldb/storage/word_index_storage.go index 9aae7ec..77dca7b 100644 --- a/glc/ldb/storage/word_index_storage.go +++ b/glc/ldb/storage/word_index_storage.go @@ -24,7 +24,7 @@ type WordIndexStorage struct { subPath string // 存储目录下的相对路径(存放数据) word string // 索引关键词 leveldb *leveldb.DB // leveldb - currentCount uint64 // 当前件数 + currentCount uint32 // 当前件数 lastTime int64 // 最后一次访问时间 closing bool // 是否关闭中状态 mu sync.Mutex // 锁 @@ -104,12 +104,12 @@ func autoCloseWordIndexStorageWhenMaxIdle(store *WordIndexStorage) { } // 日志ID添加到索引 -func (s *WordIndexStorage) Add(docId uint64) error { +func (s *WordIndexStorage) Add(docId uint32) error { // 加索引 s.lastTime = time.Now().Unix() s.currentCount++ // ID递增 - err := s.leveldb.Put(cmn.Uint64ToBytes(s.currentCount), cmn.Uint64ToBytes(docId), nil) + err := s.leveldb.Put(cmn.Uint32ToBytes(s.currentCount), cmn.Uint32ToBytes(docId), nil) if err != nil { log.Println("保存索引失败", err) return err @@ -117,14 +117,14 @@ func (s *WordIndexStorage) Add(docId uint64) error { // docId加盐为键保存索引位置(反向索引再建反向索引之意) keyDocId := fmt.Sprintf("d%d", docId) - err = s.leveldb.Put(cmn.StringToBytes(keyDocId), cmn.Uint64ToBytes(s.currentCount), nil) + err = s.leveldb.Put(cmn.StringToBytes(keyDocId), cmn.Uint32ToBytes(s.currentCount), nil) if err != nil { log.Println("保存索引失败", err) return err } // 保存建好的索引数 - s.leveldb.Put(cmn.Uint64ToBytes(0), cmn.Uint64ToBytes(s.currentCount), nil) + s.leveldb.Put(cmn.Uint32ToBytes(0), cmn.Uint32ToBytes(s.currentCount), nil) if err != nil { log.Println("保存索引件数失败", err) return err // 忽略事务问题,可下回重建 @@ -134,26 +134,26 @@ func (s *WordIndexStorage) Add(docId uint64) error { } // 按日志文档ID找索引位置(找不到返回0) -func (s *WordIndexStorage) GetPosByDocId(id uint64) uint64 { +func (s *WordIndexStorage) GetPosByDocId(id uint32) uint32 { keyDocId := fmt.Sprintf("d%d", id) idx, err := s.leveldb.Get(cmn.StringToBytes(keyDocId), nil) if err != nil { return 0 } - return cmn.BytesToUint64(idx) + return cmn.BytesToUint32(idx) } // 通过索引ID取日志ID(返回0表示有问题) -func (s *WordIndexStorage) Get(id uint64) uint64 { +func (s *WordIndexStorage) Get(id uint32) uint32 { if s.closing { return 0 } s.lastTime = time.Now().Unix() - b, err := s.leveldb.Get(cmn.Uint64ToBytes(id), nil) + b, err := s.leveldb.Get(cmn.Uint32ToBytes(id), nil) if err != nil { return 0 } - return cmn.BytesToUint64(b) + return cmn.BytesToUint32(b) } // 关闭Storage @@ -177,16 +177,16 @@ func (s *WordIndexStorage) Close() { log.Println("关闭WordIndexStorage:", s.storeName+cmn.PathSeparator()+s.subPath) } -func (s *WordIndexStorage) loadTotalCount() uint64 { - bytes, err := s.leveldb.Get(cmn.Uint64ToBytes(0), nil) +func (s *WordIndexStorage) loadTotalCount() uint32 { + bytes, err := s.leveldb.Get(cmn.Uint32ToBytes(0), nil) if err != nil || bytes == nil { return 0 } - return cmn.BytesToUint64(bytes) + return cmn.BytesToUint32(bytes) } // 总件数 -func (s *WordIndexStorage) TotalCount() uint64 { +func (s *WordIndexStorage) TotalCount() uint32 { return s.currentCount } diff --git a/glc/ldb/sysmnt/sysmnt_key_bloom.go b/glc/ldb/sysmnt/sysmnt_key_bloom.go index e11764d..0db3ec9 100644 --- a/glc/ldb/sysmnt/sysmnt_key_bloom.go +++ b/glc/ldb/sysmnt/sysmnt_key_bloom.go @@ -9,7 +9,7 @@ import ( "glc/cmn" ) -var zero6Bytes []byte = cmn.Uint64ToBytes(0) // 避免键冲突,加前缀 +var zero6Bytes []byte = cmn.Uint32ToBytes(0) // 避免键冲突,加前缀 // 检查指定关键词是否都有数据 func (s *SysmntStorage) ContainsKeyWord(kws []string) bool { diff --git a/glc/ldb/sysmnt/sysmnt_key_value.go b/glc/ldb/sysmnt/sysmnt_key_value.go index 835bfdc..a775e5d 100644 --- a/glc/ldb/sysmnt/sysmnt_key_value.go +++ b/glc/ldb/sysmnt/sysmnt_key_value.go @@ -12,8 +12,8 @@ import ( ) type SysmntData struct { - Count uint64 - Value uint64 + Count uint32 + Value uint32 Flag bool Content string } diff --git a/glc/main.go b/glc/main.go index 91d7d9f..c12296b 100644 --- a/glc/main.go +++ b/glc/main.go @@ -5,3 +5,4 @@ import "glc/onstart" func main() { onstart.Run() } + \ No newline at end of file diff --git a/glc/www/controller/log_search_controller.go b/glc/www/controller/log_search_controller.go index e01a728..df53ac6 100644 --- a/glc/www/controller/log_search_controller.go +++ b/glc/www/controller/log_search_controller.go @@ -12,7 +12,7 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult { //searchKey := tokenizer.GetSearchKey(req.GetFormParameter("searchKey")) searchKey := req.GetFormParameter("searchKey") pageSize := cmn.StringToInt(req.GetFormParameter("pageSize"), 20) - currentId := cmn.StringToUint64(req.GetFormParameter("currentId"), 36, 0) + currentId := cmn.StringToUint32(req.GetFormParameter("currentId"), 0) forward := cmn.StringToBool(req.GetFormParameter("forward"), true) eng := ldb.NewEngine(storeNmae)