software/glogcenter
1
0
Fork 0
mirror of https://github.com/gotoeasy/glogcenter.git synced 2025-09-15 12:58:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

日志ID改用uint32

Browse Source
This commit is contained in:
gotoeasy 2022-07-02 11:31:38 +08:00
parent 4067af2a60
commit 89b0fe8eba
13 changed files with 113 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
glc/cmn/cmn.go
View File

@ -25,33 +25,38 @@ func StringToInt(s string, defaultVal int) int {
return v
}
// // 字符串(10进制无符号整数形式)转uint32，超过uint32最大值会丢失精度
// // 转换失败时返回默认值
// func StringToUint32(s string, defaultVal uint32) uint32 {
// v, err := strconv.ParseUint(s, 10, 32)
// if err != nil {
// return defaultVal
// }
// return uint32(v & 0xFFFFFFFF)
// }
// 字符串(指定进制无符号整数形式)转uint64，进制base范围为2~36
// 参数错误或转换失败都返回默认值
func StringToUint64(s string, base int, defaultVal uint64) uint64 {
if s == "" || base < 2 || base > 36 {
return defaultVal
}
v, err := strconv.ParseUint(s, base, 64)
// 字符串(10进制无符号整数形式)转uint32，超过uint32最大值会丢失精度
// 转换失败时返回默认值
func StringToUint32(s string, defaultVal uint32) uint32 {
v, err := strconv.ParseUint(s, 10, 32)
if err != nil {
return defaultVal
}
return v
return uint32(v & 0xFFFFFFFF)
}
// Uint64转指定进制形式字符串
func Uint64ToString(val uint64, base int) string {
return strconv.FormatUint(val, base)
// // 字符串(指定进制无符号整数形式)转uint64，进制base范围为2~36
// // 参数错误或转换失败都返回默认值
// func StringToUint64(s string, base int, defaultVal uint64) uint64 {
// if s == "" || base < 2 || base > 36 {
// return defaultVal
// }
// v, err := strconv.ParseUint(s, base, 64)
// if err != nil {
// return defaultVal
// }
// return v
// }
// // Uint64转指定进制形式字符串
// func Uint64ToString(val uint64, base int) string {
// return strconv.FormatUint(val, base)
// }
// Uint32转字符串
func Uint32ToString(val uint32) string {
return fmt.Sprintf("%d", val)
}
func StringToBytes(s string) []byte {
@ -77,26 +82,26 @@ func BytesToString(b []byte) string {
return *(*string)(unsafe.Pointer(&b))
}
// func Uint32ToBytes(num uint32) []byte {
// bkey := make([]byte, 4)
// binary.BigEndian.PutUint32(bkey, num)
// return bkey
// }
// func BytesToUint32(bytes []byte) uint32 {
// return uint32(binary.BigEndian.Uint32(bytes))
// }
func Uint64ToBytes(num uint64) []byte {
bkey := make([]byte, 8)
binary.BigEndian.PutUint64(bkey, num)
func Uint32ToBytes(num uint32) []byte {
bkey := make([]byte, 4)
binary.BigEndian.PutUint32(bkey, num)
return bkey
}
func BytesToUint64(bytes []byte) uint64 {
return binary.BigEndian.Uint64(bytes)
func BytesToUint32(bytes []byte) uint32 {
return uint32(binary.BigEndian.Uint32(bytes))
}
// func Uint64ToBytes(num uint64) []byte {
// bkey := make([]byte, 8)
// binary.BigEndian.PutUint64(bkey, num)
// return bkey
// }
// func BytesToUint64(bytes []byte) uint64 {
// return binary.BigEndian.Uint64(bytes)
// }
func LenRune(str string) int {
return utf8.RuneCountInString(str)
}

2
glc/ldb/engine.go
View File

@ -38,7 +38,7 @@ func (e *Engine) AddTextLog(date string, logText string, system string) {
e.logStorage.AddTextLog(date, logText, system)
}
func (e *Engine) Search(searchKey string, pageSize int, currentDocId uint64, forward bool) *search.SearchResult {
func (e *Engine) Search(searchKey string, pageSize int, currentDocId uint32, forward bool) *search.SearchResult {
// 检查修正pageSize
if pageSize < 1 {

9
glc/ldb/engine_test.go
View File

@ -1,7 +1,6 @@
package ldb
import (
"fmt"
"log"
"testing"
"time"
@ -10,10 +9,10 @@ import (
func Test_all(t *testing.T) {
engine := NewDefaultEngine()
for i := 1; i <= 100; i++ {
engine.AddTextLog("date", fmt.Sprintf(`java.sql.SQLException: ddduse them aalav_%d`, i), "sssss_ssss")
}
time.Sleep(time.Duration(5) * time.Second)
// for i := 1; i <= 10000; i++ {
// engine.AddTextLog("date", fmt.Sprintf(`java.sql.SQLException: ddduse them aalav_%d`, i), "sssss_ssss")
// }
time.Sleep(time.Duration(30) * time.Second)
// for i := 1; i <= 10000; i++ {
// engine.AddTextLog(` java.sql.SQLException: them aalav`)

50
glc/ldb/search/search_keys.go
View File

@ -17,7 +17,7 @@ type SearchResult struct {
}
// 多关键词浏览日志
func Search(storeName string, kws []string, pageSize int, currentDocId uint64, forward bool) *SearchResult {
func Search(storeName string, kws []string, pageSize int, currentDocId uint32, forward bool) *SearchResult {
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
var widxs []*storage.WordIndexStorage
for _, word := range kws {
@ -27,12 +27,12 @@ func Search(storeName string, kws []string, pageSize int, currentDocId uint64, f
}
// 无关键词时走全量检索
func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward bool) *SearchResult {
func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward bool) *SearchResult {
var rs = new(SearchResult) // 检索结果
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
totalCount := storeLogData.TotalCount() // 总件数
rs.Total = cmn.Uint64ToString(totalCount, 10) // 返回的总件数用10进制字符串形式以避免出现科学计数法
rs.Total = cmn.Uint32ToString(totalCount) // 返回的总件数用10进制字符串形式以避免出现科学计数法
if totalCount == 0 {
return rs
@ -40,10 +40,10 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward
if currentDocId == 0 {
// 第一页
var min, max uint64
var min, max uint32
max = totalCount
if max > uint64(pageSize) {
min = max - uint64(pageSize) + 1
if max > uint32(pageSize) {
min = max - uint32(pageSize) + 1
} else {
min = 1
}
@ -54,14 +54,14 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward
} else if forward {
// 后一页
if currentDocId > 1 {
var min, max uint64
var min, max uint32
if currentDocId > totalCount {
max = totalCount
} else {
max = currentDocId - 1
}
if max > uint64(pageSize) {
min = max - uint64(pageSize) + 1
if max > uint32(pageSize) {
min = max - uint32(pageSize) + 1
} else {
min = 1
}
@ -73,9 +73,9 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward
} else {
// 前一页
if totalCount > currentDocId {
var min, max uint64
var min, max uint32
min = currentDocId + 1
max = min + uint64(pageSize) - 1
max = min + uint32(pageSize) - 1
if max > totalCount {
max = totalCount
}
@ -90,13 +90,13 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint64, forward
}
// 有关键词时走索引检索
func SearchWordIndex(storeName string, word string, pageSize int, currentDocId uint64, forward bool) *SearchResult {
func SearchWordIndex(storeName string, word string, pageSize int, currentDocId uint32, forward bool) *SearchResult {
var rs = new(SearchResult) // 检索结果
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
storeIndex := storage.NewWordIndexStorage(storeName, word) // 索引
totalCount := storeIndex.TotalCount() // 总件数
rs.Total = cmn.Uint64ToString(storeLogData.TotalCount(), 10) // 返回的总件数用10进制字符串形式以避免出现科学计数法
var rs = new(SearchResult) // 检索结果
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
storeIndex := storage.NewWordIndexStorage(storeName, word) // 索引
totalCount := storeIndex.TotalCount() // 总件数
rs.Total = cmn.Uint32ToString(storeLogData.TotalCount()) // 返回的总件数用10进制字符串形式以避免出现科学计数法
if totalCount == 0 {
return rs
@ -104,10 +104,10 @@ func SearchWordIndex(storeName string, word string, pageSize int, currentDocId u
if currentDocId == 0 {
// 第一页
var min, max uint64
var min, max uint32
max = totalCount
if max > uint64(pageSize) {
min = max - uint64(pageSize) + 1
if max > uint32(pageSize) {
min = max - uint32(pageSize) + 1
} else {
min = 1
}
@ -118,14 +118,14 @@ func SearchWordIndex(storeName string, word string, pageSize int, currentDocId u
} else if forward {
// 后一页
if currentDocId > 1 {
var min, max uint64
var min, max uint32
if currentDocId > totalCount {
max = totalCount
} else {
max = currentDocId - 1
}
if max > uint64(pageSize) {
min = max - uint64(pageSize) + 1
if max > uint32(pageSize) {
min = max - uint32(pageSize) + 1
} else {
min = 1
}
@ -137,9 +137,9 @@ func SearchWordIndex(storeName string, word string, pageSize int, currentDocId u
} else {
// 前一页
if totalCount > currentDocId {
var min, max uint64
var min, max uint32
min = currentDocId + 1
max = min + uint64(pageSize) - 1
max = min + uint32(pageSize) - 1
if max > totalCount {
max = totalCount
}

2
glc/ldb/search/search_same.go
View File

@ -9,7 +9,7 @@ import (
)
// 参数widxs长度要求大于1，currentDocId不传就是查第一页
func findSame(pageSize int, currentDocId uint64, forward bool, storeLogData *storage.LogDataStorageHandle, widxs ...*storage.WordIndexStorage) *SearchResult {
func findSame(pageSize int, currentDocId uint32, forward bool, storeLogData *storage.LogDataStorageHandle, widxs ...*storage.WordIndexStorage) *SearchResult {
var rs = new(SearchResult)

2
glc/ldb/storage/logdata_document.go
View File

@ -11,7 +11,7 @@ import (
// 日志的文档索引
type LogDataDocument struct {
Id uint64 `json:"id,omitempty"` // 文档ID，从1开始递增
Id uint32 `json:"id,omitempty"` // 文档ID，从1开始递增
Content string `json:"content,omitempty"` // 文档内容，内容格式自行定义
}

32
glc/ldb/storage/logdata_storage.go
View File

@ -26,7 +26,7 @@ type LogDataStorage struct {
subPath string // 存储目录下的相对路径（存放数据）
storeChan chan *LogDataModel // 存储通道
leveldb *leveldb.DB // leveldb
currentCount uint64 // 当前件数
currentCount uint32 // 当前件数
lastTime int64 // 最后一次访问时间
closing bool // 是否关闭中状态
mu sync.Mutex // 锁
@ -131,15 +131,15 @@ func readyGo(store *LogDataStorage) {
func saveLogData(store *LogDataStorage, model *LogDataModel) {
//store.wg.Done()
store.currentCount++ // ID递增
doc := new(LogDataDocument) // 文档
doc.Id = store.currentCount // 已递增好的值
model.Id = cmn.Uint64ToString(store.currentCount, 36) // 模型数据要转Json存，也得更新ID,ID用36进制字符串形式表示
doc.Content = model.ToJson() // 转json作为内容(含Id)
store.currentCount++ // ID递增
doc := new(LogDataDocument) // 文档
doc.Id = store.currentCount // 已递增好的值
model.Id = cmn.Uint32ToString(store.currentCount) // 模型数据要转Json存，也得更新ID,ID用36进制字符串形式表示
doc.Content = model.ToJson() // 转json作为内容(含Id)
// 保存
store.put(cmn.Uint64ToBytes(doc.Id), doc.ToBytes()) // 日志数据
store.leveldb.Put(cmn.Uint64ToBytes(0), cmn.Uint64ToBytes(store.currentCount), nil) // 保存日志总件数
store.put(cmn.Uint32ToBytes(doc.Id), doc.ToBytes()) // 日志数据
store.leveldb.Put(cmn.Uint32ToBytes(0), cmn.Uint32ToBytes(store.currentCount), nil) // 保存日志总件数
log.Println("保存日志数据 ", doc.Id)
}
@ -171,9 +171,9 @@ func createInvertedIndex(s *LogDataStorage) int {
// 每个关键词都创建反向索引
for _, word := range kws {
idx := NewWordIndexStorage(s.StoreName(), word)
idx.Add(cmn.StringToUint64(m.Id, 36, 0)) // 日志ID加入索引
idx.Add(cmn.StringToUint32(m.Id, 0)) // 日志ID加入索引
}
log.Println("创建日志索引：", cmn.StringToUint64(m.Id, 36, 0))
log.Println("创建日志索引：", cmn.StringToUint32(m.Id, 0))
// 保存索引信息
mnt.AddKeyWords(kws) // 关键词信息
@ -215,12 +215,12 @@ func (s *LogDataStorage) Get(key []byte) ([]byte, error) {
}
// 直接从leveldb取数据并转换为LogDataModel
func (s *LogDataStorage) GetLogDataModel(id uint64) (*LogDataModel, error) {
func (s *LogDataStorage) GetLogDataModel(id uint32) (*LogDataModel, error) {
if s.closing {
return nil, errors.New("current storage is closed") // 关闭中或已关闭时拒绝服务
}
s.lastTime = time.Now().Unix()
bts, err := s.leveldb.Get(cmn.Uint64ToBytes(id), nil)
bts, err := s.leveldb.Get(cmn.Uint32ToBytes(id), nil)
if err != nil {
return nil, err
}
@ -266,16 +266,16 @@ func (s *LogDataStorage) Close() {
log.Println("关闭LogDataStorage：", s.storeName+cmn.PathSeparator()+s.subPath)
}
func (s *LogDataStorage) loadTotalCount() uint64 {
bytes, err := s.leveldb.Get(cmn.Uint64ToBytes(0), nil)
func (s *LogDataStorage) loadTotalCount() uint32 {
bytes, err := s.leveldb.Get(cmn.Uint32ToBytes(0), nil)
if err != nil || bytes == nil {
return 0
}
return cmn.BytesToUint64(bytes)
return cmn.BytesToUint32(bytes)
}
// 总件数
func (s *LogDataStorage) TotalCount() uint64 {
func (s *LogDataStorage) TotalCount() uint32 {
return s.currentCount
}

8
glc/ldb/storage/logdata_storage_handle.go
View File

@ -75,15 +75,15 @@ func (s *LogDataStorageHandle) AddTextLog(date string, logText string, system st
// }
// 取日志（文档）
func (s *LogDataStorageHandle) GetLogDataDocument(id uint64) *LogDataDocument {
bytes, _ := s.storage.Get(cmn.Uint64ToBytes(id))
func (s *LogDataStorageHandle) GetLogDataDocument(id uint32) *LogDataDocument {
bytes, _ := s.storage.Get(cmn.Uint32ToBytes(id))
doc := new(LogDataDocument)
doc.LoadBytes(bytes)
return doc
}
// 取日志（模型）
func (s *LogDataStorageHandle) GetLogDataModel(id uint64) *LogDataModel {
func (s *LogDataStorageHandle) GetLogDataModel(id uint32) *LogDataModel {
d := s.GetLogDataDocument(id)
m := new(LogDataModel)
m.LoadJson(d.Content)
@ -91,6 +91,6 @@ func (s *LogDataStorageHandle) GetLogDataModel(id uint64) *LogDataModel {
}
// 总件数
func (s *LogDataStorageHandle) TotalCount() uint64 {
func (s *LogDataStorageHandle) TotalCount() uint32 {
return s.storage.TotalCount()
}

28
glc/ldb/storage/word_index_storage.go
View File

@ -24,7 +24,7 @@ type WordIndexStorage struct {
subPath string // 存储目录下的相对路径（存放数据）
word string // 索引关键词
leveldb *leveldb.DB // leveldb
currentCount uint64 // 当前件数
currentCount uint32 // 当前件数
lastTime int64 // 最后一次访问时间
closing bool // 是否关闭中状态
mu sync.Mutex // 锁
@ -104,12 +104,12 @@ func autoCloseWordIndexStorageWhenMaxIdle(store *WordIndexStorage) {
}
// 日志ID添加到索引
func (s *WordIndexStorage) Add(docId uint64) error {
func (s *WordIndexStorage) Add(docId uint32) error {
// 加索引
s.lastTime = time.Now().Unix()
s.currentCount++ // ID递增
err := s.leveldb.Put(cmn.Uint64ToBytes(s.currentCount), cmn.Uint64ToBytes(docId), nil)
err := s.leveldb.Put(cmn.Uint32ToBytes(s.currentCount), cmn.Uint32ToBytes(docId), nil)
if err != nil {
log.Println("保存索引失败", err)
return err
@ -117,14 +117,14 @@ func (s *WordIndexStorage) Add(docId uint64) error {
// docId加盐为键保存索引位置（反向索引再建反向索引之意）
keyDocId := fmt.Sprintf("d%d", docId)
err = s.leveldb.Put(cmn.StringToBytes(keyDocId), cmn.Uint64ToBytes(s.currentCount), nil)
err = s.leveldb.Put(cmn.StringToBytes(keyDocId), cmn.Uint32ToBytes(s.currentCount), nil)
if err != nil {
log.Println("保存索引失败", err)
return err
}
// 保存建好的索引数
s.leveldb.Put(cmn.Uint64ToBytes(0), cmn.Uint64ToBytes(s.currentCount), nil)
s.leveldb.Put(cmn.Uint32ToBytes(0), cmn.Uint32ToBytes(s.currentCount), nil)
if err != nil {
log.Println("保存索引件数失败", err)
return err // 忽略事务问题，可下回重建
@ -134,26 +134,26 @@ func (s *WordIndexStorage) Add(docId uint64) error {
}
// 按日志文档ID找索引位置(找不到返回0)
func (s *WordIndexStorage) GetPosByDocId(id uint64) uint64 {
func (s *WordIndexStorage) GetPosByDocId(id uint32) uint32 {
keyDocId := fmt.Sprintf("d%d", id)
idx, err := s.leveldb.Get(cmn.StringToBytes(keyDocId), nil)
if err != nil {
return 0
}
return cmn.BytesToUint64(idx)
return cmn.BytesToUint32(idx)
}
// 通过索引ID取日志ID（返回0表示有问题）
func (s *WordIndexStorage) Get(id uint64) uint64 {
func (s *WordIndexStorage) Get(id uint32) uint32 {
if s.closing {
return 0
}
s.lastTime = time.Now().Unix()
b, err := s.leveldb.Get(cmn.Uint64ToBytes(id), nil)
b, err := s.leveldb.Get(cmn.Uint32ToBytes(id), nil)
if err != nil {
return 0
}
return cmn.BytesToUint64(b)
return cmn.BytesToUint32(b)
}
// 关闭Storage
@ -177,16 +177,16 @@ func (s *WordIndexStorage) Close() {
log.Println("关闭WordIndexStorage：", s.storeName+cmn.PathSeparator()+s.subPath)
}
func (s *WordIndexStorage) loadTotalCount() uint64 {
bytes, err := s.leveldb.Get(cmn.Uint64ToBytes(0), nil)
func (s *WordIndexStorage) loadTotalCount() uint32 {
bytes, err := s.leveldb.Get(cmn.Uint32ToBytes(0), nil)
if err != nil || bytes == nil {
return 0
}
return cmn.BytesToUint64(bytes)
return cmn.BytesToUint32(bytes)
}
// 总件数
func (s *WordIndexStorage) TotalCount() uint64 {
func (s *WordIndexStorage) TotalCount() uint32 {
return s.currentCount
}

2
glc/ldb/sysmnt/sysmnt_key_bloom.go
View File

@ -9,7 +9,7 @@ import (
"glc/cmn"
)
var zero6Bytes []byte = cmn.Uint64ToBytes(0) // 避免键冲突，加前缀
var zero6Bytes []byte = cmn.Uint32ToBytes(0) // 避免键冲突，加前缀
// 检查指定关键词是否都有数据
func (s *SysmntStorage) ContainsKeyWord(kws []string) bool {

4
glc/ldb/sysmnt/sysmnt_key_value.go
View File

@ -12,8 +12,8 @@ import (
)
type SysmntData struct {
Count uint64
Value uint64
Count uint32
Value uint32
Flag bool
Content string
}

1
glc/main.go
View File

@ -5,3 +5,4 @@ import "glc/onstart"
func main() {
onstart.Run()
}

2
glc/www/controller/log_search_controller.go
View File

@ -12,7 +12,7 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult {
//searchKey := tokenizer.GetSearchKey(req.GetFormParameter("searchKey"))
searchKey := req.GetFormParameter("searchKey")
pageSize := cmn.StringToInt(req.GetFormParameter("pageSize"), 20)
currentId := cmn.StringToUint64(req.GetFormParameter("currentId"), 36, 0)
currentId := cmn.StringToUint32(req.GetFormParameter("currentId"), 0)
forward := cmn.StringToBool(req.GetFormParameter("forward"), true)
eng := ldb.NewEngine(storeNmae)