From 96eaee5a870c46f38cec6acc63e6446bb81d67dc Mon Sep 17 00:00:00 2001 From: gotoeasy Date: Sun, 5 Nov 2023 14:59:52 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B7=A8=E4=BB=93=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- glc/go.mod | 2 +- glc/go.sum | 2 + glc/ldb/engine.go | 15 +- glc/ldb/engine_test.go | 16 -- glc/ldb/search/search_keys.go | 106 ++++++++----- glc/ldb/sysmnt/storage_data.go | 4 +- glc/ver/version.go | 2 +- glc/www/controller/log_search_controller.go | 147 +++++++++++++++--- .../web/src/components/form/SearchForm.vue | 12 +- .../layout/page-header/PageHeader.vue | 13 +- glc/www/web/src/views/glc/search/GlcMain.vue | 69 +++++--- 11 files changed, 275 insertions(+), 113 deletions(-) diff --git a/glc/go.mod b/glc/go.mod index 377219a..6003f2d 100644 --- a/glc/go.mod +++ b/glc/go.mod @@ -6,7 +6,7 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-contrib/gzip v0.0.6 github.com/gin-gonic/gin v1.9.1 - github.com/gotoeasy/glang v0.10.18 + github.com/gotoeasy/glang v0.10.19 github.com/shirou/gopsutil v3.21.11+incompatible github.com/streadway/amqp v1.1.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 diff --git a/glc/go.sum b/glc/go.sum index 9bb8eaa..25c5746 100644 --- a/glc/go.sum +++ b/glc/go.sum @@ -270,6 +270,8 @@ github.com/gotoeasy/glang v0.10.17 h1:PG6NKA7Uy64UeCqTanaJyT81daHcnRgJRBvb2kiUUY github.com/gotoeasy/glang v0.10.17/go.mod h1:RGoWvWlVIEqRX1tOgWhyLbrvjuWMFINmnUMqZX5/vmo= github.com/gotoeasy/glang v0.10.18 h1:QPJM6pawnRpe0N3wNLhdWSdiErFC4dYDYSmuqbhQCM0= github.com/gotoeasy/glang v0.10.18/go.mod h1:RGoWvWlVIEqRX1tOgWhyLbrvjuWMFINmnUMqZX5/vmo= +github.com/gotoeasy/glang v0.10.19 h1:6LZuFyrP4o4m5wqKnketfg/A1zjcrXOoKKVQXUQ8QN4= +github.com/gotoeasy/glang v0.10.19/go.mod h1:RGoWvWlVIEqRX1tOgWhyLbrvjuWMFINmnUMqZX5/vmo= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= diff --git a/glc/ldb/engine.go b/glc/ldb/engine.go index ca4a704..7864451 100644 --- a/glc/ldb/engine.go +++ b/glc/ldb/engine.go @@ -48,13 +48,12 @@ func (e *Engine) AddLogDataModel(data *logdata.LogDataModel) { e.logStorage.AddLogDataModel(data) } -func (e *Engine) Search(searchKey string, system string, minDatetime string, maxDatetime string, loglevel string, loglevels []string, - currentDocId uint32, forward bool) *search.SearchResult { +func (e *Engine) Search(cond *search.SearchCondition) *search.SearchResult { // 分词后检索 var adds []string - adds = append(adds, system, loglevel) - kws := tokenizer.CutForSearchEx(searchKey, adds, nil) // 检索用关键词处理 + adds = append(adds, cond.System, cond.Loglevel) + kws := tokenizer.CutForSearchEx(cond.SearchKey, adds, nil) // 检索用关键词处理 // 简单检查，存在无索引数据的关键词时，直接返回 for _, word := range kws { @@ -67,14 +66,14 @@ func (e *Engine) Search(searchKey string, system string, minDatetime string, max return rs } } - - if len(kws) == 0 { + cond.Kws = kws + if len(cond.Kws) == 0 { // 无条件浏览模式（可能含多选条件） - return search.SearchLogData(e.storeName, loglevels, currentDocId, forward, minDatetime, maxDatetime) + return search.SearchLogData(e.storeName, cond) } // 多关键词查询模式 - return search.SearchWordIndex(e.storeName, kws, loglevels, currentDocId, forward, minDatetime, maxDatetime) + return search.SearchWordIndex(e.storeName, cond) } // 添加日志 diff --git a/glc/ldb/engine_test.go b/glc/ldb/engine_test.go index 7bac359..2c4aa47 100644 --- a/glc/ldb/engine_test.go +++ b/glc/ldb/engine_test.go @@ -36,20 +36,4 @@ func Test_all(t *testing.T) { } time.Sleep(time.Duration(10) * time.Second) - // for i := 1; i <= 10000; i++ { - // engine.AddTextLog(` java.sql.SQLException: them aalav`) - // } - // time.Sleep(time.Duration(5) * time.Second) - - // for i := 1; i <= 10000; i++ { - // engine.AddTextLog(` java.sql.SQLException: them`) - // } - // time.Sleep(time.Duration(5) * time.Second) - - rs := engine.Search(` them java `, "", "", "", "", make([]string, 0), 0, true) - cmn.Debug("共查到", rs.Total, "件") - for _, v := range rs.Data { - cmn.Debug(v.Id, v.Text) - } - } diff --git a/glc/ldb/search/search_keys.go b/glc/ldb/search/search_keys.go index f2cb002..12f1693 100644 --- a/glc/ldb/search/search_keys.go +++ b/glc/ldb/search/search_keys.go @@ -7,7 +7,6 @@ package search import ( - "glc/conf" "glc/ldb/storage" "glc/ldb/storage/indexdoc" "glc/ldb/storage/indexword" @@ -17,11 +16,28 @@ import ( "github.com/gotoeasy/glang/cmn" ) +type SearchCondition struct { + SearchKey string // 输入的检索文本 + StoreName string // 输入的日志仓条件 + System string // 输入的系统名条件 + DatetimeFrom string // 输入的日期范围（from）条件 + DatetimeTo string // 输入的日期范围（to）条件 + Loglevel string // 输入的日志级别（单选条件）条件【内部会过滤修改】 + Loglevels []string // 输入的日志级别（多选条件）条件【内部会过滤修改】 + CurrentStoreName string // 隐藏条件，当前日志文档ID所属的日志仓 + CurrentId uint32 // 隐藏条件，当前日志文档ID + Forward bool // 隐藏条件，是否向前检索（玩下滚动查询） + Kws []string // 【内部用】解析条件所得的检索关键词，非直接输入的检索文本 + SearchSize int // 【内部用】需要查询多少件（跨仓检索时可能多次检索，中间会内部调整） +} + type SearchResult struct { - Total string `json:"total,omitempty"` // 日志总量件数（用10进制字符串形式以避免出现科学计数法） - Count string `json:"count,omitempty"` // 当前条件最多匹配件数（用10进制字符串形式以避免出现科学计数法） - PageSize string `json:"pagesize,omitempty"` // 每次检索件数 - Data []*logdata.LogDataModel `json:"data,omitempty"` // 检索结果数据（日志文档数组） + Total string `json:"total,omitempty"` // 日志总量件数（用10进制字符串形式以避免出现科学计数法） + Count string `json:"count,omitempty"` // 当前条件最多匹配件数（用10进制字符串形式以避免出现科学计数法） + PageSize string `json:"pagesize,omitempty"` // 每次检索件数 + Data []*logdata.LogDataModel `json:"data,omitempty"` // 检索结果数据（日志文档数组） + LastStoreName string `json:"laststorename,omitempty"` // 当前检索结果中，最后一条（最久远）日志所在日志仓 + TimeMessage string `json:"timemessage,omitempty"` // 查询耗时的文本消息表示，如：耗时30毫秒 } type WidxStorage struct { @@ -31,14 +47,14 @@ type WidxStorage struct { } // 多关键词时计算关键词索引交集 -func SearchWordIndex(storeName string, kws []string, loglevels []string, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult { +func SearchWordIndex(storeName string, cond *SearchCondition) *SearchResult { storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据 // 时间条件范围判断，默认全部，有检索条件时调整范围 maxDocumentId := storeLogData.TotalCount() // 时间范围条件内的最大文档ID minDocumentId := cmn.StringToUint32("1", 1) // 时间范围条件内的最小文档ID - if !cmn.IsBlank(minDatetime) { - minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, minDatetime) // 时间范围条件内的最小文档ID，找不到时返回0 + if !cmn.IsBlank(cond.DatetimeFrom) { + minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, cond.DatetimeFrom) // 时间范围条件内的最小文档ID，找不到时返回0 if minDocumentId == 0 { // 简单判断，无匹配时直接返回 var rs = new(SearchResult) @@ -47,8 +63,8 @@ func SearchWordIndex(storeName string, kws []string, loglevels []string, current return rs } } - if !cmn.IsBlank(maxDatetime) { - maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, maxDatetime) // 时间范围条件内的最大文档ID，找不到时返回0 + if !cmn.IsBlank(cond.DatetimeTo) { + maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, cond.DatetimeTo) // 时间范围条件内的最大文档ID，找不到时返回0 if maxDocumentId == 0 { // 简单判断，无匹配时直接返回 var rs = new(SearchResult) @@ -67,7 +83,7 @@ func SearchWordIndex(storeName string, kws []string, loglevels []string, current // 汇总索引进行关联查找 var widxs []*WidxStorage - for _, word := range kws { + for _, word := range cond.Kws { widxStorage := &WidxStorage{ word: word, idxdocStorage: indexdoc.NewDocIndexStorage(storeName), @@ -75,12 +91,12 @@ func SearchWordIndex(storeName string, kws []string, loglevels []string, current } widxs = append(widxs, widxStorage) } - return findSame(currentDocId, loglevels, forward, minDocumentId, maxDocumentId, storeLogData, widxs...) + return findSame(cond, minDocumentId, maxDocumentId, storeLogData, widxs...) } // 无关键词时走全量检索 -func SearchLogData(storeName string, loglevels []string, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult { - allloglevels := cmn.Join(loglevels, ",") // 合并多选的级别条件 +func SearchLogData(storeName string, cond *SearchCondition) *SearchResult { + allloglevels := cmn.Join(cond.Loglevels, ",") // 合并多选的级别条件 noLogLevels := cmn.IsBlank(allloglevels) // 无多选条件 var rs = new(SearchResult) // 检索结果 storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据 @@ -89,17 +105,17 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo rs.Count = cmn.Uint32ToString(totalCount) // 当前条件最多匹配件数 rsCnt := 0 // 已查到的件数 - if totalCount == 0 { - return rs + if totalCount == 0 || cond.SearchSize == 0 { + return rs // 无数据或不需要检索数据 } // 时间条件范围判断，默认全部，有检索条件时调整范围 maxDocumentId := totalCount // 时间范围条件内的最大文档ID minDocumentId := cmn.StringToUint32("1", 1) // 时间范围条件内的最小文档ID - hasMin := !cmn.IsBlank(minDatetime) - hasMax := !cmn.IsBlank(maxDatetime) + hasMin := !cmn.IsBlank(cond.DatetimeFrom) + hasMax := !cmn.IsBlank(cond.DatetimeTo) if hasMin { - minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, minDatetime) // 时间范围条件内的最小文档ID + minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, cond.DatetimeFrom) // 时间范围条件内的最小文档ID if minDocumentId == 0 { // 简单判断，无匹配时直接返回 var rs = new(SearchResult) @@ -109,7 +125,7 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo } } if hasMax { - maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, maxDatetime) // 时间范围条件内的最大文档ID + maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, cond.DatetimeTo) // 时间范围条件内的最大文档ID if maxDocumentId == 0 { // 简单判断，无匹配时直接返回 var rs = new(SearchResult) @@ -128,7 +144,7 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo } // 开始检索 - if currentDocId == 0 { + if cond.CurrentId == 0 { // 第一页 var min, max uint32 max = totalCount @@ -146,19 +162,19 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) { rs.Data = append(rs.Data, md) rsCnt++ - if rsCnt >= conf.GetPageSize() { + if rsCnt >= cond.SearchSize { break } } } - } else if forward { - // 后一页 - if currentDocId > 1 { + } else if cond.Forward { + // 后一页（向下滚动触发检索） + if cond.CurrentId > 1 { var min, max uint32 - if currentDocId > totalCount { + if cond.CurrentId > totalCount { max = totalCount } else { - max = currentDocId - 1 + max = cond.CurrentId - 1 } if max > maxDocumentId { @@ -174,17 +190,17 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) { rs.Data = append(rs.Data, md) rsCnt++ - if rsCnt >= conf.GetPageSize() { + if rsCnt >= cond.SearchSize { break } } } } } else { - // 前一页 - if totalCount > currentDocId { + // 前一页（向上滚动触发检索）【暂未使用】 + if totalCount > cond.CurrentId { var min, max uint32 - min = currentDocId + 1 + min = cond.CurrentId + 1 if min < minDocumentId { min = minDocumentId // 最小不超出时间范围限制内的最小文档ID @@ -199,7 +215,7 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) { rs.Data = append(rs.Data, md) rsCnt++ - if rsCnt >= conf.GetPageSize() { + if rsCnt >= cond.SearchSize { break } } @@ -211,9 +227,9 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo return rs } -// 参数widxs长度要求大于1，currentDocId不传就是查第一页 -func findSame(currentDocId uint32, loglevels []string, forward bool, minDocumentId uint32, maxDocumentId uint32, storeLogData *storage.LogDataStorageHandle, widxs ...*WidxStorage) *SearchResult { - allloglevels := cmn.Join(loglevels, ",") // 合并多选的级别条件 +// 参数widxs长度要求大于1，currentId不传就是查第一页 +func findSame(cond *SearchCondition, minDocumentId uint32, maxDocumentId uint32, storeLogData *storage.LogDataStorageHandle, widxs ...*WidxStorage) *SearchResult { + allloglevels := cmn.Join(cond.Loglevels, ",") // 合并多选的级别条件 noLogLevels := cmn.IsBlank(allloglevels) // 无多选条件 var rs = new(SearchResult) // 查询结果 rs.Total = cmn.Uint32ToString(storeLogData.TotalCount()) // 日志总量件数 @@ -234,17 +250,21 @@ func findSame(currentDocId uint32, loglevels []string, forward bool, minDocument } rs.Count = cmn.Uint32ToString(minCount) // 当前条件最多匹配件数 + if cond.SearchSize <= 0 { + return rs // 只查关联件数，不查数据 + } + // 简单检查排除没结果的情景 totalCount := minIdx.idxwordStorage.GetTotalCount(minIdx.word) - if totalCount == 0 || (totalCount == 1 && currentDocId > 0) { + if totalCount == 0 || (totalCount == 1 && cond.CurrentId > 0) { return rs // 索引件数0、或只有1条又还要跳过，都是找不到 } // 找匹配位置并排除没结果的情景 pos := totalCount // 默认检索最新第一页 - if currentDocId > 0 { - pos = minIdx.idxdocStorage.GetWordDocSeq(minIdx.word, currentDocId) // 有相对文档ID时找相对位置 - if pos == 0 || (pos == 1 && forward) || (pos == totalCount && !forward) { + if cond.CurrentId > 0 { + pos = minIdx.idxdocStorage.GetWordDocSeq(minIdx.word, cond.CurrentId) // 有相对文档ID时找相对位置 + if pos == 0 || (pos == 1 && cond.Forward) || (pos == totalCount && !cond.Forward) { return rs // 找不到、或最后条还要向后、或最前条还要向前，都是找不到 } } @@ -252,9 +272,9 @@ func findSame(currentDocId uint32, loglevels []string, forward bool, minDocument // 位置就绪 var rsCnt int = 0 var flg bool - if currentDocId == 0 || currentDocId > 0 && forward { + if cond.CurrentId == 0 || (cond.CurrentId > 0 && cond.Forward) { // 无相对文档ID、或有且是后一页方向 - if currentDocId > 0 { + if cond.CurrentId > 0 { pos-- // 相对文档ID有的话才顺移 } @@ -287,7 +307,7 @@ func findSame(currentDocId uint32, loglevels []string, forward bool, minDocument if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) { rsCnt++ rs.Data = append(rs.Data, md) - if rsCnt >= conf.GetPageSize() { + if rsCnt >= cond.SearchSize { break // 最多找一页 } } @@ -327,7 +347,7 @@ func findSame(currentDocId uint32, loglevels []string, forward bool, minDocument if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) { rsCnt++ ary = append(ary, md) - if rsCnt >= conf.GetPageSize() { + if rsCnt >= cond.SearchSize { break // 最多找一页 } } diff --git a/glc/ldb/sysmnt/storage_data.go b/glc/ldb/sysmnt/storage_data.go index 253abfc..8f1f6f2 100644 --- a/glc/ldb/sysmnt/storage_data.go +++ b/glc/ldb/sysmnt/storage_data.go @@ -45,6 +45,7 @@ func GetStorageList() *StorageResult { var datas []*StorageModel var sum int64 + var total int64 names := com.GetStorageNames(conf.GetStorageRoot(), ".sysmnt") for _, name := range names { d := &StorageModel{ @@ -65,6 +66,7 @@ func GetStorageList() *StorageResult { sysmntStore := NewSysmntStorage() d.LogCount = sysmntStore.GetStorageDataCount(name) d.IndexCount = sysmntStore.GetStorageIndexCount(name) + total += int64(d.LogCount) } datas = append(datas, d) @@ -73,7 +75,7 @@ func GetStorageList() *StorageResult { stat, _ := disk.Usage(conf.GetStorageRoot()) rs := &StorageResult{ - Info: fmt.Sprintf("共占用空间 " + cmn.GetSizeInfo(uint64(sum)) + "，剩余空间 " + cmn.GetSizeInfo(stat.Free)), + Info: fmt.Sprintf("日志总量 " + cmn.Int64ToString(total) + " 条，共占用空间 " + cmn.GetSizeInfo(uint64(sum)) + "，剩余空间 " + cmn.GetSizeInfo(stat.Free)), Data: datas, } return rs diff --git a/glc/ver/version.go b/glc/ver/version.go index 615f327..cda5513 100644 --- a/glc/ver/version.go +++ b/glc/ver/version.go @@ -1,4 +1,4 @@ package ver // 版本号，升级版本时修改 -const VERSION = "v0.12.2" +const VERSION = "v0.12.3" diff --git a/glc/www/controller/log_search_controller.go b/glc/www/controller/log_search_controller.go index e4d4ec4..44aef70 100644 --- a/glc/www/controller/log_search_controller.go +++ b/glc/www/controller/log_search_controller.go @@ -1,13 +1,26 @@ package controller import ( + "glc/com" "glc/conf" "glc/gweb" "glc/ldb" + "glc/ldb/search" + "glc/ldb/sysmnt" + "time" "github.com/gotoeasy/glang/cmn" ) +type storageItem struct { + storeName string // 日志仓 + total uint32 // 日志件数 + isSearchRange bool // 是否条件范围的日志仓 +} + +var cacheStoreNames []string // 所有的日志仓（避免每次读磁盘，适当使用缓存） +var cacheTime time.Time // 最近一次读日志仓目录的时间点 + // 日志检索（表单提交方式） func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult { @@ -18,32 +31,122 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult { return gweb.Error403() // 登录检查 } - storeName := req.GetFormParameter("storeName") - searchKey := req.GetFormParameter("searchKey") - currentId := cmn.StringToUint32(req.GetFormParameter("currentId"), 0) - forward := cmn.StringToBool(req.GetFormParameter("forward"), true) - datetimeFrom := req.GetFormParameter("datetimeFrom") - datetimeTo := req.GetFormParameter("datetimeTo") - system := req.GetFormParameter("system") - loglevel := req.GetFormParameter("loglevel") // 单选条件 - loglevels := cmn.Split(loglevel, ",") // 多选条件 - if len(loglevels) <= 1 || len(loglevels) >= 4 { - loglevels = make([]string, 0) // 多选的单选或全选，都清空（单选走loglevel索引，全选等于没选） + startTime := time.Now() + cond := &search.SearchCondition{SearchSize: conf.GetPageSize()} + cond.StoreName = req.GetFormParameter("storeName") // 日志仓条件 + cond.SearchKey = req.GetFormParameter("searchKey") // 输入的查询关键词 + cond.CurrentStoreName = req.GetFormParameter("currentStoreName") // 滚动查询时定位用日志仓 + cond.CurrentId = cmn.StringToUint32(req.GetFormParameter("currentId"), 0) // 滚动查询时定位用ID + cond.Forward = cmn.StringToBool(req.GetFormParameter("forward"), true) // 是否向下滚动查询 + cond.DatetimeFrom = req.GetFormParameter("datetimeFrom") // 日期范围（From） + cond.DatetimeTo = req.GetFormParameter("datetimeTo") // 日期范围（To） + cond.System = req.GetFormParameter("system") // 系统 + cond.Loglevel = req.GetFormParameter("loglevel") // 单选条件 + cond.Loglevels = cmn.Split(cond.Loglevel, ",") // 多选条件 + if len(cond.Loglevels) <= 1 || len(cond.Loglevels) >= 4 { + cond.Loglevels = make([]string, 0) // 多选的单选或全选，都清空（单选走loglevel索引，全选等于没选） } - - if !cmn.IsBlank(system) { - system = "~" + cmn.Trim(system) + if !cmn.IsBlank(cond.System) { + cond.System = "~" + cmn.Trim(cond.System) // 编辑系统条件，以便精确匹配 } - if !cmn.IsBlank(loglevel) && !cmn.Contains(loglevel, ",") { - loglevel = "!" + cmn.Trim(loglevel) // 单个条件时作为索引条件 + if !cmn.IsBlank(cond.Loglevel) && !cmn.Contains(cond.Loglevel, ",") { + cond.Loglevel = "!" + cmn.Trim(cond.Loglevel) // 编辑日志级别单选条件，以便精确匹配 } else { - loglevel = "" // 多选条件时不使用，改用loglevels + cond.Loglevel = "" // 清空日志级别单选条件，以便多选配配（改用loglevels） } - eng := ldb.NewEngine(storeName) - rs := eng.Search(searchKey, system, datetimeFrom, datetimeTo, loglevel, loglevels, currentId, forward) + // 范围内的日志仓都查一遍 + // 注1）日志不断新增时，总件数可能会因为时间点原因不适最新，从而变现出点点小误差【完全可接受】 + // 注2）跨仓检索时，非本次检索的目标仓的话，只查取相关件数不做真正筛选计数以提高性能，醉打匹配件数有时可能出现较大误差【折中可接受】 + result := &search.SearchResult{PageSize: cmn.IntToString(conf.GetPageSize())} + var total uint32 + var count uint32 + storeItems := getStoreItems(cond.StoreName, cond.DatetimeFrom, cond.DatetimeTo) + sysmntStore := sysmnt.NewSysmntStorage() + for i, max := 0, len(storeItems); i < max; i++ { + item := storeItems[i] + if !item.isSearchRange { + // 不需要查数据，只查关联件数 + total += sysmntStore.GetStorageDataCount(item.storeName) // 累加总件数 + continue + } - // 检索结果后处理 - rs.PageSize = cmn.IntToString(conf.GetPageSize()) - return gweb.Result(rs) + cond.SearchSize = conf.GetPageSize() - len(result.Data) // 本次需要查多少件 + if cond.CurrentStoreName != "" && item.storeName > cond.CurrentStoreName { + cond.SearchSize = 0 // 是范围内的日志仓，但不是本次要查的，设为0不查数据，只查关联件数 + } + + eng := ldb.NewEngine(item.storeName) // 遍历日志仓检索 + rs := eng.Search(cond) // 按动态的要求件数检索 + total += cmn.StringToUint32(rs.Total, 0) // 累加总件数 + count += cmn.StringToUint32(rs.Count, 0) // 累加最大匹配件数 + if len(rs.Data) > 0 { + result.Data = append(result.Data, rs.Data...) // 累加查询结果 + result.LastStoreName = item.storeName // 设定检索结果最后一条（最久远）日志所在的日志仓，页面向下滚动继续检索时定位用 + } + + if !(cond.CurrentStoreName != "" && item.storeName > cond.CurrentStoreName) { + // 仅针对更久远的日志仓 + if len(result.Data) < conf.GetPageSize() && i < max-1 { + // 数据没查够，且后面还有日志仓待查询，准备好跨仓查询条件 + cond.CurrentId = 0 // 下一日志仓从头开始查 + cond.CurrentStoreName = "" // 从头开始所以这个条件不再适用，清空 + } + } + + } + + result.Total = cmn.Uint32ToString(total) // 总件数 + result.Count = cmn.Uint32ToString(count) // 最大匹配检索（笼统，在最大查取件数（5000件）内查完时，前端会改成精确的和结果一样的件数） + result.TimeMessage = "耗时" + cmn.GetTimeInfo(time.Since(startTime).Milliseconds()) // 查询耗时 + return gweb.Result(result) +} + +// 筛选出日志仓检索范围 +func getStoreItems(storeName string, datetimeFrom string, datetimeTo string) []*storageItem { + sysmntStore := sysmnt.NewSysmntStorage() + var items []*storageItem + if !conf.IsStoreNameAutoAddDate() { + // 单日志仓 + name := com.GeyStoreNameByDate("") + items = append(items, &storageItem{storeName: name, total: sysmntStore.GetStorageDataCount(name), isSearchRange: true}) + return items + } + + // 遍历日志仓，比较日期范围筛选日志仓 + hasDateCond := (datetimeFrom != "" && datetimeTo != "") // 是否有日期范围条件 + from := cmn.ReplaceAll(cmn.Left(datetimeFrom, 10), "-", "") // yyyymmdd或“” + to := cmn.ReplaceAll(cmn.Left(datetimeTo, 10), "-", "") // yyyymmdd或“” + if time.Since(cacheTime) >= time.Second*10 { + cacheStoreNames = com.GetStorageNames(conf.GetStorageRoot(), ".sysmnt") // 所有的日志仓，结果已排序，缓存10秒避免频繁读盘 + cacheTime = time.Now() + } + for i, max := 0, len(cacheStoreNames); i < max; i++ { + name := cacheStoreNames[i] + item := &storageItem{storeName: name, total: sysmntStore.GetStorageDataCount(name)} + date := cmn.Right(name, 8) // yyyymmdd + if storeName == "" { + // 日志仓条件空白 + if hasDateCond { + if date >= from && date <= to { + item.isSearchRange = true // 日期范围内的日志仓都是条件范围 + } + } else { + item.isSearchRange = true // 无日志仓条件、且无日期条件，全部都是条件范围了 + } + } else { + // 有日志仓条件 + if hasDateCond { + if storeName == name && date >= from && date <= to { + item.isSearchRange = true // 有日期条件，得满足日期条件，该日志仓才是条件范围 + } + } else { + if storeName == name { + item.isSearchRange = true // 没日期条件，仅该日志仓是条件范围 + } + } + } + items = append(items, item) + } + return items } diff --git a/glc/www/web/src/components/form/SearchForm.vue b/glc/www/web/src/components/form/SearchForm.vue index dd71242..aae7847 100644 --- a/glc/www/web/src/components/form/SearchForm.vue +++ b/glc/www/web/src/components/form/SearchForm.vue @@ -74,6 +74,8 @@ const formRules = ref(props.rules); const form = ref(); const moreVisible = ref(false); +$emitter.on("defaultStorageCondtion", v => defaultData.value.storage = v); + const emit = defineEmits(['search']); const fnSearch = () => { @@ -105,7 +107,15 @@ const fnReset = () => { const noMoreSearchCondition = computed(() => { for (const [key, value] of Object.entries(formData.value)) { if (key == 'searchKeys') { - continue; + continue; // 忽略不提示小蓝点 + } + if (key == 'storage') { + if (!defaultData.value.storage) { + continue; // 初始化时日志仓还没拿到，忽略不提示小蓝点 + } else if (defaultData.value.storage == value) { + continue; // 和默认的日志仓条件一样，忽略不提示小蓝点 + } + if (!value) return false; // 日志仓条件清空时，显示小蓝点 } if (value) { if (Array.isArray(value)) { diff --git a/glc/www/web/src/components/layout/page-header/PageHeader.vue b/glc/www/web/src/components/layout/page-header/PageHeader.vue index 966cb48..b430c36 100644 --- a/glc/www/web/src/components/layout/page-header/PageHeader.vue +++ b/glc/www/web/src/components/layout/page-header/PageHeader.vue @@ -83,12 +83,15 @@ function checkVersion() { $post('/v1/version/info', {}, null, { 'Content-Type': 'application/x-www-form-urlencoded' }).then(rs => { if (rs.success) { verInfo.value = rs.result.version + if (rs.result.latest && normalizeVer(rs.result.version) < normalizeVer(rs.result.latest)) { + verInfo.value = `当前版本 ${rs.result.version} ，有新版本 ${rs.result.latest} 可更新` + } // 有新版本时，左上角图标鼠标悬停显示提示（注：最新版本号的查询服务并不保证随时可用） fetch(`https://glc.gotoeasy.top/glogcenter/current/version.json?v=${verInfo.value}`) .then(response => response.json()) .then(data => { // 最新版本（服务不保证可用，可能查不到，仅查到有新版本时更新tip） - if (data.version && verInfo.value < data.version) { - verInfo.value = `当前版本 ${verInfo.value} ，有新版本 ${data.version} 可更新` + if (data.version && normalizeVer(rs.result.version) < normalizeVer(data.version)) { + verInfo.value = `当前版本 ${rs.result.version} ，有新版本 ${data.version} 可更新` } }) .catch(e => console.log(e)); @@ -97,6 +100,12 @@ function checkVersion() { } } +// 0.1.2 => v100.1001.1002 +function normalizeVer(ver) { + const ary = ver.replace("v", "").split(".") + return `v${100 + (ary[0] - 0)}.${1000 + (ary[1] - 0)}.${1000 + (ary[2] - 0)}` +} +