mirror of
https://github.com/gotoeasy/glogcenter.git
synced 2025-09-15 12:58:34 +08:00
新增跨仓查询支持
This commit is contained in:
gotoeasy
parent
581bd3f8c9
commit
96eaee5a87
@ -6,7 +6,7 @@ require (
|
||||
github.com/gin-contrib/cors v1.4.0
|
||||
github.com/gin-contrib/gzip v0.0.6
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/gotoeasy/glang v0.10.18
|
||||
github.com/gotoeasy/glang v0.10.19
|
||||
github.com/shirou/gopsutil v3.21.11+incompatible
|
||||
github.com/streadway/amqp v1.1.0
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
|
||||
|
||||
@ -270,6 +270,8 @@ github.com/gotoeasy/glang v0.10.17 h1:PG6NKA7Uy64UeCqTanaJyT81daHcnRgJRBvb2kiUUY
|
||||
github.com/gotoeasy/glang v0.10.17/go.mod h1:RGoWvWlVIEqRX1tOgWhyLbrvjuWMFINmnUMqZX5/vmo=
|
||||
github.com/gotoeasy/glang v0.10.18 h1:QPJM6pawnRpe0N3wNLhdWSdiErFC4dYDYSmuqbhQCM0=
|
||||
github.com/gotoeasy/glang v0.10.18/go.mod h1:RGoWvWlVIEqRX1tOgWhyLbrvjuWMFINmnUMqZX5/vmo=
|
||||
github.com/gotoeasy/glang v0.10.19 h1:6LZuFyrP4o4m5wqKnketfg/A1zjcrXOoKKVQXUQ8QN4=
|
||||
github.com/gotoeasy/glang v0.10.19/go.mod h1:RGoWvWlVIEqRX1tOgWhyLbrvjuWMFINmnUMqZX5/vmo=
|
||||
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
|
||||
@ -48,13 +48,12 @@ func (e *Engine) AddLogDataModel(data *logdata.LogDataModel) {
|
||||
e.logStorage.AddLogDataModel(data)
|
||||
}
|
||||
|
||||
func (e *Engine) Search(searchKey string, system string, minDatetime string, maxDatetime string, loglevel string, loglevels []string,
|
||||
currentDocId uint32, forward bool) *search.SearchResult {
|
||||
func (e *Engine) Search(cond *search.SearchCondition) *search.SearchResult {
|
||||
|
||||
// 分词后检索
|
||||
var adds []string
|
||||
adds = append(adds, system, loglevel)
|
||||
kws := tokenizer.CutForSearchEx(searchKey, adds, nil) // 检索用关键词处理
|
||||
adds = append(adds, cond.System, cond.Loglevel)
|
||||
kws := tokenizer.CutForSearchEx(cond.SearchKey, adds, nil) // 检索用关键词处理
|
||||
|
||||
// 简单检查，存在无索引数据的关键词时，直接返回
|
||||
for _, word := range kws {
|
||||
@ -67,14 +66,14 @@ func (e *Engine) Search(searchKey string, system string, minDatetime string, max
|
||||
return rs
|
||||
}
|
||||
}
|
||||
|
||||
if len(kws) == 0 {
|
||||
cond.Kws = kws
|
||||
if len(cond.Kws) == 0 {
|
||||
// 无条件浏览模式（可能含多选条件）
|
||||
return search.SearchLogData(e.storeName, loglevels, currentDocId, forward, minDatetime, maxDatetime)
|
||||
return search.SearchLogData(e.storeName, cond)
|
||||
}
|
||||
|
||||
// 多关键词查询模式
|
||||
return search.SearchWordIndex(e.storeName, kws, loglevels, currentDocId, forward, minDatetime, maxDatetime)
|
||||
return search.SearchWordIndex(e.storeName, cond)
|
||||
}
|
||||
|
||||
// 添加日志
|
||||
|
||||
@ -36,20 +36,4 @@ func Test_all(t *testing.T) {
|
||||
}
|
||||
time.Sleep(time.Duration(10) * time.Second)
|
||||
|
||||
// for i := 1; i <= 10000; i++ {
|
||||
// engine.AddTextLog(` java.sql.SQLException: them aalav`)
|
||||
// }
|
||||
// time.Sleep(time.Duration(5) * time.Second)
|
||||
|
||||
// for i := 1; i <= 10000; i++ {
|
||||
// engine.AddTextLog(` java.sql.SQLException: them`)
|
||||
// }
|
||||
// time.Sleep(time.Duration(5) * time.Second)
|
||||
|
||||
rs := engine.Search(` them java `, "", "", "", "", make([]string, 0), 0, true)
|
||||
cmn.Debug("共查到", rs.Total, "件")
|
||||
for _, v := range rs.Data {
|
||||
cmn.Debug(v.Id, v.Text)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -7,7 +7,6 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"glc/conf"
|
||||
"glc/ldb/storage"
|
||||
"glc/ldb/storage/indexdoc"
|
||||
"glc/ldb/storage/indexword"
|
||||
@ -17,11 +16,28 @@ import (
|
||||
"github.com/gotoeasy/glang/cmn"
|
||||
)
|
||||
|
||||
type SearchCondition struct {
|
||||
SearchKey string // 输入的检索文本
|
||||
StoreName string // 输入的日志仓条件
|
||||
System string // 输入的系统名条件
|
||||
DatetimeFrom string // 输入的日期范围（from）条件
|
||||
DatetimeTo string // 输入的日期范围（to）条件
|
||||
Loglevel string // 输入的日志级别（单选条件）条件【内部会过滤修改】
|
||||
Loglevels []string // 输入的日志级别（多选条件）条件【内部会过滤修改】
|
||||
CurrentStoreName string // 隐藏条件，当前日志文档ID所属的日志仓
|
||||
CurrentId uint32 // 隐藏条件，当前日志文档ID
|
||||
Forward bool // 隐藏条件，是否向前检索（玩下滚动查询）
|
||||
Kws []string // 【内部用】解析条件所得的检索关键词，非直接输入的检索文本
|
||||
SearchSize int // 【内部用】需要查询多少件（跨仓检索时可能多次检索，中间会内部调整）
|
||||
}
|
||||
|
||||
type SearchResult struct {
|
||||
Total string `json:"total,omitempty"` // 日志总量件数（用10进制字符串形式以避免出现科学计数法）
|
||||
Count string `json:"count,omitempty"` // 当前条件最多匹配件数（用10进制字符串形式以避免出现科学计数法）
|
||||
PageSize string `json:"pagesize,omitempty"` // 每次检索件数
|
||||
Data []*logdata.LogDataModel `json:"data,omitempty"` // 检索结果数据（日志文档数组）
|
||||
Total string `json:"total,omitempty"` // 日志总量件数（用10进制字符串形式以避免出现科学计数法）
|
||||
Count string `json:"count,omitempty"` // 当前条件最多匹配件数（用10进制字符串形式以避免出现科学计数法）
|
||||
PageSize string `json:"pagesize,omitempty"` // 每次检索件数
|
||||
Data []*logdata.LogDataModel `json:"data,omitempty"` // 检索结果数据（日志文档数组）
|
||||
LastStoreName string `json:"laststorename,omitempty"` // 当前检索结果中，最后一条（最久远）日志所在日志仓
|
||||
TimeMessage string `json:"timemessage,omitempty"` // 查询耗时的文本消息表示，如：耗时30毫秒
|
||||
}
|
||||
|
||||
type WidxStorage struct {
|
||||
@ -31,14 +47,14 @@ type WidxStorage struct {
|
||||
}
|
||||
|
||||
// 多关键词时计算关键词索引交集
|
||||
func SearchWordIndex(storeName string, kws []string, loglevels []string, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult {
|
||||
func SearchWordIndex(storeName string, cond *SearchCondition) *SearchResult {
|
||||
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
|
||||
|
||||
// 时间条件范围判断，默认全部，有检索条件时调整范围
|
||||
maxDocumentId := storeLogData.TotalCount() // 时间范围条件内的最大文档ID
|
||||
minDocumentId := cmn.StringToUint32("1", 1) // 时间范围条件内的最小文档ID
|
||||
if !cmn.IsBlank(minDatetime) {
|
||||
minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, minDatetime) // 时间范围条件内的最小文档ID，找不到时返回0
|
||||
if !cmn.IsBlank(cond.DatetimeFrom) {
|
||||
minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, cond.DatetimeFrom) // 时间范围条件内的最小文档ID，找不到时返回0
|
||||
if minDocumentId == 0 {
|
||||
// 简单判断，无匹配时直接返回
|
||||
var rs = new(SearchResult)
|
||||
@ -47,8 +63,8 @@ func SearchWordIndex(storeName string, kws []string, loglevels []string, current
|
||||
return rs
|
||||
}
|
||||
}
|
||||
if !cmn.IsBlank(maxDatetime) {
|
||||
maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, maxDatetime) // 时间范围条件内的最大文档ID，找不到时返回0
|
||||
if !cmn.IsBlank(cond.DatetimeTo) {
|
||||
maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, cond.DatetimeTo) // 时间范围条件内的最大文档ID，找不到时返回0
|
||||
if maxDocumentId == 0 {
|
||||
// 简单判断，无匹配时直接返回
|
||||
var rs = new(SearchResult)
|
||||
@ -67,7 +83,7 @@ func SearchWordIndex(storeName string, kws []string, loglevels []string, current
|
||||
|
||||
// 汇总索引进行关联查找
|
||||
var widxs []*WidxStorage
|
||||
for _, word := range kws {
|
||||
for _, word := range cond.Kws {
|
||||
widxStorage := &WidxStorage{
|
||||
word: word,
|
||||
idxdocStorage: indexdoc.NewDocIndexStorage(storeName),
|
||||
@ -75,12 +91,12 @@ func SearchWordIndex(storeName string, kws []string, loglevels []string, current
|
||||
}
|
||||
widxs = append(widxs, widxStorage)
|
||||
}
|
||||
return findSame(currentDocId, loglevels, forward, minDocumentId, maxDocumentId, storeLogData, widxs...)
|
||||
return findSame(cond, minDocumentId, maxDocumentId, storeLogData, widxs...)
|
||||
}
|
||||
|
||||
// 无关键词时走全量检索
|
||||
func SearchLogData(storeName string, loglevels []string, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult {
|
||||
allloglevels := cmn.Join(loglevels, ",") // 合并多选的级别条件
|
||||
func SearchLogData(storeName string, cond *SearchCondition) *SearchResult {
|
||||
allloglevels := cmn.Join(cond.Loglevels, ",") // 合并多选的级别条件
|
||||
noLogLevels := cmn.IsBlank(allloglevels) // 无多选条件
|
||||
var rs = new(SearchResult) // 检索结果
|
||||
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
|
||||
@ -89,17 +105,17 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo
|
||||
rs.Count = cmn.Uint32ToString(totalCount) // 当前条件最多匹配件数
|
||||
rsCnt := 0 // 已查到的件数
|
||||
|
||||
if totalCount == 0 {
|
||||
return rs
|
||||
if totalCount == 0 || cond.SearchSize == 0 {
|
||||
return rs // 无数据或不需要检索数据
|
||||
}
|
||||
|
||||
// 时间条件范围判断，默认全部，有检索条件时调整范围
|
||||
maxDocumentId := totalCount // 时间范围条件内的最大文档ID
|
||||
minDocumentId := cmn.StringToUint32("1", 1) // 时间范围条件内的最小文档ID
|
||||
hasMin := !cmn.IsBlank(minDatetime)
|
||||
hasMax := !cmn.IsBlank(maxDatetime)
|
||||
hasMin := !cmn.IsBlank(cond.DatetimeFrom)
|
||||
hasMax := !cmn.IsBlank(cond.DatetimeTo)
|
||||
if hasMin {
|
||||
minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, minDatetime) // 时间范围条件内的最小文档ID
|
||||
minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, cond.DatetimeFrom) // 时间范围条件内的最小文档ID
|
||||
if minDocumentId == 0 {
|
||||
// 简单判断，无匹配时直接返回
|
||||
var rs = new(SearchResult)
|
||||
@ -109,7 +125,7 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo
|
||||
}
|
||||
}
|
||||
if hasMax {
|
||||
maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, maxDatetime) // 时间范围条件内的最大文档ID
|
||||
maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, cond.DatetimeTo) // 时间范围条件内的最大文档ID
|
||||
if maxDocumentId == 0 {
|
||||
// 简单判断，无匹配时直接返回
|
||||
var rs = new(SearchResult)
|
||||
@ -128,7 +144,7 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo
|
||||
}
|
||||
|
||||
// 开始检索
|
||||
if currentDocId == 0 {
|
||||
if cond.CurrentId == 0 {
|
||||
// 第一页
|
||||
var min, max uint32
|
||||
max = totalCount
|
||||
@ -146,19 +162,19 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo
|
||||
if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) {
|
||||
rs.Data = append(rs.Data, md)
|
||||
rsCnt++
|
||||
if rsCnt >= conf.GetPageSize() {
|
||||
if rsCnt >= cond.SearchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if forward {
|
||||
// 后一页
|
||||
if currentDocId > 1 {
|
||||
} else if cond.Forward {
|
||||
// 后一页（向下滚动触发检索）
|
||||
if cond.CurrentId > 1 {
|
||||
var min, max uint32
|
||||
if currentDocId > totalCount {
|
||||
if cond.CurrentId > totalCount {
|
||||
max = totalCount
|
||||
} else {
|
||||
max = currentDocId - 1
|
||||
max = cond.CurrentId - 1
|
||||
}
|
||||
|
||||
if max > maxDocumentId {
|
||||
@ -174,17 +190,17 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo
|
||||
if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) {
|
||||
rs.Data = append(rs.Data, md)
|
||||
rsCnt++
|
||||
if rsCnt >= conf.GetPageSize() {
|
||||
if rsCnt >= cond.SearchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 前一页
|
||||
if totalCount > currentDocId {
|
||||
// 前一页（向上滚动触发检索）【暂未使用】
|
||||
if totalCount > cond.CurrentId {
|
||||
var min, max uint32
|
||||
min = currentDocId + 1
|
||||
min = cond.CurrentId + 1
|
||||
|
||||
if min < minDocumentId {
|
||||
min = minDocumentId // 最小不超出时间范围限制内的最小文档ID
|
||||
@ -199,7 +215,7 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo
|
||||
if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) {
|
||||
rs.Data = append(rs.Data, md)
|
||||
rsCnt++
|
||||
if rsCnt >= conf.GetPageSize() {
|
||||
if rsCnt >= cond.SearchSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
@ -211,9 +227,9 @@ func SearchLogData(storeName string, loglevels []string, currentDocId uint32, fo
|
||||
return rs
|
||||
}
|
||||
|
||||
// 参数widxs长度要求大于1，currentDocId不传就是查第一页
|
||||
func findSame(currentDocId uint32, loglevels []string, forward bool, minDocumentId uint32, maxDocumentId uint32, storeLogData *storage.LogDataStorageHandle, widxs ...*WidxStorage) *SearchResult {
|
||||
allloglevels := cmn.Join(loglevels, ",") // 合并多选的级别条件
|
||||
// 参数widxs长度要求大于1，currentId不传就是查第一页
|
||||
func findSame(cond *SearchCondition, minDocumentId uint32, maxDocumentId uint32, storeLogData *storage.LogDataStorageHandle, widxs ...*WidxStorage) *SearchResult {
|
||||
allloglevels := cmn.Join(cond.Loglevels, ",") // 合并多选的级别条件
|
||||
noLogLevels := cmn.IsBlank(allloglevels) // 无多选条件
|
||||
var rs = new(SearchResult) // 查询结果
|
||||
rs.Total = cmn.Uint32ToString(storeLogData.TotalCount()) // 日志总量件数
|
||||
@ -234,17 +250,21 @@ func findSame(currentDocId uint32, loglevels []string, forward bool, minDocument
|
||||
}
|
||||
rs.Count = cmn.Uint32ToString(minCount) // 当前条件最多匹配件数
|
||||
|
||||
if cond.SearchSize <= 0 {
|
||||
return rs // 只查关联件数，不查数据
|
||||
}
|
||||
|
||||
// 简单检查排除没结果的情景
|
||||
totalCount := minIdx.idxwordStorage.GetTotalCount(minIdx.word)
|
||||
if totalCount == 0 || (totalCount == 1 && currentDocId > 0) {
|
||||
if totalCount == 0 || (totalCount == 1 && cond.CurrentId > 0) {
|
||||
return rs // 索引件数0、或只有1条又还要跳过，都是找不到
|
||||
}
|
||||
|
||||
// 找匹配位置并排除没结果的情景
|
||||
pos := totalCount // 默认检索最新第一页
|
||||
if currentDocId > 0 {
|
||||
pos = minIdx.idxdocStorage.GetWordDocSeq(minIdx.word, currentDocId) // 有相对文档ID时找相对位置
|
||||
if pos == 0 || (pos == 1 && forward) || (pos == totalCount && !forward) {
|
||||
if cond.CurrentId > 0 {
|
||||
pos = minIdx.idxdocStorage.GetWordDocSeq(minIdx.word, cond.CurrentId) // 有相对文档ID时找相对位置
|
||||
if pos == 0 || (pos == 1 && cond.Forward) || (pos == totalCount && !cond.Forward) {
|
||||
return rs // 找不到、或最后条还要向后、或最前条还要向前，都是找不到
|
||||
}
|
||||
}
|
||||
@ -252,9 +272,9 @@ func findSame(currentDocId uint32, loglevels []string, forward bool, minDocument
|
||||
// 位置就绪
|
||||
var rsCnt int = 0
|
||||
var flg bool
|
||||
if currentDocId == 0 || currentDocId > 0 && forward {
|
||||
if cond.CurrentId == 0 || (cond.CurrentId > 0 && cond.Forward) {
|
||||
// 无相对文档ID、或有且是后一页方向
|
||||
if currentDocId > 0 {
|
||||
if cond.CurrentId > 0 {
|
||||
pos-- // 相对文档ID有的话才顺移
|
||||
}
|
||||
|
||||
@ -287,7 +307,7 @@ func findSame(currentDocId uint32, loglevels []string, forward bool, minDocument
|
||||
if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) {
|
||||
rsCnt++
|
||||
rs.Data = append(rs.Data, md)
|
||||
if rsCnt >= conf.GetPageSize() {
|
||||
if rsCnt >= cond.SearchSize {
|
||||
break // 最多找一页
|
||||
}
|
||||
}
|
||||
@ -327,7 +347,7 @@ func findSame(currentDocId uint32, loglevels []string, forward bool, minDocument
|
||||
if noLogLevels || cmn.ContainsIngoreCase(allloglevels, md.LogLevel) {
|
||||
rsCnt++
|
||||
ary = append(ary, md)
|
||||
if rsCnt >= conf.GetPageSize() {
|
||||
if rsCnt >= cond.SearchSize {
|
||||
break // 最多找一页
|
||||
}
|
||||
}
|
||||
|
||||
@ -45,6 +45,7 @@ func GetStorageList() *StorageResult {
|
||||
|
||||
var datas []*StorageModel
|
||||
var sum int64
|
||||
var total int64
|
||||
names := com.GetStorageNames(conf.GetStorageRoot(), ".sysmnt")
|
||||
for _, name := range names {
|
||||
d := &StorageModel{
|
||||
@ -65,6 +66,7 @@ func GetStorageList() *StorageResult {
|
||||
sysmntStore := NewSysmntStorage()
|
||||
d.LogCount = sysmntStore.GetStorageDataCount(name)
|
||||
d.IndexCount = sysmntStore.GetStorageIndexCount(name)
|
||||
total += int64(d.LogCount)
|
||||
}
|
||||
|
||||
datas = append(datas, d)
|
||||
@ -73,7 +75,7 @@ func GetStorageList() *StorageResult {
|
||||
stat, _ := disk.Usage(conf.GetStorageRoot())
|
||||
|
||||
rs := &StorageResult{
|
||||
Info: fmt.Sprintf("共占用空间 " + cmn.GetSizeInfo(uint64(sum)) + "，剩余空间 " + cmn.GetSizeInfo(stat.Free)),
|
||||
Info: fmt.Sprintf("日志总量 " + cmn.Int64ToString(total) + " 条，共占用空间 " + cmn.GetSizeInfo(uint64(sum)) + "，剩余空间 " + cmn.GetSizeInfo(stat.Free)),
|
||||
Data: datas,
|
||||
}
|
||||
return rs
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
package ver
|
||||
|
||||
// 版本号，升级版本时修改
|
||||
const VERSION = "v0.12.2"
|
||||
const VERSION = "v0.12.3"
|
||||
|
||||
@ -1,13 +1,26 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"glc/com"
|
||||
"glc/conf"
|
||||
"glc/gweb"
|
||||
"glc/ldb"
|
||||
"glc/ldb/search"
|
||||
"glc/ldb/sysmnt"
|
||||
"time"
|
||||
|
||||
"github.com/gotoeasy/glang/cmn"
|
||||
)
|
||||
|
||||
type storageItem struct {
|
||||
storeName string // 日志仓
|
||||
total uint32 // 日志件数
|
||||
isSearchRange bool // 是否条件范围的日志仓
|
||||
}
|
||||
|
||||
var cacheStoreNames []string // 所有的日志仓（避免每次读磁盘，适当使用缓存）
|
||||
var cacheTime time.Time // 最近一次读日志仓目录的时间点
|
||||
|
||||
// 日志检索（表单提交方式）
|
||||
func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult {
|
||||
|
||||
@ -18,32 +31,122 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult {
|
||||
return gweb.Error403() // 登录检查
|
||||
}
|
||||
|
||||
storeName := req.GetFormParameter("storeName")
|
||||
searchKey := req.GetFormParameter("searchKey")
|
||||
currentId := cmn.StringToUint32(req.GetFormParameter("currentId"), 0)
|
||||
forward := cmn.StringToBool(req.GetFormParameter("forward"), true)
|
||||
datetimeFrom := req.GetFormParameter("datetimeFrom")
|
||||
datetimeTo := req.GetFormParameter("datetimeTo")
|
||||
system := req.GetFormParameter("system")
|
||||
loglevel := req.GetFormParameter("loglevel") // 单选条件
|
||||
loglevels := cmn.Split(loglevel, ",") // 多选条件
|
||||
if len(loglevels) <= 1 || len(loglevels) >= 4 {
|
||||
loglevels = make([]string, 0) // 多选的单选或全选，都清空（单选走loglevel索引，全选等于没选）
|
||||
startTime := time.Now()
|
||||
cond := &search.SearchCondition{SearchSize: conf.GetPageSize()}
|
||||
cond.StoreName = req.GetFormParameter("storeName") // 日志仓条件
|
||||
cond.SearchKey = req.GetFormParameter("searchKey") // 输入的查询关键词
|
||||
cond.CurrentStoreName = req.GetFormParameter("currentStoreName") // 滚动查询时定位用日志仓
|
||||
cond.CurrentId = cmn.StringToUint32(req.GetFormParameter("currentId"), 0) // 滚动查询时定位用ID
|
||||
cond.Forward = cmn.StringToBool(req.GetFormParameter("forward"), true) // 是否向下滚动查询
|
||||
cond.DatetimeFrom = req.GetFormParameter("datetimeFrom") // 日期范围（From）
|
||||
cond.DatetimeTo = req.GetFormParameter("datetimeTo") // 日期范围（To）
|
||||
cond.System = req.GetFormParameter("system") // 系统
|
||||
cond.Loglevel = req.GetFormParameter("loglevel") // 单选条件
|
||||
cond.Loglevels = cmn.Split(cond.Loglevel, ",") // 多选条件
|
||||
if len(cond.Loglevels) <= 1 || len(cond.Loglevels) >= 4 {
|
||||
cond.Loglevels = make([]string, 0) // 多选的单选或全选，都清空（单选走loglevel索引，全选等于没选）
|
||||
}
|
||||
|
||||
if !cmn.IsBlank(system) {
|
||||
system = "~" + cmn.Trim(system)
|
||||
if !cmn.IsBlank(cond.System) {
|
||||
cond.System = "~" + cmn.Trim(cond.System) // 编辑系统条件，以便精确匹配
|
||||
}
|
||||
if !cmn.IsBlank(loglevel) && !cmn.Contains(loglevel, ",") {
|
||||
loglevel = "!" + cmn.Trim(loglevel) // 单个条件时作为索引条件
|
||||
if !cmn.IsBlank(cond.Loglevel) && !cmn.Contains(cond.Loglevel, ",") {
|
||||
cond.Loglevel = "!" + cmn.Trim(cond.Loglevel) // 编辑日志级别单选条件，以便精确匹配
|
||||
} else {
|
||||
loglevel = "" // 多选条件时不使用，改用loglevels
|
||||
cond.Loglevel = "" // 清空日志级别单选条件，以便多选配配（改用loglevels）
|
||||
}
|
||||
|
||||
eng := ldb.NewEngine(storeName)
|
||||
rs := eng.Search(searchKey, system, datetimeFrom, datetimeTo, loglevel, loglevels, currentId, forward)
|
||||
// 范围内的日志仓都查一遍
|
||||
// 注1）日志不断新增时，总件数可能会因为时间点原因不适最新，从而变现出点点小误差【完全可接受】
|
||||
// 注2）跨仓检索时，非本次检索的目标仓的话，只查取相关件数不做真正筛选计数以提高性能，醉打匹配件数有时可能出现较大误差【折中可接受】
|
||||
result := &search.SearchResult{PageSize: cmn.IntToString(conf.GetPageSize())}
|
||||
var total uint32
|
||||
var count uint32
|
||||
storeItems := getStoreItems(cond.StoreName, cond.DatetimeFrom, cond.DatetimeTo)
|
||||
sysmntStore := sysmnt.NewSysmntStorage()
|
||||
for i, max := 0, len(storeItems); i < max; i++ {
|
||||
item := storeItems[i]
|
||||
if !item.isSearchRange {
|
||||
// 不需要查数据，只查关联件数
|
||||
total += sysmntStore.GetStorageDataCount(item.storeName) // 累加总件数
|
||||
continue
|
||||
}
|
||||
|
||||
// 检索结果后处理
|
||||
rs.PageSize = cmn.IntToString(conf.GetPageSize())
|
||||
return gweb.Result(rs)
|
||||
cond.SearchSize = conf.GetPageSize() - len(result.Data) // 本次需要查多少件
|
||||
if cond.CurrentStoreName != "" && item.storeName > cond.CurrentStoreName {
|
||||
cond.SearchSize = 0 // 是范围内的日志仓，但不是本次要查的，设为0不查数据，只查关联件数
|
||||
}
|
||||
|
||||
eng := ldb.NewEngine(item.storeName) // 遍历日志仓检索
|
||||
rs := eng.Search(cond) // 按动态的要求件数检索
|
||||
total += cmn.StringToUint32(rs.Total, 0) // 累加总件数
|
||||
count += cmn.StringToUint32(rs.Count, 0) // 累加最大匹配件数
|
||||
if len(rs.Data) > 0 {
|
||||
result.Data = append(result.Data, rs.Data...) // 累加查询结果
|
||||
result.LastStoreName = item.storeName // 设定检索结果最后一条（最久远）日志所在的日志仓，页面向下滚动继续检索时定位用
|
||||
}
|
||||
|
||||
if !(cond.CurrentStoreName != "" && item.storeName > cond.CurrentStoreName) {
|
||||
// 仅针对更久远的日志仓
|
||||
if len(result.Data) < conf.GetPageSize() && i < max-1 {
|
||||
// 数据没查够，且后面还有日志仓待查询，准备好跨仓查询条件
|
||||
cond.CurrentId = 0 // 下一日志仓从头开始查
|
||||
cond.CurrentStoreName = "" // 从头开始所以这个条件不再适用，清空
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
result.Total = cmn.Uint32ToString(total) // 总件数
|
||||
result.Count = cmn.Uint32ToString(count) // 最大匹配检索（笼统，在最大查取件数（5000件）内查完时，前端会改成精确的和结果一样的件数）
|
||||
result.TimeMessage = "耗时" + cmn.GetTimeInfo(time.Since(startTime).Milliseconds()) // 查询耗时
|
||||
return gweb.Result(result)
|
||||
}
|
||||
|
||||
// 筛选出日志仓检索范围
|
||||
func getStoreItems(storeName string, datetimeFrom string, datetimeTo string) []*storageItem {
|
||||
sysmntStore := sysmnt.NewSysmntStorage()
|
||||
var items []*storageItem
|
||||
if !conf.IsStoreNameAutoAddDate() {
|
||||
// 单日志仓
|
||||
name := com.GeyStoreNameByDate("")
|
||||
items = append(items, &storageItem{storeName: name, total: sysmntStore.GetStorageDataCount(name), isSearchRange: true})
|
||||
return items
|
||||
}
|
||||
|
||||
// 遍历日志仓，比较日期范围筛选日志仓
|
||||
hasDateCond := (datetimeFrom != "" && datetimeTo != "") // 是否有日期范围条件
|
||||
from := cmn.ReplaceAll(cmn.Left(datetimeFrom, 10), "-", "") // yyyymmdd或“”
|
||||
to := cmn.ReplaceAll(cmn.Left(datetimeTo, 10), "-", "") // yyyymmdd或“”
|
||||
if time.Since(cacheTime) >= time.Second*10 {
|
||||
cacheStoreNames = com.GetStorageNames(conf.GetStorageRoot(), ".sysmnt") // 所有的日志仓，结果已排序，缓存10秒避免频繁读盘
|
||||
cacheTime = time.Now()
|
||||
}
|
||||
for i, max := 0, len(cacheStoreNames); i < max; i++ {
|
||||
name := cacheStoreNames[i]
|
||||
item := &storageItem{storeName: name, total: sysmntStore.GetStorageDataCount(name)}
|
||||
date := cmn.Right(name, 8) // yyyymmdd
|
||||
if storeName == "" {
|
||||
// 日志仓条件空白
|
||||
if hasDateCond {
|
||||
if date >= from && date <= to {
|
||||
item.isSearchRange = true // 日期范围内的日志仓都是条件范围
|
||||
}
|
||||
} else {
|
||||
item.isSearchRange = true // 无日志仓条件、且无日期条件，全部都是条件范围了
|
||||
}
|
||||
} else {
|
||||
// 有日志仓条件
|
||||
if hasDateCond {
|
||||
if storeName == name && date >= from && date <= to {
|
||||
item.isSearchRange = true // 有日期条件，得满足日期条件，该日志仓才是条件范围
|
||||
}
|
||||
} else {
|
||||
if storeName == name {
|
||||
item.isSearchRange = true // 没日期条件，仅该日志仓是条件范围
|
||||
}
|
||||
}
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
@ -74,6 +74,8 @@ const formRules = ref(props.rules);
|
||||
const form = ref();
|
||||
const moreVisible = ref(false);
|
||||
|
||||
$emitter.on("defaultStorageCondtion", v => defaultData.value.storage = v);
|
||||
|
||||
const emit = defineEmits(['search']);
|
||||
|
||||
const fnSearch = () => {
|
||||
@ -105,7 +107,15 @@ const fnReset = () => {
|
||||
const noMoreSearchCondition = computed(() => {
|
||||
for (const [key, value] of Object.entries(formData.value)) {
|
||||
if (key == 'searchKeys') {
|
||||
continue;
|
||||
continue; // 忽略不提示小蓝点
|
||||
}
|
||||
if (key == 'storage') {
|
||||
if (!defaultData.value.storage) {
|
||||
continue; // 初始化时日志仓还没拿到，忽略不提示小蓝点
|
||||
} else if (defaultData.value.storage == value) {
|
||||
continue; // 和默认的日志仓条件一样，忽略不提示小蓝点
|
||||
}
|
||||
if (!value) return false; // 日志仓条件清空时，显示小蓝点
|
||||
}
|
||||
if (value) {
|
||||
if (Array.isArray(value)) {
|
||||
|
||||
@ -83,12 +83,15 @@ function checkVersion() {
|
||||
$post('/v1/version/info', {}, null, { 'Content-Type': 'application/x-www-form-urlencoded' }).then(rs => {
|
||||
if (rs.success) {
|
||||
verInfo.value = rs.result.version
|
||||
if (rs.result.latest && normalizeVer(rs.result.version) < normalizeVer(rs.result.latest)) {
|
||||
verInfo.value = `当前版本 ${rs.result.version} ，有新版本 ${rs.result.latest} 可更新`
|
||||
}
|
||||
// 有新版本时，左上角图标鼠标悬停显示提示（注：最新版本号的查询服务并不保证随时可用）
|
||||
fetch(`https://glc.gotoeasy.top/glogcenter/current/version.json?v=${verInfo.value}`)
|
||||
.then(response => response.json())
|
||||
.then(data => { // 最新版本（服务不保证可用，可能查不到，仅查到有新版本时更新tip）
|
||||
if (data.version && verInfo.value < data.version) {
|
||||
verInfo.value = `当前版本 ${verInfo.value} ，有新版本 ${data.version} 可更新`
|
||||
if (data.version && normalizeVer(rs.result.version) < normalizeVer(data.version)) {
|
||||
verInfo.value = `当前版本 ${rs.result.version} ，有新版本 ${data.version} 可更新`
|
||||
}
|
||||
})
|
||||
.catch(e => console.log(e));
|
||||
@ -97,6 +100,12 @@ function checkVersion() {
|
||||
}
|
||||
}
|
||||
|
||||
// 0.1.2 => v100.1001.1002
|
||||
function normalizeVer(ver) {
|
||||
const ary = ver.replace("v", "").split(".")
|
||||
return `v${100 + (ary[0] - 0)}.${1000 + (ary[1] - 0)}.${1000 + (ary[2] - 0)}`
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
<style lang="scss">
|
||||
|
||||
@ -6,7 +6,8 @@
|
||||
<SearchForm :data="formData" class="c-search-form" @search="search">
|
||||
<el-row>
|
||||
<el-form-item label="选择日志仓">
|
||||
<el-select v-model="formData.storage" filterable placeholder="请选择" style="width:420px;">
|
||||
<el-select v-model="formData.storage" clearable filterable placeholder="请选择" style="width:420px;"
|
||||
@clear="reGetStorageOptions">
|
||||
<el-option v-for="item in storageOptions" :key="item.value" :label="item.label" :value="item.value" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
@ -77,6 +78,7 @@
|
||||
<script setup>
|
||||
import { useEmitter, usePageMainHooks, useTabsState } from "~/pkgs";
|
||||
import { userLogout } from "~/api";
|
||||
import { $msg } from "~/pkgs/index-pkgs";
|
||||
|
||||
const tabsState = useTabsState();
|
||||
const emitter = useEmitter(tabsState.activePath);
|
||||
@ -96,6 +98,9 @@ const info = ref(''); // 底部提示信息
|
||||
const storageOptions = ref([]) // 日志仓
|
||||
const systemSet = new Set();
|
||||
const systemOptions = ref([]) // 系统名
|
||||
const lastStoreName = ref('') // 检索结果中最久远的一条日志所属的日志仓名称
|
||||
const maxMatchCount = ref('0') // 最大匹配件数(字符串)
|
||||
const moreConditon = ref(null)
|
||||
const shortcuts = ref([
|
||||
{
|
||||
text: '近5分钟',
|
||||
@ -193,6 +198,14 @@ onMounted(() => {
|
||||
for (let i = 0; i < names.length; i++) {
|
||||
storageOptions.value.push({ value: names[i], label: `日志仓：${names[i]}` })
|
||||
}
|
||||
if (names[0]) {
|
||||
$emitter.emit("defaultStorageCondtion", names[0]); // 小蓝点提示判断用
|
||||
formData.value.storage = names[0]; // 选中第一个日志仓作为默认条件
|
||||
}
|
||||
|
||||
// 默认检索（查取好日志仓后再做检索）
|
||||
search();
|
||||
|
||||
} else if (rs.code == 403) {
|
||||
userLogout(); // 403 时登出
|
||||
router.push('/login');
|
||||
@ -214,10 +227,28 @@ onMounted(() => {
|
||||
}
|
||||
});
|
||||
|
||||
// 默认检索
|
||||
search();
|
||||
});
|
||||
|
||||
// 清除日志仓条件时，重新拉取最新日志仓列表
|
||||
function reGetStorageOptions() {
|
||||
const url = `/v1/store/names`;
|
||||
$post(url, {}, null, { 'Content-Type': 'application/x-www-form-urlencoded' }).then(rs => {
|
||||
console.log(rs)
|
||||
if (rs.success) {
|
||||
const names = rs.result || [];
|
||||
if (names.length) {
|
||||
storageOptions.value.splice(0, storageOptions.value.length)
|
||||
for (let i = 0; i < names.length; i++) {
|
||||
storageOptions.value.push({ value: names[i], label: `日志仓：${names[i]}` })
|
||||
}
|
||||
}
|
||||
if (names[0]) {
|
||||
$emitter.emit("defaultStorageCondtion", names[0]); // 小蓝点提示判断用
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 生成测试数据
|
||||
function genTestData() {
|
||||
$post('/v1/log/addTestData', {}, null, { 'Content-Type': 'application/x-www-form-urlencoded' }).then(rs => {
|
||||
@ -228,7 +259,6 @@ function genTestData() {
|
||||
function isAutoSearchMode() {
|
||||
return autoSearchMode.value
|
||||
}
|
||||
|
||||
function switchAutoSearchMode(changMode = true) {
|
||||
changMode && (autoSearchMode.value = !autoSearchMode.value);
|
||||
if (autoSearchMode.value) {
|
||||
@ -242,6 +272,8 @@ function switchAutoSearchMode(changMode = true) {
|
||||
function search() {
|
||||
autoSearchMode.value ? (showTableLoadding.value = false) : (showTableLoadding.value = true);
|
||||
const url = `/v1/log/search`;
|
||||
|
||||
// 检索条件
|
||||
const data = {};
|
||||
data.searchKey = formData.value.searchKeys;
|
||||
data.storeName = formData.value.storage;
|
||||
@ -250,6 +282,9 @@ function search() {
|
||||
data.datetimeFrom = (formData.value.datetime || ['', ''])[0];
|
||||
data.datetimeTo = (formData.value.datetime || ['', ''])[1];
|
||||
|
||||
// 保存好滚动检索的输入条件，保持和检索时一致，避免修改输入再滚动查询而出现矛盾结果
|
||||
moreConditon.value = data;
|
||||
|
||||
$post(url, data, null, { 'Content-Type': 'application/x-www-form-urlencoded' }).then(rs => {
|
||||
console.log(rs)
|
||||
if (rs.success) {
|
||||
@ -257,6 +292,8 @@ function search() {
|
||||
const pagesize = rs.result.pagesize - 0;
|
||||
tableData.value.splice(0, tableData.value.length); // 删除原全部元素，nextTick时再插入新查询结果
|
||||
document.querySelector('.c-glc-table .el-scrollbar__wrap').scrollTop = 0; // 滚动到顶部
|
||||
rs.result.laststorename && (lastStoreName.value = rs.result.laststorename); // 查到有结果时，更新
|
||||
maxMatchCount.value = rs.result.count; // 最大匹配件数
|
||||
|
||||
nextTick(() => {
|
||||
resultData.forEach(item => {
|
||||
@ -265,9 +302,9 @@ function search() {
|
||||
});
|
||||
|
||||
if (resultData.length < pagesize) {
|
||||
info.value = `日志总量 ${rs.result.total} 条，当前条件最多匹配 ${tableData.value.length} 条，正展示前 ${tableData.value.length} 条`
|
||||
info.value = `日志总量 ${rs.result.total} 条，当前条件最多匹配 ${tableData.value.length} 条，正展示前 ${tableData.value.length} 条，查询${rs.result.timemessage}`
|
||||
} else {
|
||||
info.value = `日志总量 ${rs.result.total} 条，当前条件最多匹配 ${rs.result.count} 条，正展示前 ${tableData.value.length} 条`
|
||||
info.value = `日志总量 ${rs.result.total} 条，当前条件最多匹配 ${maxMatchCount.value} 条，正展示前 ${tableData.value.length} 条，查询${rs.result.timemessage}`
|
||||
}
|
||||
});
|
||||
|
||||
@ -289,27 +326,23 @@ function searchMore() {
|
||||
}
|
||||
|
||||
const url = `/v1/log/search`;
|
||||
const data = {};
|
||||
data.searchKey = formData.value.searchKeys;
|
||||
data.storeName = formData.value.storage;
|
||||
data.system = formData.value.system;
|
||||
data.loglevel = (formData.value.loglevel || []).join(',');
|
||||
data.datetimeFrom = (formData.value.datetime || ['', ''])[0];
|
||||
data.datetimeTo = (formData.value.datetime || ['', ''])[1];
|
||||
data.forward = true
|
||||
data.currentId = tableData.value[tableData.value.length - 1].id; // 相对最后条id，继续找后面的日志
|
||||
moreConditon.value.forward = true
|
||||
moreConditon.value.currentId = tableData.value[tableData.value.length - 1].id; // 相对最后条id，继续找后面的日志
|
||||
moreConditon.value.currentStoreName = lastStoreName.value; // 相对最后条id的所属日志仓
|
||||
|
||||
$post(url, data, null, { 'Content-Type': 'application/x-www-form-urlencoded' }).then(rs => {
|
||||
$post(url, moreConditon.value, null, { 'Content-Type': 'application/x-www-form-urlencoded' }).then(rs => {
|
||||
console.log(rs)
|
||||
if (rs.success) {
|
||||
const resultData = rs.result.data || [];
|
||||
const pagesize = rs.result.pagesize - 0;
|
||||
tableData.value.push(...resultData)
|
||||
rs.result.laststorename && (lastStoreName.value = rs.result.laststorename); // 查到有结果时，更新
|
||||
|
||||
if (resultData.length < pagesize) {
|
||||
info.value = `日志总量 ${rs.result.total} 条，当前条件最多匹配 ${tableData.value.length} 条，正展示前 ${tableData.value.length} 条`
|
||||
info.value = `日志总量 ${rs.result.total} 条，当前条件最多匹配 ${tableData.value.length} 条，正展示前 ${tableData.value.length} 条，查询${rs.result.timemessage}`
|
||||
} else {
|
||||
info.value = `日志总量 ${rs.result.total} 条，当前条件最多匹配 ${rs.result.count} 条，正展示前 ${tableData.value.length} 条`
|
||||
(rs.result.count - 0 < maxMatchCount.value - 0) && (maxMatchCount.value = rs.result.count) // 控制maxMatchCount只会更小以减少误差
|
||||
info.value = `日志总量 ${rs.result.total} 条，当前条件最多匹配 ${maxMatchCount.value} 条，正展示前 ${tableData.value.length} 条，查询${rs.result.timemessage}`
|
||||
}
|
||||
|
||||
nextTick(() => {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user