0.8.8 界面优化、支持时间范围查询

This commit is contained in:
gotoeasy 2023-04-16 21:55:05 +08:00
parent 0e03507d8a
commit 0d12fdb62a
8 changed files with 416 additions and 47 deletions

View File

@ -173,14 +173,17 @@ func main() {
### 开发版`latest`
- [ ] 增加时间范围检索条件
- [ ] 界面优化
- [ ] 多语言
- [ ] 分词优化
- [ ] 日志审计
- [ ] 集群支持动态删减节点(或是页面管理删除)
### 版本`0.8.8`
- [x] 增加时间范围检索条件
- [x] 界面进一步简化优化
### 版本`0.8.7`
- [x] 修复增加特殊字符转换处理避免日志中的html标签字样无法显示

View File

@ -43,7 +43,7 @@ func (e *Engine) AddTextLog(date string, logText string, system string) {
e.logStorage.AddTextLog(date, logText, system)
}
func (e *Engine) Search(searchKey string, pageSize int, currentDocId uint32, forward bool) *search.SearchResult {
func (e *Engine) Search(searchKey string, minDatetime string, maxDatetime string, pageSize int, currentDocId uint32, forward bool) *search.SearchResult {
// 检查修正pageSize
if pageSize < 1 {
@ -76,11 +76,11 @@ func (e *Engine) Search(searchKey string, pageSize int, currentDocId uint32, for
if len(kws) == 0 {
// 无条件浏览模式
return search.SearchLogData(e.storeName, pageSize, currentDocId, forward)
return search.SearchLogData(e.storeName, pageSize, currentDocId, forward, minDatetime, maxDatetime)
}
// 多关键词查询模式
return search.SearchWordIndex(e.storeName, kws, pageSize, currentDocId, forward)
return search.SearchWordIndex(e.storeName, kws, pageSize, currentDocId, forward, minDatetime, maxDatetime)
}
// 添加日志

View File

@ -46,7 +46,7 @@ func Test_all(t *testing.T) {
// }
// time.Sleep(time.Duration(5) * time.Second)
rs := engine.Search(` them java `, 5, 0, true)
rs := engine.Search(` them java `, "", "", 5, 0, true)
cmn.Println("共查到", rs.Total, "件")
for _, v := range rs.Data {
cmn.Println(v.Id, v.Text)

View File

@ -11,6 +11,7 @@ import (
"glc/ldb/storage/indexdoc"
"glc/ldb/storage/indexword"
"glc/ldb/storage/logdata"
"strings"
"github.com/gotoeasy/glang/cmn"
)
@ -28,8 +29,41 @@ type WidxStorage struct {
}
// 多关键词时计算关键词索引交集
func SearchWordIndex(storeName string, kws []string, pageSize int, currentDocId uint32, forward bool) *SearchResult {
func SearchWordIndex(storeName string, kws []string, pageSize int, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult {
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
// 时间条件范围判断,默认全部,有检索条件时调整范围
maxDocumentId := storeLogData.TotalCount() // 时间范围条件内的最大文档ID
minDocumentId := cmn.StringToUint32("1", 1) // 时间范围条件内的最小文档ID
if !cmn.IsBlank(minDatetime) {
minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, minDatetime) // 时间范围条件内的最小文档ID找不到时返回0
if minDocumentId == 0 {
// 简单判断,无匹配时直接返回
var rs = new(SearchResult)
rs.Total = cmn.Uint32ToString(storeLogData.TotalCount())
rs.Count = "0"
return rs
}
}
if !cmn.IsBlank(maxDatetime) {
maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, maxDatetime) // 时间范围条件内的最大文档ID找不到时返回0
if maxDocumentId == 0 {
// 简单判断,无匹配时直接返回
var rs = new(SearchResult)
rs.Total = cmn.Uint32ToString(storeLogData.TotalCount())
rs.Count = "0"
return rs
}
}
if minDocumentId > maxDocumentId {
// 简单判断,无匹配时直接返回
var rs = new(SearchResult)
rs.Total = cmn.Uint32ToString(storeLogData.TotalCount())
rs.Count = "0"
return rs
}
// 汇总索引进行关联查找
var widxs []*WidxStorage
for _, word := range kws {
widxStorage := &WidxStorage{
@ -39,32 +73,76 @@ func SearchWordIndex(storeName string, kws []string, pageSize int, currentDocId
}
widxs = append(widxs, widxStorage)
}
return findSame(pageSize, currentDocId, forward, storeLogData, widxs...)
return findSame(pageSize, currentDocId, forward, minDocumentId, maxDocumentId, storeLogData, widxs...)
}
// 无关键词时走全量检索
func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward bool) *SearchResult {
func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult {
var rs = new(SearchResult) // 检索结果
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
totalCount := storeLogData.TotalCount() // 总件数
rs.Total = cmn.Uint32ToString(totalCount) // 返回的总件数用10进制字符串形式以避免出现科学计数法
rs.Total = cmn.Uint32ToString(totalCount) // 返回的日志件数用10进制字符串形式以避免出现科学计数法
rs.Count = cmn.Uint32ToString(totalCount) // 当前条件最多匹配件数
if totalCount == 0 {
return rs
}
// 时间条件范围判断,默认全部,有检索条件时调整范围
maxDocumentId := totalCount // 时间范围条件内的最大文档ID
minDocumentId := cmn.StringToUint32("1", 1) // 时间范围条件内的最小文档ID
hasMin := !cmn.IsBlank(minDatetime)
hasMax := !cmn.IsBlank(maxDatetime)
if hasMin {
minDocumentId = findMinDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, minDatetime) // 时间范围条件内的最小文档ID
if minDocumentId == 0 {
// 简单判断,无匹配时直接返回
var rs = new(SearchResult)
rs.Total = "0"
rs.Count = "0"
return rs
}
}
if hasMax {
maxDocumentId = findMaxDocumentIdByDatetime(storeLogData, minDocumentId, maxDocumentId, maxDatetime) // 时间范围条件内的最大文档ID
if maxDocumentId == 0 {
// 简单判断,无匹配时直接返回
var rs = new(SearchResult)
rs.Total = "0"
rs.Count = "0"
return rs
}
}
if hasMax || hasMin {
if minDocumentId > maxDocumentId {
// 简单判断,无匹配时直接返回
rs.Count = "0"
return rs
}
rs.Count = cmn.Uint32ToString(maxDocumentId - minDocumentId + 1) // 估算的最大匹配件数
}
// 开始检索
if currentDocId == 0 {
// 第一页
var min, max uint32
max = totalCount
if max > maxDocumentId {
max = maxDocumentId // 最大不超出时间范围限制内的最大文档ID
}
if max > uint32(pageSize) {
min = max - uint32(pageSize) + 1
} else {
min = 1
}
if min < minDocumentId {
min = minDocumentId // 最小不超出时间范围限制内的最小文档ID
}
for i := max; i >= min; i-- {
rs.Data = append(rs.Data, storeLogData.GetLogDataDocument(i).ToLogDataModel()) // 件数等同日志文档ID
}
@ -77,12 +155,21 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward
} else {
max = currentDocId - 1
}
if max > maxDocumentId {
max = maxDocumentId // 最大不超出时间范围限制内的最大文档ID
}
if max > uint32(pageSize) {
min = max - uint32(pageSize) + 1
} else {
min = 1
}
if min < minDocumentId {
min = minDocumentId // 最小不超出时间范围限制内的最小文档ID
}
for i := max; i >= min; i-- {
rs.Data = append(rs.Data, storeLogData.GetLogDataDocument(i).ToLogDataModel())
}
@ -92,11 +179,20 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward
if totalCount > currentDocId {
var min, max uint32
min = currentDocId + 1
if min < minDocumentId {
min = minDocumentId // 最小不超出时间范围限制内的最小文档ID
}
max = min + uint32(pageSize) - 1
if max > totalCount {
max = totalCount
}
if max > maxDocumentId {
max = maxDocumentId // 最大不超出时间范围限制内的最大文档ID
}
for i := max; i >= min; i-- {
rs.Data = append(rs.Data, storeLogData.GetLogDataDocument(i).ToLogDataModel())
}
@ -107,7 +203,7 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward
}
// 参数widxs长度要求大于1currentDocId不传就是查第一页
func findSame(pageSize int, currentDocId uint32, forward bool, storeLogData *storage.LogDataStorageHandle, widxs ...*WidxStorage) *SearchResult {
func findSame(pageSize int, currentDocId uint32, forward bool, minDocumentId uint32, maxDocumentId uint32, storeLogData *storage.LogDataStorageHandle, widxs ...*WidxStorage) *SearchResult {
var rs = new(SearchResult)
rs.Total = cmn.Uint32ToString(storeLogData.TotalCount()) // 日志总量件数
@ -123,6 +219,9 @@ func findSame(pageSize int, currentDocId uint32, forward bool, storeLogData *sto
minIdx = widxs[i]
}
}
if minCount > maxDocumentId-minDocumentId+1 {
minCount = maxDocumentId - minDocumentId + 1 // 最多匹配件数估算,不会超出时间条件范围,两者取其小
}
rs.Count = cmn.Uint32ToString(minCount) // 当前条件最多匹配件数
// 简单检查排除没结果的情景
@ -154,29 +253,31 @@ func findSame(pageSize int, currentDocId uint32, forward bool, storeLogData *sto
for i := tmpMinPos; i > 0; {
// 取值
docId := minIdx.idxwordStorage.GetDocId(minIdx.word, i)
// 比较
flg = true
for n := 0; n < cnt; n++ {
if widxs[n] == minIdx {
continue // 跳过比较自己
}
if docId >= minDocumentId && docId <= maxDocumentId {
// 在时间范围条件内时,继续查找比较
flg = true
for n := 0; n < cnt; n++ {
if widxs[n] == minIdx {
continue // 跳过比较自己
}
seq := widxs[n].idxdocStorage.GetWordDocSeq(widxs[n].word, docId)
if seq == 0 {
flg = false // 没找到
break
seq := widxs[n].idxdocStorage.GetWordDocSeq(widxs[n].word, docId)
if seq == 0 {
flg = false // 没找到
break
}
if seq < tmpMinPos {
tmpMinPos = seq
tmpMinIdx = widxs[n] // 当前最短索引,存起来下回比较用
}
}
if seq < tmpMinPos {
tmpMinPos = seq
tmpMinIdx = widxs[n] // 当前最短索引,存起来下回比较用
}
}
// 找到则加入结果
if flg {
rsCnt++
rs.Data = append(rs.Data, storeLogData.GetLogDataModel(docId))
if rsCnt >= pageSize {
break // 最多找一页
// 找到则加入结果
if flg {
rsCnt++
rs.Data = append(rs.Data, storeLogData.GetLogDataModel(docId))
if rsCnt >= pageSize {
break // 最多找一页
}
}
}
@ -191,6 +292,11 @@ func findSame(pageSize int, currentDocId uint32, forward bool, storeLogData *sto
for i := pos; i <= totalCount; i++ {
// 取值
docId := minIdx.idxwordStorage.GetDocId(minIdx.word, i)
if docId < minDocumentId || docId > maxDocumentId {
continue // 不在时间范围条件内,不匹配,跳过
}
// 比较
flg = true
for i := 0; i < cnt; i++ {
@ -218,6 +324,61 @@ func findSame(pageSize int, currentDocId uint32, forward bool, storeLogData *sto
}
}
rs.Total = cmn.Uint32ToString(storeLogData.TotalCount())
return rs
}
// 查找满足最小时间范围的最小文档id
func findMinDocumentIdByDatetime(storeLogData *storage.LogDataStorageHandle, uiMin uint32, uiMax uint32, minDatetime string) uint32 {
if strings.Compare(minDatetime+".000", storeLogData.GetLogDataDocument(uiMin).ToLogDataModel().Date) <= 0 {
return uiMin // 边界外输入条件常发生,特殊照顾确认边界,一定程度提高性能
}
rs := cmn.StringToUint32("0", 0)
min := uiMin + 1 // 参数的最小已检查,跳过
max := uiMax
for min <= max {
left, rigth, flg, target := findGE(storeLogData, min, max, minDatetime)
min = left
max = rigth
if flg {
rs = target
}
}
return rs
}
func findGE(storeLogData *storage.LogDataStorageHandle, min uint32, max uint32, minDatetime string) (uint32, uint32, bool, uint32) {
middle := (min + max) / 2
if strings.Compare(minDatetime+".000", storeLogData.GetLogDataDocument(middle).ToLogDataModel().Date) <= 0 {
return min, middle - 1, true, middle // 能匹配middle的日时>=minDatetime但不一定是最小匹配继续返回下次待查找的范围
}
return middle + 1, max, false, 0 // 不匹配middle的日时<minDatetime返回下次待查找的范围
}
// 查找满足最大时间范围的最大文档id
func findMaxDocumentIdByDatetime(storeLogData *storage.LogDataStorageHandle, uiMin uint32, uiMax uint32, maxDatetime string) uint32 {
if strings.Compare(storeLogData.GetLogDataDocument(uiMax).ToLogDataModel().Date, maxDatetime+".999") <= 0 {
return uiMax // 能匹配maxDatetime>=middle的日时但不一定是最小匹配继续返回下次待查找的范围
}
rs := cmn.StringToUint32("0", 0)
min := uiMin
max := uiMax - 1 // 参数的最大已检查,跳过
for min <= max {
left, rigth, flg, target := findLE(storeLogData, min, max, maxDatetime)
min = left
max = rigth
if flg {
rs = target
}
}
return rs
}
func findLE(storeLogData *storage.LogDataStorageHandle, min uint32, max uint32, maxDatetime string) (uint32, uint32, bool, uint32) {
middle := (min + max) / 2
if strings.Compare(storeLogData.GetLogDataDocument(middle).ToLogDataModel().Date, maxDatetime+".999") <= 0 {
return middle + 1, max, true, middle // 能匹配maxDatetime>=middle的日时但不一定是最小匹配继续返回下次待查找的范围
}
return min, middle - 1, false, 0 // 不匹配maxDatetime<middle的日时返回下次待查找的范围
}

View File

@ -1,3 +1,3 @@
package onstart
const VERSION = "glogcenter 0.8.7"
const VERSION = "glogcenter 0.8.8"

View File

@ -21,8 +21,10 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult {
pageSize := cmn.StringToInt(req.GetFormParameter("pageSize"), 20)
currentId := cmn.StringToUint32(req.GetFormParameter("currentId"), 0)
forward := cmn.StringToBool(req.GetFormParameter("forward"), true)
datetimeFrom := req.GetFormParameter("datetimeFrom")
datetimeTo := req.GetFormParameter("datetimeTo")
eng := ldb.NewEngine(storeName)
rs := eng.Search(searchKey, pageSize, currentId, forward)
rs := eng.Search(searchKey, datetimeFrom, datetimeTo, pageSize, currentId, forward)
return gweb.Result(rs)
}

View File

@ -32,7 +32,7 @@
:close-on-click-modal="false" :close-on-press-escape="false" :show-close="false">
<el-input placeholder="请输入用户名" v-model="username" maxlength="100"></el-input><p/>
<el-input placeholder="请输入密码" type="password" v-model="password" maxlength="100"></el-input>
<el-input placeholder="请输入密码" type="password" v-model="password" maxlength="100" autocomplete="new-password"></el-input>
<template #footer>
<span class="dialog-footer">

View File

@ -10,20 +10,81 @@
<div style="display:flex;justify-content:space-between;">
<div>
<el-select v-if="storageOptions.length > 1" v-model="storage" filterable placeholder="请选择日志仓" style="width:260px">
<el-option
v-for="item in storageOptions"
:key="item.value"
:label="item.label"
:value="item.value"
/>
</el-select>
<el-input @keyup.enter="search()" v-model="params.searchKey" placeholder="请输入关键词检索" style="width:600px">
<el-input @keyup.enter="search()" v-model="params.searchKey" placeholder="请输入关键词检索" style="width:600px;">
<template #append>
<el-button type="primary" @click="search()" class="x-search">全文检索</el-button>
<el-button type="primary" @click="search" class="x-search">
<el-icon>
<Search />
</el-icon>
<span> </span>
</el-button>
</template>
</el-input>
<el-button class="c-btn" @click="fnResetSearchForm" style="margin-left:10px;">
<el-icon>
<RefreshLeft />
</el-icon>
<span> </span>
</el-button>
<el-badge is-dot :hidden="hasMoreCondition" type="primary" style="margin-left:10px;">
<el-button circle @click="() => (showSearchPanel = !showSearchPanel)">
<el-icon>
<ArrowUp v-if="showSearchPanel" />
<ArrowDown v-else />
</el-icon>
</el-button>
</el-badge>
<div v-show="showSearchPanel" class="c-down-panel">
<el-form ref="form" :inline="true" label-width="100">
<el-row>
<el-form-item label="选择日志仓">
<el-select v-if="storageOptions.length > 0" v-model="storage" filterable placeholder="请选择" style="width:420px;">
<el-option
v-for="item in storageOptions"
:key="item.value"
:label="item.label"
:value="item.value"
/>
</el-select>
</el-form-item>
</el-row>
<el-row>
<el-form-item label="时间范围">
<el-date-picker
v-model="params.datetime"
type="datetimerange"
:shortcuts="shortcuts"
range-separator=""
value-format="YYYY-MM-DD HH:mm:ss"
start-placeholder="开始时间"
end-placeholder="结束时间"
/>
</el-form-item>
</el-row>
</el-form>
<el-divider style="margin: 0 0 10px;" />
<el-row justify="center">
<el-button type="primary" class="x-search" @click="search">
<el-icon size="14">
<Search />
</el-icon>
<span> </span>
</el-button>
<el-button class="c-btn" @click="() => (showSearchPanel = false)">
<el-icon size="14">
<ArrowUp />
</el-icon>
<span> </span>
</el-button>
</el-row>
</div>
</div>
</div>
</div>
@ -70,6 +131,7 @@
<script>
import api from '../api'
import { ref } from 'vue'
import { Search, RefreshLeft, ArrowUp, ArrowDown } from '@element-plus/icons-vue'
const FixHeight = 215 // 177
@ -83,6 +145,7 @@ export default {
params: {
storeName: '',
searchKey: '',
datetime: null,
pageSize: 100,
currentId: '',
forward: true,
@ -91,6 +154,90 @@ export default {
info: '',
storage: ref(''),
storageOptions: [],
showSearchPanel: ref(false),
shortcuts: [
{
text: '近5分钟',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 5 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
{
text: '近10分钟',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 10 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
{
text: '近15分钟',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 15 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
{
text: '最近20分钟',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 20 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
{
text: '最近30分钟',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 30 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
{
text: '近1小时',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 60 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
{
text: '近2小时',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 2 * 60 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
{
text: '近3小时',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 3 * 60 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
{
text: '近4小时',
value: () => {
const start = new Date()
start.setTime(start.getTime() - 4 * 60 * 60 * 1000)
const end = new Date()
return [start, end]
},
},
],
}
},
created(){
@ -128,7 +275,18 @@ export default {
this.search()
},
computed:{
hasMoreCondition(){
return !this.params.datetime && !this.storage;
}
},
methods: {
fnResetSearchForm(){
this.params.searchKey = '';
this.params.datetime = null;
this.storage = '';
this.search();
},
searchMore() {
if (this.data.length >= 5000) {
if (this.info.indexOf('请考虑') < 0){
@ -164,6 +322,8 @@ export default {
this.params.storeName = this.storage
this.params.currentId = ''
this.params.datetimeFrom = (this.params.datetime || ['', ''])[0]
this.params.datetimeTo = (this.params.datetime || ['', ''])[1]
// console.info("----------this.params",this.params)
api.search(this.params).then(rs => {
let res = rs.data
@ -183,6 +343,8 @@ export default {
}).finally(() => {
this.loading = false
})
this.showSearchPanel = false;
},
},
}
@ -245,4 +407,45 @@ button.el-button.x-search{
.el-popper.is-dark{
display: none;
}
.c-down-panel {
position: absolute;
z-index: 100;
width: 550px;
padding: 20px;
margin-top: 10px;
margin-left: 425px;
background-color: white;
box-shadow: 0 2px 12px 0 rgb(0 0 0 / 30%);
}
button.el-button:focus {
color: var(--el-button-text-color);
background-color: var(--el-button-bg-color);
border-color: var(--el-button-border-color);
}
button.el-button:hover {
color: var(--el-button-hover-text-color);
background-color: var(--el-button-hover-bg-color);
border-color: var(--el-button-hover-border-color);
}
button.el-button:active {
color: var(--el-button-active-text-color);
background-color: var(--el-button-active-bg-color);
border-color: var(--el-button-active-border-color);
}
button.el-button.is-link:focus {
color: var(--el-button-focus-link-text-color);
}
button.el-button.is-link:hover {
color: var(--el-button-hover-link-text-color);
}
button.el-button.is-link:active {
color: var(--el-button-active-link-text-color);
}
</style>