software/glogcenter
1
0
Fork 0
mirror of https://github.com/gotoeasy/glogcenter.git synced 2025-09-15 12:58:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

pageSize配置化

Browse Source
This commit is contained in:
gotoeasy 2023-08-17 12:49:35 +08:00
parent 3165a2a535
commit 3a8d571292
6 changed files with 33 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -87,6 +87,7 @@ docker run -d -p 8080:8080 -e GLC_CLUSTER_MODE=true -e GLC_SERVER_URL=http://172
- [x] `GLC_ENABLE_SECURITY_KEY`日志添加的接口是否开启API秘钥校验，默认`false`
- [x] `GLC_SECURITY_KEY`API秘钥，默认`glogcenter`，秘钥的`header`键名为`X-GLC-AUTH`
- [ ] `GLC_ENABLE_CORS`是否允许跨域，默认`false`
- [ ] `GLC_PAGE_SIZE`每次检索件数，默认`100`（有效范围`1~1000`）
- [x] `GLC_ENABLE_AMQP_CONSUME`是否开启`rabbitMq`消费者接收日志，默认`false`
- [x] `GLC_AMQP_ADDR`消息队列`rabbitMq`连接地址，例：`amqp://user:password@ip:port/`，默认空白
- [x] `GLC_AMQP_JSON_FORMAT`消息队列`rabbitMq`消息文本是否为`json`格式，默认`true`

15
glc/conf/config.go
View File

@ -45,6 +45,7 @@ var minioBucket string
var enableUploadMinio bool
var goMaxProcess int
var enableCors bool
var pageSize int
func init() {
cmn.SetLogLevel(cmn.GetEnvStr("GLC_LOG_LEVEL", "INFO")) // 默认INFO级别日志
@ -91,6 +92,20 @@ func UpdateConfigByEnv() {
enableUploadMinio = cmn.GetEnvBool("GLC_ENABLE_UPLOAD_MINIO", false) // 是否开启上传备份至MINIO服务器，默认false
goMaxProcess = getGoMaxProcessConf(cmn.GetEnvInt("GLC_GOMAXPROCS", -1)) // 使用的最大CPU数量，默认是最大CPU数量（设定值不在实际数量范围是按最大看待）
enableCors = cmn.GetEnvBool("GLC_ENABLE_CORS", false) // 是否允许跨域，默认false
pageSize = getPageSizeConf(cmn.GetEnvInt("GLC_PAGE_SIZE", 100)) // 每次检索件数，默认100（有效范围1~1000）
}
func GetPageSize() int {
return pageSize
}
func getPageSizeConf(n int) int {
if n < 1 {
n = 1
}
if n > 1000 {
n = 1000
}
return n
}
// 取配置： 是否允许跨域，可通过环境变量“GLC_ENABLE_CROSS”设定，默认false

14
glc/ldb/engine.go
View File

@ -49,15 +49,7 @@ func (e *Engine) AddLogDataModel(data *logdata.LogDataModel) {
}
func (e *Engine) Search(searchKey string, system string, minDatetime string, maxDatetime string, loglevel string,
pageSize int, currentDocId uint32, forward bool) *search.SearchResult {
// 检查修正pageSize
if pageSize < 1 {
pageSize = 1
}
if pageSize > 1000 {
pageSize = 1000
}
currentDocId uint32, forward bool) *search.SearchResult {
// 分词后检索
var adds []string
@ -84,11 +76,11 @@ func (e *Engine) Search(searchKey string, system string, minDatetime string, max
if len(kws) == 0 {
// 无条件浏览模式
return search.SearchLogData(e.storeName, pageSize, currentDocId, forward, minDatetime, maxDatetime)
return search.SearchLogData(e.storeName, currentDocId, forward, minDatetime, maxDatetime)
}
// 多关键词查询模式
return search.SearchWordIndex(e.storeName, kws, pageSize, currentDocId, forward, minDatetime, maxDatetime)
return search.SearchWordIndex(e.storeName, kws, currentDocId, forward, minDatetime, maxDatetime)
}
// 添加日志

2
glc/ldb/engine_test.go
View File

@ -46,7 +46,7 @@ func Test_all(t *testing.T) {
// }
// time.Sleep(time.Duration(5) * time.Second)
rs := engine.Search(` them java `, "", "", "", "", 5, 0, true)
rs := engine.Search(` them java `, "", "", "", "", 0, true)
cmn.Println("共查到", rs.Total, "件")
for _, v := range rs.Data {
cmn.Println(v.Id, v.Text)

23
glc/ldb/search/search_keys.go
View File

@ -7,6 +7,7 @@
package search
import (
"glc/conf"
"glc/ldb/storage"
"glc/ldb/storage/indexdoc"
"glc/ldb/storage/indexword"
@ -29,7 +30,7 @@ type WidxStorage struct {
}
// 多关键词时计算关键词索引交集
func SearchWordIndex(storeName string, kws []string, pageSize int, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult {
func SearchWordIndex(storeName string, kws []string, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult {
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
// 时间条件范围判断，默认全部，有检索条件时调整范围
@ -73,11 +74,11 @@ func SearchWordIndex(storeName string, kws []string, pageSize int, currentDocId
}
widxs = append(widxs, widxStorage)
}
return findSame(pageSize, currentDocId, forward, minDocumentId, maxDocumentId, storeLogData, widxs...)
return findSame(currentDocId, forward, minDocumentId, maxDocumentId, storeLogData, widxs...)
}
// 无关键词时走全量检索
func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult {
func SearchLogData(storeName string, currentDocId uint32, forward bool, minDatetime string, maxDatetime string) *SearchResult {
var rs = new(SearchResult) // 检索结果
storeLogData := storage.NewLogDataStorageHandle(storeName) // 数据
@ -133,8 +134,8 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward
max = maxDocumentId // 最大不超出时间范围限制内的最大文档ID
}
if max > uint32(pageSize) {
min = max - uint32(pageSize) + 1
if max > uint32(conf.GetPageSize()) {
min = max - uint32(conf.GetPageSize()) + 1
} else {
min = 1
}
@ -160,8 +161,8 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward
max = maxDocumentId // 最大不超出时间范围限制内的最大文档ID
}
if max > uint32(pageSize) {
min = max - uint32(pageSize) + 1
if max > uint32(conf.GetPageSize()) {
min = max - uint32(conf.GetPageSize()) + 1
} else {
min = 1
}
@ -184,7 +185,7 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward
min = minDocumentId // 最小不超出时间范围限制内的最小文档ID
}
max = min + uint32(pageSize) - 1
max = min + uint32(conf.GetPageSize()) - 1
if max > totalCount {
max = totalCount
}
@ -203,7 +204,7 @@ func SearchLogData(storeName string, pageSize int, currentDocId uint32, forward
}
// 参数widxs长度要求大于1，currentDocId不传就是查第一页
func findSame(pageSize int, currentDocId uint32, forward bool, minDocumentId uint32, maxDocumentId uint32, storeLogData *storage.LogDataStorageHandle, widxs ...*WidxStorage) *SearchResult {
func findSame(currentDocId uint32, forward bool, minDocumentId uint32, maxDocumentId uint32, storeLogData *storage.LogDataStorageHandle, widxs ...*WidxStorage) *SearchResult {
var rs = new(SearchResult)
rs.Total = cmn.Uint32ToString(storeLogData.TotalCount()) // 日志总量件数
@ -275,7 +276,7 @@ func findSame(pageSize int, currentDocId uint32, forward bool, minDocumentId uin
if flg {
rsCnt++
rs.Data = append(rs.Data, storeLogData.GetLogDataModel(docId))
if rsCnt >= pageSize {
if rsCnt >= conf.GetPageSize() {
break // 最多找一页
}
}
@ -312,7 +313,7 @@ func findSame(pageSize int, currentDocId uint32, forward bool, minDocumentId uin
if flg {
rsCnt++
ary = append(ary, storeLogData.GetLogDataModel(docId))
if rsCnt >= pageSize {
if rsCnt >= conf.GetPageSize() {
break // 最多找一页
}
}

4
glc/www/controller/log_search_controller.go
View File

@ -16,9 +16,7 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult {
}
storeName := req.GetFormParameter("storeName")
//searchKey := tokenizer.GetSearchKey(req.GetFormParameter("searchKey"))
searchKey := req.GetFormParameter("searchKey")
pageSize := cmn.StringToInt(req.GetFormParameter("pageSize"), 20)
currentId := cmn.StringToUint32(req.GetFormParameter("currentId"), 0)
forward := cmn.StringToBool(req.GetFormParameter("forward"), true)
datetimeFrom := req.GetFormParameter("datetimeFrom")
@ -34,6 +32,6 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult {
}
eng := ldb.NewEngine(storeName)
rs := eng.Search(searchKey, system, datetimeFrom, datetimeTo, loglevel, pageSize, currentId, forward)
rs := eng.Search(searchKey, system, datetimeFrom, datetimeTo, loglevel, currentId, forward)
return gweb.Result(rs)
}