From 7df31233cf110aaaa2c02f8506fd3ba56935f287 Mon Sep 17 00:00:00 2001 From: gotoeasy Date: Wed, 15 Nov 2023 19:58:49 +0800 Subject: [PATCH] 0.14.0 --- glc-logback-appender/pom.xml | 2 +- .../framework/glc/filter/GlcFilter.java | 17 ++++++---- .../glc/logback/appender/GlcAmqpAppender.java | 4 +++ .../logback/appender/GlcHttpJsonAppender.java | 4 +++ .../glc/logback/appender/MdcUtil.java | 9 +++++ glc/ldb/engine.go | 2 +- glc/ldb/search/search_keys.go | 13 +++---- glc/ldb/storage/logdata/logdata_model.go | 34 +++++++++---------- glc/ldb/storage/logdata/logdata_storage.go | 11 +++--- glc/ver/version.go | 2 +- .../controller/log_add_testdata_controller.go | 2 ++ glc/www/controller/log_search_controller.go | 10 ++++-- glc/www/web/src/views/glc/search/GlcMain.vue | 10 ++++++ 13 files changed, 80 insertions(+), 40 deletions(-) diff --git a/glc-logback-appender/pom.xml b/glc-logback-appender/pom.xml index 26c32ab..cb2df3b 100644 --- a/glc-logback-appender/pom.xml +++ b/glc-logback-appender/pom.xml @@ -5,7 +5,7 @@ top.gotoeasy glc-logback-appender - 0.13.0 + 0.14.0 logback appender for glogcenter diff --git a/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/filter/GlcFilter.java b/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/filter/GlcFilter.java index 6d4d10d..05fe175 100644 --- a/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/filter/GlcFilter.java +++ b/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/filter/GlcFilter.java @@ -25,11 +25,8 @@ public class GlcFilter implements Filter { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { - // 设定日志中心相关的的traceid、clientip - HttpServletRequest httpServletRequest = (HttpServletRequest)request; - String traceid = httpServletRequest.getHeader(MdcUtil.TRACE_ID); - MdcUtil.setTraceId((traceid == null || traceid.length() == 0) ? MdcUtil.generateTraceId() : traceid); - MdcUtil.setClientIp(getIpAddr(httpServletRequest)); + // 设定日志中心相关的的traceid、clientip，若需要设定user，可继承setMdcKeyValues添加相关逻辑 + setMdcKeyValues(request); chain.doFilter(request, response); } @@ -38,7 +35,15 @@ public class GlcFilter implements Filter { public void destroy() { } - private static String getIpAddr(HttpServletRequest request) { + protected void setMdcKeyValues(ServletRequest request) { + // 设定日志中心相关的的traceid、clientip + HttpServletRequest httpServletRequest = (HttpServletRequest)request; + String traceid = httpServletRequest.getHeader(MdcUtil.TRACE_ID); + MdcUtil.setTraceId((traceid == null || traceid.length() == 0) ? MdcUtil.generateTraceId() : traceid); + MdcUtil.setClientIp(getIpAddr(httpServletRequest)); + } + + protected String getIpAddr(HttpServletRequest request) { String[] headerNames = { "X-Forwarded-For", "X-Real-IP", "Proxy-Client-IP", "WL-Proxy-Client-IP", "HTTP_CLIENT_IP", "HTTP_X_FORWARDED_FOR" }; for (String headerName : headerNames) { diff --git a/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/GlcAmqpAppender.java b/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/GlcAmqpAppender.java index 98db82a..b72142f 100644 --- a/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/GlcAmqpAppender.java +++ b/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/GlcAmqpAppender.java @@ -97,6 +97,7 @@ public class GlcAmqpAppender extends AppenderBase { String traceid = event.getMDCPropertyMap().get(MdcUtil.TRACE_ID); String clientip = event.getMDCPropertyMap().get(MdcUtil.CLIENT_IP); + String user = event.getMDCPropertyMap().get(MdcUtil.USER); String body = "{\"text\":" + Util.encodeStr(text.trim()); body += ",\"date\":\"" + Util.getDateString() + "\""; @@ -110,6 +111,9 @@ public class GlcAmqpAppender extends AppenderBase { if (clientip != null && !"".equals(clientip)) { body += ",\"clientip\":" + Util.encodeStr(clientip); } + if (user != null && !"".equals(user)) { + body += ",\"user\":" + Util.encodeStr(user); + } body += "}"; channel.basicPublish("", "glc-log-queue", null, body.getBytes("utf-8")); diff --git a/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/GlcHttpJsonAppender.java b/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/GlcHttpJsonAppender.java index 9fd7ed7..3d003dc 100644 --- a/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/GlcHttpJsonAppender.java +++ b/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/GlcHttpJsonAppender.java @@ -67,6 +67,7 @@ public class GlcHttpJsonAppender extends AppenderBase { try { String traceid = event.getMDCPropertyMap().get(MdcUtil.TRACE_ID); String clientip = event.getMDCPropertyMap().get(MdcUtil.CLIENT_IP); + String user = event.getMDCPropertyMap().get(MdcUtil.USER); body = "{\"text\":" + Util.encodeStr(text.trim()); body += ",\"date\":\"" + Util.getDateString() + "\""; @@ -80,6 +81,9 @@ public class GlcHttpJsonAppender extends AppenderBase { if (clientip != null && !"".equals(clientip)) { body += ",\"clientip\":" + Util.encodeStr(clientip); } + if (user != null && !"".equals(user)) { + body += ",\"user\":" + Util.encodeStr(user); + } body += "}"; URL url = new URL(glcApiUrl); diff --git a/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/MdcUtil.java b/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/MdcUtil.java index a4bc539..7c17738 100644 --- a/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/MdcUtil.java +++ b/glc-logback-appender/src/main/java/top/gotoeasy/framework/glc/logback/appender/MdcUtil.java @@ -8,6 +8,11 @@ public class MdcUtil { public static final String TRACE_ID = "traceid"; public static final String CLIENT_IP = "clientip"; + public static final String USER = "user"; + + public static void setUser(String user) { + MDC.put(USER, user); + } public static void setClientIp(String clientip) { MDC.put(CLIENT_IP, clientip); @@ -17,6 +22,10 @@ public class MdcUtil { MDC.put(TRACE_ID, traceid); } + public static void removeUser() { + MDC.remove(USER); + } + public static void removeClientIp() { MDC.remove(CLIENT_IP); } diff --git a/glc/ldb/engine.go b/glc/ldb/engine.go index 72837a5..3ac75e6 100644 --- a/glc/ldb/engine.go +++ b/glc/ldb/engine.go @@ -52,7 +52,7 @@ func (e *Engine) Search(cond *search.SearchCondition) *search.SearchResult { // 分词后检索 var adds []string - adds = append(adds, cond.OrgSystem, cond.Loglevel) + adds = append(adds, cond.OrgSystem, cond.Loglevel, cond.User) kws := tokenizer.CutForSearchEx(cond.SearchKey, adds, nil) // 检索用关键词处理 // 【快速检查1】，存在无索引数据的关键词时，直接返回 diff --git a/glc/ldb/search/search_keys.go b/glc/ldb/search/search_keys.go index 8c2ed0e..edb5936 100644 --- a/glc/ldb/search/search_keys.go +++ b/glc/ldb/search/search_keys.go @@ -24,6 +24,7 @@ type SearchCondition struct { DatetimeTo string // 输入的日期范围（to）条件 Loglevel string // 输入的日志级别（单选条件）条件【内部会过滤修改】 Loglevels []string // 输入的日志级别（多选条件）条件【内部会过滤修改】 + User string // 输入的用户条件 CurrentStoreName string // 隐藏条件，当前日志文档ID所属的日志仓 CurrentId uint32 // 隐藏条件，当前日志文档ID Forward bool // 隐藏条件，是否向前检索（玩下滚动查询） @@ -400,7 +401,7 @@ func findSame(cond *SearchCondition, minDocumentId uint32, maxDocumentId uint32, idxdocStorage := indexdoc.NewDocIndexStorage(storeLogData.GetStoreName()) // 判断系统权限用 has := false for j, max2 := 0, len(cond.Systems); j < max2; j++ { - if (idxdocStorage.GetWordDocSeq(cond.Systems[j], i)) > 0 { + if (idxdocStorage.GetWordDocSeq(cond.Systems[j], docId)) > 0 { has = true break } @@ -410,12 +411,12 @@ func findSame(cond *SearchCondition, minDocumentId uint32, maxDocumentId uint32, } } - if cond.Loglevel == "" && len(cond.Loglevels) > 0 { + if flg && cond.Loglevel == "" && len(cond.Loglevels) > 0 { // 日志级别范围内作过滤检查 idxdocStorage := indexdoc.NewDocIndexStorage(storeLogData.GetStoreName()) has := false for j, max2 := 0, len(cond.Loglevels); j < max2; j++ { - if (idxdocStorage.GetWordDocSeq("!"+cond.Loglevels[j], i)) > 0 { + if (idxdocStorage.GetWordDocSeq("!"+cond.Loglevels[j], docId)) > 0 { has = true break } @@ -471,7 +472,7 @@ func findSame(cond *SearchCondition, minDocumentId uint32, maxDocumentId uint32, idxdocStorage := indexdoc.NewDocIndexStorage(storeLogData.GetStoreName()) // 判断系统权限用 has := false for j, max2 := 0, len(cond.Systems); j < max2; j++ { - if (idxdocStorage.GetWordDocSeq(cond.Systems[j], i)) > 0 { + if (idxdocStorage.GetWordDocSeq(cond.Systems[j], docId)) > 0 { has = true break } @@ -481,12 +482,12 @@ func findSame(cond *SearchCondition, minDocumentId uint32, maxDocumentId uint32, } } - if cond.Loglevel == "" && len(cond.Loglevels) > 0 { + if flg && cond.Loglevel == "" && len(cond.Loglevels) > 0 { // 日志级别范围内作过滤检查 idxdocStorage := indexdoc.NewDocIndexStorage(storeLogData.GetStoreName()) has := false for j, max2 := 0, len(cond.Loglevels); j < max2; j++ { - if (idxdocStorage.GetWordDocSeq("!"+cond.Loglevels[j], i)) > 0 { + if (idxdocStorage.GetWordDocSeq("!"+cond.Loglevels[j], docId)) > 0 { has = true break } diff --git a/glc/ldb/storage/logdata/logdata_model.go b/glc/ldb/storage/logdata/logdata_model.go index bf83dfd..5d23fec 100644 --- a/glc/ldb/storage/logdata/logdata_model.go +++ b/glc/ldb/storage/logdata/logdata_model.go @@ -14,23 +14,23 @@ import ( // 其中Tags是空格分隔的标签，日期外各属性值会按空格分词 // 对应的json属性统一全小写 type LogDataModel struct { - Id string `json:"id,omitempty"` // 从1开始递增(36进制字符串) - Text string `json:"text,omitempty"` // 【必须】日志内容，多行时仅为首行，直接显示用，是全文检索对象 - Date string `json:"date,omitempty"` // 日期（格式YYYY-MM-DD HH:MM:SS.SSS） - System string `json:"system,omitempty"` // 系统名 - ServerName string `json:"servername,omitempty"` // 服务器名 - ServerIp string `json:"serverip,omitempty"` // 服务器IP - ClientIp string `json:"clientip,omitempty"` // 客户端IP - TraceId string `json:"traceid,omitempty"` // 跟踪ID - LogType string `json:"logtype,omitempty"` // 日志类型（1:登录日志、2:操作日志） - LogLevel string `json:"loglevel,omitempty"` // 日志级别（debug、info、error等） - User string `json:"user,omitempty"` // 用户 - Module string `json:"module,omitempty"` // 模块 - Operation string `json:"action,omitempty"` // 操作 - Detail string `json:"detail,omitempty"` // 多行时的详细日志信息，通常是包含错误堆栈等的日志内容（这部分内容不做索引处理） - Tags []string `json:"tags,omitempty"` // 自定义标签，都作为关键词看待处理 - Keywords []string `json:"keywords,omitempty"` // 自定义的关键词 - Sensitives []string `json:"sensitives,omitempty"` // 要删除的敏感词 + Id string `json:"id,omitempty"` // 从1开始递增(36进制字符串) + Text string `json:"text,omitempty"` // 【必须】日志内容，多行时仅为首行，直接显示用，是全文检索对象 + Date string `json:"date,omitempty"` // 日期（格式YYYY-MM-DD HH:MM:SS.SSS） + System string `json:"system,omitempty"` // 系统名 + ServerName string `json:"servername,omitempty"` // 服务器名 + ServerIp string `json:"serverip,omitempty"` // 服务器IP + ClientIp string `json:"clientip,omitempty"` // 客户端IP + TraceId string `json:"traceid,omitempty"` // 跟踪ID + // LogType string `json:"logtype,omitempty"` // 【X】日志类型（1:登录日志、2:操作日志） + LogLevel string `json:"loglevel,omitempty"` // 日志级别（debug、info、warn、error） + User string `json:"user,omitempty"` // 用户 + // Module string `json:"module,omitempty"` // 【X】模块 + // Operation string `json:"action,omitempty"` // 【X】操作 + Detail string `json:"detail,omitempty"` // 【内部字段】多行时的详细日志信息，通常是包含错误堆栈等的日志内容 + // Tags []string `json:"tags,omitempty"` // 【X】自定义标签，都作为关键词看待处理 + // Keywords []string `json:"keywords,omitempty"` // 【X】自定义的关键词 + // Sensitives []string `json:"sensitives,omitempty"` // 【X】要删除的敏感词 } func (d *LogDataModel) ToJson() string { diff --git a/glc/ldb/storage/logdata/logdata_storage.go b/glc/ldb/storage/logdata/logdata_storage.go index 44a4837..2313e7b 100644 --- a/glc/ldb/storage/logdata/logdata_storage.go +++ b/glc/ldb/storage/logdata/logdata_storage.go @@ -187,23 +187,24 @@ func (s *LogDataStorage) createInvertedIndex() int { } // 整理生成关键词 - adds := docm.Keywords - adds = append(adds, docm.Tags...) + var adds []string if docm.System != "" { adds = append(adds, "~"+docm.System) } if docm.LogLevel != "" { adds = append(adds, "!"+docm.LogLevel) } + if docm.User != "" { + adds = append(adds, "@"+docm.User) + } - tgtStr := docm.System + " " + docm.ServerName + " " + docm.ServerIp + - " " + docm.ClientIp + " " + docm.TraceId + " " + docm.LogLevel + " " + docm.User + " " + docm.Module + " " + docm.Operation + tgtStr := docm.System + " " + docm.ServerName + " " + docm.ServerIp + " " + docm.ClientIp + " " + docm.TraceId + " " + docm.LogLevel + " " + docm.User if docm.Detail != "" && conf.IsMulitLineSearch() { tgtStr = tgtStr + " " + docm.Detail // 支持日志列全部行作为索引检索对象 } else { tgtStr = tgtStr + " " + docm.Text // 日志列仅第一行作为索引检索对象 } - kws := tokenizer.CutForSearchEx(tgtStr, adds, docm.Sensitives) // 两数组参数的元素可以重复或空白，会被判断整理 + kws := tokenizer.CutForSearchEx(tgtStr, adds, nil) // 两数组参数的元素可以重复或空白，会被判断整理 // 每个关键词都创建反向索引 for _, word := range kws { diff --git a/glc/ver/version.go b/glc/ver/version.go index f4cc316..4f8eee9 100644 --- a/glc/ver/version.go +++ b/glc/ver/version.go @@ -1,4 +1,4 @@ package ver // 版本号，升级版本时修改 -const VERSION = "v0.13.0" +const VERSION = "v0.14.0" diff --git a/glc/www/controller/log_add_testdata_controller.go b/glc/www/controller/log_add_testdata_controller.go index 995a8cf..aaa889c 100644 --- a/glc/www/controller/log_add_testdata_controller.go +++ b/glc/www/controller/log_add_testdata_controller.go @@ -28,6 +28,7 @@ func JsonLogAddTestDataController(req *gweb.HttpRequest) *gweb.HttpResult { ClientIp: "127.0.0.1", TraceId: traceId, LogLevel: "INFO", + User: "tuser-" + cmn.RandomString(1), } addDataModelLog(md) @@ -44,6 +45,7 @@ func JsonLogAddTestDataController(req *gweb.HttpRequest) *gweb.HttpResult { ClientIp: "127.0.0.1", TraceId: traceId, LogLevel: "DEBUG", + User: "tuser-" + cmn.RandomString(1), } addDataModelLog(md2) diff --git a/glc/www/controller/log_search_controller.go b/glc/www/controller/log_search_controller.go index eb91d62..69cd613 100644 --- a/glc/www/controller/log_search_controller.go +++ b/glc/www/controller/log_search_controller.go @@ -34,7 +34,7 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult { catchSession.Set(token, username) // 会话重新计时 } - // 准备好各种场景的检索条件 + // 准备好各种场景的检索条件（系统【~】、日志级别【!】、用户【@】） startTime := time.Now() mnt := sysmnt.NewSysmntStorage() cond := &search.SearchCondition{SearchSize: conf.GetPageSize()} @@ -46,8 +46,12 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult { cond.DatetimeFrom = req.GetFormParameter("datetimeFrom") // 日期范围（From） cond.DatetimeTo = req.GetFormParameter("datetimeTo") // 日期范围（To） cond.OrgSystem = cmn.Trim(req.GetFormParameter("system")) // 系统 - cond.Loglevel = req.GetFormParameter("loglevel") // 单选条件 + cond.User = cmn.ToLower(cmn.Trim(req.GetFormParameter("user"))) // 用户 + cond.Loglevel = cmn.ToLower(req.GetFormParameter("loglevel")) // 单选条件 cond.Loglevels = cmn.Split(cond.Loglevel, ",") // 多选条件 + if cond.User != "" { + cond.User = "@" + cond.User // 有指定用户条件 + } if len(cond.Loglevels) <= 1 || len(cond.Loglevels) >= 4 { cond.Loglevels = make([]string, 0) // 多选的单选或全选，都清空（单选走loglevel索引，全选等于没选） } @@ -80,7 +84,7 @@ func LogSearchController(req *gweb.HttpRequest) *gweb.HttpResult { cond.OrgSystem = "~" + cond.OrgSystem // 多个系统权限，按输入的系统作条件 } } else { - ary := cmn.Split(user.Systems, ",") + ary := cmn.Split(cmn.ToLower(user.Systems), ",") okSystem := false for i := 0; i < len(ary); i++ { cond.OrgSystems = append(cond.OrgSystems, "~"+ary[i]) // 仅设定的系统有访问权限 diff --git a/glc/www/web/src/views/glc/search/GlcMain.vue b/glc/www/web/src/views/glc/search/GlcMain.vue index 050da26..05cb03f 100644 --- a/glc/www/web/src/views/glc/search/GlcMain.vue +++ b/glc/www/web/src/views/glc/search/GlcMain.vue @@ -31,6 +31,10 @@ value-format="YYYY-MM-DD HH:mm:ss" start-placeholder="开始时间" end-placeholder="结束时间" popper-class="c-datapicker" /> + + + @@ -290,6 +294,7 @@ function search() { data.loglevel = (formData.value.loglevel || []).join(','); data.datetimeFrom = (formData.value.datetime || ['', ''])[0]; data.datetimeTo = (formData.value.datetime || ['', ''])[1]; + data.user = formData.value.user; // 保存好滚动检索的输入条件，保持和检索时一致，避免修改输入再滚动查询而出现矛盾结果 moreConditon.value = data; @@ -419,6 +424,11 @@ function fnDownload() { --el-input-width: 100%; } +.c-search-form .el-form-item--small .el-form-item__label { + height: 30px; + line-height: 30px; +} + .c-datapicker.el-popper.is-pure { margin-left: -100px; }