mirror of
https://github.com/gotoeasy/glogcenter.git
synced 2025-09-15 12:58:34 +08:00
重构优化
This commit is contained in:
gotoeasy
parent
5804b8a730
commit
72c6784f92
@ -7,7 +7,11 @@ import (
|
||||
"fmt"
|
||||
"glc/conf"
|
||||
"hash/crc32"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@ -89,7 +93,13 @@ func Uint32ToBytes(num uint32) []byte {
|
||||
}
|
||||
|
||||
func BytesToUint32(bytes []byte) uint32 {
|
||||
return uint32(binary.BigEndian.Uint32(bytes))
|
||||
return binary.BigEndian.Uint32(bytes)
|
||||
}
|
||||
|
||||
func Uint16ToBytes(num uint16) []byte {
|
||||
bkey := make([]byte, 2)
|
||||
binary.BigEndian.PutUint16(bkey, num)
|
||||
return bkey
|
||||
}
|
||||
|
||||
// func Uint64ToBytes(num uint64) []byte {
|
||||
@ -191,3 +201,46 @@ func SubStringRune(str string, start int, end int) string {
|
||||
func JoinBytes(bts ...[]byte) []byte {
|
||||
return bytes.Join(bts, []byte(""))
|
||||
}
|
||||
|
||||
func GetStorageNames(path string, excludes ...string) []string {
|
||||
fileinf, err := ioutil.ReadDir(path)
|
||||
if err != nil {
|
||||
log.Println("读取目录失败", err)
|
||||
return []string{}
|
||||
}
|
||||
|
||||
mapDir := make(map[string]string)
|
||||
for _, v := range fileinf {
|
||||
if v.IsDir() {
|
||||
mapDir[v.Name()] = ""
|
||||
}
|
||||
}
|
||||
for i := 0; i < len(excludes); i++ {
|
||||
delete(mapDir, excludes[i])
|
||||
}
|
||||
|
||||
var rs []string
|
||||
for k := range mapDir {
|
||||
rs = append(rs, k)
|
||||
}
|
||||
|
||||
// 倒序
|
||||
sort.Slice(rs, func(i, j int) bool {
|
||||
return rs[i] > rs[j]
|
||||
})
|
||||
|
||||
return rs
|
||||
}
|
||||
|
||||
func GetDirInfo(path string) (uint32, int64, error) {
|
||||
var count uint32
|
||||
var size int64
|
||||
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
|
||||
if !info.IsDir() {
|
||||
size += info.Size()
|
||||
count++
|
||||
}
|
||||
return err
|
||||
})
|
||||
return count, size, err
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ require github.com/syndtr/goleveldb v1.0.0
|
||||
|
||||
require (
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-playground/locales v0.14.0 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.10.0 // indirect
|
||||
@ -16,10 +17,12 @@ require (
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/ugorji/go/codec v1.2.7 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 // indirect
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
|
||||
golang.org/x/text v0.3.6 // indirect
|
||||
google.golang.org/protobuf v1.28.0 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
@ -29,6 +32,7 @@ require (
|
||||
github.com/gin-contrib/gzip v0.0.5
|
||||
github.com/gin-gonic/gin v1.8.1
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.22.6
|
||||
github.com/streadway/amqp v1.0.0
|
||||
github.com/wangbin/jiebago v0.3.2
|
||||
)
|
||||
|
||||
21
glc/go.sum
21
glc/go.sum
@ -9,6 +9,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm
|
||||
github.com/gin-gonic/gin v1.7.4/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
|
||||
github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8=
|
||||
github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
|
||||
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
|
||||
@ -27,6 +29,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
|
||||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
@ -41,6 +45,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
|
||||
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
|
||||
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
@ -56,18 +61,26 @@ github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV
|
||||
github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/shirou/gopsutil/v3 v3.22.6 h1:FnHOFOh+cYAM0C30P+zysPISzlknLC5Z1G4EAElznfQ=
|
||||
github.com/shirou/gopsutil/v3 v3.22.6/go.mod h1:EdIubSnZhbAvBS1yJ7Xi+AShB/hxwLHOMz4MCYz7yMs=
|
||||
github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo=
|
||||
github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
|
||||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
|
||||
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
|
||||
github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
|
||||
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
|
||||
@ -75,6 +88,8 @@ github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0
|
||||
github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
|
||||
github.com/wangbin/jiebago v0.3.2 h1:reQKp0xTXWFK7eQ19L6Ofq5xODSR2hcam55qcdCCNpw=
|
||||
github.com/wangbin/jiebago v0.3.2/go.mod h1:PAqQLauF0qAzy/63jBvO7Goh0oYBq1ocr0OXHLlujwQ=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
|
||||
@ -87,12 +102,17 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069 h1:siQdpVirKtzPhKl3lZWozZraCFObP8S1v6PRp0bLrtU=
|
||||
golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
@ -117,3 +137,4 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@ -20,7 +20,7 @@ func NewEngine(storeName string) *Engine {
|
||||
return &Engine{
|
||||
storeName: storeName,
|
||||
logStorage: storage.NewLogDataStorageHandle(storeName),
|
||||
idxwStorage: indexword.NewWordIndexStorage(storeName, " "), // 特殊（空格做关键词参数传入）
|
||||
idxwStorage: indexword.NewWordIndexStorage(storeName),
|
||||
}
|
||||
}
|
||||
|
||||
@ -29,7 +29,7 @@ func NewDefaultEngine() *Engine {
|
||||
return &Engine{
|
||||
storeName: storeName,
|
||||
logStorage: storage.NewLogDataStorageHandle(storeName),
|
||||
idxwStorage: indexword.NewWordIndexStorage(storeName, " "), // 特殊（空格做关键词参数传入）
|
||||
idxwStorage: indexword.NewWordIndexStorage(storeName),
|
||||
}
|
||||
}
|
||||
|
||||
@ -59,7 +59,7 @@ func (e *Engine) Search(searchKey string, pageSize int, currentDocId uint32, for
|
||||
|
||||
// 简单检查，存在无索引数据的关键词时，直接返回
|
||||
for _, word := range kws {
|
||||
idxw := indexword.NewWordIndexStorage(e.storeName, word)
|
||||
idxw := indexword.NewWordIndexStorage(e.storeName)
|
||||
if idxw.GetTotalCount(word) < 1 {
|
||||
log.Println("关键词", word, "没有索引数据，直接返回空结果")
|
||||
rs := new(search.SearchResult)
|
||||
|
||||
@ -2,11 +2,22 @@ package ldb
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"glc/ldb/sysmnt"
|
||||
"log"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Test_GetSubDirs(t *testing.T) {
|
||||
s := sysmnt.GetSysmntStorage("default")
|
||||
rs := s.GetStorageData()
|
||||
|
||||
for i := 0; i < len(rs); i++ {
|
||||
log.Println(rs[i])
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func Test_all(t *testing.T) {
|
||||
|
||||
engine := NewDefaultEngine()
|
||||
|
||||
@ -34,8 +34,8 @@ func SearchWordIndex(storeName string, kws []string, pageSize int, currentDocId
|
||||
for _, word := range kws {
|
||||
widxStorage := &WidxStorage{
|
||||
word: word,
|
||||
idxdocStorage: indexdoc.NewDocIndexStorage(storeName, word),
|
||||
idxwordStorage: indexword.NewWordIndexStorage(storeName, word),
|
||||
idxdocStorage: indexdoc.NewDocIndexStorage(storeName),
|
||||
idxwordStorage: indexword.NewWordIndexStorage(storeName),
|
||||
}
|
||||
widxs = append(widxs, widxStorage)
|
||||
}
|
||||
|
||||
@ -44,7 +44,7 @@ func getStorage(cacheName string) *DocIndexStorage {
|
||||
}
|
||||
|
||||
// 获取存储对象，线程安全（带缓存无则创建有则直取）
|
||||
func NewDocIndexStorage(storeName string, word string) *DocIndexStorage { // 存储器，文档，自定义对象
|
||||
func NewDocIndexStorage(storeName string) *DocIndexStorage { // 存储器，文档，自定义对象
|
||||
|
||||
// 缓存有则取用
|
||||
subPath := "inverted" + cmn.PathSeparator() + "d"
|
||||
|
||||
@ -20,15 +20,17 @@ import (
|
||||
)
|
||||
|
||||
type WordIndexStorage struct {
|
||||
storeName string // 存储目录
|
||||
subPath string // 存储目录下的相对路径（存放数据）
|
||||
leveldb *leveldb.DB // leveldb
|
||||
lastTime int64 // 最后一次访问时间
|
||||
closing bool // 是否关闭中状态
|
||||
mu sync.Mutex // 锁
|
||||
storeName string // 存储目录
|
||||
subPath string // 存储目录下的相对路径（存放数据）
|
||||
leveldb *leveldb.DB // leveldb
|
||||
lastTime int64 // 最后一次访问时间
|
||||
indexedCount uint32 // 已建索引件数
|
||||
closing bool // 是否关闭中状态
|
||||
mu sync.Mutex // 锁
|
||||
}
|
||||
|
||||
var zeroUint32Bytes []byte = cmn.Uint32ToBytes(0)
|
||||
var zeroUint16Bytes []byte = cmn.Uint16ToBytes(0) // 索引件数的key
|
||||
|
||||
var idxMu sync.Mutex
|
||||
var mapStorage map[string](*WordIndexStorage)
|
||||
@ -47,7 +49,7 @@ func getStorage(cacheName string) *WordIndexStorage {
|
||||
}
|
||||
|
||||
// 获取存储对象，线程安全（带缓存无则创建有则直取）
|
||||
func NewWordIndexStorage(storeName string, word string) *WordIndexStorage { // 存储器，文档，自定义对象
|
||||
func NewWordIndexStorage(storeName string) *WordIndexStorage { // 存储器，文档，自定义对象
|
||||
|
||||
// 缓存有则取用
|
||||
subPath := "inverted" + cmn.PathSeparator() + "k"
|
||||
@ -81,6 +83,7 @@ func NewWordIndexStorage(storeName string, word string) *WordIndexStorage { //
|
||||
panic(err)
|
||||
}
|
||||
store.leveldb = db
|
||||
store.loadIndexedCount() // 加载已建索引件数
|
||||
mapStorage[cacheName] = store // 缓存起来
|
||||
|
||||
// 逐秒判断，若闲置超时则自动关闭
|
||||
@ -104,6 +107,24 @@ func (s *WordIndexStorage) autoCloseWhenMaxIdle() {
|
||||
}
|
||||
}
|
||||
|
||||
// 取已建索引件数
|
||||
func (s *WordIndexStorage) GetIndexedCount() uint32 {
|
||||
return s.indexedCount
|
||||
}
|
||||
|
||||
// 初期加载已建索引件数
|
||||
func (s *WordIndexStorage) loadIndexedCount() {
|
||||
bt, err := s.leveldb.Get(zeroUint16Bytes, nil)
|
||||
if err == nil {
|
||||
s.indexedCount = cmn.BytesToUint32(bt)
|
||||
}
|
||||
}
|
||||
|
||||
// 保存已建索引件数
|
||||
func (s *WordIndexStorage) SavetIndexedCount(count uint32) error {
|
||||
return s.leveldb.Put(zeroUint16Bytes, cmn.Uint32ToBytes(count), nil)
|
||||
}
|
||||
|
||||
// 取关键词索引当前的文档数
|
||||
func (s *WordIndexStorage) GetTotalCount(word string) uint32 {
|
||||
wordBytes := cmn.StringToBytes(word)
|
||||
@ -133,7 +154,7 @@ func (s *WordIndexStorage) Add(word string, docId uint32) error {
|
||||
}
|
||||
|
||||
// 添加文档反向索引
|
||||
diStorage := indexdoc.NewDocIndexStorage(s.storeName, word)
|
||||
diStorage := indexdoc.NewDocIndexStorage(s.storeName)
|
||||
err = diStorage.AddWordDocSeq(word, docId, seq)
|
||||
if err != nil {
|
||||
log.Println("保存日志反向索引失败", err)
|
||||
|
||||
@ -11,7 +11,6 @@ import (
|
||||
"glc/cmn"
|
||||
"glc/conf"
|
||||
"glc/ldb/storage/indexword"
|
||||
"glc/ldb/sysmnt"
|
||||
"glc/ldb/tokenizer"
|
||||
"glc/onexit"
|
||||
"log"
|
||||
@ -187,7 +186,7 @@ func (s *LogDataStorage) createInvertedIndex() int {
|
||||
|
||||
// 每个关键词都创建反向索引
|
||||
for _, word := range kws {
|
||||
idxw := indexword.NewWordIndexStorage(s.StoreName(), word)
|
||||
idxw := indexword.NewWordIndexStorage(s.StoreName())
|
||||
idxw.Add(word, cmn.StringToUint32(docm.Id, 0)) // 日志ID加入索引
|
||||
}
|
||||
//log.Println("创建日志索引：", cmn.StringToUint32(docm.Id, 0))
|
||||
@ -282,7 +281,7 @@ func (s *LogDataStorage) Close() {
|
||||
|
||||
func (s *LogDataStorage) loadMetaData() {
|
||||
|
||||
// 初始化：当前索引件数
|
||||
// 初始化：当前日志件数
|
||||
bytes, err := s.leveldb.Get(zeroUint32Bytes, nil)
|
||||
if err != nil || bytes == nil {
|
||||
s.currentCount = 0
|
||||
@ -292,10 +291,8 @@ func (s *LogDataStorage) loadMetaData() {
|
||||
}
|
||||
|
||||
// 初始化：已建索引件数
|
||||
mntKey := "INDEX:" + s.StoreName()
|
||||
sysStorage := sysmnt.GetSysmntStorage(s.StoreName())
|
||||
sysmntData := sysStorage.GetSysmntData(mntKey)
|
||||
s.indexedCount = sysmntData.Count
|
||||
idxw := indexword.NewWordIndexStorage(s.StoreName())
|
||||
s.indexedCount = idxw.GetIndexedCount()
|
||||
s.savedIndexedCount = s.indexedCount
|
||||
}
|
||||
|
||||
@ -309,11 +306,8 @@ func (s *LogDataStorage) saveMetaData() {
|
||||
|
||||
if s.savedIndexedCount < s.indexedCount {
|
||||
s.savedIndexedCount = s.indexedCount
|
||||
mntKey := "INDEX:" + s.StoreName()
|
||||
sysStorage := sysmnt.GetSysmntStorage(s.StoreName())
|
||||
sysmntData := new(sysmnt.SysmntData)
|
||||
sysmntData.Count = s.savedIndexedCount
|
||||
sysStorage.SetSysmntData(mntKey, sysmntData)
|
||||
idxw := indexword.NewWordIndexStorage(s.StoreName())
|
||||
idxw.SavetIndexedCount(s.savedIndexedCount)
|
||||
log.Println("保存LogDataStorage已建索引件数:", s.savedIndexedCount)
|
||||
}
|
||||
}
|
||||
|
||||
37
glc/ldb/sysmnt/storage_data.go
Normal file
37
glc/ldb/sysmnt/storage_data.go
Normal file
@ -0,0 +1,37 @@
|
||||
/**
|
||||
* 日志仓信息
|
||||
*/
|
||||
package sysmnt
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"glc/cmn"
|
||||
"glc/conf"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type StorageData struct {
|
||||
Name string
|
||||
DataCount uint32
|
||||
IndexCount uint32
|
||||
FileCount uint32
|
||||
TotalSize string
|
||||
}
|
||||
|
||||
func (s *SysmntStorage) GetStorageData() []*StorageData {
|
||||
|
||||
var rs []*StorageData
|
||||
names := cmn.GetStorageNames(conf.GetStorageRoot(), ".sysmnt")
|
||||
for _, name := range names {
|
||||
d := &StorageData{
|
||||
Name: name,
|
||||
}
|
||||
|
||||
cnt, size, _ := cmn.GetDirInfo(conf.GetStorageRoot() + cmn.PathSeparator() + name)
|
||||
sizem, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", float64(size)/1024/1024), 64)
|
||||
d.TotalSize = fmt.Sprintf("%fM", sizem)
|
||||
d.FileCount = cnt
|
||||
rs = append(rs, d)
|
||||
}
|
||||
return rs
|
||||
}
|
||||
@ -37,8 +37,8 @@ func init() {
|
||||
func GetSysmntStorage(storeName string) *SysmntStorage { // 存储器，文档，自定义对象
|
||||
|
||||
// 缓存有则取用
|
||||
subPath := "sysmnt"
|
||||
cacheName := storeName + cmn.PathSeparator() + subPath
|
||||
subPath := ".sysmnt"
|
||||
cacheName := subPath
|
||||
if sysmntStorage != nil && !sysmntStorage.IsClose() { // 尝试用缓存实例存储器
|
||||
return sysmntStorage
|
||||
}
|
||||
@ -102,7 +102,7 @@ func (s *SysmntStorage) Close() {
|
||||
s.leveldb.Close()
|
||||
sysmntStorage = nil
|
||||
|
||||
log.Println("关闭SysmntStorage：", s.storeName+cmn.PathSeparator()+s.subPath)
|
||||
log.Println("关闭SysmntStorage：", s.subPath)
|
||||
}
|
||||
|
||||
// 直接存入数据到leveldb
|
||||
|
||||
Loading…
Reference in New Issue
Block a user