diff --git a/.gitignore b/.gitignore index b884ebb..923345b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ data.db config.yaml ltest gtest -mtest \ No newline at end of file +mtest +log.bleve \ No newline at end of file diff --git a/boot/boot.go b/boot/boot.go index 8c56d00..aebfc56 100644 --- a/boot/boot.go +++ b/boot/boot.go @@ -25,7 +25,6 @@ func init() { initArgs() initLogHandle() initLog() - initEs() initLogHanler() initWaitCond() initProcess() @@ -82,10 +81,6 @@ func initLog() { logger.InitLog() } -func initEs() { - logic.EsLogic.InitEs() -} - func initProcess() { logic.ProcessCtlLogic.ProcessInit() } diff --git a/config/config.go b/config/config.go index fa1c405..3e2efaa 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,7 @@ var CF = new(configuration) type configuration struct { LogLevel string `default:"debug" describe:"日志等级[debug,info]"` Listen string `default:":8797" describe:"监听端口"` - EsEnable bool `default:"false" describe:"启用Elasticsearch"` + StorgeType string `default:"sqlite" describe:"存储引擎[sqlite、es、bleve]"` EsUrl string `default:"" describe:"Elasticsearch url"` EsIndex string `default:"server_log_v1" describe:"Elasticsearch index"` EsUsername string `default:"" describe:"Elasticsearch用户名"` diff --git a/go.mod b/go.mod index 9dbc060..1efe399 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,35 @@ require ( github.com/containerd/cgroups/v3 v3.0.4 github.com/gorilla/websocket v1.5.1 github.com/opencontainers/runtime-spec v1.2.0 + github.com/vcaesar/gse-bleve v0.40.0 +) + +require ( + github.com/RoaringBitmap/roaring/v2 v2.4.5 // indirect + github.com/bits-and-blooms/bitset v1.22.0 // indirect + github.com/blevesearch/bleve_index_api v1.2.8 // indirect + github.com/blevesearch/geo v0.2.3 // indirect + github.com/blevesearch/go-faiss v1.0.25 // indirect + github.com/blevesearch/go-porterstemmer v1.0.3 // indirect + github.com/blevesearch/gtreap v0.1.1 // indirect + github.com/blevesearch/mmap-go v1.0.4 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.3.10 // indirect + github.com/blevesearch/segment v0.9.1 // indirect + github.com/blevesearch/snowballstem v0.9.0 // indirect + github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect + github.com/blevesearch/vellum v1.1.0 // indirect + github.com/blevesearch/zapx/v11 v11.4.2 // indirect + github.com/blevesearch/zapx/v12 v12.4.2 // indirect + github.com/blevesearch/zapx/v13 v13.4.2 // indirect + github.com/blevesearch/zapx/v14 v14.4.2 // indirect + github.com/blevesearch/zapx/v15 v15.4.2 // indirect + github.com/blevesearch/zapx/v16 v16.2.4 // indirect + github.com/go-ego/gse v0.70.2 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/mschoch/smat v0.2.0 // indirect + github.com/vcaesar/cedar v0.20.1 // indirect + go.etcd.io/bbolt v1.4.0 // indirect ) require ( @@ -48,7 +77,7 @@ require ( github.com/gdamore/encoding v1.0.0 // indirect github.com/gdamore/tcell/v2 v2.7.1 // indirect github.com/glebarez/go-sqlite v1.21.2 // indirect - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect @@ -86,13 +115,14 @@ require ( go.uber.org/multierr v1.10.0 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.29.0 // indirect - golang.org/x/sys v0.27.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.22.0 // indirect google.golang.org/protobuf v1.35.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( + github.com/blevesearch/bleve/v2 v2.5.2 github.com/creack/pty v1.1.21 github.com/gin-gonic/gin v1.9.1 github.com/glebarez/sqlite v1.11.0 diff --git a/go.sum b/go.sum index c79c63e..3b0fe91 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,46 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2JW2gggRdg= +github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.22.0 h1:Tquv9S8+SGaS3EhyA+up3FXzmkhxPGjQQCkcs2uw7w4= +github.com/bits-and-blooms/bitset v1.22.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/blevesearch/bleve/v2 v2.5.2 h1:Ab0r0MODV2C5A6BEL87GqLBySqp/s9xFgceCju6BQk8= +github.com/blevesearch/bleve/v2 v2.5.2/go.mod h1:5Dj6dUQxZM6aqYT3eutTD/GpWKGFSsV8f7LDidFbwXo= +github.com/blevesearch/bleve_index_api v1.2.8 h1:Y98Pu5/MdlkRyLM0qDHostYo7i+Vv1cDNhqTeR4Sy6Y= +github.com/blevesearch/bleve_index_api v1.2.8/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0= +github.com/blevesearch/geo v0.2.3 h1:K9/vbGI9ehlXdxjxDRJtoAMt7zGAsMIzc6n8zWcwnhg= +github.com/blevesearch/geo v0.2.3/go.mod h1:K56Q33AzXt2YExVHGObtmRSFYZKYGv0JEN5mdacJJR8= +github.com/blevesearch/go-faiss v1.0.25 h1:lel1rkOUGbT1CJ0YgzKwC7k+XH0XVBHnCVWahdCXk4U= +github.com/blevesearch/go-faiss v1.0.25/go.mod h1:OMGQwOaRRYxrmeNdMrXJPvVx8gBnvE5RYrr0BahNnkk= +github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= +github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= +github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= +github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= +github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= +github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= +github.com/blevesearch/scorch_segment_api/v2 v2.3.10 h1:Yqk0XD1mE0fDZAJXTjawJ8If/85JxnLd8v5vG/jWE/s= +github.com/blevesearch/scorch_segment_api/v2 v2.3.10/go.mod h1:Z3e6ChN3qyN35yaQpl00MfI5s8AxUJbpTR/DL8QOQ+8= +github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= +github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= +github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= +github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs= +github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMGZzVrdmaozG2MfoB+A= +github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= +github.com/blevesearch/vellum v1.1.0 h1:CinkGyIsgVlYf8Y2LUQHvdelgXr6PYuvoDIajq6yR9w= +github.com/blevesearch/vellum v1.1.0/go.mod h1:QgwWryE8ThtNPxtgWJof5ndPfx0/YMBh+W2weHKPw8Y= +github.com/blevesearch/zapx/v11 v11.4.2 h1:l46SV+b0gFN+Rw3wUI1YdMWdSAVhskYuvxlcgpQFljs= +github.com/blevesearch/zapx/v11 v11.4.2/go.mod h1:4gdeyy9oGa/lLa6D34R9daXNUvfMPZqUYjPwiLmekwc= +github.com/blevesearch/zapx/v12 v12.4.2 h1:fzRbhllQmEMUuAQ7zBuMvKRlcPA5ESTgWlDEoB9uQNE= +github.com/blevesearch/zapx/v12 v12.4.2/go.mod h1:TdFmr7afSz1hFh/SIBCCZvcLfzYvievIH6aEISCte58= +github.com/blevesearch/zapx/v13 v13.4.2 h1:46PIZCO/ZuKZYgxI8Y7lOJqX3Irkc3N8W82QTK3MVks= +github.com/blevesearch/zapx/v13 v13.4.2/go.mod h1:knK8z2NdQHlb5ot/uj8wuvOq5PhDGjNYQQy0QDnopZk= +github.com/blevesearch/zapx/v14 v14.4.2 h1:2SGHakVKd+TrtEqpfeq8X+So5PShQ5nW6GNxT7fWYz0= +github.com/blevesearch/zapx/v14 v14.4.2/go.mod h1:rz0XNb/OZSMjNorufDGSpFpjoFKhXmppH9Hi7a877D8= +github.com/blevesearch/zapx/v15 v15.4.2 h1:sWxpDE0QQOTjyxYbAVjt3+0ieu8NCE0fDRaFxEsp31k= +github.com/blevesearch/zapx/v15 v15.4.2/go.mod h1:1pssev/59FsuWcgSnTa0OeEpOzmhtmr/0/11H0Z8+Nw= +github.com/blevesearch/zapx/v16 v16.2.4 h1:tGgfvleXTAkwsD5mEzgM3zCS/7pgocTCnO1oyAUjlww= +github.com/blevesearch/zapx/v16 v16.2.4/go.mod h1:Rti/REtuuMmzwsI8/C/qIzRaEoSK/wiFYw5e5ctUKKs= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= @@ -45,6 +86,8 @@ github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9g github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= +github.com/go-ego/gse v0.70.2 h1:y2UMOHJMtI+0b2GjxTtQfKON5DMmlyX1hOQHTo8UVVs= +github.com/go-ego/gse v0.70.2/go.mod h1:kesekpZfcFQ/kwd9b27VZHUOH5dQUjaaQUZ4OGt4Hj4= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -72,15 +115,21 @@ github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0kt github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/iamacarpet/go-winpty v1.0.2 h1:jwPVTYrjAHZx6Mcm6K5i9G4opMp5TblEHH5EQCl/Gzw= @@ -141,6 +190,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= @@ -183,8 +234,9 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= @@ -193,9 +245,17 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/vcaesar/cedar v0.20.1 h1:cDOmYWdprO7ZW8cngJrDi8Zivnscj9dA/y8Y+2SB1P0= +github.com/vcaesar/cedar v0.20.1/go.mod h1:iMDweyuW76RvSrCkQeZeQk4iCbshiPzcCvcGCtpM7iI= +github.com/vcaesar/gse-bleve v0.40.0 h1:Qnv/9v8uyqNkFHxdK4VLN4EwZ0WLxudSgBcUzd2uxQs= +github.com/vcaesar/gse-bleve v0.40.0/go.mod h1:ORZ+jIAEIQeC6PEai24UJr8wisHAWrSukKXTPVVPXzc= +github.com/vcaesar/tt v0.20.0 h1:9t2Ycb9RNHcP0WgQgIaRKJBB+FrRdejuaL6uWIHuoBA= +github.com/vcaesar/tt v0.20.0/go.mod h1:GHPxQYhn+7OgKakRusH7KJ0M5MhywoeLb8Fcffs/Gtg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk= +go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= @@ -237,8 +297,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= -golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -259,12 +319,16 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/datatypes v1.2.4 h1:uZmGAcK/QZ0uyfCuVg0VQY1ZmV9h1fuG0tMwKByO1z4= diff --git a/internal/app/logic/bleve.go b/internal/app/logic/bleve.go new file mode 100644 index 0000000..3dd5d5d --- /dev/null +++ b/internal/app/logic/bleve.go @@ -0,0 +1,135 @@ +package logic + +import ( + "time" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/search" + "github.com/google/uuid" + + "github.com/lzh-1625/go_process_manager/internal/app/model" + logger "github.com/lzh-1625/go_process_manager/log" + gse "github.com/vcaesar/gse-bleve" +) + +type bleveLogic struct { + index bleve.Index +} + +var BleveLogic = new(bleveLogic) + +func (b *bleveLogic) InitBleve() { + opt := gse.Option{ + Dicts: "embed, zh", + Stop: "", + Opt: "search-hmm", + Trim: "trim", + } + indexMapping, err := gse.NewMapping(opt) + if err != nil { + logger.Logger.Errorw("bleve init fail", "err", err) + return + } + mapping := bleve.NewDocumentMapping() + log := bleve.NewTextFieldMapping() + log.Index = true + time := bleve.NewNumericFieldMapping() + time.Index = true + name := bleve.NewKeywordFieldMapping() + name.Index = true + using := bleve.NewKeywordFieldMapping() + using.Index = true + mapping.AddFieldMappingsAt("log", log) + mapping.AddFieldMappingsAt("time", time) + mapping.AddFieldMappingsAt("name", name) + mapping.AddFieldMappingsAt("using", using) + indexMapping.AddDocumentMapping("server_log_v1", mapping) + index, err := bleve.Open("log.bleve") + if err != nil { + index, err = bleve.New("log.bleve", indexMapping) + if err != nil { + logger.Logger.Errorw("bleve init error", "err", err) + return + } + } + b.index = index +} + +func (b *bleveLogic) Insert(logContent string, processName string, using string, ts int64) { + if err := b.index.Index(uuid.NewString(), model.ProcessLog{ + Log: logContent, + Name: processName, + Using: using, + Time: ts, + }); err != nil { + logger.Logger.Warnw("bleve log insert failed", "err", err) + } +} + +func (b *bleveLogic) Search(req model.GetLogReq, filterProcessName ...string) (result model.LogResp) { + buildQuery := bleve.NewBooleanQuery() + if req.Match.Log != "" { + logQuery := bleve.NewMatchQuery(req.Match.Log) + logQuery.SetField("log") + buildQuery.AddMust(logQuery) + } + if req.Match.Name != "" { + nameQuery := bleve.NewTermQuery(req.Match.Name) + nameQuery.SetField("name") + buildQuery.AddMust(nameQuery) + } + if req.Match.Using != "" { + usingQuery := bleve.NewWildcardQuery("*" + req.Match.Using + "*") + usingQuery.SetField("using") + buildQuery.AddMust(usingQuery) + } + if req.TimeRange.EndTime != 0 || req.TimeRange.StartTime != 0 { + st := float64(req.TimeRange.StartTime) + et := float64(req.TimeRange.EndTime) + timeQuery := bleve.NewNumericRangeQuery(&st, &et) + buildQuery.AddMust(timeQuery) + } else { + st := float64(0) + et := float64(time.Now().UnixMilli()) + timeQuery := bleve.NewNumericRangeQuery(&st, &et) + buildQuery.AddMust(timeQuery) + } + if len(filterProcessName) != 0 { + for _, v := range filterProcessName { + filterQuery := bleve.NewTermQuery(v) + filterQuery.SetField("name") + buildQuery.AddShould(filterQuery) + } + } + sortArgs := ([]string{"-_score"}) + if req.Sort == "desc" { + sortArgs = append(sortArgs, "-time") + } + if req.Sort == "asc" { + sortArgs = append(sortArgs, "time") + } + res, err := b.index.Search(&bleve.SearchRequest{ + Query: buildQuery, + Fields: []string{"log", "name", "using", "time"}, + From: req.Page.From, + Size: req.Page.Size, + Sort: search.ParseSortOrderStrings(sortArgs), + }) + if err != nil { + logger.Logger.Warnw("bleve search failed", "err", err) + return + } + data := []*model.ProcessLog{} + for _, v := range res.Hits { + data = append(data, &model.ProcessLog{ + Log: v.Fields["log"].(string), + Time: int64(v.Fields["time"].(float64)), + Using: v.Fields["using"].(string), + Name: v.Fields["name"].(string), + }) + } + + result.Data = data + result.Total = int64(res.Total) + return +} diff --git a/internal/app/logic/es.go b/internal/app/logic/es.go index b52f264..63e43c5 100644 --- a/internal/app/logic/es.go +++ b/internal/app/logic/es.go @@ -24,11 +24,6 @@ var ( ) func (e *esLogic) InitEs() bool { - if !config.CF.EsEnable { - log.Logger.Debug("不使用es") - return false - } - var err error EsLogic.esClient, err = elastic.NewClient( elastic.SetURL(config.CF.EsUrl), @@ -42,7 +37,6 @@ func (e *esLogic) InitEs() bool { }), ) if err != nil { - config.CF.EsEnable = false log.Logger.Warnw("Failed to connect to es", "err", err) return false } diff --git a/internal/app/logic/log_handler.go b/internal/app/logic/log_handler.go index 4501c92..6f203d6 100644 --- a/internal/app/logic/log_handler.go +++ b/internal/app/logic/log_handler.go @@ -20,7 +20,7 @@ var ( func InitLogHandle() { Loghandler.antsPool, _ = ants.NewPool(config.CF.LogHandlerPoolSize, ants.WithNonblocking(true), ants.WithExpiryDuration(3*time.Second), ants.WithPanicHandler(func(i interface{}) { - log.Logger.Error("es消息储存失败") + log.Logger.Warnw("日志储存失败", "err", i) })) } diff --git a/internal/app/logic/log_logic.go b/internal/app/logic/log_logic.go index 771149d..e1e2ff1 100644 --- a/internal/app/logic/log_logic.go +++ b/internal/app/logic/log_logic.go @@ -4,6 +4,7 @@ import ( "github.com/lzh-1625/go_process_manager/config" "github.com/lzh-1625/go_process_manager/internal/app/model" "github.com/lzh-1625/go_process_manager/internal/app/repository" + "github.com/lzh-1625/go_process_manager/log" ) type LogLogic interface { @@ -14,10 +15,18 @@ type LogLogic interface { var LogLogicImpl LogLogic func InitLog() { - if config.CF.EsEnable { - LogLogicImpl = LogEs - } else { + switch config.CF.StorgeType { + case "sqlite": LogLogicImpl = LogSqlite + log.Logger.Infow("使用sqlite作为日志存储") + case "es": + LogLogicImpl = LogEs + EsLogic.InitEs() + log.Logger.Infow("使用es作为日志存储") + case "bleve": + LogLogicImpl = BleveLogic + BleveLogic.InitBleve() + log.Logger.Infow("使用bleve作为日志存储") } } diff --git a/internal/app/logic/process_pty_linux.go b/internal/app/logic/process_pty_linux.go index dae8ce2..96221e5 100644 --- a/internal/app/logic/process_pty_linux.go +++ b/internal/app/logic/process_pty_linux.go @@ -114,7 +114,9 @@ func (p *ProcessPty) readInit() { } func (p *ProcessPty) ReadCache(ws ConnectInstance) { - ws.Write(p.cacheBytesBuf.Bytes()) + if p.cacheBytesBuf != nil { + ws.Write(p.cacheBytesBuf.Bytes()) + } } func (p *ProcessPty) bufHanle(b []byte) {