diff --git a/hook/go.mod b/hook/go.mod index f923acc..38fc157 100644 --- a/hook/go.mod +++ b/hook/go.mod @@ -5,29 +5,12 @@ go 1.16 require ( github.com/opencontainers/runtime-spec v1.0.2 github.com/prashantv/gostub v1.1.0 - huawei.com/npu-exporter v0.2.14 + huawei.com/mindx/common/hwlog v0.0.0 mindxcheckutils v1.0.0 ) replace ( + huawei.com/mindx/common/hwlog => codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3 + huawei.com/mindx/common/utils => codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6 mindxcheckutils => ../mindxcheckutils - huawei.com/kmc => codehub-dg-y.huawei.com/it-edge-native/edge-native-core/coastguard.git v1.0.6 - huawei.com/npu-exporter => codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/npu-exporter.git v0.2.13 - k8s.io/api v0.0.0 => codehub-dg-y.huawei.com/OpenSourceCenter/kubernetes.git/staging/src/k8s.io/api v1.19.4-h4 - k8s.io/apimachinery => codehub-dg-y.huawei.com/OpenSourceCenter/kubernetes.git/staging/src/k8s.io/apimachinery v1.19.4-h4 - k8s.io/client-go => codehub-dg-y.huawei.com/OpenSourceCenter/kubernetes.git/staging/src/k8s.io/client-go v1.19.4-h4 - k8s.io/cri-api => codehub-dg-y.huawei.com/OpenSourceCenter/kubernetes.git/staging/src/k8s.io/cri-api v1.19.4-h4 - github.com/agiledragon/gomonkey/v2 => github.com/agiledragon/gomonkey/v2 v2.2.0 - github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify v1.5.1 - github.com/golang/protobuf => github.com/golang/protobuf v1.5.1 - github.com/patrickmn/go-cache => github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible - github.com/pkg/errors => github.com/pkg/errors v0.9.1 - github.com/prashantv/gostub => github.com/prashantv/gostub v1.0.1-0.20191007164320-bbe3712b9c4a - github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.11.1 - github.com/smartystreets/goconvey => github.com/smartystreets/goconvey v1.6.4 - github.com/stretchr/testify => github.com/stretchr/testify v1.7.0 - go.uber.org/zap => go.uber.org/zap v1.16.0 - golang.org/x/crypto => golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b - google.golang.org/grpc => google.golang.org/grpc v1.28.0 - gopkg.in/natefinch/lumberjack.v2 => gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) diff --git a/hook/go.sum b/hook/go.sum index 9345f6a..af039e2 100644 --- a/hook/go.sum +++ b/hook/go.sum @@ -1,15 +1,39 @@ +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3 h1:Dq2tZTuClykBv1tS6C2XU6hY/waVK0K5lyr/8tJcT/I= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3/go.mod h1:tuvVYh4aDcHqpiyNKZNiBgt1XBehJ130VubLidKWNjY= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6 h1:nGlHkoj7L6DdSYFFLjceD6t5XrNAKrM+Gk3wT0iOfWE= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6/go.mod h1:gzaoiHwloaNTtm46GVe93KtXMcXVHJ8LBVqWEAjmh+0= +github.com/agiledragon/gomonkey/v2 v2.8.0 h1:u2K2nNGyk0ippzklz1CWalllEB9ptD+DtSXeCX5O000= +github.com/agiledragon/gomonkey/v2 v2.8.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hook/main.go b/hook/main.go index 7e5aea1..4467329 100644 --- a/hook/main.go +++ b/hook/main.go @@ -6,6 +6,7 @@ package main import ( "bufio" + "context" "encoding/json" "fmt" "log" @@ -18,10 +19,10 @@ import ( "syscall" "github.com/opencontainers/runtime-spec/specs-go" - "go.uber.org/zap" - "huawei.com/npu-exporter/hwlog" "mindxcheckutils" + + "huawei.com/mindx/common/hwlog" ) const ( @@ -39,7 +40,6 @@ const ( borderNum = 2 kvPairSize = 2 - maxLogLength = 1024 maxCommandLength = 65535 ) @@ -61,7 +61,7 @@ type containerConfig struct { Env []string } -func initLogModule(stopCh <-chan struct{}) error { +func initLogModule(ctx context.Context) error { const backups = 2 const logMaxAge = 365 runLogConfig := hwlog.LogConfig{ @@ -69,8 +69,9 @@ func initLogModule(stopCh <-chan struct{}) error { LogLevel: 0, MaxBackups: backups, MaxAge: logMaxAge, + OnlyToFile: true, } - if err := hwlog.InitRunLogger(&runLogConfig, stopCh); err != nil { + if err := hwlog.InitRunLogger(&runLogConfig, ctx); err != nil { fmt.Printf("hwlog init failed, error is %v", err) return err } @@ -79,8 +80,9 @@ func initLogModule(stopCh <-chan struct{}) error { LogLevel: 0, MaxBackups: backups, MaxAge: logMaxAge, + OnlyToFile: true, } - if err := hwlog.InitOperateLogger(&operateLogConfig, stopCh); err != nil { + if err := hwlog.InitOperateLogger(&operateLogConfig, ctx); err != nil { fmt.Printf("hwlog init failed, error is %v", err) return err } @@ -105,6 +107,7 @@ func removeDuplication(devices []int) []int { func parseDevices(visibleDevices string) ([]int, error) { devices := make([]int, 0) + const maxDevice = 128 for _, d := range strings.Split(visibleDevices, ",") { d = strings.TrimSpace(d) @@ -118,12 +121,12 @@ func parseDevices(visibleDevices string) ([]int, error) { borders[1] = strings.TrimSpace(borders[1]) left, err := strconv.Atoi(borders[0]) - if err != nil { + if err != nil || left < 0 { return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0]) } right, err := strconv.Atoi(borders[1]) - if err != nil { + if err != nil || right > maxDevice { return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1]) } @@ -152,6 +155,10 @@ func parseMounts(mounts string) []string { if mounts == "" { return []string{baseConfig} } + const maxMountLength = 128 + if len(mounts) > maxMountLength { + return []string{baseConfig} + } mountConfigs := make([]string, 0) for _, m := range strings.Split(mounts, ",") { @@ -179,11 +186,15 @@ func parseRuntimeOptions(runtimeOptions string) ([]string, error) { if runtimeOptions == "" { return parsedOptions, nil } + const maxLength = 128 + if len(runtimeOptions) > maxLength { + return nil, fmt.Errorf("invalid runtime option") + } for _, option := range strings.Split(runtimeOptions, ",") { option = strings.TrimSpace(option) if !isRuntimeOptionValid(option) { - return nil, fmt.Errorf("invalid runtime option %s", option) + return nil, fmt.Errorf("invalid runtime option") } parsedOptions = append(parsedOptions, option) @@ -224,7 +235,7 @@ var getContainerConfig = func() (*containerConfig, error) { } configPath := path.Join(state.Bundle, "config.json") - if _, err := mindxcheckutils.RealFileChecker(configPath, false, true, mindxcheckutils.DefaultSize); err != nil { + if _, err := mindxcheckutils.RealFileChecker(configPath, true, true, mindxcheckutils.DefaultSize); err != nil { return nil, err } @@ -247,14 +258,14 @@ var getContainerConfig = func() (*containerConfig, error) { return ret, nil } -func getValueByKey(data []string, key string) string { +func getValueByKey(data []string, name string) string { for _, s := range data { p := strings.SplitN(s, "=", 2) if len(p) != kvPairSize { log.Panicln("environment error") } - if p[0] == key { + if p[0] == name && len(p) == kvPairSize { return p[1] } } @@ -288,12 +299,16 @@ func readMountConfig(dir string, name string) ([]string, []string, error) { } defer f.Close() - fileMountList := make([]string, 0) - dirMountList := make([]string, 0) - + fileMountList, dirMountList := make([]string, 0), make([]string, 0) + const maxEntryNumber = 128 + entryCount := 0 scanner := bufio.NewScanner(f) for scanner.Scan() { mountPath := scanner.Text() + entryCount = entryCount + 1 + if entryCount > maxEntryNumber { + return nil, nil, fmt.Errorf("mount list too long") + } absMountPath, err := filepath.Abs(mountPath) if err != nil { continue // skipping files/dirs with any problems @@ -305,7 +320,7 @@ func readMountConfig(dir string, name string) ([]string, []string, error) { continue // skipping files/dirs with any problems } - if stat.Mode().IsRegular() || stat.Mode()&os.ModeSocket != 0 { + if stat.Mode().IsRegular() { fileMountList = append(fileMountList, mountPath) } else if stat.Mode().IsDir() { dirMountList = append(dirMountList, mountPath) @@ -341,6 +356,20 @@ func readConfigsOfDir(dir string, configs []string) ([]string, []string, error) return fileMountList, dirMountList, nil } +func getArgs(cliPath string, devices []int, containerConfig *containerConfig, + fileMountList []string, dirMountList []string) []string { + args := append([]string{cliPath}, + "--devices", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(devices)), ","), "[]"), + "--pid", fmt.Sprintf("%d", containerConfig.Pid), "--rootfs", containerConfig.Rootfs) + for _, filePath := range fileMountList { + args = append(args, "--mount-file", filePath) + } + for _, dirPath := range dirMountList { + args = append(args, "--mount-dir", dirPath) + } + return args +} + func doPrestartHook() error { containerConfig, err := getContainerConfig() if err != nil { @@ -381,23 +410,14 @@ func doPrestartHook() error { if _, err := mindxcheckutils.RealFileChecker(cliPath, true, false, mindxcheckutils.DefaultSize); err != nil { return err } - - args := append([]string{cliPath}, - "--devices", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(devices)), ","), "[]"), - "--pid", fmt.Sprintf("%d", containerConfig.Pid), - "--rootfs", containerConfig.Rootfs) - - for _, filePath := range fileMountList { - args = append(args, "--mount-file", filePath) - } - - for _, dirPath := range dirMountList { - args = append(args, "--mount-dir", dirPath) - } - + args := getArgs(cliPath, devices, containerConfig, fileMountList, dirMountList) if len(parsedOptions) > 0 { args = append(args, "--options", strings.Join(parsedOptions, ",")) } + hwlog.OpLog.Infof("ascend docker hook success, will start cli") + if err := mindxcheckutils.ChangeRuntimeLogMode("hook-run-", "hook-operate-"); err != nil { + return err + } if err := doExec(cliPath, args, os.Environ()); err != nil { return fmt.Errorf("failed to exec ascend-docker-cli %v: %v", args, err) } @@ -412,34 +432,30 @@ func main() { }() log.SetPrefix(loggingPrefix) - stopCh := make(chan struct{}) - if err := initLogModule(stopCh); err != nil { - close(stopCh) + ctx, _ := context.WithCancel(context.Background()) + if err := initLogModule(ctx); err != nil { log.Fatal(err) } logPrefixWords, err := mindxcheckutils.GetLogPrefix() if err != nil { - close(stopCh) log.Fatal(err) } - hwlog.RunLog.ZapLogger = hwlog.RunLog.ZapLogger.With(zap.String("user-info", logPrefixWords)) - hwlog.OpLog.ZapLogger = hwlog.OpLog.ZapLogger.With(zap.String("user-info", logPrefixWords)) + defer func() { + if err := mindxcheckutils.ChangeRuntimeLogMode("hook-run-", "hook-operate-"); err != nil { + fmt.Println("defer changeFileMode function failed") + } + }() + hwlog.OpLog.Infof("%v ascend docker hook starting, try to setup container", logPrefixWords) hwlog.RunLog.Infof("ascend docker hook starting") if !mindxcheckutils.StringChecker(strings.Join(os.Args, " "), 0, maxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { - close(stopCh) + hwlog.RunLog.Errorf("ascend docker hook failed") + hwlog.OpLog.Errorf("%v ascend docker hook failed", logPrefixWords) log.Fatal("command error") } - logWords := fmt.Sprintf("running %v", os.Args) - if len(logWords) > maxLogLength { - logWords = logWords[0:maxLogLength-1] + "..." - } - hwlog.OpLog.Infof(logWords) if err := doPrestartHook(); err != nil { hwlog.RunLog.Errorf("ascend docker hook failed") - hwlog.OpLog.Errorf("failed: err %v", err) - close(stopCh) - log.Fatal(err) + hwlog.OpLog.Errorf("%v ascend docker hook failed", logPrefixWords) + log.Fatal(fmt.Errorf("failed in runtime.doProcess ")) } - close(stopCh) } diff --git a/install/deb/src/go.mod b/install/deb/src/go.mod index 5d8e2d2..d5b1dd7 100644 --- a/install/deb/src/go.mod +++ b/install/deb/src/go.mod @@ -3,7 +3,12 @@ module main go 1.16 require ( + huawei.com/mindx/common/hwlog v0.0.0 mindxcheckutils v1.0.0 ) -replace mindxcheckutils => ../../../mindxcheckutils +replace ( + huawei.com/mindx/common/hwlog => codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3 + huawei.com/mindx/common/utils => codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6 + mindxcheckutils => ../../../mindxcheckutils +) diff --git a/install/deb/src/go.sum b/install/deb/src/go.sum new file mode 100644 index 0000000..3e15b58 --- /dev/null +++ b/install/deb/src/go.sum @@ -0,0 +1,25 @@ +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3 h1:Dq2tZTuClykBv1tS6C2XU6hY/waVK0K5lyr/8tJcT/I= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3/go.mod h1:tuvVYh4aDcHqpiyNKZNiBgt1XBehJ130VubLidKWNjY= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6 h1:nGlHkoj7L6DdSYFFLjceD6t5XrNAKrM+Gk3wT0iOfWE= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6/go.mod h1:gzaoiHwloaNTtm46GVe93KtXMcXVHJ8LBVqWEAjmh+0= +github.com/agiledragon/gomonkey/v2 v2.8.0 h1:u2K2nNGyk0ippzklz1CWalllEB9ptD+DtSXeCX5O000= +github.com/agiledragon/gomonkey/v2 v2.8.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/install/deb/src/main.go b/install/deb/src/main.go index 327d0e9..90ff7be 100644 --- a/install/deb/src/main.go +++ b/install/deb/src/main.go @@ -4,6 +4,7 @@ package main import ( + "context" "encoding/json" "flag" "fmt" @@ -11,6 +12,9 @@ import ( "log" "os" "path/filepath" + "strings" + + "huawei.com/mindx/common/hwlog" "mindxcheckutils" ) @@ -33,25 +37,52 @@ const ( rmCommandLength = 3 addCommandLength = 4 addCommand = "add" + maxCommandLength = 65535 + logPath = "/var/log/ascend-docker-runtime/installer.log" rmCommand = "rm" maxFileSize = 1024 * 1024 * 10 ) func main() { - defer func() { - if err := recover(); err != nil { - log.Fatal(err) - } - }() - log.SetPrefix("LOG: ") - log.Printf("running %s", os.Args) - err := process() - if err != nil { - log.Printf("run %s failed", os.Args) + ctx, _ := context.WithCancel(context.Background()) + if err := initLogModule(ctx); err != nil { log.Fatal(err) - } else { - log.Printf("run %s success", os.Args) } + logPrefixWords, err := mindxcheckutils.GetLogPrefix() + if err != nil { + log.Fatal(err) + } + hwlog.OpLog.Infof("%v installer started", logPrefixWords) + + if !mindxcheckutils.StringChecker(strings.Join(os.Args, " "), 0, + maxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { + hwlog.OpLog.Infof("%v run failed", logPrefixWords) + log.Fatal("command error") + } + + err = process() + if err != nil { + hwlog.OpLog.Infof("%v run %s failed", logPrefixWords, os.Args) + log.Fatal(fmt.Errorf("error in installation")) + } else { + hwlog.OpLog.Infof("%v run %s success", logPrefixWords, os.Args) + } +} + +func initLogModule(ctx context.Context) error { + const backups = 2 + const logMaxAge = 365 + logConfig := hwlog.LogConfig{ + LogFileName: logPath, + LogLevel: 0, + MaxBackups: backups, + MaxAge: logMaxAge, + } + if err := hwlog.InitOperateLogger(&logConfig, ctx); err != nil { + fmt.Printf("hwlog init failed, error is %v", err) + return err + } + return nil } func process() error { @@ -101,14 +132,14 @@ func process() error { } // check file permission - writeContent, err := createJSONString(srcFilePath, runtimeFilePath, action) + writeContent, err := createJsonString(srcFilePath, runtimeFilePath, action) if err != nil { return err } - return writeJSON(destFilePath, writeContent) + return writeJson(destFilePath, writeContent) } -func createJSONString(srcFilePath, runtimeFilePath, action string) ([]byte, error) { +func createJsonString(srcFilePath, runtimeFilePath, action string) ([]byte, error) { var writeContent []byte if _, err := os.Stat(srcFilePath); err == nil { daemon, err := modifyDaemon(srcFilePath, runtimeFilePath, action) @@ -128,7 +159,7 @@ func createJSONString(srcFilePath, runtimeFilePath, action string) ([]byte, erro return writeContent, nil } -func writeJSON(destFilePath string, writeContent []byte) error { +func writeJson(destFilePath string, writeContent []byte) error { if _, err := os.Stat(destFilePath); os.IsNotExist(err) { const perm = 0600 file, err := os.OpenFile(destFilePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm) @@ -145,13 +176,14 @@ func writeJSON(destFilePath string, writeContent []byte) error { return fmt.Errorf("close target file failed") } return nil + } else { + return fmt.Errorf("target file already existed") } - return fmt.Errorf("target file already existed") } func modifyDaemon(srcFilePath, runtimeFilePath, action string) (map[string]interface{}, error) { // existed... - daemon, err := loadOriginJSON(srcFilePath) + daemon, err := loadOriginJson(srcFilePath) if err != nil { return nil, err } @@ -190,7 +222,7 @@ func modifyDaemon(srcFilePath, runtimeFilePath, action string) (map[string]inter return daemon, nil } -func loadOriginJSON(srcFilePath string) (map[string]interface{}, error) { +func loadOriginJson(srcFilePath string) (map[string]interface{}, error) { if fileInfo, err := os.Stat(srcFilePath); err != nil { return nil, err } else if fileInfo.Size() > maxFileSize { diff --git a/mindxcheckutils/mindxcheckutils.go b/mindxcheckutils/mindxcheckutils.go index 2643165..9418f7c 100644 --- a/mindxcheckutils/mindxcheckutils.go +++ b/mindxcheckutils/mindxcheckutils.go @@ -24,7 +24,11 @@ const ( // DefaultStringSize default string max length DefaultStringSize = 256 // DefaultPathSize default string max length - DefaultPathSize = 4096 + DefaultPathSize = 4096 + runLogDir = "/var/log/ascend-docker-runtime/" + backupLogFileMode os.FileMode = 0400 + runLogFileMode os.FileMode = 0750 + maxFileNum = 32 ) var logPrefix = "" @@ -206,3 +210,37 @@ func StringChecker(text string, minLength, maxLength int, whiteList string) bool } return true } + +// ChangeRuntimeLogMode change log mode +func ChangeRuntimeLogMode(runLog, operLog string) error { + runLogDirLen := len(runLogDir) + var logMode os.FileMode + counter := 0 + err := filepath.Walk(runLogDir, func(fileOrPath string, fileInfo os.FileInfo, err error) error { + counter += 1 + if counter > maxFileNum { + return fmt.Errorf("the counter file is over maxFileNum") + } + if err != nil { + fmt.Printf("prevent panic by handling failure accessing a path %q: %v\n", fileOrPath, err) + return err + } + hasLogPrefix := strings.HasPrefix(fileOrPath[runLogDirLen:], + runLog) || strings.HasPrefix(fileOrPath[runLogDirLen:], operLog) + if !hasLogPrefix { + return nil + } + logMode = backupLogFileMode + if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { + return fmt.Errorf("the file or path is symlink") + } + if errChmod := os.Chmod(fileOrPath, logMode); errChmod != nil { + return fmt.Errorf("set file mode %s failed", fileOrPath) + } + return nil + }) + if err != nil { + return fmt.Errorf("traversal runLogDir failed") + } + return nil +} diff --git a/runtime/go.mod b/runtime/go.mod index f923acc..bb1ddad 100644 --- a/runtime/go.mod +++ b/runtime/go.mod @@ -5,29 +5,14 @@ go 1.16 require ( github.com/opencontainers/runtime-spec v1.0.2 github.com/prashantv/gostub v1.1.0 - huawei.com/npu-exporter v0.2.14 + github.com/stretchr/testify v1.8.0 // indirect + huawei.com/mindx/common/hwlog v0.0.0 mindxcheckutils v1.0.0 ) replace ( - mindxcheckutils => ../mindxcheckutils - huawei.com/kmc => codehub-dg-y.huawei.com/it-edge-native/edge-native-core/coastguard.git v1.0.6 - huawei.com/npu-exporter => codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/npu-exporter.git v0.2.13 - k8s.io/api v0.0.0 => codehub-dg-y.huawei.com/OpenSourceCenter/kubernetes.git/staging/src/k8s.io/api v1.19.4-h4 - k8s.io/apimachinery => codehub-dg-y.huawei.com/OpenSourceCenter/kubernetes.git/staging/src/k8s.io/apimachinery v1.19.4-h4 - k8s.io/client-go => codehub-dg-y.huawei.com/OpenSourceCenter/kubernetes.git/staging/src/k8s.io/client-go v1.19.4-h4 - k8s.io/cri-api => codehub-dg-y.huawei.com/OpenSourceCenter/kubernetes.git/staging/src/k8s.io/cri-api v1.19.4-h4 - github.com/agiledragon/gomonkey/v2 => github.com/agiledragon/gomonkey/v2 v2.2.0 - github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify v1.5.1 - github.com/golang/protobuf => github.com/golang/protobuf v1.5.1 - github.com/patrickmn/go-cache => github.com/patrickmn/go-cache v2.1.1-0.20191004192108-46f407853014+incompatible - github.com/pkg/errors => github.com/pkg/errors v0.9.1 github.com/prashantv/gostub => github.com/prashantv/gostub v1.0.1-0.20191007164320-bbe3712b9c4a - github.com/prometheus/client_golang => github.com/prometheus/client_golang v1.11.1 - github.com/smartystreets/goconvey => github.com/smartystreets/goconvey v1.6.4 - github.com/stretchr/testify => github.com/stretchr/testify v1.7.0 - go.uber.org/zap => go.uber.org/zap v1.16.0 - golang.org/x/crypto => golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b - google.golang.org/grpc => google.golang.org/grpc v1.28.0 - gopkg.in/natefinch/lumberjack.v2 => gopkg.in/natefinch/lumberjack.v2 v2.0.0 + huawei.com/mindx/common/hwlog => codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3 + huawei.com/mindx/common/utils => codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6 + mindxcheckutils => ../mindxcheckutils ) diff --git a/runtime/go.sum b/runtime/go.sum index deb268c..7e0fec5 100644 --- a/runtime/go.sum +++ b/runtime/go.sum @@ -1,19 +1,43 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3 h1:Dq2tZTuClykBv1tS6C2XU6hY/waVK0K5lyr/8tJcT/I= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/hwlog v0.0.3/go.mod h1:tuvVYh4aDcHqpiyNKZNiBgt1XBehJ130VubLidKWNjY= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6 h1:nGlHkoj7L6DdSYFFLjceD6t5XrNAKrM+Gk3wT0iOfWE= +codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/common-utils.git/utils v0.0.6/go.mod h1:gzaoiHwloaNTtm46GVe93KtXMcXVHJ8LBVqWEAjmh+0= +github.com/agiledragon/gomonkey/v2 v2.8.0 h1:u2K2nNGyk0ippzklz1CWalllEB9ptD+DtSXeCX5O000= +github.com/agiledragon/gomonkey/v2 v2.8.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= -github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/prashantv/gostub v1.0.1-0.20191007164320-bbe3712b9c4a h1:tkiehFUSAXqIwMzuwKutcjSIZhpc3OKax/c9oKDz5mY= +github.com/prashantv/gostub v1.0.1-0.20191007164320-bbe3712b9c4a/go.mod h1:dP1v6T1QzyGJJKFocwAU0lSZKpfjstjH8TlhkEU0on0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/npu-exporter.git v0.2.7 h1:yKnqOCGmdknfWhfDBqSKYWuioQzdiXz7VeKh56EGhjw= -codehub-dg-y.huawei.com/MindX_DL/AtlasEnableWarehouse/npu-exporter.git v0.2.7/go.mod h1:sUdarWqC4+C7fr0rpGUNvx1n1JixILyKOlS1gDfpi/w= -codehub-dg-y.huawei.com/it-edge-native/edge-native-core/coastguard.git v1.0.6 h1:QO4JTYr7hsyjd8VeE9ARivO3iIes/1GS6DkLiS5w3Xo= -codehub-dg-y.huawei.com/it-edge-native/edge-native-core/coastguard.git v1.0.6/go.mod h1:mbUvvh/0ox+lNtLoraCsf+NMpTCL2h4L/NtfboraaH4= \ No newline at end of file +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/runtime/main.go b/runtime/main.go index c2fedd5..5a57a3d 100644 --- a/runtime/main.go +++ b/runtime/main.go @@ -5,6 +5,7 @@ package main import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -18,9 +19,8 @@ import ( "syscall" "github.com/opencontainers/runtime-spec/specs-go" - "go.uber.org/zap" - "huawei.com/npu-exporter/hwlog" + "huawei.com/mindx/common/hwlog" "mindxcheckutils" ) @@ -66,7 +66,7 @@ func getArgs() (*args, error) { return args, nil } -func initLogModule(stopCh <-chan struct{}) error { +func initLogModule(ctx context.Context) error { const backups = 2 const logMaxAge = 365 runLogConfig := hwlog.LogConfig{ @@ -76,7 +76,7 @@ func initLogModule(stopCh <-chan struct{}) error { MaxAge: logMaxAge, OnlyToFile: true, } - if err := hwlog.InitRunLogger(&runLogConfig, stopCh); err != nil { + if err := hwlog.InitRunLogger(&runLogConfig, ctx); err != nil { fmt.Printf("hwlog init failed, error is %v", err) return err } @@ -87,7 +87,7 @@ func initLogModule(stopCh <-chan struct{}) error { MaxAge: logMaxAge, OnlyToFile: true, } - if err := hwlog.InitOperateLogger(&operateLogConfig, stopCh); err != nil { + if err := hwlog.InitOperateLogger(&operateLogConfig, ctx); err != nil { fmt.Printf("hwlog init failed, error is %v", err) return err } @@ -110,6 +110,10 @@ var execRunc = func() error { return err } + hwlog.OpLog.Infof("ascend docker runtime success, will start runc") + if err := mindxcheckutils.ChangeRuntimeLogMode("runtime-run-", "runtime-operate-"); err != nil { + return err + } if err = syscall.Exec(runcPath, append([]string{runcPath}, os.Args[1:]...), os.Environ()); err != nil { return fmt.Errorf("failed to exec runc: %v", err) } @@ -282,34 +286,35 @@ func main() { log.Fatal(err) } }() - stopCh := make(chan struct{}) - if err := initLogModule(stopCh); err != nil { - close(stopCh) + ctx, _ := context.WithCancel(context.Background()) + if err := initLogModule(ctx); err != nil { log.Fatal(err) } logPrefixWords, err := mindxcheckutils.GetLogPrefix() if err != nil { - close(stopCh) log.Fatal(err) } - hwlog.RunLog.ZapLogger = hwlog.RunLog.ZapLogger.With(zap.String("user-info", logPrefixWords)) - hwlog.OpLog.ZapLogger = hwlog.OpLog.ZapLogger.With(zap.String("user-info", logPrefixWords)) + defer func() { + if err := mindxcheckutils.ChangeRuntimeLogMode("runtime-run-", "runtime-operate-"); err != nil { + fmt.Println("defer changeFileMode function failed") + } + }() hwlog.RunLog.Infof("ascend docker runtime starting") + hwlog.OpLog.Infof("%v ascend docker runtime starting, try to setup container", logPrefixWords) if !mindxcheckutils.StringChecker(strings.Join(os.Args, " "), 0, maxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { - close(stopCh) + hwlog.RunLog.Infof("%v ascend docker runtime failed", logPrefixWords) + hwlog.OpLog.Errorf("%v ascend docker runtime failed", logPrefixWords) log.Fatal("command error") } - logWords := fmt.Sprintf("running %v", os.Args) + logWords := fmt.Sprintf("%v running %v", logPrefixWords, os.Args) if len(logWords) > maxLogLength { logWords = logWords[0:maxLogLength-1] + "..." } hwlog.OpLog.Infof(logWords) if err := doProcess(); err != nil { hwlog.RunLog.Errorf("%v ascend docker runtime failed", logPrefixWords) - hwlog.OpLog.Errorf("%v failed", logWords) - close(stopCh) + hwlog.OpLog.Errorf("%v %v failed", logPrefixWords, logWords) log.Fatal(err) } - close(stopCh) }