Match-id-612f96a0f3e7311aa7703fa29b0f0820baef3f46

cli/src/main.c

@@ -305,7 +305,6 @@ static struct {
     const char c;
     CmdArgParser parser;
 } g_cmdArgParsers[NUM_OF_CMD_ARGS] = {
-    {'d', DevicesCmdArgParser},
     {'p', PidCmdArgParser},
     {'r', RootfsCmdArgParser},
     {'o', OptionsCmdArgParser},
@@ -343,7 +342,7 @@ static inline bool IsCmdArgsValid(const struct CmdArgs *args)
         Logger("args pointer is null!", LEVEL_ERROR, SCREEN_YES);
         return false;
     }
-    return (strlen(args->devices) > 0) && (strlen(args->rootfs) > 0) && (args->pid > 0);
+    return (strlen(args->rootfs) > 0) && (args->pid > 0);
 }
 
 static int ParseDeviceIDs(size_t *idList, size_t *idListSize, char *devices)
@@ -476,8 +475,7 @@ int SetupContainer(struct CmdArgs *args)
         close(config.originNsFd);
         return -1;
     }
-    Logger("setup up cgroup", LEVEL_INFO, SCREEN_YES);
+    
-    ret = SetupCgroup(&config);
     if (ret < 0) {
         Logger("failed to set up cgroup.", LEVEL_ERROR, SCREEN_YES);
         close(config.originNsFd);

cli/src/u_mount.c

@@ -413,19 +413,6 @@ int DoMounting(const struct ParsedConfig *config)
     }
 
     int ret;
-    ret = DoDeviceMounting(config->rootfs,
-                           (IsVirtual() ? VDEVICE_NAME : DEVICE_NAME),
-                           config->devices, config->devicesNr);
-    if (ret < 0) {
-        Logger("failed to mount devices.", LEVEL_ERROR, SCREEN_YES);
-        return -1;
-    }
-
-    ret = DoCtrlDeviceMounting(config->rootfs);
-    if (ret < 0) {
-        Logger("failed to mount ctrl devices.", LEVEL_ERROR, SCREEN_YES);
-        return -1;
-    }
 
     if (IsOptionNoDrvSet()) {
         return 0;

hook/main.go

@@ -24,8 +24,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"sort"
-	"strconv"
 	"strings"
 	"syscall"
 
@@ -40,7 +38,6 @@ const (
 	loggingPrefix          = "ascend-docker-hook"
 	runLogPath             = "/var/log/ascend-docker-runtime/hook-run.log"
 	operateLogPath         = "/var/log/ascend-docker-runtime/hook-operate.log"
-	ascendVisibleDevices   = "ASCEND_VISIBLE_DEVICES"
 	ascendRuntimeOptions   = "ASCEND_RUNTIME_OPTIONS"
 	ascendRuntimeMounts    = "ASCEND_RUNTIME_MOUNTS"
 	ascendDockerCli        = "ascend-docker-cli"
@@ -102,68 +99,6 @@ func initLogModule(ctx context.Context) error {
 	return nil
 }
 
-func removeDuplication(devices []int) []int {
-	list := make([]int, 0, len(devices))
-	prev := -1
-
-	for _, device := range devices {
-		if device == prev {
-			continue
-		}
-
-		list = append(list, device)
-		prev = device
-	}
-
-	return list
-}
-
-func parseDevices(visibleDevices string) ([]int, error) {
-	devices := make([]int, 0)
-	const maxDevice = 128
-
-	for _, d := range strings.Split(visibleDevices, ",") {
-		d = strings.TrimSpace(d)
-		if strings.Contains(d, "-") {
-			borders := strings.Split(d, "-")
-			if len(borders) != borderNum {
-				return nil, fmt.Errorf("invalid device range: %s", d)
-			}
-
-			borders[0] = strings.TrimSpace(borders[0])
-			borders[1] = strings.TrimSpace(borders[1])
-
-			left, err := strconv.Atoi(borders[0])
-			if err != nil || left < 0 {
-				return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0])
-			}
-
-			right, err := strconv.Atoi(borders[1])
-			if err != nil || right > maxDevice {
-				return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1])
-			}
-
-			if left > right {
-				return nil, fmt.Errorf("left boarder (%d) should not be larger than the right one(%d)", left, right)
-			}
-
-			for n := left; n <= right; n++ {
-				devices = append(devices, n)
-			}
-		} else {
-			n, err := strconv.Atoi(d)
-			if err != nil {
-				return nil, fmt.Errorf("invalid single device parameter: %s", d)
-			}
-
-			devices = append(devices, n)
-		}
-	}
-
-	sort.Slice(devices, func(i, j int) bool { return i < j })
-	return removeDuplication(devices), nil
-}
-
 func parseMounts(mounts string) []string {
 	if mounts == "" {
 		return []string{baseConfig}
@@ -372,10 +307,10 @@ func readConfigsOfDir(dir string, configs []string) ([]string, []string, error)
 	return fileMountList, dirMountList, nil
 }
 
-func getArgs(cliPath string, devices []int, containerConfig *containerConfig,
+func getArgs(cliPath string, containerConfig *containerConfig, fileMountList []string,
-	fileMountList []string, dirMountList []string) []string {
+	dirMountList []string) []string {
 	args := append([]string{cliPath},
-		"--devices", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(devices)), ","), "[]"),
+
 		"--pid", fmt.Sprintf("%d", containerConfig.Pid), "--rootfs", containerConfig.Rootfs)
 	for _, filePath := range fileMountList {
 		args = append(args, "--mount-file", filePath)
@@ -392,16 +327,6 @@ func doPrestartHook() error {
 		return fmt.Errorf("failed to get container config: %v", err)
 	}
 
-	visibleDevices := getValueByKey(containerConfig.Env, ascendVisibleDevices)
-	if visibleDevices == "" {
-		return nil
-	}
-
-	devices, err := parseDevices(visibleDevices)
-	if err != nil {
-		return fmt.Errorf("failed to parse device setting: %v", err)
-	}
-
 	mountConfigs := parseMounts(getValueByKey(containerConfig.Env, ascendRuntimeMounts))
 
 	fileMountList, dirMountList, err := readConfigsOfDir(configDir, mountConfigs)
@@ -426,7 +351,7 @@ func doPrestartHook() error {
 	if _, err := mindxcheckutils.RealFileChecker(cliPath, true, false, mindxcheckutils.DefaultSize); err != nil {
 		return err
 	}
-	args := getArgs(cliPath, devices, containerConfig, fileMountList, dirMountList)
+	args := getArgs(cliPath, containerConfig, fileMountList, dirMountList)
 	if len(parsedOptions) > 0 {
 		args = append(args, "--options", strings.Join(parsedOptions, ","))
 	}

runtime/go.mod

@@ -3,6 +3,7 @@ module main
 go 1.17
 
 require (
+	github.com/containerd/containerd v1.6.19
 	github.com/opencontainers/runtime-spec v1.0.3-0.20220718201635-a8106e99982b
 	github.com/prashantv/gostub v1.1.0
 	huawei.com/npu-exporter/v3 v3.0.0
@@ -10,12 +11,31 @@ require (
 )
 
 require (
+	github.com/Microsoft/go-winio v0.5.2 // indirect
+	github.com/Microsoft/hcsshim v0.9.7 // indirect
+	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/containerd/continuity v0.3.0 // indirect
+	github.com/containerd/ttrpc v1.1.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
-	github.com/jtolds/gls v4.20.0+incompatible // indirect
+	github.com/klauspost/compress v1.11.13 // indirect
+	github.com/moby/sys/mountinfo v0.5.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
+	github.com/opencontainers/runc v1.1.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/smartystreets/assertions v1.13.0 // indirect
 	github.com/stretchr/testify v1.8.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect
+	google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
+	google.golang.org/grpc v1.47.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
 )
 
 replace (

runtime/main.go

@@ -25,9 +25,12 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"sort"
+	"strconv"
 	"strings"
 	"syscall"
 
+	"github.com/containerd/containerd/oci"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"huawei.com/npu-exporter/v3/common-utils/hwlog"
 
@@ -38,18 +41,27 @@ import (
 const (
 	runLogPath          = "/var/log/ascend-docker-runtime/runtime-run.log"
 	operateLogPath      = "/var/log/ascend-docker-runtime/runtime-operate.log"
-	maxLogLength        = 1024
+	hookDefaultFilePath = "/usr/local/bin/ascend-docker-hook"
+	devicePath          = "/dev/"
+	davinciName         = "davinci"
+	davinciManager      = "davinci_manager"
+	devmmSvm            = "devmm_svm"
+	hisiHdc             = "hisi_hdc"
 	maxCommandLength    = 65535
 	hookCli             = "ascend-docker-hook"
 	destroyHookCli      = "ascend-docker-destroy"
-	hookDefaultFilePath = "/usr/local/bin/ascend-docker-hook"
 	dockerRuncFile      = "docker-runc"
 	runcFile            = "runc"
 	envLength           = 2
+	kvPairSize          = 2
+	borderNum           = 2
+	vdeviceIdlen        = 3
 
 	// ENV for device-plugin to identify ascend-docker-runtime
-	useAscendDocker = "ASCEND_DOCKER_RUNTIME=True"
+	useAscendDocker      = "ASCEND_DOCKER_RUNTIME=True"
-	devicePlugin    = "ascend-device-plugin"
+	devicePlugin         = "ascend-device-plugin"
+	ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES"
+	ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS"
 )
 
 var (
@@ -212,16 +224,174 @@ func addHook(spec *specs.Spec) error {
 	return nil
 }
 
+func removeDuplication(devices []int) []int {
+	list := make([]int, 0, len(devices))
+	prev := -1
+
+	for _, device := range devices {
+		if device == prev {
+			continue
+		}
+
+		list = append(list, device)
+		prev = device
+	}
+
+	return list
+}
+
+func parseDevices(visibleDevices string) ([]int, error) {
+	devices := make([]int, 0)
+	const maxDevice = 128
+
+	for _, d := range strings.Split(visibleDevices, ",") {
+		d = strings.TrimSpace(d)
+		if strings.Contains(d, "-") {
+			borders := strings.Split(d, "-")
+			if len(borders) != borderNum {
+				return nil, fmt.Errorf("invalid device range: %s", d)
+			}
+
+			borders[0] = strings.TrimSpace(borders[0])
+			borders[1] = strings.TrimSpace(borders[1])
+
+			left, err := strconv.Atoi(borders[0])
+			if err != nil || left < 0 {
+				return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0])
+			}
+
+			right, err := strconv.Atoi(borders[1])
+			if err != nil || right > maxDevice {
+				return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1])
+			}
+
+			if left > right {
+				return nil, fmt.Errorf("left boarder (%d) should not be larger than the right one(%d)", left, right)
+			}
+
+			for n := left; n <= right; n++ {
+				devices = append(devices, n)
+			}
+		} else {
+			n, err := strconv.Atoi(d)
+			if err != nil {
+				return nil, fmt.Errorf("invalid single device parameter: %s", d)
+			}
+
+			devices = append(devices, n)
+		}
+	}
+
+	sort.Slice(devices, func(i, j int) bool { return i < j })
+	return removeDuplication(devices), nil
+}
+
+func getValueByKey(data []string, name string) string {
+	for _, envLine := range data {
+		words := strings.SplitN(envLine, "=", kvPairSize)
+		if len(words) != kvPairSize {
+			hwlog.RunLog.Errorf("environment error")
+			return ""
+		}
+
+		if words[0] == name {
+			return words[1]
+		}
+	}
+
+	return ""
+}
+
+func addDeviceToSpec(spec *specs.Spec, dPath string, vdevice bool) error {
+	device, err := oci.DeviceFromPath(dPath)
+	if err != nil {
+		return fmt.Errorf("failed to get device info : %#v", err)
+	}
+
+	lenPath := len(dPath)
+	if vdevice {
+		vPath := devicePath + davinciName + dPath[lenPath-vdeviceIdlen:]
+		device.Path = vPath
+	}
+
+	spec.Linux.Devices = append(spec.Linux.Devices, *device)
+	newDeviceCgroup := specs.LinuxDeviceCgroup{
+		Allow:  true,
+		Type:   device.Type,
+		Major:  &device.Major,
+		Minor:  &device.Minor,
+		Access: "rwm",
+	}
+	spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, newDeviceCgroup)
+	return nil
+}
+
+func addManagerDevice(spec *specs.Spec) error {
+	managerPath := devicePath + davinciManager
+	if err := addDeviceToSpec(spec, managerPath, false); err != nil {
+		return fmt.Errorf("failed to add manager device to spec : %#v", err)
+	}
+
+	svmPath := devicePath + devmmSvm
+	if _, err := os.Stat(svmPath); err == nil {
+		if err = addDeviceToSpec(spec, svmPath, false); err != nil {
+			return fmt.Errorf("failed to add devmm_svm to spec : %#v", err)
+		}
+	}
+
+	hdcPath := devicePath + hisiHdc
+	if _, err := os.Stat(hdcPath); err == nil {
+		if err = addDeviceToSpec(spec, hdcPath, false); err != nil {
+			return fmt.Errorf("failed to add hisi_hdc device to spec : %#v", err)
+		}
+	}
+
+	return nil
+}
+
+func addDevice(spec *specs.Spec) error {
+	visibleDevices := getValueByKey(spec.Process.Env, ascendVisibleDevices)
+	if visibleDevices == "" {
+		return nil
+	}
+	hwlog.RunLog.Infof("getValueByKey visibleDevices: %#v", visibleDevices)
+
+	devices, err := parseDevices(visibleDevices)
+	if err != nil {
+		return fmt.Errorf("failed to parse device : %#v", err)
+	}
+	hwlog.RunLog.Infof("devices is: %#v", devices)
+	devciename := davinciName
+	vdevice := false
+	virtual := getValueByKey(spec.Process.Env, ascendRuntimeOptions)
+	if strings.Contains(virtual, "VIRTUAL") {
+		devciename = "v" + devciename
+		vdevice = true
+	}
+	for _, deviceId := range devices {
+		dPath := devicePath + devciename + strconv.Itoa(deviceId)
+		if err = addDeviceToSpec(spec, dPath, vdevice); err != nil {
+			return fmt.Errorf("failed to add davinci device to spec: %v", err)
+		}
+	}
+
+	if err = addManagerDevice(spec); err != nil {
+		return fmt.Errorf("failed to add Manager device to spec: %v", err)
+	}
+
+	return nil
+}
+
 func updateEnvAndPostHook(spec *specs.Spec, vdevice dcmi.VDeviceInfo) {
 	newEnv := make([]string, 0)
 	needAddVirtualFlag := true
 	for _, line := range spec.Process.Env {
 		words := strings.Split(line, "=")
-		if len(words) == envLength && strings.TrimSpace(words[0]) == "ASCEND_VISIBLE_DEVICES" {
+		if len(words) == envLength && strings.TrimSpace(words[0]) == ascendVisibleDevices {
 			newEnv = append(newEnv, fmt.Sprintf("ASCEND_VISIBLE_DEVICES=%d", vdevice.VdeviceID))
 			continue
 		}
-		if len(words) == envLength && strings.TrimSpace(words[0]) == "ASCEND_RUNTIME_OPTIONS" {
+		if len(words) == envLength && strings.TrimSpace(words[0]) == ascendRuntimeOptions {
 			needAddVirtualFlag = false
 			if strings.Contains(words[1], "VIRTUAL") {
 				newEnv = append(newEnv, line)
@@ -284,6 +454,10 @@ func modifySpecFile(path string) error {
 		return fmt.Errorf("failed to inject hook: %v", err)
 	}
 
+	if err = addDevice(&spec); err != nil {
+		return fmt.Errorf("failed to add device to env: %v", err)
+	}
+
 	addEnvToDevicePlugin(&spec)
 
 	jsonOutput, err := json.Marshal(spec)