zishuo/ascend-docker-runtime
1
0
Fork 0
mirror of https://github.com/Ascend/ascend-docker-runtime.git synced 2025-10-12 07:10:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Match-id-612f96a0f3e7311aa7703fa29b0f0820baef3f46

Browse Source
This commit is contained in:
BianTanggui
2023-03-10 22:48:18 +08:00
parent 7794dc2680
commit e9b3dbd230
5 changed files with 207 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cli/src/main.c
View File

@@ -305,7 +305,6 @@ static struct {
const char c;
CmdArgParser parser;
} g_cmdArgParsers[NUM_OF_CMD_ARGS] = {
{'d', DevicesCmdArgParser},
{'p', PidCmdArgParser},
{'r', RootfsCmdArgParser},
{'o', OptionsCmdArgParser},
@@ -343,7 +342,7 @@ static inline bool IsCmdArgsValid(const struct CmdArgs *args)
Logger("args pointer is null!", LEVEL_ERROR, SCREEN_YES);
return false;
}
return (strlen(args->devices) > 0) && (strlen(args->rootfs) > 0) && (args->pid > 0);
return (strlen(args->rootfs) > 0) && (args->pid > 0);
}
static int ParseDeviceIDs(size_t *idList, size_t *idListSize, char *devices)
@@ -476,8 +475,7 @@ int SetupContainer(struct CmdArgs *args)
close(config.originNsFd);
return -1;
}
Logger("setup up cgroup", LEVEL_INFO, SCREEN_YES);
ret = SetupCgroup(&config);
if (ret < 0) {
Logger("failed to set up cgroup.", LEVEL_ERROR, SCREEN_YES);
close(config.originNsFd);

13
cli/src/u_mount.c
View File

@@ -413,19 +413,6 @@ int DoMounting(const struct ParsedConfig *config)
}
int ret;
ret = DoDeviceMounting(config->rootfs,
(IsVirtual() ? VDEVICE_NAME : DEVICE_NAME),
config->devices, config->devicesNr);
if (ret < 0) {
Logger("failed to mount devices.", LEVEL_ERROR, SCREEN_YES);
return -1;
}
ret = DoCtrlDeviceMounting(config->rootfs);
if (ret < 0) {
Logger("failed to mount ctrl devices.", LEVEL_ERROR, SCREEN_YES);
return -1;
}
if (IsOptionNoDrvSet()) {
return 0;

83
hook/main.go
View File

@@ -24,8 +24,6 @@ import (
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
@@ -40,7 +38,6 @@ const (
loggingPrefix = "ascend-docker-hook"
runLogPath = "/var/log/ascend-docker-runtime/hook-run.log"
operateLogPath = "/var/log/ascend-docker-runtime/hook-operate.log"
ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES"
ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS"
ascendRuntimeMounts = "ASCEND_RUNTIME_MOUNTS"
ascendDockerCli = "ascend-docker-cli"
@@ -102,68 +99,6 @@ func initLogModule(ctx context.Context) error {
return nil
}
func removeDuplication(devices []int) []int {
list := make([]int, 0, len(devices))
prev := -1
for _, device := range devices {
if device == prev {
continue
}
list = append(list, device)
prev = device
}
return list
}
func parseDevices(visibleDevices string) ([]int, error) {
devices := make([]int, 0)
const maxDevice = 128
for _, d := range strings.Split(visibleDevices, ",") {
d = strings.TrimSpace(d)
if strings.Contains(d, "-") {
borders := strings.Split(d, "-")
if len(borders) != borderNum {
return nil, fmt.Errorf("invalid device range: %s", d)
}
borders[0] = strings.TrimSpace(borders[0])
borders[1] = strings.TrimSpace(borders[1])
left, err := strconv.Atoi(borders[0])
if err != nil || left < 0 {
return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0])
}
right, err := strconv.Atoi(borders[1])
if err != nil || right > maxDevice {
return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1])
}
if left > right {
return nil, fmt.Errorf("left boarder (%d) should not be larger than the right one(%d)", left, right)
}
for n := left; n <= right; n++ {
devices = append(devices, n)
}
} else {
n, err := strconv.Atoi(d)
if err != nil {
return nil, fmt.Errorf("invalid single device parameter: %s", d)
}
devices = append(devices, n)
}
}
sort.Slice(devices, func(i, j int) bool { return i < j })
return removeDuplication(devices), nil
}
func parseMounts(mounts string) []string {
if mounts == "" {
return []string{baseConfig}
@@ -372,10 +307,10 @@ func readConfigsOfDir(dir string, configs []string) ([]string, []string, error)
return fileMountList, dirMountList, nil
}
func getArgs(cliPath string, devices []int, containerConfig *containerConfig,
fileMountList []string, dirMountList []string) []string {
func getArgs(cliPath string, containerConfig *containerConfig, fileMountList []string,
dirMountList []string) []string {
args := append([]string{cliPath},
"--devices", strings.Trim(strings.Join(strings.Fields(fmt.Sprint(devices)), ","), "[]"),
"--pid", fmt.Sprintf("%d", containerConfig.Pid), "--rootfs", containerConfig.Rootfs)
for _, filePath := range fileMountList {
args = append(args, "--mount-file", filePath)
@@ -392,16 +327,6 @@ func doPrestartHook() error {
return fmt.Errorf("failed to get container config: %v", err)
}
visibleDevices := getValueByKey(containerConfig.Env, ascendVisibleDevices)
if visibleDevices == "" {
return nil
}
devices, err := parseDevices(visibleDevices)
if err != nil {
return fmt.Errorf("failed to parse device setting: %v", err)
}
mountConfigs := parseMounts(getValueByKey(containerConfig.Env, ascendRuntimeMounts))
fileMountList, dirMountList, err := readConfigsOfDir(configDir, mountConfigs)
@@ -426,7 +351,7 @@ func doPrestartHook() error {
if _, err := mindxcheckutils.RealFileChecker(cliPath, true, false, mindxcheckutils.DefaultSize); err != nil {
return err
}
args := getArgs(cliPath, devices, containerConfig, fileMountList, dirMountList)
args := getArgs(cliPath, containerConfig, fileMountList, dirMountList)
if len(parsedOptions) > 0 {
args = append(args, "--options", strings.Join(parsedOptions, ","))
}

22
runtime/go.mod
View File

@@ -3,6 +3,7 @@ module main
go 1.17
require (
github.com/containerd/containerd v1.6.19
github.com/opencontainers/runtime-spec v1.0.3-0.20220718201635-a8106e99982b
github.com/prashantv/gostub v1.1.0
huawei.com/npu-exporter/v3 v3.0.0
@@ -10,12 +11,31 @@ require (
)
require (
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Microsoft/hcsshim v0.9.7 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/containerd/continuity v0.3.0 // indirect
github.com/containerd/ttrpc v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/gopherjs/gopherjs v1.17.2 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/klauspost/compress v1.11.13 // indirect
github.com/moby/sys/mountinfo v0.5.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/opencontainers/runc v1.1.2 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/smartystreets/assertions v1.13.0 // indirect
github.com/stretchr/testify v1.8.0 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
google.golang.org/grpc v1.47.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
)
replace (

182
runtime/main.go
View File

@@ -25,9 +25,12 @@ import (
"os/exec"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"github.com/containerd/containerd/oci"
"github.com/opencontainers/runtime-spec/specs-go"
"huawei.com/npu-exporter/v3/common-utils/hwlog"
@@ -38,18 +41,27 @@ import (
const (
runLogPath = "/var/log/ascend-docker-runtime/runtime-run.log"
operateLogPath = "/var/log/ascend-docker-runtime/runtime-operate.log"
maxLogLength = 1024
hookDefaultFilePath = "/usr/local/bin/ascend-docker-hook"
devicePath = "/dev/"
davinciName = "davinci"
davinciManager = "davinci_manager"
devmmSvm = "devmm_svm"
hisiHdc = "hisi_hdc"
maxCommandLength = 65535
hookCli = "ascend-docker-hook"
destroyHookCli = "ascend-docker-destroy"
hookDefaultFilePath = "/usr/local/bin/ascend-docker-hook"
dockerRuncFile = "docker-runc"
runcFile = "runc"
envLength = 2
kvPairSize = 2
borderNum = 2
vdeviceIdlen = 3
// ENV for device-plugin to identify ascend-docker-runtime
useAscendDocker = "ASCEND_DOCKER_RUNTIME=True"
devicePlugin = "ascend-device-plugin"
ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES"
ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS"
)
var (
@@ -212,16 +224,174 @@ func addHook(spec *specs.Spec) error {
return nil
}
func removeDuplication(devices []int) []int {
list := make([]int, 0, len(devices))
prev := -1
for _, device := range devices {
if device == prev {
continue
}
list = append(list, device)
prev = device
}
return list
}
func parseDevices(visibleDevices string) ([]int, error) {
devices := make([]int, 0)
const maxDevice = 128
for _, d := range strings.Split(visibleDevices, ",") {
d = strings.TrimSpace(d)
if strings.Contains(d, "-") {
borders := strings.Split(d, "-")
if len(borders) != borderNum {
return nil, fmt.Errorf("invalid device range: %s", d)
}
borders[0] = strings.TrimSpace(borders[0])
borders[1] = strings.TrimSpace(borders[1])
left, err := strconv.Atoi(borders[0])
if err != nil || left < 0 {
return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0])
}
right, err := strconv.Atoi(borders[1])
if err != nil || right > maxDevice {
return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1])
}
if left > right {
return nil, fmt.Errorf("left boarder (%d) should not be larger than the right one(%d)", left, right)
}
for n := left; n <= right; n++ {
devices = append(devices, n)
}
} else {
n, err := strconv.Atoi(d)
if err != nil {
return nil, fmt.Errorf("invalid single device parameter: %s", d)
}
devices = append(devices, n)
}
}
sort.Slice(devices, func(i, j int) bool { return i < j })
return removeDuplication(devices), nil
}
func getValueByKey(data []string, name string) string {
for _, envLine := range data {
words := strings.SplitN(envLine, "=", kvPairSize)
if len(words) != kvPairSize {
hwlog.RunLog.Errorf("environment error")
return ""
}
if words[0] == name {
return words[1]
}
}
return ""
}
func addDeviceToSpec(spec *specs.Spec, dPath string, vdevice bool) error {
device, err := oci.DeviceFromPath(dPath)
if err != nil {
return fmt.Errorf("failed to get device info : %#v", err)
}
lenPath := len(dPath)
if vdevice {
vPath := devicePath + davinciName + dPath[lenPath-vdeviceIdlen:]
device.Path = vPath
}
spec.Linux.Devices = append(spec.Linux.Devices, *device)
newDeviceCgroup := specs.LinuxDeviceCgroup{
Allow: true,
Type: device.Type,
Major: &device.Major,
Minor: &device.Minor,
Access: "rwm",
}
spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, newDeviceCgroup)
return nil
}
func addManagerDevice(spec *specs.Spec) error {
managerPath := devicePath + davinciManager
if err := addDeviceToSpec(spec, managerPath, false); err != nil {
return fmt.Errorf("failed to add manager device to spec : %#v", err)
}
svmPath := devicePath + devmmSvm
if _, err := os.Stat(svmPath); err == nil {
if err = addDeviceToSpec(spec, svmPath, false); err != nil {
return fmt.Errorf("failed to add devmm_svm to spec : %#v", err)
}
}
hdcPath := devicePath + hisiHdc
if _, err := os.Stat(hdcPath); err == nil {
if err = addDeviceToSpec(spec, hdcPath, false); err != nil {
return fmt.Errorf("failed to add hisi_hdc device to spec : %#v", err)
}
}
return nil
}
func addDevice(spec *specs.Spec) error {
visibleDevices := getValueByKey(spec.Process.Env, ascendVisibleDevices)
if visibleDevices == "" {
return nil
}
hwlog.RunLog.Infof("getValueByKey visibleDevices: %#v", visibleDevices)
devices, err := parseDevices(visibleDevices)
if err != nil {
return fmt.Errorf("failed to parse device : %#v", err)
}
hwlog.RunLog.Infof("devices is: %#v", devices)
devciename := davinciName
vdevice := false
virtual := getValueByKey(spec.Process.Env, ascendRuntimeOptions)
if strings.Contains(virtual, "VIRTUAL") {
devciename = "v" + devciename
vdevice = true
}
for _, deviceId := range devices {
dPath := devicePath + devciename + strconv.Itoa(deviceId)
if err = addDeviceToSpec(spec, dPath, vdevice); err != nil {
return fmt.Errorf("failed to add davinci device to spec: %v", err)
}
}
if err = addManagerDevice(spec); err != nil {
return fmt.Errorf("failed to add Manager device to spec: %v", err)
}
return nil
}
func updateEnvAndPostHook(spec *specs.Spec, vdevice dcmi.VDeviceInfo) {
newEnv := make([]string, 0)
needAddVirtualFlag := true
for _, line := range spec.Process.Env {
words := strings.Split(line, "=")
if len(words) == envLength && strings.TrimSpace(words[0]) == "ASCEND_VISIBLE_DEVICES" {
if len(words) == envLength && strings.TrimSpace(words[0]) == ascendVisibleDevices {
newEnv = append(newEnv, fmt.Sprintf("ASCEND_VISIBLE_DEVICES=%d", vdevice.VdeviceID))
continue
}
if len(words) == envLength && strings.TrimSpace(words[0]) == "ASCEND_RUNTIME_OPTIONS" {
if len(words) == envLength && strings.TrimSpace(words[0]) == ascendRuntimeOptions {
needAddVirtualFlag = false
if strings.Contains(words[1], "VIRTUAL") {
newEnv = append(newEnv, line)
@@ -284,6 +454,10 @@ func modifySpecFile(path string) error {
return fmt.Errorf("failed to inject hook: %v", err)
}
if err = addDevice(&spec); err != nil {
return fmt.Errorf("failed to add device to env: %v", err)
}
addEnvToDevicePlugin(&spec)
jsonOutput, err := json.Marshal(spec)