diff --git a/build/build.sh b/build/build.sh index 169e926..c460b26 100644 --- a/build/build.sh +++ b/build/build.sh @@ -75,9 +75,10 @@ function build_run_package() /bin/cp -f {${RUNTIMESRCDIR},${HOOKSRCDIR},${INSTALLHELPERSRCDIR},${CLISRCDIR}}/build/ascend-docker* run_pkg /bin/cp -f scripts/uninstall.sh run_pkg + /bin/cp -f scripts/base.list run_pkg FILECNT=`ls -l run_pkg |grep "^-"|wc -l` echo "prepare package $FILECNT bins" - if [ $FILECNT -ne 5 ]; then + if [ $FILECNT -ne 6 ]; then exit 1 fi diff --git a/build/scripts/base.list b/build/scripts/base.list new file mode 100644 index 0000000..cf1c56f --- /dev/null +++ b/build/scripts/base.list @@ -0,0 +1,7 @@ +/usr/local/Ascend/driver/lib64 +/usr/local/Ascend/driver/tools +/usr/local/Ascend/driver/include +/usr/local/Ascend/add-ons +/usr/local/dcmi +/usr/local/bin/npu-smi +/var/log/npu/conf/slog/slog.conf \ No newline at end of file diff --git a/build/scripts/run_main.sh b/build/scripts/run_main.sh index abe9692..48bd27f 100644 --- a/build/scripts/run_main.sh +++ b/build/scripts/run_main.sh @@ -1,5 +1,6 @@ #!/bin/bash +ASCEND_RUNTIME_CONFIG_DIR=/etc/ascend-docker-runtime.d DOCKER_CONFIG_DIR=/etc/docker INSTALL_PATH=/usr/local/Ascend/Ascend-Docker-Runtime @@ -24,6 +25,13 @@ function install() cp -f ./uninstall.sh ${INSTALL_PATH}/script/uninstall.sh chmod 550 ${INSTALL_PATH}/script/uninstall.sh + if [ -d "${ASCEND_RUNTIME_CONFIG_DIR}" ]; then + rm -rf ${ASCEND_RUNTIME_CONFIG_DIR} + fi + mkdir -p ${ASCEND_RUNTIME_CONFIG_DIR} + cp -f ./base.list ${ASCEND_RUNTIME_CONFIG_DIR}/base.list + chmod 440 ${ASCEND_RUNTIME_CONFIG_DIR}/base.list + echo 'install executable files success' if [ ! -d "${DOCKER_CONFIG_DIR}" ]; then @@ -70,16 +78,23 @@ function upgrade() exit 1 fi + if [ ! -d "${ASCEND_RUNTIME_CONFIG_DIR}" ]; then + echo 'ERROR: the configuration directory does not exist' + exit 1 + fi + cp -f ./ascend-docker-runtime ${INSTALL_PATH}/ascend-docker-runtime cp -f ./ascend-docker-hook ${INSTALL_PATH}/ascend-docker-hook cp -f ./ascend-docker-cli ${INSTALL_PATH}/ascend-docker-cli cp -f ./ascend-docker-plugin-install-helper ${INSTALL_PATH}/ascend-docker-plugin-install-helper cp -f ./uninstall.sh ${INSTALL_PATH}/script/uninstall.sh + cp -f ./base.list ${ASCEND_RUNTIME_CONFIG_DIR}/base.list chmod 550 ${INSTALL_PATH}/ascend-docker-runtime chmod 550 ${INSTALL_PATH}/ascend-docker-hook chmod 550 ${INSTALL_PATH}/ascend-docker-cli chmod 550 ${INSTALL_PATH}/ascend-docker-plugin-install-helper chmod 550 ${INSTALL_PATH}/script/uninstall.sh + chmod 440 ${ASCEND_RUNTIME_CONFIG_DIR}/base.list echo 'upgrade ascend docker runtime success' } diff --git a/build/scripts/uninstall.sh b/build/scripts/uninstall.sh index 93ba8a2..cb4fbad 100644 --- a/build/scripts/uninstall.sh +++ b/build/scripts/uninstall.sh @@ -3,6 +3,7 @@ ROOT=$(cd `dirname $0`; pwd)/.. DST='/etc/docker/daemon.json' SRC="${DST}.${PPID}" +ASCEND_RUNTIME_CONFIG_DIR=/etc/ascend-docker-runtime.d if [ ! -f "${DST}" ]; then exit 0 @@ -15,3 +16,5 @@ if [ "$?" != "0" ]; then fi mv ${SRC} ${DST} + +rm -rf ${ASCEND_RUNTIME_CONFIG_DIR} diff --git a/cli/src/basic.h b/cli/src/basic.h index c97b389..3a628ac 100644 --- a/cli/src/basic.h +++ b/cli/src/basic.h @@ -7,23 +7,17 @@ #include #include +#include #define DEVICE_NAME "davinci" #define DAVINCI_MANAGER "davinci_manager" #define DEVMM_SVM "devmm_svm" #define HISI_HDC "hisi_hdc" -#define ASCEND_DRIVER_LIB64_PATH "/usr/local/Ascend/driver/lib64" -#define ASCEND_DRIVER_TOOLS_PATH "/usr/local/Ascend/driver/tools" -#define ASCEND_DRIVER_INC_PATH "/usr/local/Ascend/driver/include" -#define ASCEND_ADDONS_PATH "/usr/local/Ascend/add-ons" -#define ASCEND_DCMI_PATH "/usr/local/dcmi" -#define ASCEND_NPU_SMI_PATH "/usr/local/bin/npu-smi" -#define ASCEND_NPU_SMI_PATH_OLD "/usr/local/sbin/npu-smi" -#define ASCEND_SLOG_CONF_PATH "/var/log/npu/conf/slog/slog.conf" #define DEFAULT_DIR_MODE 0755 #define BUF_SIZE 1024 #define MAX_DEVICE_NR 1024 #define DEFAULT_LOG_FILE "/var/log/ascend-docker-runtime.log" +#define MAX_MOUNT_NR 512 #define ALLOW_PATH "/devices.allow" #define ROOT_GAP 4 @@ -38,6 +32,11 @@ struct PathInfo { size_t dstLen; }; +struct MountList { + unsigned int count; + char list[MAX_MOUNT_NR][PATH_MAX]; +}; + struct ParsedConfig { char rootfs[BUF_SIZE]; unsigned int devices[MAX_DEVICE_NR]; @@ -45,6 +44,8 @@ struct ParsedConfig { char containerNsPath[BUF_SIZE]; char cgroupPath[BUF_SIZE]; int originNsFd; + const struct MountList *files; + const struct MountList *dirs; }; void InitParsedConfig(struct ParsedConfig *parsedConfig); diff --git a/cli/src/main.c b/cli/src/main.c index b312edc..88898bf 100644 --- a/cli/src/main.c +++ b/cli/src/main.c @@ -28,6 +28,8 @@ struct CmdArgs { char rootfs[BUF_SIZE]; int pid; char options[BUF_SIZE]; + struct MountList files; + struct MountList dirs; }; static struct option g_cmdOpts[] = { @@ -35,6 +37,8 @@ static struct option g_cmdOpts[] = { {"pid", required_argument, 0, 'p'}, {"rootfs", required_argument, 0, 'r'}, {"options", required_argument, 0, 'o'}, + {"mount-file", required_argument, 0, 'f'}, + {"mount-dir", required_argument, 0, 'i'}, {0, 0, 0, 0} }; @@ -90,7 +94,41 @@ static bool OptionsCmdArgParser(struct CmdArgs *args, const char *arg) return true; } -#define NUM_OF_CMD_ARGS 4 +static bool MountFileCmdArgParser(struct CmdArgs *args, const char *arg) +{ + if (args->files.count == MAX_MOUNT_NR) { + LOG_ERROR("error: too many files to mount, max number is %u", MAX_MOUNT_NR); + return -1; + } + + char *dst = &args->files.list[args->files.count++][0]; + errno_t err = strcpy_s(dst, PATH_MAX, arg); + if (err != EOK) { + LOG_ERROR("error: failed to copy mount file path: %s", arg); + return false; + } + + return true; +} + +static bool MountDirCmdArgParser(struct CmdArgs *args, const char *arg) +{ + if (args->dirs.count == MAX_MOUNT_NR) { + LOG_ERROR("error: too many directories to mount, max number is %u", MAX_MOUNT_NR); + return -1; + } + + char *dst = &args->dirs.list[args->dirs.count++][0]; + errno_t err = strcpy_s(dst, PATH_MAX, arg); + if (err != EOK) { + LOG_ERROR("error: failed to copy mount directory path: %s", arg); + return false; + } + + return true; +} + +#define NUM_OF_CMD_ARGS 6 static struct { const char c; @@ -99,7 +137,9 @@ static struct { {'d', DevicesCmdArgParser}, {'p', PidCmdArgParser}, {'r', RootfsCmdArgParser}, - {'o', OptionsCmdArgParser} + {'o', OptionsCmdArgParser}, + {'f', MountFileCmdArgParser}, + {'i', MountDirCmdArgParser} }; static int ParseOneCmdArg(struct CmdArgs *args, char indicator, const char *value) @@ -201,6 +241,9 @@ int DoPrepare(const struct CmdArgs *args, struct ParsedConfig *config) return -1; } + config->files = (const struct MountList *)&args->files; + config->dirs = (const struct MountList *)&args->dirs; + return 0; } @@ -258,7 +301,7 @@ int Process(int argc, char **argv) int optionIndex; struct CmdArgs args = {0}; - while ((c = getopt_long(argc, argv, "d:p:r:o", g_cmdOpts, &optionIndex)) != -1) { + while ((c = getopt_long(argc, argv, "d:p:r:o:f:i", g_cmdOpts, &optionIndex)) != -1) { ret = ParseOneCmdArg(&args, (char)c, optarg); if (ret < 0) { LOG_ERROR("error: failed to parse cmd args."); diff --git a/cli/src/mount.c b/cli/src/mount.c index 333b303..c3e8f79 100644 --- a/cli/src/mount.c +++ b/cli/src/mount.c @@ -151,11 +151,6 @@ int MountFile(const char *rootfs, const char *filepath) return 0; } - if (!S_ISREG(srcStat.st_mode)) { - LOG_ERROR("error: this should be a regular file to be mounted: %s.", filepath); - return -1; - } - ret = CreateFile(dst, srcStat.st_mode); if (ret < 0) { LOG_ERROR("error: failed to create mount dst file: %s.", dst); @@ -188,11 +183,6 @@ int MountDir(const char *rootfs, const char *src) return 0; } - if (!S_ISDIR(srcStat.st_mode)) { - LOG_ERROR("error: this should be a directory to be mounted: %s.", src); - return -1; - } - ret = MakeDirWithParent(dst, DEFAULT_DIR_MODE); if (ret < 0) { LOG_ERROR("error: failed to make dir: %s.", dst); @@ -232,60 +222,31 @@ int DoCtrlDeviceMounting(const char *rootfs) return 0; } -int DoDirectoryMounting(const char *rootfs) +int DoDirectoryMounting(const char *rootfs, const struct MountList *list) { - /* directory */ - int ret = MountDir(rootfs, ASCEND_DRIVER_LIB64_PATH); - if (ret < 0) { - LOG_ERROR("error: failed to do mount %s.", ASCEND_DRIVER_LIB64_PATH); - return -1; - } + int ret; - ret = MountDir(rootfs, ASCEND_DRIVER_TOOLS_PATH); - if (ret < 0) { - LOG_ERROR("error: failed to do mount %s.", ASCEND_DRIVER_TOOLS_PATH); - return -1; - } - - ret = MountDir(rootfs, ASCEND_DRIVER_INC_PATH); - if (ret < 0) { - LOG_ERROR("error: failed to do mount %s.", ASCEND_DRIVER_INC_PATH); - return -1; - } - - ret = MountDir(rootfs, ASCEND_ADDONS_PATH); - if (ret < 0) { - LOG_ERROR("error: failed to do mount %s.", ASCEND_ADDONS_PATH); - return -1; - } - - ret = MountDir(rootfs, ASCEND_DCMI_PATH); - if (ret < 0) { - LOG_ERROR("error: failed to do mount %s.", ASCEND_DCMI_PATH); - return -1; + for (unsigned int i = 0; i < list->count; i++) { + ret = MountDir(rootfs, (const char *)&list->list[i][0]); + if (ret < 0) { + LOG_ERROR("error: failed to do directory mounting for %s.", &list->list[i][0]); + return -1; + } } return 0; } -int DoFileMounting(const char *rootfs) +int DoFileMounting(const char *rootfs, const struct MountList *list) { - int ret = MountFile(rootfs, ASCEND_NPU_SMI_PATH); - if (ret < 0) { - LOG_ERROR("error: failed to do mount %s.", ASCEND_NPU_SMI_PATH); - return -1; - } + int ret; - ret = MountFile(rootfs, ASCEND_NPU_SMI_PATH_OLD); - if (ret < 0) { - LOG_ERROR("error: failed to do mount %s.", ASCEND_NPU_SMI_PATH_OLD); - return -1; - } - - ret = MountFile(rootfs, ASCEND_SLOG_CONF_PATH); - if (ret < 0) { - LOG_ERROR("error: failed to do mount %s.", ASCEND_SLOG_CONF_PATH); - return -1; + for (unsigned int i = 0; i < list->count; i++) { + ret = MountFile(rootfs, (const char *)&list->list[i][0]); + if (ret < 0) { + LOG_ERROR("error: failed to do file mounting for %s.", &list->list[i][0]); + return -1; + } } return 0; @@ -311,13 +272,13 @@ int DoMounting(const struct ParsedConfig *config) return 0; } - ret = DoFileMounting(config->rootfs); + ret = DoFileMounting(config->rootfs, config->files); if (ret < 0) { LOG_ERROR("error: failed to mount files."); return -1; } - ret = DoDirectoryMounting(config->rootfs); + ret = DoDirectoryMounting(config->rootfs, config->dirs); if (ret < 0) { LOG_ERROR("error: failed to do mount directories."); return -1; diff --git a/cli/test/dt/testcase/gtest_mytestcase.cpp b/cli/test/dt/testcase/gtest_mytestcase.cpp index c04b2df..beea9c9 100644 --- a/cli/test/dt/testcase/gtest_mytestcase.cpp +++ b/cli/test/dt/testcase/gtest_mytestcase.cpp @@ -6,6 +6,7 @@ #include #include "gtest/gtest.h" #include "mockcpp/mockcpp.hpp" +#include #include using namespace std; @@ -14,6 +15,7 @@ using namespace testing; #define DAVINCI_MANAGER_PATH "/dev/davinci_manager" #define BUF_SIZE 1024 #define MAX_DEVICE_NR 1024 +#define MAX_MOUNT_NR 512 typedef char *(*ParseFileLine)(char *, const char *); extern "C" int IsStrEqual(const char *s1, const char *s2); extern "C" int GetNsPath(const int pid, const char *nsType, char *buf, size_t bufSize); @@ -47,18 +49,25 @@ extern "C" int GetCgroupPath(const struct CmdArgs *args, char *effPath, const si extern "C" int SetupCgroup(const struct ParsedConfig *config); extern "C" int SetupContainer(struct CmdArgs *args); extern "C" int Process(int argc, char **argv); -extern "C" int DoFileMounting(const char *rootfs); +extern "C" int DoFileMounting(const char *rootfs, const struct MountList *list); extern "C" int DoMounting(const struct ParsedConfig *config); -extern "C" int DoDirectoryMounting(const char *rootfs); +extern "C" int DoDirectoryMounting(const char *rootfs, const struct MountList *list); extern "C" int DoPrepare(const struct CmdArgs *args, struct ParsedConfig *config); extern "C" int ParseRuntimeOptions(const char *options); extern "C" bool IsOptionNoDrvSet(); +struct MountList { + unsigned int count; + char list[MAX_MOUNT_NR][PATH_MAX]; +}; + struct CmdArgs { char devices[BUF_SIZE]; char rootfs[BUF_SIZE]; int pid; char options[BUF_SIZE]; + struct MountList files; + struct MountList dirs; }; struct ParsedConfig { @@ -68,6 +77,8 @@ struct ParsedConfig { char containerNsPath[BUF_SIZE]; char cgroupPath[BUF_SIZE]; int originNsFd; + const struct MountList *files; + const struct MountList *dirs; }; int stub_setns(int fd, int nstype) @@ -222,22 +233,22 @@ int Stub_DoCtrlDeviceMounting_Failed(const char *rootfs) return -1; } -int Stub_DoDirectoryMounting_Success(const char *rootfs) +int Stub_DoDirectoryMounting_Success(const char *rootfs, const struct MountList *list) { return 0; } -int Stub_DoDirectoryMounting_Failed(const char *rootfs) +int Stub_DoDirectoryMounting_Failed(const char *rootfs, const struct MountList *list) { return -1; } -int Stub_DoFileMounting_Success(const char *rootfs) +int Stub_DoFileMounting_Success(const char *rootfs, const struct MountList *list) { return 0; } -int Stub_DoFileMounting_Failed(const char *rootfs) +int Stub_DoFileMounting_Failed(const char *rootfs, const struct MountList *list) { return -1; } diff --git a/hook/main.go b/hook/main.go index ab388e8..61fc22b 100644 --- a/hook/main.go +++ b/hook/main.go @@ -5,12 +5,14 @@ package main import ( + "bufio" "encoding/json" "flag" "fmt" "log" "os" "path" + "path/filepath" "sort" "strconv" "strings" @@ -23,8 +25,12 @@ const ( loggingPrefix = "ascend-docker-hook" ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES" ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS" + ascendRuntimeMounts = "ASCEND_RUNTIME_MOUNTS" ascendDockerCli = "ascend-docker-cli" defaultAscendDockerCli = "/usr/local/bin/ascend-docker-cli" + configDir = "/etc/ascend-docker-runtime.d" + baseConfig = "base" + configFileSuffix = "list" borderNum = 2 kvPairSize = 2 @@ -37,7 +43,7 @@ var ( defaultAscendDockerCliName = defaultAscendDockerCli ) -var validRuntimeOptions = [...]string { +var validRuntimeOptions = [...]string{ "NODRV", "VERBOSE", } @@ -109,6 +115,22 @@ func parseDevices(visibleDevices string) ([]int, error) { return removeDuplication(devices), nil } +func parseMounts(mounts string) []string { + mountConfigs := make([]string, 0) + + if mounts == "" { + return nil + } + + for _, m := range strings.Split(mounts, ",") { + m = strings.TrimSpace(m) + m = strings.ToLower(m) + mountConfigs = append(mountConfigs, m) + } + + return mountConfigs +} + func isRuntimeOptionValid(option string) bool { for _, validOption := range validRuntimeOptions { if option == validOption { @@ -161,7 +183,7 @@ func parseOciSpecFile(file string) (*specs.Spec, error) { return spec, nil } -var getContainerConfig = func () (*containerConfig, error) { +var getContainerConfig = func() (*containerConfig, error) { state := new(specs.State) decoder := json.NewDecoder(containerConfigInputStream) @@ -199,6 +221,89 @@ func getValueByKey(data []string, key string) string { return "" } +func readMountConfig(dir string, name string) ([]string, []string, error) { + configFileName := fmt.Sprintf("%s.%s", name, configFileSuffix) + baseConfigFilePath, err := filepath.Abs(filepath.Join(dir, configFileName)) + if err != nil { + return nil, nil, fmt.Errorf("failed to assemble base config file path: %w", err) + } + + fileInfo, err := os.Stat(baseConfigFilePath) + if err != nil { + return nil, nil, fmt.Errorf("cannot stat base configuration file %s : %w", baseConfigFilePath, err) + } + + if !fileInfo.Mode().IsRegular() { + return nil, nil, fmt.Errorf("base configuration file damaged because is not a regular file") + } + + f, err := os.Open(baseConfigFilePath) + if err != nil { + return nil, nil, fmt.Errorf("failed to open base configuration file %s: %w", baseConfigFilePath, err) + } + defer f.Close() + + fileMountList := make([]string, 0) + dirMountList := make([]string, 0) + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + mountPath := scanner.Text() + absMountPath, err := filepath.Abs(mountPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to get absolute path from %s: %w", mountPath) + } + mountPath = absMountPath + + stat, err := os.Stat(mountPath) + if err != nil { + return nil, nil, fmt.Errorf("failed to stat %s: %w", mountPath, err) + } + + if stat.Mode().IsRegular() || stat.Mode()&os.ModeSocket != 0 { + fileMountList = append(fileMountList, mountPath) + } else if stat.Mode().IsDir() { + dirMountList = append(dirMountList, mountPath) + } else { + return nil, nil, fmt.Errorf("%s is not a file nor a directory, which is illegal", mountPath) + } + } + + return fileMountList, dirMountList, nil +} + +func readConfigsOfDir(dir string, mountConfigs []string) ([]string, []string, error) { + fileInfo, err := os.Stat(dir) + if err != nil { + return nil, nil, fmt.Errorf("cannot stat configuration directory %s : %w", dir, err) + } + + if !fileInfo.Mode().IsDir() { + return nil, nil, fmt.Errorf("%s should be a dir for ascend docker runtime, but now it is not", dir) + } + + fileMountList := make([]string, 0) + dirMountList := make([]string, 0) + + configs := []string{baseConfig} + + if mountConfigs != nil { + configs = append(configs, mountConfigs...) + } + + for _, config := range configs { + fileList, dirList, err := readMountConfig(dir, config) + if err != nil { + return nil, nil, fmt.Errorf("failed to process config %s: %w", config, err) + } + + fileMountList = append(fileMountList, fileList...) + dirMountList = append(dirMountList, dirList...) + } + + return fileMountList, dirMountList, nil +} + func doPrestartHook() error { containerConfig, err := getContainerConfig() if err != nil { @@ -215,6 +320,13 @@ func doPrestartHook() error { return fmt.Errorf("failed to parse device setting: %w", err) } + mountConfigs := parseMounts(getValueByKey(containerConfig.Env, ascendRuntimeMounts)) + + fileMountList, dirMountList, err := readConfigsOfDir(configDir, mountConfigs) + if err != nil { + return fmt.Errorf("failed to read configuration from config directory: %w", err) + } + parsedOptions, err := parseRuntimeOptions(getValueByKey(containerConfig.Env, ascendRuntimeOptions)) if err != nil { return fmt.Errorf("failed to parse runtime options: %w", err) @@ -235,6 +347,14 @@ func doPrestartHook() error { "--pid", fmt.Sprintf("%d", containerConfig.Pid), "--rootfs", containerConfig.Rootfs) + for _, filePath := range fileMountList { + args = append(args, "--mount-file", filePath) + } + + for _, dirPath := range dirMountList { + args = append(args, "--mount-dir", dirPath) + } + if len(parsedOptions) > 0 { args = append(args, "--options", strings.Join(parsedOptions, ",")) }