zishuo/ascend-docker-runtime
1
0
Fork 0
mirror of https://github.com/Ascend/ascend-docker-runtime.git synced 2025-10-13 02:23:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Match-id-1dfa3a0229c36252b00c078413911cac87225f79

Browse Source
This commit is contained in:
BianTanggui
2020-10-19 11:59:21 +08:00
parent 74edb62e28
commit a70c4c07b0
9 changed files with 239 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
build/build.sh
View File

@@ -75,9 +75,10 @@ function build_run_package()
/bin/cp -f {${RUNTIMESRCDIR},${HOOKSRCDIR},${INSTALLHELPERSRCDIR},${CLISRCDIR}}/build/ascend-docker* run_pkg
/bin/cp -f scripts/uninstall.sh run_pkg
/bin/cp -f scripts/base.list run_pkg
FILECNT=`ls -l run_pkg |grep "^-"|wc -l`
echo "prepare package $FILECNT bins"
if [ $FILECNT -ne 5 ]; then
if [ $FILECNT -ne 6 ]; then
exit 1
fi

7
build/scripts/base.list Normal file
View File

@@ -0,0 +1,7 @@
/usr/local/Ascend/driver/lib64
/usr/local/Ascend/driver/tools
/usr/local/Ascend/driver/include
/usr/local/Ascend/add-ons
/usr/local/dcmi
/usr/local/bin/npu-smi
/var/log/npu/conf/slog/slog.conf

15
build/scripts/run_main.sh
View File

@@ -1,5 +1,6 @@
#!/bin/bash
ASCEND_RUNTIME_CONFIG_DIR=/etc/ascend-docker-runtime.d
DOCKER_CONFIG_DIR=/etc/docker
INSTALL_PATH=/usr/local/Ascend/Ascend-Docker-Runtime
@@ -24,6 +25,13 @@ function install()
cp -f ./uninstall.sh ${INSTALL_PATH}/script/uninstall.sh
chmod 550 ${INSTALL_PATH}/script/uninstall.sh
if [ -d "${ASCEND_RUNTIME_CONFIG_DIR}" ]; then
rm -rf ${ASCEND_RUNTIME_CONFIG_DIR}
fi
mkdir -p ${ASCEND_RUNTIME_CONFIG_DIR}
cp -f ./base.list ${ASCEND_RUNTIME_CONFIG_DIR}/base.list
chmod 440 ${ASCEND_RUNTIME_CONFIG_DIR}/base.list
echo 'install executable files success'
if [ ! -d "${DOCKER_CONFIG_DIR}" ]; then
@@ -70,16 +78,23 @@ function upgrade()
exit 1
fi
if [ ! -d "${ASCEND_RUNTIME_CONFIG_DIR}" ]; then
echo 'ERROR: the configuration directory does not exist'
exit 1
fi
cp -f ./ascend-docker-runtime ${INSTALL_PATH}/ascend-docker-runtime
cp -f ./ascend-docker-hook ${INSTALL_PATH}/ascend-docker-hook
cp -f ./ascend-docker-cli ${INSTALL_PATH}/ascend-docker-cli
cp -f ./ascend-docker-plugin-install-helper ${INSTALL_PATH}/ascend-docker-plugin-install-helper
cp -f ./uninstall.sh ${INSTALL_PATH}/script/uninstall.sh
cp -f ./base.list ${ASCEND_RUNTIME_CONFIG_DIR}/base.list
chmod 550 ${INSTALL_PATH}/ascend-docker-runtime
chmod 550 ${INSTALL_PATH}/ascend-docker-hook
chmod 550 ${INSTALL_PATH}/ascend-docker-cli
chmod 550 ${INSTALL_PATH}/ascend-docker-plugin-install-helper
chmod 550 ${INSTALL_PATH}/script/uninstall.sh
chmod 440 ${ASCEND_RUNTIME_CONFIG_DIR}/base.list
echo 'upgrade ascend docker runtime success'
}

3
build/scripts/uninstall.sh
View File

@@ -3,6 +3,7 @@
ROOT=$(cd `dirname $0`; pwd)/..
DST='/etc/docker/daemon.json'
SRC="${DST}.${PPID}"
ASCEND_RUNTIME_CONFIG_DIR=/etc/ascend-docker-runtime.d
if [ ! -f "${DST}" ]; then
exit 0
@@ -15,3 +16,5 @@ if [ "$?" != "0" ]; then
fi
mv ${SRC} ${DST}
rm -rf ${ASCEND_RUNTIME_CONFIG_DIR}

17
cli/src/basic.h
View File

@@ -7,23 +7,17 @@
#include <sys/types.h>
#include <stdint.h>
#include <limits.h>
#define DEVICE_NAME "davinci"
#define DAVINCI_MANAGER "davinci_manager"
#define DEVMM_SVM "devmm_svm"
#define HISI_HDC "hisi_hdc"
#define ASCEND_DRIVER_LIB64_PATH "/usr/local/Ascend/driver/lib64"
#define ASCEND_DRIVER_TOOLS_PATH "/usr/local/Ascend/driver/tools"
#define ASCEND_DRIVER_INC_PATH "/usr/local/Ascend/driver/include"
#define ASCEND_ADDONS_PATH "/usr/local/Ascend/add-ons"
#define ASCEND_DCMI_PATH "/usr/local/dcmi"
#define ASCEND_NPU_SMI_PATH "/usr/local/bin/npu-smi"
#define ASCEND_NPU_SMI_PATH_OLD "/usr/local/sbin/npu-smi"
#define ASCEND_SLOG_CONF_PATH "/var/log/npu/conf/slog/slog.conf"
#define DEFAULT_DIR_MODE 0755
#define BUF_SIZE 1024
#define MAX_DEVICE_NR 1024
#define DEFAULT_LOG_FILE "/var/log/ascend-docker-runtime.log"
#define MAX_MOUNT_NR 512
#define ALLOW_PATH "/devices.allow"
#define ROOT_GAP 4
@@ -38,6 +32,11 @@ struct PathInfo {
size_t dstLen;
};
struct MountList {
unsigned int count;
char list[MAX_MOUNT_NR][PATH_MAX];
};
struct ParsedConfig {
char rootfs[BUF_SIZE];
unsigned int devices[MAX_DEVICE_NR];
@@ -45,6 +44,8 @@ struct ParsedConfig {
char containerNsPath[BUF_SIZE];
char cgroupPath[BUF_SIZE];
int originNsFd;
const struct MountList *files;
const struct MountList *dirs;
};
void InitParsedConfig(struct ParsedConfig *parsedConfig);

49
cli/src/main.c
View File

@@ -28,6 +28,8 @@ struct CmdArgs {
char rootfs[BUF_SIZE];
int pid;
char options[BUF_SIZE];
struct MountList files;
struct MountList dirs;
};
static struct option g_cmdOpts[] = {
@@ -35,6 +37,8 @@ static struct option g_cmdOpts[] = {
{"pid", required_argument, 0, 'p'},
{"rootfs", required_argument, 0, 'r'},
{"options", required_argument, 0, 'o'},
{"mount-file", required_argument, 0, 'f'},
{"mount-dir", required_argument, 0, 'i'},
{0, 0, 0, 0}
};
@@ -90,7 +94,41 @@ static bool OptionsCmdArgParser(struct CmdArgs *args, const char *arg)
return true;
}
#define NUM_OF_CMD_ARGS 4
static bool MountFileCmdArgParser(struct CmdArgs *args, const char *arg)
{
if (args->files.count == MAX_MOUNT_NR) {
LOG_ERROR("error: too many files to mount, max number is %u", MAX_MOUNT_NR);
return -1;
}
char *dst = &args->files.list[args->files.count++][0];
errno_t err = strcpy_s(dst, PATH_MAX, arg);
if (err != EOK) {
LOG_ERROR("error: failed to copy mount file path: %s", arg);
return false;
}
return true;
}
static bool MountDirCmdArgParser(struct CmdArgs *args, const char *arg)
{
if (args->dirs.count == MAX_MOUNT_NR) {
LOG_ERROR("error: too many directories to mount, max number is %u", MAX_MOUNT_NR);
return -1;
}
char *dst = &args->dirs.list[args->dirs.count++][0];
errno_t err = strcpy_s(dst, PATH_MAX, arg);
if (err != EOK) {
LOG_ERROR("error: failed to copy mount directory path: %s", arg);
return false;
}
return true;
}
#define NUM_OF_CMD_ARGS 6
static struct {
const char c;
@@ -99,7 +137,9 @@ static struct {
{'d', DevicesCmdArgParser},
{'p', PidCmdArgParser},
{'r', RootfsCmdArgParser},
{'o', OptionsCmdArgParser}
{'o', OptionsCmdArgParser},
{'f', MountFileCmdArgParser},
{'i', MountDirCmdArgParser}
};
static int ParseOneCmdArg(struct CmdArgs *args, char indicator, const char *value)
@@ -201,6 +241,9 @@ int DoPrepare(const struct CmdArgs *args, struct ParsedConfig *config)
return -1;
}
config->files = (const struct MountList *)&args->files;
config->dirs = (const struct MountList *)&args->dirs;
return 0;
}
@@ -258,7 +301,7 @@ int Process(int argc, char **argv)
int optionIndex;
struct CmdArgs args = {0};
while ((c = getopt_long(argc, argv, "d:p:r:o", g_cmdOpts, &optionIndex)) != -1) {
while ((c = getopt_long(argc, argv, "d:p:r:o:f:i", g_cmdOpts, &optionIndex)) != -1) {
ret = ParseOneCmdArg(&args, (char)c, optarg);
if (ret < 0) {
LOG_ERROR("error: failed to parse cmd args.");

75
cli/src/mount.c
View File

@@ -151,11 +151,6 @@ int MountFile(const char *rootfs, const char *filepath)
return 0;
}
if (!S_ISREG(srcStat.st_mode)) {
LOG_ERROR("error: this should be a regular file to be mounted: %s.", filepath);
return -1;
}
ret = CreateFile(dst, srcStat.st_mode);
if (ret < 0) {
LOG_ERROR("error: failed to create mount dst file: %s.", dst);
@@ -188,11 +183,6 @@ int MountDir(const char *rootfs, const char *src)
return 0;
}
if (!S_ISDIR(srcStat.st_mode)) {
LOG_ERROR("error: this should be a directory to be mounted: %s.", src);
return -1;
}
ret = MakeDirWithParent(dst, DEFAULT_DIR_MODE);
if (ret < 0) {
LOG_ERROR("error: failed to make dir: %s.", dst);
@@ -232,60 +222,31 @@ int DoCtrlDeviceMounting(const char *rootfs)
return 0;
}
int DoDirectoryMounting(const char *rootfs)
int DoDirectoryMounting(const char *rootfs, const struct MountList *list)
{
/* directory */
int ret = MountDir(rootfs, ASCEND_DRIVER_LIB64_PATH);
if (ret < 0) {
LOG_ERROR("error: failed to do mount %s.", ASCEND_DRIVER_LIB64_PATH);
return -1;
}
int ret;
ret = MountDir(rootfs, ASCEND_DRIVER_TOOLS_PATH);
if (ret < 0) {
LOG_ERROR("error: failed to do mount %s.", ASCEND_DRIVER_TOOLS_PATH);
return -1;
}
ret = MountDir(rootfs, ASCEND_DRIVER_INC_PATH);
if (ret < 0) {
LOG_ERROR("error: failed to do mount %s.", ASCEND_DRIVER_INC_PATH);
return -1;
}
ret = MountDir(rootfs, ASCEND_ADDONS_PATH);
if (ret < 0) {
LOG_ERROR("error: failed to do mount %s.", ASCEND_ADDONS_PATH);
return -1;
}
ret = MountDir(rootfs, ASCEND_DCMI_PATH);
if (ret < 0) {
LOG_ERROR("error: failed to do mount %s.", ASCEND_DCMI_PATH);
return -1;
for (unsigned int i = 0; i < list->count; i++) {
ret = MountDir(rootfs, (const char *)&list->list[i][0]);
if (ret < 0) {
LOG_ERROR("error: failed to do directory mounting for %s.", &list->list[i][0]);
return -1;
}
}
return 0;
}
int DoFileMounting(const char *rootfs)
int DoFileMounting(const char *rootfs, const struct MountList *list)
{
int ret = MountFile(rootfs, ASCEND_NPU_SMI_PATH);
if (ret < 0) {
LOG_ERROR("error: failed to do mount %s.", ASCEND_NPU_SMI_PATH);
return -1;
}
int ret;
ret = MountFile(rootfs, ASCEND_NPU_SMI_PATH_OLD);
if (ret < 0) {
LOG_ERROR("error: failed to do mount %s.", ASCEND_NPU_SMI_PATH_OLD);
return -1;
}
ret = MountFile(rootfs, ASCEND_SLOG_CONF_PATH);
if (ret < 0) {
LOG_ERROR("error: failed to do mount %s.", ASCEND_SLOG_CONF_PATH);
return -1;
for (unsigned int i = 0; i < list->count; i++) {
ret = MountFile(rootfs, (const char *)&list->list[i][0]);
if (ret < 0) {
LOG_ERROR("error: failed to do file mounting for %s.", &list->list[i][0]);
return -1;
}
}
return 0;
@@ -311,13 +272,13 @@ int DoMounting(const struct ParsedConfig *config)
return 0;
}
ret = DoFileMounting(config->rootfs);
ret = DoFileMounting(config->rootfs, config->files);
if (ret < 0) {
LOG_ERROR("error: failed to mount files.");
return -1;
}
ret = DoDirectoryMounting(config->rootfs);
ret = DoDirectoryMounting(config->rootfs, config->dirs);
if (ret < 0) {
LOG_ERROR("error: failed to do mount directories.");
return -1;

23
cli/test/dt/testcase/gtest_mytestcase.cpp
View File

@@ -6,6 +6,7 @@
#include <iostream>
#include "gtest/gtest.h"
#include "mockcpp/mockcpp.hpp"
#include <limits.h>
#include <sys/mount.h>
using namespace std;
@@ -14,6 +15,7 @@ using namespace testing;
#define DAVINCI_MANAGER_PATH "/dev/davinci_manager"
#define BUF_SIZE 1024
#define MAX_DEVICE_NR 1024
#define MAX_MOUNT_NR 512
typedef char *(*ParseFileLine)(char *, const char *);
extern "C" int IsStrEqual(const char *s1, const char *s2);
extern "C" int GetNsPath(const int pid, const char *nsType, char *buf, size_t bufSize);
@@ -47,18 +49,25 @@ extern "C" int GetCgroupPath(const struct CmdArgs *args, char *effPath, const si
extern "C" int SetupCgroup(const struct ParsedConfig *config);
extern "C" int SetupContainer(struct CmdArgs *args);
extern "C" int Process(int argc, char **argv);
extern "C" int DoFileMounting(const char *rootfs);
extern "C" int DoFileMounting(const char *rootfs, const struct MountList *list);
extern "C" int DoMounting(const struct ParsedConfig *config);
extern "C" int DoDirectoryMounting(const char *rootfs);
extern "C" int DoDirectoryMounting(const char *rootfs, const struct MountList *list);
extern "C" int DoPrepare(const struct CmdArgs *args, struct ParsedConfig *config);
extern "C" int ParseRuntimeOptions(const char *options);
extern "C" bool IsOptionNoDrvSet();
struct MountList {
unsigned int count;
char list[MAX_MOUNT_NR][PATH_MAX];
};
struct CmdArgs {
char devices[BUF_SIZE];
char rootfs[BUF_SIZE];
int pid;
char options[BUF_SIZE];
struct MountList files;
struct MountList dirs;
};
struct ParsedConfig {
@@ -68,6 +77,8 @@ struct ParsedConfig {
char containerNsPath[BUF_SIZE];
char cgroupPath[BUF_SIZE];
int originNsFd;
const struct MountList *files;
const struct MountList *dirs;
};
int stub_setns(int fd, int nstype)
@@ -222,22 +233,22 @@ int Stub_DoCtrlDeviceMounting_Failed(const char *rootfs)
return -1;
}
int Stub_DoDirectoryMounting_Success(const char *rootfs)
int Stub_DoDirectoryMounting_Success(const char *rootfs, const struct MountList *list)
{
return 0;
}
int Stub_DoDirectoryMounting_Failed(const char *rootfs)
int Stub_DoDirectoryMounting_Failed(const char *rootfs, const struct MountList *list)
{
return -1;
}
int Stub_DoFileMounting_Success(const char *rootfs)
int Stub_DoFileMounting_Success(const char *rootfs, const struct MountList *list)
{
return 0;
}
int Stub_DoFileMounting_Failed(const char *rootfs)
int Stub_DoFileMounting_Failed(const char *rootfs, const struct MountList *list)
{
return -1;
}

124
hook/main.go
View File

@@ -5,12 +5,14 @@
package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"log"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
@@ -23,8 +25,12 @@ const (
loggingPrefix = "ascend-docker-hook"
ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES"
ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS"
ascendRuntimeMounts = "ASCEND_RUNTIME_MOUNTS"
ascendDockerCli = "ascend-docker-cli"
defaultAscendDockerCli = "/usr/local/bin/ascend-docker-cli"
configDir = "/etc/ascend-docker-runtime.d"
baseConfig = "base"
configFileSuffix = "list"
borderNum = 2
kvPairSize = 2
@@ -37,7 +43,7 @@ var (
defaultAscendDockerCliName = defaultAscendDockerCli
)
var validRuntimeOptions = [...]string {
var validRuntimeOptions = [...]string{
"NODRV",
"VERBOSE",
}
@@ -109,6 +115,22 @@ func parseDevices(visibleDevices string) ([]int, error) {
return removeDuplication(devices), nil
}
func parseMounts(mounts string) []string {
mountConfigs := make([]string, 0)
if mounts == "" {
return nil
}
for _, m := range strings.Split(mounts, ",") {
m = strings.TrimSpace(m)
m = strings.ToLower(m)
mountConfigs = append(mountConfigs, m)
}
return mountConfigs
}
func isRuntimeOptionValid(option string) bool {
for _, validOption := range validRuntimeOptions {
if option == validOption {
@@ -161,7 +183,7 @@ func parseOciSpecFile(file string) (*specs.Spec, error) {
return spec, nil
}
var getContainerConfig = func () (*containerConfig, error) {
var getContainerConfig = func() (*containerConfig, error) {
state := new(specs.State)
decoder := json.NewDecoder(containerConfigInputStream)
@@ -199,6 +221,89 @@ func getValueByKey(data []string, key string) string {
return ""
}
func readMountConfig(dir string, name string) ([]string, []string, error) {
configFileName := fmt.Sprintf("%s.%s", name, configFileSuffix)
baseConfigFilePath, err := filepath.Abs(filepath.Join(dir, configFileName))
if err != nil {
return nil, nil, fmt.Errorf("failed to assemble base config file path: %w", err)
}
fileInfo, err := os.Stat(baseConfigFilePath)
if err != nil {
return nil, nil, fmt.Errorf("cannot stat base configuration file %s : %w", baseConfigFilePath, err)
}
if !fileInfo.Mode().IsRegular() {
return nil, nil, fmt.Errorf("base configuration file damaged because is not a regular file")
}
f, err := os.Open(baseConfigFilePath)
if err != nil {
return nil, nil, fmt.Errorf("failed to open base configuration file %s: %w", baseConfigFilePath, err)
}
defer f.Close()
fileMountList := make([]string, 0)
dirMountList := make([]string, 0)
scanner := bufio.NewScanner(f)
for scanner.Scan() {
mountPath := scanner.Text()
absMountPath, err := filepath.Abs(mountPath)
if err != nil {
return nil, nil, fmt.Errorf("failed to get absolute path from %s: %w", mountPath)
}
mountPath = absMountPath
stat, err := os.Stat(mountPath)
if err != nil {
return nil, nil, fmt.Errorf("failed to stat %s: %w", mountPath, err)
}
if stat.Mode().IsRegular() || stat.Mode()&os.ModeSocket != 0 {
fileMountList = append(fileMountList, mountPath)
} else if stat.Mode().IsDir() {
dirMountList = append(dirMountList, mountPath)
} else {
return nil, nil, fmt.Errorf("%s is not a file nor a directory, which is illegal", mountPath)
}
}
return fileMountList, dirMountList, nil
}
func readConfigsOfDir(dir string, mountConfigs []string) ([]string, []string, error) {
fileInfo, err := os.Stat(dir)
if err != nil {
return nil, nil, fmt.Errorf("cannot stat configuration directory %s : %w", dir, err)
}
if !fileInfo.Mode().IsDir() {
return nil, nil, fmt.Errorf("%s should be a dir for ascend docker runtime, but now it is not", dir)
}
fileMountList := make([]string, 0)
dirMountList := make([]string, 0)
configs := []string{baseConfig}
if mountConfigs != nil {
configs = append(configs, mountConfigs...)
}
for _, config := range configs {
fileList, dirList, err := readMountConfig(dir, config)
if err != nil {
return nil, nil, fmt.Errorf("failed to process config %s: %w", config, err)
}
fileMountList = append(fileMountList, fileList...)
dirMountList = append(dirMountList, dirList...)
}
return fileMountList, dirMountList, nil
}
func doPrestartHook() error {
containerConfig, err := getContainerConfig()
if err != nil {
@@ -215,6 +320,13 @@ func doPrestartHook() error {
return fmt.Errorf("failed to parse device setting: %w", err)
}
mountConfigs := parseMounts(getValueByKey(containerConfig.Env, ascendRuntimeMounts))
fileMountList, dirMountList, err := readConfigsOfDir(configDir, mountConfigs)
if err != nil {
return fmt.Errorf("failed to read configuration from config directory: %w", err)
}
parsedOptions, err := parseRuntimeOptions(getValueByKey(containerConfig.Env, ascendRuntimeOptions))
if err != nil {
return fmt.Errorf("failed to parse runtime options: %w", err)
@@ -235,6 +347,14 @@ func doPrestartHook() error {
"--pid", fmt.Sprintf("%d", containerConfig.Pid),
"--rootfs", containerConfig.Rootfs)
for _, filePath := range fileMountList {
args = append(args, "--mount-file", filePath)
}
for _, dirPath := range dirMountList {
args = append(args, "--mount-dir", dirPath)
}
if len(parsedOptions) > 0 {
args = append(args, "--options", strings.Join(parsedOptions, ","))
}