mirror of
https://github.com/kubenetworks/kubevpn.git
synced 2025-10-05 23:36:59 +08:00
529 lines
17 KiB
Go
529 lines
17 KiB
Go
package handler
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"net"
|
|
"time"
|
|
|
|
log "github.com/sirupsen/logrus"
|
|
admissionv1 "k8s.io/api/admissionregistration/v1"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
v1 "k8s.io/api/core/v1"
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/fields"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/util/cert"
|
|
cmdutil "k8s.io/kubectl/pkg/cmd/util"
|
|
"k8s.io/kubectl/pkg/polymorphichelpers"
|
|
"k8s.io/kubectl/pkg/util/podutils"
|
|
"k8s.io/utils/pointer"
|
|
"k8s.io/utils/ptr"
|
|
|
|
"github.com/wencaiwulue/kubevpn/v2/pkg/config"
|
|
"github.com/wencaiwulue/kubevpn/v2/pkg/util"
|
|
)
|
|
|
|
func createOutboundPod(ctx context.Context, factory cmdutil.Factory, clientset *kubernetes.Clientset, namespace string, gvisor bool, imagePullSecretName string) (err error) {
|
|
innerIpv4CIDR := net.IPNet{IP: config.RouterIP, Mask: config.CIDR.Mask}
|
|
innerIpv6CIDR := net.IPNet{IP: config.RouterIP6, Mask: config.CIDR6.Mask}
|
|
|
|
service, err := clientset.CoreV1().Services(namespace).Get(ctx, config.ConfigMapPodTrafficManager, metav1.GetOptions{})
|
|
if err == nil {
|
|
var pod *v1.Pod
|
|
pod, err = polymorphichelpers.AttachablePodForObjectFn(factory, service, 2*time.Second)
|
|
if err == nil && pod.DeletionTimestamp.IsZero() && podutils.IsPodReady(pod) {
|
|
log.Infoln("Use exist traffic manager")
|
|
return
|
|
}
|
|
}
|
|
var deleteResource = func(ctx context.Context) {
|
|
options := metav1.DeleteOptions{}
|
|
_ = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(ctx, config.ConfigMapPodTrafficManager+"."+namespace, options)
|
|
_ = clientset.RbacV1().RoleBindings(namespace).Delete(ctx, config.ConfigMapPodTrafficManager, options)
|
|
_ = clientset.RbacV1().Roles(namespace).Delete(ctx, config.ConfigMapPodTrafficManager, options)
|
|
_ = clientset.CoreV1().ServiceAccounts(namespace).Delete(ctx, config.ConfigMapPodTrafficManager, options)
|
|
_ = clientset.CoreV1().Services(namespace).Delete(ctx, config.ConfigMapPodTrafficManager, options)
|
|
_ = clientset.AppsV1().Deployments(namespace).Delete(ctx, config.ConfigMapPodTrafficManager, options)
|
|
}
|
|
defer func() {
|
|
if err != nil {
|
|
deleteResource(context.Background())
|
|
}
|
|
}()
|
|
deleteResource(ctx)
|
|
|
|
// 1) label namespace
|
|
log.Infof("Labeling Namespace %s", namespace)
|
|
ns, err := clientset.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{})
|
|
if err != nil {
|
|
log.Errorf("Get Namespace error: %s", err.Error())
|
|
return err
|
|
}
|
|
if ns.Labels == nil {
|
|
ns.Labels = map[string]string{}
|
|
}
|
|
ns.Labels["ns"] = namespace
|
|
_, err = clientset.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{})
|
|
if err != nil {
|
|
log.Infof("Labeling Namespace error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 2) create serviceAccount
|
|
log.Infof("Creating ServiceAccount %s", config.ConfigMapPodTrafficManager)
|
|
_, err = clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, &v1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
AutomountServiceAccountToken: pointer.Bool(true),
|
|
}, metav1.CreateOptions{})
|
|
if err != nil {
|
|
log.Infof("Creating ServiceAccount error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 3) create roles
|
|
log.Infof("Creating Roles %s", config.ConfigMapPodTrafficManager)
|
|
_, err = clientset.RbacV1().Roles(namespace).Create(ctx, &rbacv1.Role{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Rules: []rbacv1.PolicyRule{{
|
|
Verbs: []string{"get", "list", "watch", "create", "update", "patch", "delete"},
|
|
APIGroups: []string{""},
|
|
Resources: []string{"configmaps", "secrets"},
|
|
ResourceNames: []string{config.ConfigMapPodTrafficManager},
|
|
}},
|
|
}, metav1.CreateOptions{})
|
|
if err != nil {
|
|
log.Errorf("Creating Roles error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 4) create roleBinding
|
|
log.Infof("Creating RoleBinding %s", config.ConfigMapPodTrafficManager)
|
|
_, err = clientset.RbacV1().RoleBindings(namespace).Create(ctx, &rbacv1.RoleBinding{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Subjects: []rbacv1.Subject{{
|
|
Kind: "ServiceAccount",
|
|
//APIGroup: "rbac.authorization.k8s.io",
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
}},
|
|
RoleRef: rbacv1.RoleRef{
|
|
APIGroup: "rbac.authorization.k8s.io",
|
|
Kind: "Role",
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
},
|
|
}, metav1.CreateOptions{})
|
|
if err != nil {
|
|
log.Errorf("Creating RoleBinding error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 5) create service
|
|
log.Infof("Creating Service %s", config.ConfigMapPodTrafficManager)
|
|
udp8422 := "8422-for-udp"
|
|
tcp10800 := "10800-for-tcp"
|
|
tcp9002 := "9002-for-envoy"
|
|
tcp80 := "80-for-webhook"
|
|
udp53 := "53-for-dns"
|
|
_, err = clientset.CoreV1().Services(namespace).Create(ctx, &v1.Service{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: v1.ServiceSpec{
|
|
Ports: []v1.ServicePort{{
|
|
Name: udp8422,
|
|
Protocol: v1.ProtocolUDP,
|
|
Port: 8422,
|
|
TargetPort: intstr.FromInt32(8422),
|
|
}, {
|
|
Name: tcp10800,
|
|
Protocol: v1.ProtocolTCP,
|
|
Port: 10800,
|
|
TargetPort: intstr.FromInt32(10800),
|
|
}, {
|
|
Name: tcp9002,
|
|
Protocol: v1.ProtocolTCP,
|
|
Port: 9002,
|
|
TargetPort: intstr.FromInt32(9002),
|
|
}, {
|
|
Name: tcp80,
|
|
Protocol: v1.ProtocolTCP,
|
|
Port: 80,
|
|
TargetPort: intstr.FromInt32(80),
|
|
}, {
|
|
Name: udp53,
|
|
Protocol: v1.ProtocolUDP,
|
|
Port: 53,
|
|
TargetPort: intstr.FromInt32(53),
|
|
}},
|
|
Selector: map[string]string{"app": config.ConfigMapPodTrafficManager},
|
|
Type: v1.ServiceTypeClusterIP,
|
|
},
|
|
}, metav1.CreateOptions{})
|
|
if err != nil {
|
|
log.Errorf("Creating Service error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
domain := util.GetTlsDomain(namespace)
|
|
var crt, key []byte
|
|
crt, key, err = cert.GenerateSelfSignedCertKey(domain, nil, nil)
|
|
if err != nil {
|
|
log.Errorf("Generate self signed cert and key error: %s", err.Error())
|
|
return err
|
|
}
|
|
// reason why not use v1.SecretTypeTls is because it needs key called tls.crt and tls.key, but tls.key can not as env variable
|
|
// ➜ ~ export tls.key=a
|
|
//export: not valid in this context: tls.key
|
|
secret := &v1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Data: map[string][]byte{
|
|
config.TLSCertKey: crt,
|
|
config.TLSPrivateKeyKey: key,
|
|
},
|
|
Type: v1.SecretTypeOpaque,
|
|
}
|
|
_, err = clientset.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
|
|
if err != nil && !k8serrors.IsAlreadyExists(err) {
|
|
log.Errorf("Creating secret error: %s", err.Error())
|
|
return err
|
|
}
|
|
|
|
// 6) create mutatingWebhookConfigurations
|
|
log.Infof("Creating MutatingWebhookConfiguration %s", config.ConfigMapPodTrafficManager)
|
|
_, err = clientset.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(ctx, &admissionv1.MutatingWebhookConfiguration{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager + "." + namespace,
|
|
Namespace: namespace,
|
|
},
|
|
Webhooks: []admissionv1.MutatingWebhook{{
|
|
Name: config.ConfigMapPodTrafficManager + ".naison.io", // no sense
|
|
ClientConfig: admissionv1.WebhookClientConfig{
|
|
Service: &admissionv1.ServiceReference{
|
|
Namespace: namespace,
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Path: pointer.String("/pods"),
|
|
Port: pointer.Int32(80),
|
|
},
|
|
CABundle: crt,
|
|
},
|
|
Rules: []admissionv1.RuleWithOperations{{
|
|
Operations: []admissionv1.OperationType{admissionv1.Create, admissionv1.Delete},
|
|
Rule: admissionv1.Rule{
|
|
APIGroups: []string{""},
|
|
APIVersions: []string{"v1"},
|
|
Resources: []string{"pods"},
|
|
Scope: ptr.To(admissionv1.NamespacedScope),
|
|
},
|
|
}},
|
|
FailurePolicy: ptr.To(admissionv1.Ignore),
|
|
// same as above label ns
|
|
NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"ns": namespace}},
|
|
SideEffects: ptr.To(admissionv1.SideEffectClassNone),
|
|
TimeoutSeconds: ptr.To[int32](15),
|
|
AdmissionReviewVersions: []string{"v1", "v1beta1"},
|
|
ReinvocationPolicy: ptr.To(admissionv1.NeverReinvocationPolicy),
|
|
/*// needs to enable featureGate=AdmissionWebhookMatchConditions
|
|
MatchConditions: []admissionv1.MatchCondition{
|
|
{
|
|
Name: "",
|
|
Expression: fmt.Sprintf(
|
|
"container_name.exists(c, c == '%s') && environment_variable.find(e, e == '%s').exists()",
|
|
config.ContainerSidecarVPN, config.EnvInboundPodTunIPv4,
|
|
),
|
|
},
|
|
},*/
|
|
}},
|
|
}, metav1.CreateOptions{})
|
|
if err != nil && !k8serrors.IsForbidden(err) && !k8serrors.IsAlreadyExists(err) {
|
|
return fmt.Errorf("failed to create MutatingWebhookConfigurations: %v", err)
|
|
}
|
|
|
|
// 7) create deployment
|
|
log.Infof("Creating Deployment %s", config.ConfigMapPodTrafficManager)
|
|
deploy := genDeploySpec(namespace, innerIpv4CIDR, innerIpv6CIDR, udp8422, tcp10800, tcp9002, udp53, tcp80, gvisor, imagePullSecretName)
|
|
deploy, err = clientset.AppsV1().Deployments(namespace).Create(ctx, deploy, metav1.CreateOptions{})
|
|
if err != nil {
|
|
log.Errorf("Failed to create deployment for %s: %v", config.ConfigMapPodTrafficManager, err)
|
|
return err
|
|
}
|
|
str := fields.OneTermEqualSelector("app", config.ConfigMapPodTrafficManager).String()
|
|
_, selector, err := polymorphichelpers.SelectorsForObject(deploy)
|
|
if err == nil {
|
|
str = selector.String()
|
|
}
|
|
watchStream, err := clientset.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{
|
|
LabelSelector: str,
|
|
})
|
|
if err != nil {
|
|
log.Errorf("Failed to create watch for %s: %v", config.ConfigMapPodTrafficManager, err)
|
|
return err
|
|
}
|
|
defer watchStream.Stop()
|
|
var ok bool
|
|
var last string
|
|
ctx2, cancelFunc := context.WithTimeout(ctx, time.Minute*60)
|
|
defer cancelFunc()
|
|
log.Infoln()
|
|
wait.UntilWithContext(ctx2, func(ctx context.Context) {
|
|
podList, err := clientset.CoreV1().Pods(namespace).List(ctx2, metav1.ListOptions{
|
|
LabelSelector: fields.OneTermEqualSelector("app", config.ConfigMapPodTrafficManager).String(),
|
|
})
|
|
if err != nil {
|
|
log.Errorf("Failed to list pods for %s: %v", config.ConfigMapPodTrafficManager, err)
|
|
return
|
|
}
|
|
|
|
for _, podT := range podList.Items {
|
|
podT := &podT
|
|
if podT.DeletionTimestamp != nil {
|
|
continue
|
|
}
|
|
var sb = bytes.NewBuffer(nil)
|
|
sb.WriteString(fmt.Sprintf("Pod %s is %s...\n", podT.Name, podT.Status.Phase))
|
|
if podT.Status.Reason != "" {
|
|
sb.WriteString(fmt.Sprintf(" reason %s", podT.Status.Reason))
|
|
}
|
|
if podT.Status.Message != "" {
|
|
sb.WriteString(fmt.Sprintf(" message %s", podT.Status.Message))
|
|
}
|
|
util.PrintStatus(podT, sb)
|
|
if last != sb.String() {
|
|
log.Infof(sb.String())
|
|
}
|
|
last = sb.String()
|
|
|
|
if podutils.IsPodReady(podT) && func() bool {
|
|
for _, status := range podT.Status.ContainerStatuses {
|
|
if !status.Ready {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}() {
|
|
cancelFunc()
|
|
ok = true
|
|
}
|
|
}
|
|
}, time.Second*3)
|
|
if !ok {
|
|
log.Errorf("Wait pod %s to be ready timeout", config.ConfigMapPodTrafficManager)
|
|
return errors.New(fmt.Sprintf("wait pod %s to be ready timeout", config.ConfigMapPodTrafficManager))
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func genDeploySpec(namespace string, innerIpv4CIDR net.IPNet, innerIpv6CIDR net.IPNet, udp8422 string, tcp10800 string, tcp9002 string, udp53 string, tcp80 string, gvisor bool, imagePullSecretName string) *appsv1.Deployment {
|
|
var resourcesSmall = v1.ResourceRequirements{
|
|
Requests: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("100m"),
|
|
v1.ResourceMemory: resource.MustParse("128Mi"),
|
|
},
|
|
Limits: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("200m"),
|
|
v1.ResourceMemory: resource.MustParse("256Mi"),
|
|
},
|
|
}
|
|
var resourcesLarge = v1.ResourceRequirements{
|
|
Requests: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("500m"),
|
|
v1.ResourceMemory: resource.MustParse("512Mi"),
|
|
},
|
|
Limits: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("2000m"),
|
|
v1.ResourceMemory: resource.MustParse("2048Mi"),
|
|
},
|
|
}
|
|
|
|
deploy := &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: appsv1.DeploymentSpec{
|
|
Replicas: pointer.Int32(1),
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: map[string]string{"app": config.ConfigMapPodTrafficManager},
|
|
},
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{"app": config.ConfigMapPodTrafficManager},
|
|
Annotations: map[string]string{
|
|
"sidecar.istio.io/inject": "false",
|
|
},
|
|
},
|
|
Spec: v1.PodSpec{
|
|
ServiceAccountName: config.ConfigMapPodTrafficManager,
|
|
Volumes: []v1.Volume{{
|
|
Name: config.VolumeEnvoyConfig,
|
|
VolumeSource: v1.VolumeSource{
|
|
ConfigMap: &v1.ConfigMapVolumeSource{
|
|
LocalObjectReference: v1.LocalObjectReference{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
},
|
|
Items: []v1.KeyToPath{
|
|
{
|
|
Key: config.KeyEnvoy,
|
|
Path: "envoy-config.yaml",
|
|
},
|
|
},
|
|
Optional: pointer.Bool(false),
|
|
},
|
|
},
|
|
}},
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: config.ContainerSidecarVPN,
|
|
Image: config.Image,
|
|
Command: []string{"/bin/sh", "-c"},
|
|
Args: []string{util.If(
|
|
gvisor,
|
|
`
|
|
kubevpn serve -L "tcp://:10800" -L "gtcp://:10801" -L "gudp://:10802" --debug=true`,
|
|
`
|
|
sysctl -w net.ipv4.ip_forward=1
|
|
sysctl -w net.ipv6.conf.all.disable_ipv6=0
|
|
sysctl -w net.ipv6.conf.all.forwarding=1
|
|
update-alternatives --set iptables /usr/sbin/iptables-legacy
|
|
iptables -F
|
|
ip6tables -F
|
|
iptables -P INPUT ACCEPT
|
|
ip6tables -P INPUT ACCEPT
|
|
iptables -P FORWARD ACCEPT
|
|
ip6tables -P FORWARD ACCEPT
|
|
iptables -t nat -A POSTROUTING -s ${CIDR4} -o eth0 -j MASQUERADE
|
|
ip6tables -t nat -A POSTROUTING -s ${CIDR6} -o eth0 -j MASQUERADE
|
|
kubevpn serve -L "tcp://:10800" -L "tun://:8422?net=${TunIPv4}" -L "gtcp://:10801" -L "gudp://:10802" --debug=true`,
|
|
)},
|
|
EnvFrom: []v1.EnvFromSource{{
|
|
SecretRef: &v1.SecretEnvSource{
|
|
LocalObjectReference: v1.LocalObjectReference{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
},
|
|
},
|
|
}},
|
|
Env: []v1.EnvVar{
|
|
{
|
|
Name: "CIDR4",
|
|
Value: config.CIDR.String(),
|
|
},
|
|
{
|
|
Name: "CIDR6",
|
|
Value: config.CIDR6.String(),
|
|
},
|
|
{
|
|
Name: config.EnvInboundPodTunIPv4,
|
|
Value: innerIpv4CIDR.String(),
|
|
},
|
|
{
|
|
Name: config.EnvInboundPodTunIPv6,
|
|
Value: innerIpv6CIDR.String(),
|
|
},
|
|
},
|
|
Ports: []v1.ContainerPort{{
|
|
Name: udp8422,
|
|
ContainerPort: 8422,
|
|
Protocol: v1.ProtocolUDP,
|
|
}, {
|
|
Name: tcp10800,
|
|
ContainerPort: 10800,
|
|
Protocol: v1.ProtocolTCP,
|
|
}},
|
|
Resources: resourcesLarge,
|
|
ImagePullPolicy: v1.PullIfNotPresent,
|
|
SecurityContext: &v1.SecurityContext{
|
|
Capabilities: &v1.Capabilities{
|
|
Add: util.If(gvisor,
|
|
[]v1.Capability{},
|
|
[]v1.Capability{
|
|
"NET_ADMIN",
|
|
//"SYS_MODULE",
|
|
},
|
|
),
|
|
},
|
|
RunAsUser: pointer.Int64(0),
|
|
RunAsGroup: pointer.Int64(0),
|
|
Privileged: util.If(gvisor, pointer.Bool(false), pointer.Bool(true)),
|
|
},
|
|
},
|
|
{
|
|
Name: config.ContainerSidecarControlPlane,
|
|
Image: config.Image,
|
|
Command: []string{"kubevpn"},
|
|
Args: []string{"control-plane", "--watchDirectoryFilename", "/etc/envoy/envoy-config.yaml"},
|
|
Ports: []v1.ContainerPort{
|
|
{
|
|
Name: tcp9002,
|
|
ContainerPort: 9002,
|
|
Protocol: v1.ProtocolTCP,
|
|
},
|
|
{
|
|
Name: udp53,
|
|
ContainerPort: 53,
|
|
Protocol: v1.ProtocolUDP,
|
|
},
|
|
},
|
|
VolumeMounts: []v1.VolumeMount{
|
|
{
|
|
Name: config.VolumeEnvoyConfig,
|
|
ReadOnly: true,
|
|
MountPath: "/etc/envoy",
|
|
},
|
|
},
|
|
ImagePullPolicy: v1.PullIfNotPresent,
|
|
Resources: resourcesSmall,
|
|
},
|
|
{
|
|
Name: "webhook",
|
|
Image: config.Image,
|
|
Command: []string{"kubevpn"},
|
|
Args: []string{"webhook"},
|
|
Ports: []v1.ContainerPort{{
|
|
Name: tcp80,
|
|
ContainerPort: 80,
|
|
Protocol: v1.ProtocolTCP,
|
|
}},
|
|
EnvFrom: []v1.EnvFromSource{{
|
|
SecretRef: &v1.SecretEnvSource{
|
|
LocalObjectReference: v1.LocalObjectReference{
|
|
Name: config.ConfigMapPodTrafficManager,
|
|
},
|
|
},
|
|
}},
|
|
Env: []v1.EnvVar{},
|
|
ImagePullPolicy: v1.PullIfNotPresent,
|
|
Resources: resourcesSmall,
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyAlways,
|
|
ImagePullSecrets: []v1.LocalObjectReference{{
|
|
Name: imagePullSecretName,
|
|
}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
return deploy
|
|
}
|