package e2eqemu import ( "flag" "fmt" "io" "os" "os/exec" "path/filepath" "runtime" "strconv" "strings" "testing" "time" e2e_utils "github.com/containers/gvisor-tap-vsock/test-utils" "github.com/onsi/ginkgo" "github.com/onsi/gomega" log "github.com/sirupsen/logrus" ) func TestSuite(t *testing.T) { gomega.RegisterFailHandler(ginkgo.Fail) ginkgo.RunSpecs(t, "gvisor-tap-vsock suite") } const ( sock = "/tmp/gvproxy-api.sock" qemuPort = 5555 sshPort = 2222 ignitionUser = "test" qconLog = "qcon.log" podmanSock = "/run/user/1001/podman/podman.sock" podmanRootSock = "/run/podman/podman.sock" // #nosec "test" (for manual usage) ignitionPasswordHash = "$y$j9T$TqJWt3/mKJbH0sYi6B/LD1$QjVRuUgntjTHjAdAkqhkr4F73m.Be4jBXdAaKw98sPC" ) var ( tmpDir string binDir string host *exec.Cmd client *exec.Cmd privateKeyFile string publicKeyFile string ignFile string forwardSock string forwardRootSock string ) func init() { flag.StringVar(&tmpDir, "tmpDir", "../tmp", "temporary working directory") flag.StringVar(&binDir, "bin", "../bin", "directory with compiled binaries") privateKeyFile = filepath.Join(tmpDir, "id_test") publicKeyFile = privateKeyFile + ".pub" ignFile = filepath.Join(tmpDir, "test.ign") forwardSock = filepath.Join(tmpDir, "podman-remote.sock") forwardRootSock = filepath.Join(tmpDir, "podman-root-remote.sock") } var _ = ginkgo.BeforeSuite(func() { gomega.Expect(os.MkdirAll(filepath.Join(tmpDir, "disks"), os.ModePerm)).Should(gomega.Succeed()) downloader, err := e2e_utils.NewFcosDownloader(filepath.Join(tmpDir, "disks")) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) qemuImage, err := downloader.DownloadImage("qemu", "qcow2.xz") gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) publicKey, err := e2e_utils.CreateSSHKeys(publicKeyFile, privateKeyFile) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) err = e2e_utils.CreateIgnition(ignFile, publicKey, ignitionUser, ignitionPasswordHash) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) outer: for panics := 0; ; panics++ { _ = os.Remove(sock) // #nosec host = exec.Command(filepath.Join(binDir, "gvproxy"), fmt.Sprintf("--listen=unix://%s", sock), fmt.Sprintf("--listen-qemu=tcp://127.0.0.1:%d", qemuPort), fmt.Sprintf("--forward-sock=%s", forwardSock), fmt.Sprintf("--forward-dest=%s", podmanSock), fmt.Sprintf("--forward-user=%s", ignitionUser), fmt.Sprintf("--forward-identity=%s", privateKeyFile), fmt.Sprintf("--forward-sock=%s", forwardRootSock), fmt.Sprintf("--forward-dest=%s", podmanRootSock), fmt.Sprintf("--forward-user=%s", "root"), fmt.Sprintf("--forward-identity=%s", privateKeyFile)) host.Stderr = os.Stderr host.Stdout = os.Stdout gomega.Expect(host.Start()).Should(gomega.Succeed()) go func() { if err := host.Wait(); err != nil { log.Error(err) } }() for { _, err := os.Stat(sock) if os.IsNotExist(err) { log.Info("waiting for socket") time.Sleep(100 * time.Millisecond) continue } break } efiArgs, err := efiArgs() gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) template := `%s%s-m 2048 -nographic -serial file:%s -drive if=virtio,file=%s,snapshot=on -fw_cfg name=opt/com.coreos/config,file=%s -netdev socket,id=vlan,connect=127.0.0.1:%d -device virtio-net-pci,netdev=vlan,mac=5a:94:ef:e4:0c:ee` // #nosec client = exec.Command(qemuExecutable(), strings.Split(fmt.Sprintf(template, qemuArgs(), efiArgs, qconLog, qemuImage, ignFile, qemuPort), " ")...) client.Stderr = os.Stderr client.Stdout = os.Stdout gomega.Expect(client.Start()).Should(gomega.Succeed()) go func() { if err := client.Wait(); err != nil { log.Error(err) } }() for { _, err := sshExec("whoami") if err == nil { break outer } // Check for panic didPanic, err := panicCheck(qconLog) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) if didPanic { gomega.Expect(panics).ToNot(gomega.BeNumerically(">", 15), "No more than 15 panics allowed") log.Info("Detected Kernel panic, retrying...") _ = client.Process.Kill() _ = host.Process.Kill() _ = os.Remove(qconLog) continue outer } log.Infof("waiting for client to connect: %v", err) time.Sleep(time.Second) } } err = scp(filepath.Join(binDir, "test-companion"), "/tmp/test-companion") gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) // start an embedded DNS and http server in the VM. Wait a bit for the server to start. cmd := sshCommand("sudo /tmp/test-companion") gomega.Expect(cmd.Start()).ShouldNot(gomega.HaveOccurred()) time.Sleep(5 * time.Second) }) func qemuExecutable() string { qemuBinaries := []string{"qemu-kvm", fmt.Sprintf("qemu-system-%s", e2e_utils.CoreosArch())} for _, binary := range qemuBinaries { path, err := exec.LookPath(binary) if err == nil && path != "" { return path } } return "" } func qemuArgs() string { accel := "kvm" if runtime.GOOS == "darwin" { accel = "hvf" } machine := "q35" switch runtime.GOARCH { case "amd64": machine = "q35" case "arm64": machine = "virt" default: panic(fmt.Sprintf("unsupported arch: %s", runtime.GOARCH)) } return fmt.Sprintf("-machine %s,accel=%s:tcg -smp 4 -cpu host ", machine, accel) } func scp(src, dst string) error { sshCmd := exec.Command("scp", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "IdentitiesOnly=yes", "-i", privateKeyFile, "-P", strconv.Itoa(sshPort), src, fmt.Sprintf("%s@127.0.0.1:%s", ignitionUser, dst)) // #nosec G204 sshCmd.Stderr = os.Stderr sshCmd.Stdout = os.Stdout return sshCmd.Run() } func sshExec(cmd ...string) ([]byte, error) { return sshCommand(cmd...).Output() } func sshCommand(cmd ...string) *exec.Cmd { sshCmd := exec.Command("ssh", "-o", "UserKnownHostsFile=/dev/null", "-o", "StrictHostKeyChecking=no", "-o", "IdentitiesOnly=yes", "-i", privateKeyFile, "-p", strconv.Itoa(sshPort), fmt.Sprintf("%s@127.0.0.1", ignitionUser), "--", strings.Join(cmd, " ")) // #nosec G204 return sshCmd } func panicCheck(con string) (bool, error) { file, err := os.Open(con) if err != nil { return false, err } _, _ = file.Seek(-500, io.SeekEnd) // Ignore seek errors (not enough content yet) contents := make([]byte, 500) _, err = io.ReadAtLeast(file, contents, len(contents)) if err != nil && err != io.ErrUnexpectedEOF { return false, err } return strings.Contains(string(contents), "end Kernel panic"), nil } var _ = ginkgo.AfterSuite(func() { if host != nil { if err := host.Process.Kill(); err != nil { log.Error(err) } } if client != nil { if err := client.Process.Kill(); err != nil { log.Error(err) } } })