From e98e074b57cb03246036f39730a72caa761444e4 Mon Sep 17 00:00:00 2001 From: Quentin Renard Date: Mon, 30 Dec 2019 15:41:29 +0100 Subject: [PATCH] Added archive --- archive.go | 209 +++++++++++++++++++++++++++++++++++++++++++ archive_test.go | 48 ++++++++++ astikit.go | 8 ++ astikit_test.go | 71 +++++++++++++++ http.go | 4 +- http_test.go | 40 ++------- os.go | 2 +- os_test.go | 25 ++---- testdata/archive/d/f | 1 + testdata/archive/f | 1 + 10 files changed, 355 insertions(+), 54 deletions(-) create mode 100644 archive.go create mode 100644 archive_test.go create mode 100644 astikit.go create mode 100644 astikit_test.go create mode 100644 testdata/archive/d/f create mode 100644 testdata/archive/f diff --git a/archive.go b/archive.go new file mode 100644 index 0000000..845e318 --- /dev/null +++ b/archive.go @@ -0,0 +1,209 @@ +package astikit + +import ( + "archive/zip" + "context" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" +) + +// internal shouldn't lead with a "/" +func zipInternalPath(p string) (external, internal string) { + if items := strings.Split(p, ".zip"); len(items) > 1 { + external = items[0] + ".zip" + internal = strings.TrimPrefix(strings.Join(items[1:], ".zip"), string(os.PathSeparator)) + return + } + external = p + return +} + +// Zip zips a src into a dst +// Possible dst formats are: +// - /path/to/zip.zip +// - /path/to/zip.zip/root/path +func Zip(ctx context.Context, dst, src string) (err error) { + // Get external/internal path + externalPath, internalPath := zipInternalPath(dst) + + // Make sure the directory exists + if err = os.MkdirAll(filepath.Dir(externalPath), DefaultDirMode); err != nil { + return fmt.Errorf("astikit: mkdirall %s failed: %w", filepath.Dir(externalPath), err) + } + + // Create destination file + var dstFile *os.File + if dstFile, err = os.Create(externalPath); err != nil { + return fmt.Errorf("astikit: creating %s failed: %w", externalPath, err) + } + defer dstFile.Close() + + // Create zip writer + var zw = zip.NewWriter(dstFile) + defer zw.Close() + + // Walk + if err = filepath.Walk(src, func(path string, info os.FileInfo, e error) (err error) { + // Process error + if e != nil { + err = e + return + } + + // Init header + var h *zip.FileHeader + if h, err = zip.FileInfoHeader(info); err != nil { + return fmt.Errorf("astikit: initializing zip header failed: %w", err) + } + + // Set header info + h.Name = filepath.Join(internalPath, strings.TrimPrefix(path, src)) + if info.IsDir() { + h.Name += string(os.PathSeparator) + } else { + h.Method = zip.Deflate + } + + // Create writer + var w io.Writer + if w, err = zw.CreateHeader(h); err != nil { + return fmt.Errorf("astikit: creating zip header failed: %w", err) + } + + // If path is dir, stop here + if info.IsDir() { + return + } + + // Open path + var walkFile *os.File + if walkFile, err = os.Open(path); err != nil { + return fmt.Errorf("astikit: opening %s failed: %w", path, err) + } + defer walkFile.Close() + + // Copy + if _, err = Copy(ctx, w, walkFile); err != nil { + return fmt.Errorf("astikit: copying failed: %w", err) + } + return + }); err != nil { + return fmt.Errorf("astikit: walking failed: %w", err) + } + return +} + +// Unzip unzips a src into a dst +// Possible src formats are: +// - /path/to/zip.zip +// - /path/to/zip.zip/root/path +func Unzip(ctx context.Context, dst, src string) (err error) { + // Get external/internal path + externalPath, internalPath := zipInternalPath(src) + + // Make sure the destination exists + if err = os.MkdirAll(dst, DefaultDirMode); err != nil { + return fmt.Errorf("astikit: mkdirall %s failed: %w", dst, err) + } + + // Open overall reader + var r *zip.ReadCloser + if r, err = zip.OpenReader(externalPath); err != nil { + return fmt.Errorf("astikit: opening overall zip reader on %s failed: %w", externalPath, err) + } + defer r.Close() + + // Loop through files to determine their type + var dirs, files, symlinks = make(map[string]*zip.File), make(map[string]*zip.File), make(map[string]*zip.File) + for _, f := range r.File { + // Validate internal path + if internalPath != "" && !strings.HasPrefix(f.Name, internalPath) { + continue + } + var p = filepath.Join(dst, strings.TrimPrefix(f.Name, internalPath)) + + // Check file type + if f.FileInfo().Mode()&os.ModeSymlink != 0 { + symlinks[p] = f + } else if f.FileInfo().IsDir() { + dirs[p] = f + } else { + files[p] = f + } + } + + // Create dirs + for p, f := range dirs { + if err = os.MkdirAll(p, f.FileInfo().Mode().Perm()); err != nil { + return fmt.Errorf("astikit: mkdirall %s failed: %w", p, err) + } + } + + // Create files + for p, f := range files { + if err = createZipFile(ctx, f, p); err != nil { + return fmt.Errorf("astikit: creating zip file into %s failed: %w", p, err) + } + } + + // Create symlinks + for p, f := range symlinks { + if err = createZipSymlink(f, p); err != nil { + return fmt.Errorf("astikit: creating zip symlink into %s failed: %w", p, err) + } + } + return +} + +func createZipFile(ctx context.Context, f *zip.File, p string) (err error) { + // Open file reader + var fr io.ReadCloser + if fr, err = f.Open(); err != nil { + return fmt.Errorf("astikit: opening zip reader on file %s failed: %w", f.Name, err) + } + defer fr.Close() + + // Since dirs don't always come up we make sure the directory of the file exists with default + // file mode + if err = os.MkdirAll(filepath.Dir(p), DefaultDirMode); err != nil { + return fmt.Errorf("astikit: mkdirall %s failed: %w", filepath.Dir(p), err) + } + + // Open the file + var fl *os.File + if fl, err = os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.FileInfo().Mode().Perm()); err != nil { + return fmt.Errorf("astikit: opening file %s failed: %w", p, err) + } + defer fl.Close() + + // Copy + if _, err = Copy(ctx, fl, fr); err != nil { + return fmt.Errorf("astikit: copying %s into %s failed: %w", f.Name, p, err) + } + return +} + +func createZipSymlink(f *zip.File, p string) (err error) { + // Open file reader + var fr io.ReadCloser + if fr, err = f.Open(); err != nil { + return fmt.Errorf("astikit: opening zip reader on file %s failed: %w", f.Name, err) + } + defer fr.Close() + + // If file is a symlink we retrieve the target path that is in the content of the file + var b []byte + if b, err = ioutil.ReadAll(fr); err != nil { + return fmt.Errorf("astikit: ioutil.Readall on %s failed: %w", f.Name, err) + } + + // Create the symlink + if err = os.Symlink(string(b), p); err != nil { + return fmt.Errorf("astikit: creating symlink from %s to %s failed: %w", string(b), p, err) + } + return +} diff --git a/archive_test.go b/archive_test.go new file mode 100644 index 0000000..3289048 --- /dev/null +++ b/archive_test.go @@ -0,0 +1,48 @@ +package astikit + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func TestZip(t *testing.T) { + // Create temp dir + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("creating temp dir failed: %w", err) + } + + // Make sure to delete temp dir + defer os.RemoveAll(dir) + + // With internal path + i := "testdata/archive" + f := filepath.Join(dir, "with-internal", "f.zip/root") + err = Zip(context.Background(), f, i) + if err != nil { + t.Errorf("expected no error, got %+v", err) + } + d := filepath.Join(dir, "with-internal", "d") + err = Unzip(context.Background(), d, f) + if err != nil { + t.Errorf("expected no error, got %+v", err) + } + compareDir(t, i, d) + + // Without internal path + i = "testdata/archive" + f = filepath.Join(dir, "without-internal", "f.zip") + err = Zip(context.Background(), f, i) + if err != nil { + t.Errorf("expected no error, got %+v", err) + } + d = filepath.Join(dir, "without-internal", "d") + err = Unzip(context.Background(), d, f) + if err != nil { + t.Errorf("expected no error, got %+v", err) + } + compareDir(t, i, d) +} diff --git a/astikit.go b/astikit.go new file mode 100644 index 0000000..cb6909a --- /dev/null +++ b/astikit.go @@ -0,0 +1,8 @@ +package astikit + +import "os" + +// Default modes +var ( + DefaultDirMode os.FileMode = 0755 +) diff --git a/astikit_test.go b/astikit_test.go new file mode 100644 index 0000000..e50a38a --- /dev/null +++ b/astikit_test.go @@ -0,0 +1,71 @@ +package astikit + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func fileContent(t *testing.T, path string) string { + b, err := ioutil.ReadFile(path) + if err != nil { + t.Errorf("expected no error, got %+v", err) + } + return string(b) +} + +func checkFile(t *testing.T, p string, e string) { + if g := fileContent(t, p); e != g { + t.Errorf("expected %s, got %s", e, g) + } +} + +func compareFile(t *testing.T, expectedPath, gotPath string) { + if e, g := fileContent(t, expectedPath), fileContent(t, gotPath); e != g { + t.Errorf("expected %s, got %s", e, g) + } +} + +func dirContent(t *testing.T, dir string) (o map[string]string) { + o = make(map[string]string) + err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) (err error) { + // Check error + if e != nil { + return e + } + + // Don't process dirs + if info.IsDir() { + return + } + + // Read + var b []byte + if b, err = ioutil.ReadFile(path); err != nil { + return + } + + // Add to map + o[strings.TrimPrefix(path, dir)] = string(b) + return + }) + if err != nil { + t.Errorf("expected no error, got %+v", err) + } + return +} + +func checkDir(t *testing.T, p string, e map[string]string) { + if g := dirContent(t, p); !reflect.DeepEqual(e, g) { + t.Errorf("expected %s, got %s", e, g) + } +} + +func compareDir(t *testing.T, ePath, gPath string) { + if e, g := dirContent(t, ePath), dirContent(t, gPath); !reflect.DeepEqual(e, g) { + t.Errorf("expected %+v, got %+v", e, g) + } +} diff --git a/http.go b/http.go index ed4ded1..c4765ca 100644 --- a/http.go +++ b/http.go @@ -309,7 +309,7 @@ func (d *HTTPDownloader) DownloadInDirectory(ctx context.Context, dst string, sr defer buf.Close() // Make sure destination directory exists - if err = os.MkdirAll(dst, 0755); err != nil { + if err = os.MkdirAll(dst, DefaultDirMode); err != nil { err = fmt.Errorf("astikit: mkdirall %s failed: %w", dst, err) return } @@ -414,7 +414,7 @@ func (d *HTTPDownloader) DownloadInWriter(ctx context.Context, dst io.Writer, sr // maintaining the initial order func (d *HTTPDownloader) DownloadInFile(ctx context.Context, dst string, srcs ...HTTPDownloaderSrc) (err error) { // Make sure destination directory exists - if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + if err = os.MkdirAll(filepath.Dir(dst), DefaultDirMode); err != nil { err = fmt.Errorf("astikit: mkdirall %s failed: %w", filepath.Dir(dst), err) return } diff --git a/http_test.go b/http_test.go index 5b6a87e..5291a25 100644 --- a/http_test.go +++ b/http_test.go @@ -8,7 +8,6 @@ import ( "net/http" "os" "path/filepath" - "reflect" "testing" "time" ) @@ -25,7 +24,7 @@ func TestServeHTTP(t *testing.T) { Addr: ln.Addr().String(), Handler: http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { w.Stop() - time.Sleep(100*time.Millisecond) + time.Sleep(100 * time.Millisecond) i++ }), }) @@ -114,7 +113,7 @@ func TestHTTPDownloader(t *testing.T) { // In case of DownloadInWriter we want to check if the order is kept event // if downloaded order is messed up if req.URL.EscapedPath() == "/path/to/2" { - time.Sleep(100*time.Millisecond) + time.Sleep(100 * time.Millisecond) } resp = &http.Response{ Body: ioutil.NopCloser(bytes.NewBufferString(req.URL.EscapedPath())), @@ -135,38 +134,11 @@ func TestHTTPDownloader(t *testing.T) { if err != nil { t.Errorf("expected no error, got %+v", err) } - dt := make(map[string]string) - err = filepath.Walk(dir, func(path string, info os.FileInfo, e error) (err error) { - // Check error - if e != nil { - return e - } - - // Don't process root - if path == dir { - return - } - - // Read - var b []byte - if b, err = ioutil.ReadFile(path); err != nil { - return - } - - // Add to map - dt[filepath.Base(path)] = string(b) - return + checkDir(t, dir, map[string]string{ + "/1": "/path/to/1", + "/2": "/path/to/2", + "/3": "/path/to/3", }) - if err != nil { - t.Errorf("expected no error, got %+v", err) - } - if e := map[string]string{ - "1": "/path/to/1", - "2": "/path/to/2", - "3": "/path/to/3", - }; !reflect.DeepEqual(e, dt) { - t.Errorf("expected %+v, got %+v", e, dt) - } // Download in writer w := &bytes.Buffer{} diff --git a/os.go b/os.go index 58e5b95..2b8d67f 100644 --- a/os.go +++ b/os.go @@ -79,7 +79,7 @@ func LocalCopyFileFunc(ctx context.Context, dst string, srcStat os.FileInfo, src } // Create the destination folder - if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + if err = os.MkdirAll(filepath.Dir(dst), DefaultDirMode); err != nil { err = fmt.Errorf("astikit: mkdirall %s failed: %w", filepath.Dir(dst), err) return } diff --git a/os_test.go b/os_test.go index e2cccad..9007cc1 100644 --- a/os_test.go +++ b/os_test.go @@ -8,16 +8,6 @@ import ( "testing" ) -func checkFile(t *testing.T, p string, e string) { - b, err := ioutil.ReadFile(p) - if err != nil { - t.Errorf("expected no error, got %+v", err) - } - if g := string(b); e != g { - t.Errorf("expected %s, got %s", e, g) - } -} - func TestCopyFile(t *testing.T) { // Create temporary dir p, err := ioutil.TempDir("", "") @@ -35,19 +25,20 @@ func TestCopyFile(t *testing.T) { }() // Copy file - err = CopyFile(context.Background(), filepath.Join(p, "f"), "testdata/os/f", LocalCopyFileFunc) + e := "testdata/os/f" + g := filepath.Join(p, "f") + err = CopyFile(context.Background(), g, e, LocalCopyFileFunc) if err != nil { t.Errorf("expected no error, got %+v", err) } - checkFile(t, filepath.Join(p, "f"), "0") + compareFile(t, e, g) // Copy dir - err = CopyFile(context.Background(), filepath.Join(p, "d"), "testdata/os/d", LocalCopyFileFunc) + e = "testdata/os/d" + g = filepath.Join(p, "d") + err = CopyFile(context.Background(), g, e, LocalCopyFileFunc) if err != nil { t.Errorf("expected no error, got %+v", err) } - checkFile(t, filepath.Join(p, "d", "f1"), "1") - checkFile(t, filepath.Join(p, "d", "d1", "f11"), "2") - checkFile(t, filepath.Join(p, "d", "d2", "f21"), "3") - checkFile(t, filepath.Join(p, "d", "d2", "d21", "f211"), "4") + compareDir(t, e, g) } diff --git a/testdata/archive/d/f b/testdata/archive/d/f new file mode 100644 index 0000000..56a6051 --- /dev/null +++ b/testdata/archive/d/f @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/testdata/archive/f b/testdata/archive/f new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/testdata/archive/f @@ -0,0 +1 @@ +0 \ No newline at end of file