Added archive

archive.go

@@ -0,0 +1,209 @@
+package astikit
+
+import (
+	"archive/zip"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// internal shouldn't lead with a "/"
+func zipInternalPath(p string) (external, internal string) {
+	if items := strings.Split(p, ".zip"); len(items) > 1 {
+		external = items[0] + ".zip"
+		internal = strings.TrimPrefix(strings.Join(items[1:], ".zip"), string(os.PathSeparator))
+		return
+	}
+	external = p
+	return
+}
+
+// Zip zips a src into a dst
+// Possible dst formats are:
+//   - /path/to/zip.zip
+//   - /path/to/zip.zip/root/path
+func Zip(ctx context.Context, dst, src string) (err error) {
+	// Get external/internal path
+	externalPath, internalPath := zipInternalPath(dst)
+
+	// Make sure the directory exists
+	if err = os.MkdirAll(filepath.Dir(externalPath), DefaultDirMode); err != nil {
+		return fmt.Errorf("astikit: mkdirall %s failed: %w", filepath.Dir(externalPath), err)
+	}
+
+	// Create destination file
+	var dstFile *os.File
+	if dstFile, err = os.Create(externalPath); err != nil {
+		return fmt.Errorf("astikit: creating %s failed: %w", externalPath, err)
+	}
+	defer dstFile.Close()
+
+	// Create zip writer
+	var zw = zip.NewWriter(dstFile)
+	defer zw.Close()
+
+	// Walk
+	if err = filepath.Walk(src, func(path string, info os.FileInfo, e error) (err error) {
+		// Process error
+		if e != nil {
+			err = e
+			return
+		}
+
+		// Init header
+		var h *zip.FileHeader
+		if h, err = zip.FileInfoHeader(info); err != nil {
+			return fmt.Errorf("astikit: initializing zip header failed: %w", err)
+		}
+
+		// Set header info
+		h.Name = filepath.Join(internalPath, strings.TrimPrefix(path, src))
+		if info.IsDir() {
+			h.Name += string(os.PathSeparator)
+		} else {
+			h.Method = zip.Deflate
+		}
+
+		// Create writer
+		var w io.Writer
+		if w, err = zw.CreateHeader(h); err != nil {
+			return fmt.Errorf("astikit: creating zip header failed: %w", err)
+		}
+
+		// If path is dir, stop here
+		if info.IsDir() {
+			return
+		}
+
+		// Open path
+		var walkFile *os.File
+		if walkFile, err = os.Open(path); err != nil {
+			return fmt.Errorf("astikit: opening %s failed: %w", path, err)
+		}
+		defer walkFile.Close()
+
+		// Copy
+		if _, err = Copy(ctx, w, walkFile); err != nil {
+			return fmt.Errorf("astikit: copying failed: %w", err)
+		}
+		return
+	}); err != nil {
+		return fmt.Errorf("astikit: walking failed: %w", err)
+	}
+	return
+}
+
+// Unzip unzips a src into a dst
+// Possible src formats are:
+//   - /path/to/zip.zip
+//   - /path/to/zip.zip/root/path
+func Unzip(ctx context.Context, dst, src string) (err error) {
+	// Get external/internal path
+	externalPath, internalPath := zipInternalPath(src)
+
+	// Make sure the destination exists
+	if err = os.MkdirAll(dst, DefaultDirMode); err != nil {
+		return fmt.Errorf("astikit: mkdirall %s failed: %w", dst, err)
+	}
+
+	// Open overall reader
+	var r *zip.ReadCloser
+	if r, err = zip.OpenReader(externalPath); err != nil {
+		return fmt.Errorf("astikit: opening overall zip reader on %s failed: %w", externalPath, err)
+	}
+	defer r.Close()
+
+	// Loop through files to determine their type
+	var dirs, files, symlinks = make(map[string]*zip.File), make(map[string]*zip.File), make(map[string]*zip.File)
+	for _, f := range r.File {
+		// Validate internal path
+		if internalPath != "" && !strings.HasPrefix(f.Name, internalPath) {
+			continue
+		}
+		var p = filepath.Join(dst, strings.TrimPrefix(f.Name, internalPath))
+
+		// Check file type
+		if f.FileInfo().Mode()&os.ModeSymlink != 0 {
+			symlinks[p] = f
+		} else if f.FileInfo().IsDir() {
+			dirs[p] = f
+		} else {
+			files[p] = f
+		}
+	}
+
+	// Create dirs
+	for p, f := range dirs {
+		if err = os.MkdirAll(p, f.FileInfo().Mode().Perm()); err != nil {
+			return fmt.Errorf("astikit: mkdirall %s failed: %w", p, err)
+		}
+	}
+
+	// Create files
+	for p, f := range files {
+		if err = createZipFile(ctx, f, p); err != nil {
+			return fmt.Errorf("astikit: creating zip file into %s failed: %w", p, err)
+		}
+	}
+
+	// Create symlinks
+	for p, f := range symlinks {
+		if err = createZipSymlink(f, p); err != nil {
+			return fmt.Errorf("astikit: creating zip symlink into %s failed: %w", p, err)
+		}
+	}
+	return
+}
+
+func createZipFile(ctx context.Context, f *zip.File, p string) (err error) {
+	// Open file reader
+	var fr io.ReadCloser
+	if fr, err = f.Open(); err != nil {
+		return fmt.Errorf("astikit: opening zip reader on file %s failed: %w", f.Name, err)
+	}
+	defer fr.Close()
+
+	// Since dirs don't always come up we make sure the directory of the file exists with default
+	// file mode
+	if err = os.MkdirAll(filepath.Dir(p), DefaultDirMode); err != nil {
+		return fmt.Errorf("astikit: mkdirall %s failed: %w", filepath.Dir(p), err)
+	}
+
+	// Open the file
+	var fl *os.File
+	if fl, err = os.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.FileInfo().Mode().Perm()); err != nil {
+		return fmt.Errorf("astikit: opening file %s failed: %w", p, err)
+	}
+	defer fl.Close()
+
+	// Copy
+	if _, err = Copy(ctx, fl, fr); err != nil {
+		return fmt.Errorf("astikit: copying %s into %s failed: %w", f.Name, p, err)
+	}
+	return
+}
+
+func createZipSymlink(f *zip.File, p string) (err error) {
+	// Open file reader
+	var fr io.ReadCloser
+	if fr, err = f.Open(); err != nil {
+		return fmt.Errorf("astikit: opening zip reader on file %s failed: %w", f.Name, err)
+	}
+	defer fr.Close()
+
+	// If file is a symlink we retrieve the target path that is in the content of the file
+	var b []byte
+	if b, err = ioutil.ReadAll(fr); err != nil {
+		return fmt.Errorf("astikit: ioutil.Readall on %s failed: %w", f.Name, err)
+	}
+
+	// Create the symlink
+	if err = os.Symlink(string(b), p); err != nil {
+		return fmt.Errorf("astikit: creating symlink from %s to %s failed: %w", string(b), p, err)
+	}
+	return
+}

archive_test.go

@@ -0,0 +1,48 @@
+package astikit
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestZip(t *testing.T) {
+	// Create temp dir
+	dir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("creating temp dir failed: %w", err)
+	}
+
+	// Make sure to delete temp dir
+	defer os.RemoveAll(dir)
+
+	// With internal path
+	i := "testdata/archive"
+	f := filepath.Join(dir, "with-internal", "f.zip/root")
+	err = Zip(context.Background(), f, i)
+	if err != nil {
+		t.Errorf("expected no error, got %+v", err)
+	}
+	d := filepath.Join(dir, "with-internal", "d")
+	err = Unzip(context.Background(), d, f)
+	if err != nil {
+		t.Errorf("expected no error, got %+v", err)
+	}
+	compareDir(t, i, d)
+
+	// Without internal path
+	i = "testdata/archive"
+	f = filepath.Join(dir, "without-internal", "f.zip")
+	err = Zip(context.Background(), f, i)
+	if err != nil {
+		t.Errorf("expected no error, got %+v", err)
+	}
+	d = filepath.Join(dir, "without-internal", "d")
+	err = Unzip(context.Background(), d, f)
+	if err != nil {
+		t.Errorf("expected no error, got %+v", err)
+	}
+	compareDir(t, i, d)
+}

astikit.go

@@ -0,0 +1,8 @@
+package astikit
+
+import "os"
+
+// Default modes
+var (
+	DefaultDirMode os.FileMode = 0755
+)

astikit_test.go

@@ -0,0 +1,71 @@
+package astikit
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func fileContent(t *testing.T, path string) string {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		t.Errorf("expected no error, got %+v", err)
+	}
+	return string(b)
+}
+
+func checkFile(t *testing.T, p string, e string) {
+	if g := fileContent(t, p); e != g {
+		t.Errorf("expected %s, got %s", e, g)
+	}
+}
+
+func compareFile(t *testing.T, expectedPath, gotPath string) {
+	if e, g := fileContent(t, expectedPath), fileContent(t, gotPath); e != g {
+		t.Errorf("expected %s, got %s", e, g)
+	}
+}
+
+func dirContent(t *testing.T, dir string) (o map[string]string) {
+	o = make(map[string]string)
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) (err error) {
+		// Check error
+		if e != nil {
+			return e
+		}
+
+		// Don't process dirs
+		if info.IsDir() {
+			return
+		}
+
+		// Read
+		var b []byte
+		if b, err = ioutil.ReadFile(path); err != nil {
+			return
+		}
+
+		// Add to map
+		o[strings.TrimPrefix(path, dir)] = string(b)
+		return
+	})
+	if err != nil {
+		t.Errorf("expected no error, got %+v", err)
+	}
+	return
+}
+
+func checkDir(t *testing.T, p string, e map[string]string) {
+	if g := dirContent(t, p); !reflect.DeepEqual(e, g) {
+		t.Errorf("expected %s, got %s", e, g)
+	}
+}
+
+func compareDir(t *testing.T, ePath, gPath string) {
+	if e, g := dirContent(t, ePath), dirContent(t, gPath); !reflect.DeepEqual(e, g) {
+		t.Errorf("expected %+v, got %+v", e, g)
+	}
+}

http.go

@@ -309,7 +309,7 @@ func (d *HTTPDownloader) DownloadInDirectory(ctx context.Context, dst string, sr
 		defer buf.Close()
 
 		// Make sure destination directory exists
-		if err = os.MkdirAll(dst, 0755); err != nil {
+		if err = os.MkdirAll(dst, DefaultDirMode); err != nil {
 			err = fmt.Errorf("astikit: mkdirall %s failed: %w", dst, err)
 			return
 		}
@@ -414,7 +414,7 @@ func (d *HTTPDownloader) DownloadInWriter(ctx context.Context, dst io.Writer, sr
 // maintaining the initial order
 func (d *HTTPDownloader) DownloadInFile(ctx context.Context, dst string, srcs ...HTTPDownloaderSrc) (err error) {
 	// Make sure destination directory exists
-	if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+	if err = os.MkdirAll(filepath.Dir(dst), DefaultDirMode); err != nil {
 		err = fmt.Errorf("astikit: mkdirall %s failed: %w", filepath.Dir(dst), err)
 		return
 	}

http_test.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"reflect"
 	"testing"
 	"time"
 )
@@ -25,7 +24,7 @@ func TestServeHTTP(t *testing.T) {
 		Addr: ln.Addr().String(),
 		Handler: http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
 			w.Stop()
-			time.Sleep(100*time.Millisecond)
+			time.Sleep(100 * time.Millisecond)
 			i++
 		}),
 	})
@@ -114,7 +113,7 @@ func TestHTTPDownloader(t *testing.T) {
 				// In case of DownloadInWriter we want to check if the order is kept event
 				// if downloaded order is messed up
 				if req.URL.EscapedPath() == "/path/to/2" {
-					time.Sleep(100*time.Millisecond)
+					time.Sleep(100 * time.Millisecond)
 				}
 				resp = &http.Response{
 					Body:       ioutil.NopCloser(bytes.NewBufferString(req.URL.EscapedPath())),
@@ -135,38 +134,11 @@ func TestHTTPDownloader(t *testing.T) {
 	if err != nil {
 		t.Errorf("expected no error, got %+v", err)
 	}
-	dt := make(map[string]string)
-	err = filepath.Walk(dir, func(path string, info os.FileInfo, e error) (err error) {
-		// Check error
-		if e != nil {
-			return e
-		}
-
-		// Don't process root
-		if path == dir {
-			return
-		}
-
-		// Read
-		var b []byte
-		if b, err = ioutil.ReadFile(path); err != nil {
-			return
-		}
-
-		// Add to map
-		dt[filepath.Base(path)] = string(b)
-		return
+	checkDir(t, dir, map[string]string{
+		"/1": "/path/to/1",
+		"/2": "/path/to/2",
+		"/3": "/path/to/3",
 	})
-	if err != nil {
-		t.Errorf("expected no error, got %+v", err)
-	}
-	if e := map[string]string{
-		"1": "/path/to/1",
-		"2": "/path/to/2",
-		"3": "/path/to/3",
-	}; !reflect.DeepEqual(e, dt) {
-		t.Errorf("expected %+v, got %+v", e, dt)
-	}
 
 	// Download in writer
 	w := &bytes.Buffer{}

os.go

@@ -79,7 +79,7 @@ func LocalCopyFileFunc(ctx context.Context, dst string, srcStat os.FileInfo, src
 	}
 
 	// Create the destination folder
-	if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+	if err = os.MkdirAll(filepath.Dir(dst), DefaultDirMode); err != nil {
 		err = fmt.Errorf("astikit: mkdirall %s failed: %w", filepath.Dir(dst), err)
 		return
 	}

os_test.go

@@ -8,16 +8,6 @@ import (
 	"testing"
 )
 
-func checkFile(t *testing.T, p string, e string) {
-	b, err := ioutil.ReadFile(p)
-	if err != nil {
-		t.Errorf("expected no error, got %+v", err)
-	}
-	if g := string(b); e != g {
-		t.Errorf("expected %s, got %s", e, g)
-	}
-}
-
 func TestCopyFile(t *testing.T) {
 	// Create temporary dir
 	p, err := ioutil.TempDir("", "")
@@ -35,19 +25,20 @@ func TestCopyFile(t *testing.T) {
 	}()
 
 	// Copy file
-	err = CopyFile(context.Background(), filepath.Join(p, "f"), "testdata/os/f", LocalCopyFileFunc)
+	e := "testdata/os/f"
+	g := filepath.Join(p, "f")
+	err = CopyFile(context.Background(), g, e, LocalCopyFileFunc)
 	if err != nil {
 		t.Errorf("expected no error, got %+v", err)
 	}
-	checkFile(t, filepath.Join(p, "f"), "0")
+	compareFile(t, e, g)
 
 	// Copy dir
-	err = CopyFile(context.Background(), filepath.Join(p, "d"), "testdata/os/d", LocalCopyFileFunc)
+	e = "testdata/os/d"
+	g = filepath.Join(p, "d")
+	err = CopyFile(context.Background(), g, e, LocalCopyFileFunc)
 	if err != nil {
 		t.Errorf("expected no error, got %+v", err)
 	}
-	checkFile(t, filepath.Join(p, "d", "f1"), "1")
-	checkFile(t, filepath.Join(p, "d", "d1", "f11"), "2")
-	checkFile(t, filepath.Join(p, "d", "d2", "f21"), "3")
-	checkFile(t, filepath.Join(p, "d", "d2", "d21", "f211"), "4")
+	compareDir(t, e, g)
 }

testdata/archive/d/f

@@ -0,0 +1 @@
+1

testdata/archive/f

@@ -0,0 +1 @@
+0