From 4f8e7c131a2ea771d41587d2b51dd68de0bbdfcd Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Mon, 16 May 2022 23:59:28 +0200 Subject: [PATCH] WebDAV: Upload of videos, RAWs, moments, months, and states #2293 --- internal/query/albums.go | 47 ++++++++++ internal/query/file_selection.go | 30 ++++-- internal/query/file_selection_test.go | 121 +++++++++++++++++++++++++ internal/query/photo_selection.go | 7 ++ internal/query/photo_selection_test.go | 79 ++++++++-------- internal/search/photos.go | 7 ++ internal/workers/share.go | 21 +++-- 7 files changed, 252 insertions(+), 60 deletions(-) create mode 100644 internal/query/file_selection_test.go diff --git a/internal/query/albums.go b/internal/query/albums.go index 657026e02..58eb969ee 100644 --- a/internal/query/albums.go +++ b/internal/query/albums.go @@ -110,3 +110,50 @@ func AlbumEntryFound(uid string) error { return UnscopedDb().Exec(`UPDATE photos_albums SET missing = 0 WHERE photo_uid = ?`, uid).Error } } + +// AlbumsPhotoUIDs returns up to 10000 photo UIDs that belong to the specified albums. +func AlbumsPhotoUIDs(albums []string, includeDefault, includePrivate bool) (photos []string, err error) { + for _, albumUid := range albums { + a, err := AlbumByUID(albumUid) + + if err != nil { + log.Warnf("query: album %s not found (%s)", albumUid, err.Error()) + continue + } + + if a.IsDefault() && !includeDefault { + continue + } + + frm := form.SearchPhotos{ + Album: a.AlbumUID, + Filter: a.AlbumFilter, + Count: 10000, + Offset: 0, + Public: !includePrivate, + Hidden: false, + Archived: false, + Quality: 1, + } + + res, count, err := search.PhotoIds(frm) + + if err != nil { + return photos, err + } else if count == 0 { + continue + } + + ids := make([]string, 0, count) + + for _, r := range res { + ids = append(ids, r.PhotoUID) + } + + if len(ids) > 0 { + photos = append(photos, ids...) + } + } + + return photos, nil +} diff --git a/internal/query/file_selection.go b/internal/query/file_selection.go index 1e1b7ea38..d202542ca 100644 --- a/internal/query/file_selection.go +++ b/internal/query/file_selection.go @@ -46,10 +46,13 @@ func DownloadSelection(mediaRaw, mediaSidecar, originals bool) FileSelection { } // ShareSelection selects files to share, for example for upload via WebDAV. -func ShareSelection(primary bool) FileSelection { +func ShareSelection(originals bool) FileSelection { return FileSelection{ - Originals: !primary, - Primary: primary, + Originals: originals, + Primary: !originals, + Hidden: false, + Private: false, + Archived: false, MaxSize: 1024 * MegaByte, } } @@ -60,6 +63,13 @@ func SelectedFiles(f form.Selection, o FileSelection) (results entity.Files, err return results, errors.New("no items selected") } + // Resolve photos in smart albums. + if photoIds, err := AlbumsPhotoUIDs(f.Albums, false, o.Private); err != nil { + log.Warnf("query: %s", err.Error()) + } else if len(photoIds) > 0 { + f.Photos = append(f.Photos, photoIds...) + } + var concat string switch DbDialect() { case MySQL: @@ -93,37 +103,37 @@ func SelectedFiles(f form.Selection, o FileSelection) (results entity.Files, err // File size limit? if o.MaxSize > 0 { - s = s.Where("file_size < ?", o.MaxSize) + s = s.Where("files.file_size < ?", o.MaxSize) } // Specific media types only? if len(o.Media) > 0 { - s = s.Where("media_type IN (?)", o.Media) + s = s.Where("files.media_type IN (?)", o.Media) } // Exclude media types? if len(o.OmitMedia) > 0 { - s = s.Where("media_type NOT IN (?)", o.OmitMedia) + s = s.Where("files.media_type NOT IN (?)", o.OmitMedia) } // Specific file types only? if len(o.Types) > 0 { - s = s.Where("file_type IN (?)", o.Types) + s = s.Where("files.file_type IN (?)", o.Types) } // Exclude file types? if len(o.OmitTypes) > 0 { - s = s.Where("file_type NOT IN (?)", o.OmitTypes) + s = s.Where("files.file_type NOT IN (?)", o.OmitTypes) } // Primary files only? if o.Primary { - s = s.Where("file_primary = 1") + s = s.Where("files.file_primary = 1") } // Files in originals only? if o.Originals { - s = s.Where("file_root = '/'") + s = s.Where("files.file_root = '/'") } // Exclude private? diff --git a/internal/query/file_selection_test.go b/internal/query/file_selection_test.go new file mode 100644 index 000000000..1230b2974 --- /dev/null +++ b/internal/query/file_selection_test.go @@ -0,0 +1,121 @@ +package query + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/photoprism/photoprism/internal/form" +) + +func TestFileSelection(t *testing.T) { + none := form.Selection{Photos: []string{}} + + one := form.Selection{Photos: []string{"pt9jtdre2lvl0yh8"}} + + two := form.Selection{Photos: []string{"pt9jtdre2lvl0yh7", "pt9jtdre2lvl0yh8"}} + + albums := form.Selection{Albums: []string{"at9lxuqxpogaaba9", "at6axuzitogaaiax", "at9lxuqxpogaaba8", "at9lxuqxpogaaba7"}} + + months := form.Selection{Albums: []string{"at1lxuqipogaabj9"}} + + folders := form.Selection{Albums: []string{"at1lxuqipogaaba1", "at1lxuqipogaabj8"}} + + states := form.Selection{Albums: []string{"at1lxuqipogaab11", "at1lxuqipotaab12", "at1lxuqipotaab19"}} + + many := form.Selection{ + Files: []string{"ft8es39w45bnlqdw"}, + Photos: []string{"pt9jtdre2lvl0y21", "pt9jtdre2lvl0y19", "pr2xu7myk7wrbk38", "pt9jtdre2lvl0yh7", "pt9jtdre2lvl0yh8"}, + } + + t.Run("EmptySelection", func(t *testing.T) { + sel := DownloadSelection(true, false, true) + if results, err := SelectedFiles(none, sel); err == nil { + t.Fatal("error expected") + } else { + assert.Empty(t, results) + } + }) + t.Run("DownloadSelectionRawSidecarPrivate", func(t *testing.T) { + sel := DownloadSelection(true, true, false) + if results, err := SelectedFiles(one, sel); err != nil { + t.Fatal(err) + } else { + assert.Len(t, results, 2) + } + }) + t.Run("DownloadSelectionRawOriginals", func(t *testing.T) { + sel := DownloadSelection(true, false, true) + if results, err := SelectedFiles(two, sel); err != nil { + t.Fatal(err) + } else { + assert.Len(t, results, 2) + } + }) + t.Run("ShareSelectionOriginals", func(t *testing.T) { + sel := ShareSelection(false) + if results, err := SelectedFiles(many, sel); err != nil { + t.Fatal(err) + } else { + assert.Len(t, results, 4) + } + }) + t.Run("ShareSelectionPrimary", func(t *testing.T) { + sel := ShareSelection(true) + if results, err := SelectedFiles(many, sel); err != nil { + t.Fatal(err) + } else { + assert.Len(t, results, 6) + } + }) + t.Run("ShareAlbums", func(t *testing.T) { + sel := ShareSelection(true) + if results, err := SelectedFiles(albums, sel); err != nil { + t.Fatal(err) + } else { + assert.Len(t, results, 8) + } + }) + t.Run("ShareMonths", func(t *testing.T) { + sel := ShareSelection(true) + if results, err := SelectedFiles(months, sel); err != nil { + t.Fatal(err) + } else { + assert.Len(t, results, 0) + } + }) + t.Run("ShareFoldersOriginals", func(t *testing.T) { + sel := ShareSelection(true) + if results, err := SelectedFiles(folders, sel); err != nil { + t.Fatal(err) + } else { + assert.Len(t, results, 4) + } + }) + t.Run("ShareFolders", func(t *testing.T) { + sel := ShareSelection(false) + if results, err := SelectedFiles(folders, sel); err != nil { + t.Fatal(err) + } else { + log.Debugf("ShareFolders Results: %#v", results) + assert.Len(t, results, 2) + } + }) + t.Run("ShareStatesOriginals", func(t *testing.T) { + sel := ShareSelection(true) + if results, err := SelectedFiles(states, sel); err != nil { + t.Fatal(err) + } else { + assert.Len(t, results, 3) + } + }) + t.Run("ShareStates", func(t *testing.T) { + sel := ShareSelection(false) + if results, err := SelectedFiles(states, sel); err != nil { + t.Fatal(err) + } else { + log.Debugf("ShareStates Result: %#v", results[0]) + assert.Len(t, results, 1) + } + }) +} diff --git a/internal/query/photo_selection.go b/internal/query/photo_selection.go index 0621b981d..bb4c3ec46 100644 --- a/internal/query/photo_selection.go +++ b/internal/query/photo_selection.go @@ -14,6 +14,13 @@ func SelectedPhotos(f form.Selection) (results entity.Photos, err error) { return results, errors.New("no items selected") } + // Resolve photos in smart albums. + if photoIds, err := AlbumsPhotoUIDs(f.Albums, false, false); err != nil { + log.Warnf("query: %s", err.Error()) + } else if len(photoIds) > 0 { + f.Photos = append(f.Photos, photoIds...) + } + var concat string switch DbDialect() { diff --git a/internal/query/photo_selection_test.go b/internal/query/photo_selection_test.go index 24936d8f0..d2141198e 100644 --- a/internal/query/photo_selection_test.go +++ b/internal/query/photo_selection_test.go @@ -3,12 +3,21 @@ package query import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/form" - "github.com/stretchr/testify/assert" ) func TestPhotoSelection(t *testing.T) { + albums := form.Selection{Albums: []string{"at9lxuqxpogaaba9", "at6axuzitogaaiax", "at9lxuqxpogaaba8", "at9lxuqxpogaaba7"}} + + months := form.Selection{Albums: []string{"at1lxuqipogaabj9"}} + + folders := form.Selection{Albums: []string{"at1lxuqipogaaba1", "at1lxuqipogaabj8"}} + + states := form.Selection{Albums: []string{"at1lxuqipogaab11", "at1lxuqipotaab12", "at1lxuqipotaab19"}} + t.Run("no items selected", func(t *testing.T) { f := form.Selection{ Photos: []string{}, @@ -33,58 +42,44 @@ func TestPhotoSelection(t *testing.T) { assert.Equal(t, 2, len(r)) assert.IsType(t, entity.Photos{}, r) }) -} + t.Run("FindAlbums", func(t *testing.T) { + r, err := SelectedPhotos(albums) -func TestFileSelection(t *testing.T) { - none := form.Selection{Photos: []string{}} - - one := form.Selection{Photos: []string{"pt9jtdre2lvl0yh8"}} - - two := form.Selection{Photos: []string{"pt9jtdre2lvl0yh7", "pt9jtdre2lvl0yh8"}} - - many := form.Selection{ - Files: []string{"ft8es39w45bnlqdw"}, - Photos: []string{"pt9jtdre2lvl0y21", "pt9jtdre2lvl0y19", "pr2xu7myk7wrbk38", "pt9jtdre2lvl0yh7", "pt9jtdre2lvl0yh8"}, - } - - t.Run("EmptySelection", func(t *testing.T) { - sel := DownloadSelection(true, false, true) - if results, err := SelectedFiles(none, sel); err == nil { - t.Fatal("error expected") - } else { - assert.Empty(t, results) - } - }) - t.Run("DownloadSelectionRawSidecarPrivate", func(t *testing.T) { - sel := DownloadSelection(true, true, false) - if results, err := SelectedFiles(one, sel); err != nil { + if err != nil { t.Fatal(err) - } else { - assert.Len(t, results, 2) } + + assert.Equal(t, 6, len(r)) + assert.IsType(t, entity.Photos{}, r) }) - t.Run("DownloadSelectionRawOriginals", func(t *testing.T) { - sel := DownloadSelection(true, false, true) - if results, err := SelectedFiles(two, sel); err != nil { + t.Run("FindMonths", func(t *testing.T) { + r, err := SelectedPhotos(months) + + if err != nil { t.Fatal(err) - } else { - assert.Len(t, results, 2) } + + assert.Equal(t, 0, len(r)) + assert.IsType(t, entity.Photos{}, r) }) - t.Run("ShareSelectionOriginals", func(t *testing.T) { - sel := ShareSelection(false) - if results, err := SelectedFiles(many, sel); err != nil { + t.Run("FindFolders", func(t *testing.T) { + r, err := SelectedPhotos(folders) + + if err != nil { t.Fatal(err) - } else { - assert.Len(t, results, 6) } + + assert.Equal(t, 2, len(r)) + assert.IsType(t, entity.Photos{}, r) }) - t.Run("ShareSelectionPrimary", func(t *testing.T) { - sel := ShareSelection(true) - if results, err := SelectedFiles(many, sel); err != nil { + t.Run("FindStates", func(t *testing.T) { + r, err := SelectedPhotos(states) + + if err != nil { t.Fatal(err) - } else { - assert.Len(t, results, 4) } + + assert.Equal(t, 1, len(r)) + assert.IsType(t, entity.Photos{}, r) }) } diff --git a/internal/search/photos.go b/internal/search/photos.go index 9d17bb66e..a7c8596be 100644 --- a/internal/search/photos.go +++ b/internal/search/photos.go @@ -31,6 +31,13 @@ func Photos(f form.SearchPhotos) (results PhotoResults, count int, err error) { return searchPhotos(f, PhotosColsAll) } +// PhotoIds finds photo and file ids based on the search form provided and returns them as PhotoResults. +func PhotoIds(f form.SearchPhotos) (files PhotoResults, count int, err error) { + f.Merged = false + f.Primary = true + return searchPhotos(f, "photos.id, photos.photo_uid, files.file_uid") +} + // photos searches for photos based on a Form and returns PhotoResults ([]Photo). func searchPhotos(f form.SearchPhotos, resultCols string) (results PhotoResults, count int, err error) { start := time.Now() diff --git a/internal/workers/share.go b/internal/workers/share.go index 7b626404d..83d395eb3 100644 --- a/internal/workers/share.go +++ b/internal/workers/share.go @@ -5,6 +5,8 @@ import ( "path/filepath" "runtime/debug" + "github.com/photoprism/photoprism/pkg/fs" + "github.com/photoprism/photoprism/internal/config" "github.com/photoprism/photoprism/internal/entity" "github.com/photoprism/photoprism/internal/form" @@ -78,6 +80,16 @@ func (worker *Share) Start() (err error) { continue } + size := thumb.Size{} + + if a.ShareSize != "" { + if s, ok := thumb.Sizes[thumb.Name(a.ShareSize)]; ok { + size = s + } else { + size = thumb.Sizes[thumb.Fit2048] + } + } + client := webdav.New(a.AccURL, a.AccUser, a.AccPass, webdav.Timeout(a.AccTimeout)) existingDirs := make(map[string]string) @@ -97,14 +109,7 @@ func (worker *Share) Start() (err error) { srcFileName := photoprism.FileName(file.File.FileRoot, file.File.FileName) - if a.ShareSize != "" { - size, ok := thumb.Sizes[thumb.Name(a.ShareSize)] - - if !ok { - log.Errorf("share: invalid size %s", a.ShareSize) - continue - } - + if fs.ImageJPEG.Equal(file.File.FileType) && size.Width > 0 && size.Height > 0 { srcFileName, err = thumb.FromFile(srcFileName, file.File.FileHash, worker.conf.ThumbCachePath(), size.Width, size.Height, file.File.FileOrientation, size.Options...) if err != nil {