update
This commit is contained in:
parent
aa69771b1d
commit
56177f5b3c
|
@ -23,6 +23,7 @@ require (
|
|||
github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0
|
||||
github.com/xor-gate/goexif2 v1.1.0
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
|
||||
golang.org/x/exp v0.0.0-20230116083435-1de6713980de
|
||||
golang.org/x/image v0.0.0-20220617043117-41969df76e82
|
||||
gopkg.in/vansante/go-ffprobe.v2 v2.0.3
|
||||
gorm.io/driver/mysql v1.3.4
|
||||
|
@ -31,7 +32,7 @@ require (
|
|||
gorm.io/gorm v1.23.7
|
||||
)
|
||||
|
||||
require golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect
|
||||
require golang.org/x/sys v0.1.0 // indirect
|
||||
|
||||
require (
|
||||
github.com/agnivade/levenshtein v1.1.1 // indirect
|
||||
|
|
|
@ -214,6 +214,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
|
|||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/exp v0.0.0-20230116083435-1de6713980de h1:DBWn//IJw30uYCgERoxCg84hWtA97F4wMiKOIh00Uf0=
|
||||
golang.org/x/exp v0.0.0-20230116083435-1de6713980de/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.0.0-20220617043117-41969df76e82 h1:KpZB5pUSBvrHltNEdK/tw0xlPeD13M6M6aGP32gKqiw=
|
||||
golang.org/x/image v0.0.0-20220617043117-41969df76e82/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
|
||||
|
@ -246,8 +248,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
|
|||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY=
|
||||
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
|
|
|
@ -1,9 +1,12 @@
|
|||
package scanner_queue
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"path"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
@ -20,7 +23,8 @@ import (
|
|||
|
||||
// ScannerJob describes a job on the queue to be run by the scanner over a single album
|
||||
type ScannerJob struct {
|
||||
ctx scanner_task.TaskContext
|
||||
ctx scanner_task.TaskContext
|
||||
media_paths []string
|
||||
// album *models.Album
|
||||
// cache *scanner_cache.AlbumScannerCache
|
||||
}
|
||||
|
@ -28,6 +32,7 @@ type ScannerJob struct {
|
|||
func NewScannerJob(ctx scanner_task.TaskContext) ScannerJob {
|
||||
return ScannerJob{
|
||||
ctx,
|
||||
make([]string, 0),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -150,7 +155,7 @@ func (queue *ScannerQueue) processQueue(notifyThrottle *utils.Throttle) {
|
|||
// Delete finished job from queue
|
||||
queue.mutex.Lock()
|
||||
for i, x := range queue.in_progress {
|
||||
if x == nextJob {
|
||||
if x.ctx == nextJob.ctx && reflect.DeepEqual(x.media_paths, nextJob.media_paths) {
|
||||
queue.in_progress[i] = queue.in_progress[len(queue.in_progress)-1]
|
||||
queue.in_progress = queue.in_progress[0 : len(queue.in_progress)-1]
|
||||
break
|
||||
|
@ -246,6 +251,47 @@ func AddUserToQueue(user *models.User) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func AddMediaToQueue(mediaPath string) error {
|
||||
var media *models.Media
|
||||
|
||||
if err := global_scanner_queue.db.Preload("Album").Where("path = ?", mediaPath).Find(&media).Error; err != nil {
|
||||
return errors.Wrap(err, "media by path database query")
|
||||
}
|
||||
|
||||
// add album to the queue
|
||||
var album *models.Album
|
||||
var subalbumPath string
|
||||
if media == nil {
|
||||
albumPath := path.Base(mediaPath)
|
||||
for album == nil {
|
||||
if err := global_scanner_queue.db.Where("path = ?", albumPath).Find(&album).Error; err != nil {
|
||||
return errors.Wrap(err, "album by path database query")
|
||||
}
|
||||
subalbumPath = albumPath
|
||||
albumPath = path.Base(albumPath)
|
||||
}
|
||||
if album == nil {
|
||||
return errors.New("No root album found")
|
||||
}
|
||||
}
|
||||
|
||||
var userAlbumOwner []*models.User
|
||||
if err := global_scanner_queue.db.Model(&album).Association("Owners").Find(&userAlbumOwner); err != nil {
|
||||
return errors.Wrap(err, "find owners for album")
|
||||
}
|
||||
|
||||
scanQueue := list.New()
|
||||
scanQueue.PushBack(scanner.ScanInfo{
|
||||
Path: subalbumPath,
|
||||
Parent: album,
|
||||
Ignore: nil,
|
||||
})
|
||||
album_cache := scanner_cache.MakeAlbumCache()
|
||||
scanner.ProcessUserAlbums(scanQueue, global_scanner_queue.db, userAlbumOwner, album_cache)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Queue should be locked prior to calling this function
|
||||
func (queue *ScannerQueue) addJob(job *ScannerJob) error {
|
||||
if exists, err := queue.jobOnQueue(job); exists || err != nil {
|
||||
|
|
|
@ -15,6 +15,7 @@ import (
|
|||
"github.com/photoview/photoview/api/utils"
|
||||
"github.com/pkg/errors"
|
||||
ignore "github.com/sabhiram/go-gitignore"
|
||||
"golang.org/x/exp/slices"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
|
@ -43,6 +44,12 @@ func getPhotoviewIgnore(ignorePath string) ([]string, error) {
|
|||
return photoviewIgnore, scanner.Err()
|
||||
}
|
||||
|
||||
type ScanInfo struct {
|
||||
Path string
|
||||
Parent *models.Album
|
||||
Ignore []string
|
||||
}
|
||||
|
||||
func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cache.AlbumScannerCache) ([]*models.Album, []error) {
|
||||
|
||||
if err := user.FillAlbums(db); err != nil {
|
||||
|
@ -61,12 +68,6 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
|
|||
|
||||
scanErrors := make([]error, 0)
|
||||
|
||||
type scanInfo struct {
|
||||
path string
|
||||
parent *models.Album
|
||||
ignore []string
|
||||
}
|
||||
|
||||
scanQueue := list.New()
|
||||
|
||||
for _, album := range userRootAlbums {
|
||||
|
@ -78,26 +79,44 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
|
|||
scanErrors = append(scanErrors, errors.Errorf("Could not read album directory for user '%s': %s\n", user.Username, album.Path))
|
||||
}
|
||||
} else {
|
||||
scanQueue.PushBack(scanInfo{
|
||||
path: album.Path,
|
||||
parent: nil,
|
||||
ignore: nil,
|
||||
scanQueue.PushBack(ScanInfo{
|
||||
Path: album.Path,
|
||||
Parent: nil,
|
||||
Ignore: nil,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
userAlbums, err2 := ProcessUserAlbums(scanQueue, db, []*models.User{user}, album_cache)
|
||||
if err2 != nil {
|
||||
scanErrors = append(scanErrors, err2...)
|
||||
}
|
||||
|
||||
deleteErrors := cleanup_tasks.DeleteOldUserAlbums(db, userAlbums, user)
|
||||
scanErrors = append(scanErrors, deleteErrors...)
|
||||
|
||||
return userAlbums, scanErrors
|
||||
}
|
||||
|
||||
func ProcessUserAlbums(scanQueue *list.List, db *gorm.DB, users []*models.User, album_cache *scanner_cache.AlbumScannerCache) ([]*models.Album, []error) {
|
||||
userAlbums := make([]*models.Album, 0)
|
||||
var scanErrors []error
|
||||
|
||||
userIds := make([]int, len(users))
|
||||
for i, user := range users {
|
||||
userIds[i] = user.ID
|
||||
}
|
||||
|
||||
for scanQueue.Front() != nil {
|
||||
albumInfo := scanQueue.Front().Value.(scanInfo)
|
||||
albumInfo := scanQueue.Front().Value.(ScanInfo)
|
||||
scanQueue.Remove(scanQueue.Front())
|
||||
|
||||
albumPath := albumInfo.path
|
||||
albumParent := albumInfo.parent
|
||||
albumIgnore := albumInfo.ignore
|
||||
albumPath := albumInfo.Path
|
||||
albumParent := albumInfo.Parent
|
||||
albumIgnore := albumInfo.Ignore
|
||||
|
||||
// Read path
|
||||
dirContent, err := ioutil.ReadDir(albumPath)
|
||||
dirContent, err := os.ReadDir(albumPath)
|
||||
if err != nil {
|
||||
scanErrors = append(scanErrors, errors.Wrapf(err, "read directory (%s)", albumPath))
|
||||
continue
|
||||
|
@ -166,14 +185,20 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
|
|||
|
||||
// Add user as an owner of the album if not already
|
||||
var userAlbumOwner []models.User
|
||||
if err := tx.Model(&album).Association("Owners").Find(&userAlbumOwner, "user_albums.user_id = ?", user.ID); err != nil {
|
||||
if err := tx.Model(&album).Association("Owners").Find(&userAlbumOwner, "user_albums.user_id in (?)", userIds); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(userAlbumOwner) == 0 {
|
||||
newUser := models.User{}
|
||||
newUser.ID = user.ID
|
||||
if err := tx.Model(&album).Association("Owners").Append(&newUser); err != nil {
|
||||
return err
|
||||
if len(userAlbumOwner) != len(userIds) {
|
||||
for userId := range userIds {
|
||||
i := slices.IndexFunc(userAlbumOwner, func(user models.User) bool { return user.ID == userId })
|
||||
if i != -1 {
|
||||
continue
|
||||
}
|
||||
newUser := models.User{}
|
||||
newUser.ID = userId
|
||||
if err := tx.Model(&album).Association("Owners").Append(&newUser); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -207,18 +232,15 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
|
|||
}
|
||||
|
||||
if (item.IsDir() || isDirSymlink) && directoryContainsPhotos(subalbumPath, album_cache, albumIgnore) {
|
||||
scanQueue.PushBack(scanInfo{
|
||||
path: subalbumPath,
|
||||
parent: album,
|
||||
ignore: albumIgnore,
|
||||
scanQueue.PushBack(ScanInfo{
|
||||
Path: subalbumPath,
|
||||
Parent: album,
|
||||
Ignore: albumIgnore,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
deleteErrors := cleanup_tasks.DeleteOldUserAlbums(db, userAlbums, user)
|
||||
scanErrors = append(scanErrors, deleteErrors...)
|
||||
|
||||
return userAlbums, scanErrors
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,101 @@
|
|||
package watcher_scanner
|
||||
|
||||
import (
|
||||
"github.com/fsnotify/fsnotify"
|
||||
"github.com/photoview/photoview/api/graphql/models"
|
||||
"github.com/photoview/photoview/api/scanner/scanner_queue"
|
||||
"github.com/pkg/errors"
|
||||
"gorm.io/gorm"
|
||||
"log"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var mainWatcherScanner *watcherScanner = nil
|
||||
|
||||
type watcherScanner struct {
|
||||
watcherChanged chan bool
|
||||
watcher *fsnotify.Watcher
|
||||
mutex *sync.Mutex
|
||||
db *gorm.DB
|
||||
}
|
||||
|
||||
func InitializeWatcherScanner(db *gorm.DB) error {
|
||||
if mainWatcherScanner != nil {
|
||||
panic("watcher scanner has already been initialized")
|
||||
}
|
||||
|
||||
// Create new watcher
|
||||
watcher, err := fsnotify.NewWatcher()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
mainWatcherScanner = &watcherScanner{
|
||||
db: db,
|
||||
watcherChanged: make(chan bool),
|
||||
watcher: watcher,
|
||||
mutex: &sync.Mutex{},
|
||||
}
|
||||
|
||||
success, errs := mainWatcherScanner.addPathsToWatch()
|
||||
if len(errs) != 0 {
|
||||
log.Println("watcher errors")
|
||||
log.Println(errs)
|
||||
if !success {
|
||||
return errors.New("errors found during watcher scanner setup")
|
||||
}
|
||||
}
|
||||
|
||||
go mainWatcherScanner.processWatchEvents()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ws watcherScanner) addPathsToWatch() (bool, []error) {
|
||||
var allAlbumPaths []*models.Album
|
||||
|
||||
if err := ws.db.Select("path").Find(&allAlbumPaths).Error; err != nil {
|
||||
return false, []error{errors.Wrap(err, "watcher scanner find albums query")}
|
||||
}
|
||||
|
||||
errs := make([]error, 0)
|
||||
for _, album := range allAlbumPaths {
|
||||
// log.Println("add path", album.Path)
|
||||
err := ws.watcher.Add(album.Path)
|
||||
if err != nil {
|
||||
errs = append(errs, errors.Wrap(err, "add path watcher scanner"))
|
||||
}
|
||||
}
|
||||
return len(allAlbumPaths) != len(errs), errs
|
||||
}
|
||||
|
||||
func (ws watcherScanner) processWatchEvents() {
|
||||
log.Println("watching for events")
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-ws.watcher.Events:
|
||||
if !ok {
|
||||
//log.Println("not ok", event)
|
||||
continue
|
||||
}
|
||||
log.Println("event:", event)
|
||||
var media *models.Media
|
||||
if event.Has(fsnotify.Create) {
|
||||
scanner_queue.AddMediaToQueue(event.Name)
|
||||
log.Println("create event ", event.Name, event.Op)
|
||||
} else if event.Has(fsnotify.Remove) {
|
||||
ws.db.Where("path = ?", event.Name).Delete(&media)
|
||||
log.Println("remove event ", event.Name, event.Op)
|
||||
} else if event.Has(fsnotify.Rename) {
|
||||
ws.db.Where("path = ?", event.Name).Delete(&media)
|
||||
log.Println("rename event ", event.Name, event.Op)
|
||||
}
|
||||
case err, ok := <-ws.watcher.Errors:
|
||||
if !ok {
|
||||
//log.Println("not ok, error", err)
|
||||
continue
|
||||
}
|
||||
log.Println("error:", err)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package fsnotify_test
|
||||
package watcher_scanner_test
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
@ -43,7 +43,7 @@ func TestFSNotify(t *testing.T) {
|
|||
}()
|
||||
|
||||
// Add a path.
|
||||
err = watcher.Add("/tmp")
|
||||
err = watcher.Add("/Users/amithanda/photos")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"github.com/photoview/photoview/api/scanner/watcher_scanner"
|
||||
"log"
|
||||
"net/http"
|
||||
"path"
|
||||
|
@ -54,6 +55,10 @@ func main() {
|
|||
log.Panicf("Could not initialize periodic scanner: %s", err)
|
||||
}
|
||||
|
||||
if err := watcher_scanner.InitializeWatcherScanner(db); err != nil {
|
||||
log.Panicf("Could not initialize watcher scanner: %s", err)
|
||||
}
|
||||
|
||||
executable_worker.InitializeExecutableWorkers()
|
||||
|
||||
exif.InitializeEXIFParser()
|
||||
|
|
Loading…
Reference in New Issue