update

api/go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0
 	github.com/xor-gate/goexif2 v1.1.0
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
+	golang.org/x/exp v0.0.0-20230116083435-1de6713980de
 	golang.org/x/image v0.0.0-20220617043117-41969df76e82
 	gopkg.in/vansante/go-ffprobe.v2 v2.0.3
 	gorm.io/driver/mysql v1.3.4
@@ -31,7 +32,7 @@ require (
 	gorm.io/gorm v1.23.7
 )
 
-require golang.org/x/sys v0.0.0-20220908164124-27713097b956 // indirect
+require golang.org/x/sys v0.1.0 // indirect
 
 require (
 	github.com/agnivade/levenshtein v1.1.1 // indirect

api/go.sum

@@ -214,6 +214,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/exp v0.0.0-20230116083435-1de6713980de h1:DBWn//IJw30uYCgERoxCg84hWtA97F4wMiKOIh00Uf0=
+golang.org/x/exp v0.0.0-20230116083435-1de6713980de/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/image v0.0.0-20220617043117-41969df76e82 h1:KpZB5pUSBvrHltNEdK/tw0xlPeD13M6M6aGP32gKqiw=
 golang.org/x/image v0.0.0-20220617043117-41969df76e82/go.mod h1:doUCurBvlfPMKfmIpRIywoHmhN3VyhnoFDbvIEWF4hY=
@@ -246,8 +248,9 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220908164124-27713097b956 h1:XeJjHH1KiLpKGb6lvMiksZ9l0fVUh+AmGcm0nOMEBOY=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

api/scanner/scanner_queue/queue.go

@@ -1,9 +1,12 @@
 package scanner_queue
 
 import (
+	"container/list"
 	"context"
 	"fmt"
 	"log"
+	"path"
+	"reflect"
 	"sync"
 	"time"
 
@@ -21,6 +24,7 @@ import (
 // ScannerJob describes a job on the queue to be run by the scanner over a single album
 type ScannerJob struct {
 	ctx         scanner_task.TaskContext
+	media_paths []string
 	// album *models.Album
 	// cache *scanner_cache.AlbumScannerCache
 }
@@ -28,6 +32,7 @@ type ScannerJob struct {
 func NewScannerJob(ctx scanner_task.TaskContext) ScannerJob {
 	return ScannerJob{
 		ctx,
+		make([]string, 0),
 	}
 }
 
@@ -150,7 +155,7 @@ func (queue *ScannerQueue) processQueue(notifyThrottle *utils.Throttle) {
 			// Delete finished job from queue
 			queue.mutex.Lock()
 			for i, x := range queue.in_progress {
-				if x == nextJob {
+				if x.ctx == nextJob.ctx && reflect.DeepEqual(x.media_paths, nextJob.media_paths) {
 					queue.in_progress[i] = queue.in_progress[len(queue.in_progress)-1]
 					queue.in_progress = queue.in_progress[0 : len(queue.in_progress)-1]
 					break
@@ -246,6 +251,47 @@ func AddUserToQueue(user *models.User) error {
 	return nil
 }
 
+func AddMediaToQueue(mediaPath string) error {
+	var media *models.Media
+
+	if err := global_scanner_queue.db.Preload("Album").Where("path = ?", mediaPath).Find(&media).Error; err != nil {
+		return errors.Wrap(err, "media by path database query")
+	}
+
+	// add album to the queue
+	var album *models.Album
+	var subalbumPath string
+	if media == nil {
+		albumPath := path.Base(mediaPath)
+		for album == nil {
+			if err := global_scanner_queue.db.Where("path = ?", albumPath).Find(&album).Error; err != nil {
+				return errors.Wrap(err, "album by path database query")
+			}
+			subalbumPath = albumPath
+			albumPath = path.Base(albumPath)
+		}
+		if album == nil {
+			return errors.New("No root album found")
+		}
+	}
+
+	var userAlbumOwner []*models.User
+	if err := global_scanner_queue.db.Model(&album).Association("Owners").Find(&userAlbumOwner); err != nil {
+		return errors.Wrap(err, "find owners for album")
+	}
+
+	scanQueue := list.New()
+	scanQueue.PushBack(scanner.ScanInfo{
+		Path:   subalbumPath,
+		Parent: album,
+		Ignore: nil,
+	})
+	album_cache := scanner_cache.MakeAlbumCache()
+	scanner.ProcessUserAlbums(scanQueue, global_scanner_queue.db, userAlbumOwner, album_cache)
+
+	return nil
+}
+
 // Queue should be locked prior to calling this function
 func (queue *ScannerQueue) addJob(job *ScannerJob) error {
 	if exists, err := queue.jobOnQueue(job); exists || err != nil {

api/scanner/scanner_user.go

@@ -15,6 +15,7 @@ import (
 	"github.com/photoview/photoview/api/utils"
 	"github.com/pkg/errors"
 	ignore "github.com/sabhiram/go-gitignore"
+	"golang.org/x/exp/slices"
 	"gorm.io/gorm"
 )
 
@@ -43,6 +44,12 @@ func getPhotoviewIgnore(ignorePath string) ([]string, error) {
 	return photoviewIgnore, scanner.Err()
 }
 
+type ScanInfo struct {
+	Path   string
+	Parent *models.Album
+	Ignore []string
+}
+
 func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cache.AlbumScannerCache) ([]*models.Album, []error) {
 
 	if err := user.FillAlbums(db); err != nil {
@@ -61,12 +68,6 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
 
 	scanErrors := make([]error, 0)
 
-	type scanInfo struct {
-		path   string
-		parent *models.Album
-		ignore []string
-	}
-
 	scanQueue := list.New()
 
 	for _, album := range userRootAlbums {
@@ -78,26 +79,44 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
 				scanErrors = append(scanErrors, errors.Errorf("Could not read album directory for user '%s': %s\n", user.Username, album.Path))
 			}
 		} else {
-			scanQueue.PushBack(scanInfo{
+			scanQueue.PushBack(ScanInfo{
-				path:   album.Path,
+				Path:   album.Path,
-				parent: nil,
+				Parent: nil,
-				ignore: nil,
+				Ignore: nil,
 			})
 		}
 	}
 
+	userAlbums, err2 := ProcessUserAlbums(scanQueue, db, []*models.User{user}, album_cache)
+	if err2 != nil {
+		scanErrors = append(scanErrors, err2...)
+	}
+
+	deleteErrors := cleanup_tasks.DeleteOldUserAlbums(db, userAlbums, user)
+	scanErrors = append(scanErrors, deleteErrors...)
+
+	return userAlbums, scanErrors
+}
+
+func ProcessUserAlbums(scanQueue *list.List, db *gorm.DB, users []*models.User, album_cache *scanner_cache.AlbumScannerCache) ([]*models.Album, []error) {
 	userAlbums := make([]*models.Album, 0)
+	var scanErrors []error
+
+	userIds := make([]int, len(users))
+	for i, user := range users {
+		userIds[i] = user.ID
+	}
 
 	for scanQueue.Front() != nil {
-		albumInfo := scanQueue.Front().Value.(scanInfo)
+		albumInfo := scanQueue.Front().Value.(ScanInfo)
 		scanQueue.Remove(scanQueue.Front())
 
-		albumPath := albumInfo.path
+		albumPath := albumInfo.Path
-		albumParent := albumInfo.parent
+		albumParent := albumInfo.Parent
-		albumIgnore := albumInfo.ignore
+		albumIgnore := albumInfo.Ignore
 
 		// Read path
-		dirContent, err := ioutil.ReadDir(albumPath)
+		dirContent, err := os.ReadDir(albumPath)
 		if err != nil {
 			scanErrors = append(scanErrors, errors.Wrapf(err, "read directory (%s)", albumPath))
 			continue
@@ -166,16 +185,22 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
 
 				// Add user as an owner of the album if not already
 				var userAlbumOwner []models.User
-				if err := tx.Model(&album).Association("Owners").Find(&userAlbumOwner, "user_albums.user_id = ?", user.ID); err != nil {
+				if err := tx.Model(&album).Association("Owners").Find(&userAlbumOwner, "user_albums.user_id in (?)", userIds); err != nil {
 					return err
 				}
-				if len(userAlbumOwner) == 0 {
+				if len(userAlbumOwner) != len(userIds) {
+					for userId := range userIds {
+						i := slices.IndexFunc(userAlbumOwner, func(user models.User) bool { return user.ID == userId })
+						if i != -1 {
+							continue
+						}
 						newUser := models.User{}
-					newUser.ID = user.ID
+						newUser.ID = userId
 						if err := tx.Model(&album).Association("Owners").Append(&newUser); err != nil {
 							return err
 						}
 					}
+				}
 
 				// Update album ignore
 				album_cache.InsertAlbumIgnore(albumPath, albumIgnore)
@@ -207,18 +232,15 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
 			}
 
 			if (item.IsDir() || isDirSymlink) && directoryContainsPhotos(subalbumPath, album_cache, albumIgnore) {
-				scanQueue.PushBack(scanInfo{
+				scanQueue.PushBack(ScanInfo{
-					path:   subalbumPath,
+					Path:   subalbumPath,
-					parent: album,
+					Parent: album,
-					ignore: albumIgnore,
+					Ignore: albumIgnore,
 				})
 			}
 		}
 	}
 
-	deleteErrors := cleanup_tasks.DeleteOldUserAlbums(db, userAlbums, user)
-	scanErrors = append(scanErrors, deleteErrors...)
-
 	return userAlbums, scanErrors
 }
 

api/scanner/watcher_scanner/watcher_scanner.go

@@ -0,0 +1,101 @@
+package watcher_scanner
+
+import (
+	"github.com/fsnotify/fsnotify"
+	"github.com/photoview/photoview/api/graphql/models"
+	"github.com/photoview/photoview/api/scanner/scanner_queue"
+	"github.com/pkg/errors"
+	"gorm.io/gorm"
+	"log"
+	"sync"
+)
+
+var mainWatcherScanner *watcherScanner = nil
+
+type watcherScanner struct {
+	watcherChanged chan bool
+	watcher        *fsnotify.Watcher
+	mutex          *sync.Mutex
+	db             *gorm.DB
+}
+
+func InitializeWatcherScanner(db *gorm.DB) error {
+	if mainWatcherScanner != nil {
+		panic("watcher scanner has already been initialized")
+	}
+
+	// Create new watcher
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	mainWatcherScanner = &watcherScanner{
+		db:             db,
+		watcherChanged: make(chan bool),
+		watcher:        watcher,
+		mutex:          &sync.Mutex{},
+	}
+
+	success, errs := mainWatcherScanner.addPathsToWatch()
+	if len(errs) != 0 {
+		log.Println("watcher errors")
+		log.Println(errs)
+		if !success {
+			return errors.New("errors found during watcher scanner setup")
+		}
+	}
+
+	go mainWatcherScanner.processWatchEvents()
+
+	return nil
+}
+
+func (ws watcherScanner) addPathsToWatch() (bool, []error) {
+	var allAlbumPaths []*models.Album
+
+	if err := ws.db.Select("path").Find(&allAlbumPaths).Error; err != nil {
+		return false, []error{errors.Wrap(err, "watcher scanner find albums query")}
+	}
+
+	errs := make([]error, 0)
+	for _, album := range allAlbumPaths {
+		// log.Println("add path", album.Path)
+		err := ws.watcher.Add(album.Path)
+		if err != nil {
+			errs = append(errs, errors.Wrap(err, "add path watcher scanner"))
+		}
+	}
+	return len(allAlbumPaths) != len(errs), errs
+}
+
+func (ws watcherScanner) processWatchEvents() {
+	log.Println("watching for events")
+	for {
+		select {
+		case event, ok := <-ws.watcher.Events:
+			if !ok {
+				//log.Println("not ok", event)
+				continue
+			}
+			log.Println("event:", event)
+			var media *models.Media
+			if event.Has(fsnotify.Create) {
+				scanner_queue.AddMediaToQueue(event.Name)
+				log.Println("create event ", event.Name, event.Op)
+			} else if event.Has(fsnotify.Remove) {
+				ws.db.Where("path = ?", event.Name).Delete(&media)
+				log.Println("remove event ", event.Name, event.Op)
+			} else if event.Has(fsnotify.Rename) {
+				ws.db.Where("path = ?", event.Name).Delete(&media)
+				log.Println("rename event ", event.Name, event.Op)
+			}
+		case err, ok := <-ws.watcher.Errors:
+			if !ok {
+				//log.Println("not ok, error", err)
+				continue
+			}
+			log.Println("error:", err)
+		}
+	}
+}

api/scanner/watcher_scanner/watcher_scanner_test.go

@@ -1,4 +1,4 @@
-package fsnotify_test
+package watcher_scanner_test
 
 import (
 	"log"
@@ -43,7 +43,7 @@ func TestFSNotify(t *testing.T) {
 	}()
 
 	// Add a path.
-	err = watcher.Add("/tmp")
+	err = watcher.Add("/Users/amithanda/photos")
 	if err != nil {
 		log.Fatal(err)
 	}

api/server.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/photoview/photoview/api/scanner/watcher_scanner"
 	"log"
 	"net/http"
 	"path"
@@ -54,6 +55,10 @@ func main() {
 		log.Panicf("Could not initialize periodic scanner: %s", err)
 	}
 
+	if err := watcher_scanner.InitializeWatcherScanner(db); err != nil {
+		log.Panicf("Could not initialize watcher scanner: %s", err)
+	}
+
 	executable_worker.InitializeExecutableWorkers()
 
 	exif.InitializeEXIFParser()