2020-06-22 21:52:53 +02:00
|
|
|
package scanner
|
|
|
|
|
|
|
|
import (
|
2020-06-23 15:13:07 +02:00
|
|
|
"fmt"
|
2020-06-22 21:52:53 +02:00
|
|
|
"io/ioutil"
|
2021-02-26 20:34:08 +01:00
|
|
|
"log"
|
2020-06-22 21:52:53 +02:00
|
|
|
"path"
|
2020-06-23 15:13:07 +02:00
|
|
|
"time"
|
2020-06-22 21:52:53 +02:00
|
|
|
|
2020-12-17 22:51:43 +01:00
|
|
|
"github.com/photoview/photoview/api/graphql/models"
|
|
|
|
"github.com/photoview/photoview/api/graphql/notification"
|
2021-02-15 17:35:28 +01:00
|
|
|
"github.com/photoview/photoview/api/scanner/face_detection"
|
2020-12-17 22:51:43 +01:00
|
|
|
"github.com/photoview/photoview/api/utils"
|
2020-11-23 19:59:01 +01:00
|
|
|
"github.com/pkg/errors"
|
2021-03-16 18:26:51 +01:00
|
|
|
ignore "github.com/sabhiram/go-gitignore"
|
2020-11-23 19:39:44 +01:00
|
|
|
"gorm.io/gorm"
|
2020-06-22 21:52:53 +02:00
|
|
|
)
|
|
|
|
|
2020-12-22 01:14:43 +01:00
|
|
|
func NewRootAlbum(db *gorm.DB, rootPath string, owner *models.User) (*models.Album, error) {
|
|
|
|
|
|
|
|
owners := []models.User{
|
|
|
|
*owner,
|
|
|
|
}
|
|
|
|
|
2020-12-31 00:37:11 +01:00
|
|
|
var matchedAlbums []models.Album
|
2021-01-17 12:45:23 +01:00
|
|
|
if err := db.Where("path_hash = ?", models.MD5Hash(rootPath)).Find(&matchedAlbums).Error; err != nil {
|
2020-12-22 01:14:43 +01:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2020-12-31 00:37:11 +01:00
|
|
|
if len(matchedAlbums) > 0 {
|
|
|
|
album := matchedAlbums[0]
|
|
|
|
|
|
|
|
if err := db.Model(&owner).Association("Albums").Append(&album); err != nil {
|
|
|
|
return nil, errors.Wrap(err, "failed to add owner to already existing album")
|
|
|
|
}
|
|
|
|
|
|
|
|
return &album, nil
|
|
|
|
} else {
|
|
|
|
album := models.Album{
|
|
|
|
Title: path.Base(rootPath),
|
|
|
|
Path: rootPath,
|
|
|
|
Owners: owners,
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := db.Create(&album).Error; err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return &album, nil
|
|
|
|
}
|
2020-12-22 01:14:43 +01:00
|
|
|
}
|
|
|
|
|
2020-11-23 19:39:44 +01:00
|
|
|
func scanAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB) {
|
2020-06-23 15:13:07 +02:00
|
|
|
|
|
|
|
album_notify_key := utils.GenerateToken()
|
|
|
|
notifyThrottle := utils.NewThrottle(500 * time.Millisecond)
|
|
|
|
notifyThrottle.Trigger(nil)
|
|
|
|
|
2020-06-22 23:52:41 +02:00
|
|
|
// Scan for photos
|
2021-02-15 17:35:28 +01:00
|
|
|
albumMedia, err := findMediaForAlbum(album, cache, db, func(photo *models.Media, newPhoto bool) {
|
2020-06-23 15:13:07 +02:00
|
|
|
if newPhoto {
|
|
|
|
notifyThrottle.Trigger(func() {
|
|
|
|
notification.BroadcastNotification(&models.Notification{
|
|
|
|
Key: album_notify_key,
|
|
|
|
Type: models.NotificationTypeMessage,
|
2020-07-11 14:21:10 +02:00
|
|
|
Header: fmt.Sprintf("Found new media in album '%s'", album.Title),
|
|
|
|
Content: fmt.Sprintf("Found %s", photo.Path),
|
2020-06-23 15:13:07 +02:00
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
2020-06-22 23:52:41 +02:00
|
|
|
})
|
|
|
|
if err != nil {
|
2020-07-11 14:21:10 +02:00
|
|
|
ScannerError("Failed to find media for album (%s): %s", album.Path, err)
|
2020-06-22 23:52:41 +02:00
|
|
|
}
|
|
|
|
|
2020-06-23 15:13:07 +02:00
|
|
|
album_has_changes := false
|
2021-02-15 17:35:28 +01:00
|
|
|
for count, media := range albumMedia {
|
2021-03-16 22:27:27 +01:00
|
|
|
processing_was_needed := false
|
2020-06-23 00:40:47 +02:00
|
|
|
|
2020-11-30 21:29:49 +01:00
|
|
|
transactionError := db.Transaction(func(tx *gorm.DB) error {
|
2021-03-16 22:27:27 +01:00
|
|
|
log.Printf("Process media transaction enter (%s)", media.Path)
|
|
|
|
processing_was_needed, err = ProcessMedia(tx, media)
|
2020-11-23 19:59:01 +01:00
|
|
|
if err != nil {
|
2021-02-15 17:35:28 +01:00
|
|
|
return errors.Wrapf(err, "failed to process photo (%s)", media.Path)
|
2020-11-23 19:59:01 +01:00
|
|
|
}
|
2020-06-22 23:52:41 +02:00
|
|
|
|
2020-11-23 19:59:01 +01:00
|
|
|
if processing_was_needed {
|
|
|
|
album_has_changes = true
|
2021-02-15 17:35:28 +01:00
|
|
|
progress := float64(count) / float64(len(albumMedia)) * 100.0
|
2020-11-23 19:59:01 +01:00
|
|
|
notification.BroadcastNotification(&models.Notification{
|
|
|
|
Key: album_notify_key,
|
|
|
|
Type: models.NotificationTypeProgress,
|
|
|
|
Header: fmt.Sprintf("Processing media for album '%s'", album.Title),
|
2021-02-15 17:35:28 +01:00
|
|
|
Content: fmt.Sprintf("Processed media at %s", media.Path),
|
2020-11-23 19:59:01 +01:00
|
|
|
Progress: &progress,
|
|
|
|
})
|
|
|
|
}
|
2020-06-23 15:13:07 +02:00
|
|
|
|
2021-03-16 22:27:27 +01:00
|
|
|
log.Printf("Process media transaction exit (%s)", media.Path)
|
2020-11-23 19:59:01 +01:00
|
|
|
return nil
|
|
|
|
})
|
|
|
|
|
2020-11-30 21:29:49 +01:00
|
|
|
if transactionError != nil {
|
|
|
|
ScannerError("Failed to begin database transaction: %s", transactionError)
|
2020-06-23 00:40:47 +02:00
|
|
|
}
|
2021-03-16 22:27:27 +01:00
|
|
|
|
|
|
|
if processing_was_needed && media.Type == models.MediaTypePhoto {
|
|
|
|
go func(media *models.Media) {
|
|
|
|
if err := face_detection.GlobalFaceDetector.DetectFaces(db, media); err != nil {
|
|
|
|
ScannerError("Error detecting faces in image (%s): %s", media.Path, err)
|
|
|
|
}
|
|
|
|
}(media)
|
|
|
|
}
|
2020-06-23 15:13:07 +02:00
|
|
|
}
|
2020-06-23 00:40:47 +02:00
|
|
|
|
2021-02-15 17:35:28 +01:00
|
|
|
cleanup_errors := CleanupMedia(db, album.ID, albumMedia)
|
2020-07-24 19:37:58 +02:00
|
|
|
for _, err := range cleanup_errors {
|
|
|
|
ScannerError("Failed to delete old media: %s", err)
|
|
|
|
}
|
|
|
|
|
2020-06-23 15:13:07 +02:00
|
|
|
if album_has_changes {
|
|
|
|
timeoutDelay := 2000
|
|
|
|
notification.BroadcastNotification(&models.Notification{
|
|
|
|
Key: album_notify_key,
|
|
|
|
Type: models.NotificationTypeMessage,
|
|
|
|
Positive: true,
|
2020-07-11 14:21:10 +02:00
|
|
|
Header: fmt.Sprintf("Done processing media for album '%s'", album.Title),
|
|
|
|
Content: fmt.Sprintf("All media have been processed"),
|
2020-06-23 15:13:07 +02:00
|
|
|
Timeout: &timeoutDelay,
|
|
|
|
})
|
2020-06-22 23:52:41 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-11-23 19:39:44 +01:00
|
|
|
func findMediaForAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB, onScanPhoto func(photo *models.Media, newPhoto bool)) ([]*models.Media, error) {
|
2020-06-22 21:52:53 +02:00
|
|
|
|
2020-07-10 14:26:19 +02:00
|
|
|
albumPhotos := make([]*models.Media, 0)
|
2020-06-22 21:52:53 +02:00
|
|
|
|
|
|
|
dirContent, err := ioutil.ReadDir(album.Path)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
2021-03-10 21:51:36 +01:00
|
|
|
// Get ignore data
|
2021-03-16 18:26:51 +01:00
|
|
|
albumIgnore := ignore.CompileIgnoreLines(*cache.GetAlbumIgnore(album.Path)...)
|
2021-02-26 20:34:08 +01:00
|
|
|
|
2020-06-22 21:52:53 +02:00
|
|
|
for _, item := range dirContent {
|
|
|
|
photoPath := path.Join(album.Path, item.Name())
|
|
|
|
|
2021-03-10 21:51:36 +01:00
|
|
|
if !item.IsDir() && isPathMedia(photoPath, cache) {
|
|
|
|
// Match file against ignore data
|
2021-03-16 18:26:51 +01:00
|
|
|
if albumIgnore.MatchesPath(item.Name()) {
|
2021-03-10 21:51:36 +01:00
|
|
|
log.Printf("File %s ignored\n", item.Name())
|
|
|
|
continue
|
2021-02-26 20:34:08 +01:00
|
|
|
}
|
|
|
|
|
2020-12-09 11:40:37 +01:00
|
|
|
// Skip the JPEGs that are compressed version of raw files
|
|
|
|
counterpartFile := scanForRawCounterpartFile(photoPath)
|
|
|
|
if counterpartFile != nil {
|
|
|
|
continue
|
|
|
|
}
|
2020-06-22 21:52:53 +02:00
|
|
|
|
2020-12-10 12:30:10 +01:00
|
|
|
err := db.Transaction(func(tx *gorm.DB) error {
|
2021-02-15 17:35:28 +01:00
|
|
|
media, isNewMedia, err := ScanMedia(tx, photoPath, album.ID, cache)
|
2020-11-23 19:59:01 +01:00
|
|
|
if err != nil {
|
|
|
|
return errors.Wrapf(err, "Scanning media error (%s)", photoPath)
|
|
|
|
}
|
2020-06-22 21:52:53 +02:00
|
|
|
|
2021-02-15 17:35:28 +01:00
|
|
|
onScanPhoto(media, isNewMedia)
|
2020-06-22 21:52:53 +02:00
|
|
|
|
2021-02-15 17:35:28 +01:00
|
|
|
albumPhotos = append(albumPhotos, media)
|
2020-11-23 19:59:01 +01:00
|
|
|
|
|
|
|
return nil
|
|
|
|
})
|
2020-06-22 21:52:53 +02:00
|
|
|
|
|
|
|
if err != nil {
|
2020-12-14 10:41:00 +01:00
|
|
|
ScannerError("Error scanning media for album (%d): %s\n", album.ID, err)
|
2020-06-22 21:52:53 +02:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-23 15:18:13 +02:00
|
|
|
return albumPhotos, nil
|
2020-06-22 21:52:53 +02:00
|
|
|
}
|