mirror
/
photoview
mirror of https://github.com/photoview/photoview.git
1
Fork 0
Code

make the processing idempotent

This commit is contained in:
amit handa 2023-01-24 18:25:15 -08:00
parent 3048c41c3e
commit 880ef195fb
8 changed files with 42 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
api/scanner/scanner_album.go
View File

@ -6,6 +6,7 @@ import (
"log" "log"
"os" "os"
"path" "path"
"time"
"github.com/photoview/photoview/api/graphql/models" "github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner/media_encoding" "github.com/photoview/photoview/api/scanner/media_encoding"
@ -85,6 +86,11 @@ func ValidRootPath(rootPath string) bool {
return true return true
} }
type ChangedMedia struct {
media *models.Media
newModTime time.Time
}
func ScanAlbum(ctx scanner_task.TaskContext) error { func ScanAlbum(ctx scanner_task.TaskContext) error {
newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx) newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx)
@ -94,16 +100,16 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
ctx = newCtx ctx = newCtx
// Scan for photos // Scan for photos
albumMedia, err := findMediaForAlbum(ctx) albumUpdatedMedia, err := findMediaForAlbum(ctx)
if err != nil { if err != nil {
return errors.Wrapf(err, "find media for album (%s): %s", ctx.GetAlbum().Path, err) return errors.Wrapf(err, "find media for album (%s): %s", ctx.GetAlbum().Path, err)
} }
changedMedia := make([]*models.Media, 0) changedMedia := make([]*models.Media, 0)
for i, media := range albumMedia { for i, media := range albumUpdatedMedia {
updatedURLs := []*models.MediaURL{} updatedURLs := []*models.MediaURL{}
mediaData := media_encoding.NewEncodeMediaData(media) mediaData := media_encoding.NewEncodeMediaData(media.media)
// define new ctx for scope of for-loop // define new ctx for scope of for-loop
ctx, err := scanner_tasks.Tasks.BeforeProcessMedia(ctx, &mediaData) ctx, err := scanner_tasks.Tasks.BeforeProcessMedia(ctx, &mediaData)
@ -112,15 +118,18 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
} }
transactionError := ctx.DatabaseTransaction(func(ctx scanner_task.TaskContext) error { transactionError := ctx.DatabaseTransaction(func(ctx scanner_task.TaskContext) error {
updatedURLs, err = processMedia(ctx, &mediaData) updatedURLs, err = processMedia(ctx, &mediaData, media.newModTime)
if err != nil { if err != nil {
return errors.Wrapf(err, "process media (%s)", media.Path) return errors.Wrapf(err, "process media (%s)", media.media.Path)
} }
if len(updatedURLs) > 0 { if len(updatedURLs) > 0 {
changedMedia = append(changedMedia, media) changedMedia = append(changedMedia, media.media)
} }
if media.media.UpdatedAt.Before(media.newModTime) {
ctx.GetDB().Save(mediaData.Media)
}
return nil return nil
}) })
@ -128,11 +137,15 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
return errors.Wrap(err, "process media database transaction") return errors.Wrap(err, "process media database transaction")
} }
if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumMedia)); err != nil { if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumUpdatedMedia)); err != nil {
return errors.Wrap(err, "after process media") return errors.Wrap(err, "after process media")
} }
} }
albumMedia := make([]*models.Media, 0)
for _, media := range albumUpdatedMedia {
albumMedia = append(albumMedia, media.media)
}
if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, changedMedia, albumMedia); err != nil { if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, changedMedia, albumMedia); err != nil {
return errors.Wrap(err, "after scan album") return errors.Wrap(err, "after scan album")
} }
@ -140,9 +153,9 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
return nil return nil
} }
func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) { func findMediaForAlbum(ctx scanner_task.TaskContext) ([]ChangedMedia, error) {
albumMedia := make([]*models.Media, 0) albumMedia := make([]ChangedMedia, 0)
dirContent, err := ioutil.ReadDir(ctx.GetAlbum().Path) dirContent, err := ioutil.ReadDir(ctx.GetAlbum().Path)
if err != nil { if err != nil {
@ -189,7 +202,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
return err return err
} }
albumMedia = append(albumMedia, media) albumMedia = append(albumMedia, ChangedMedia{media, item.ModTime()})
return nil return nil
}) })
@ -205,7 +218,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
return albumMedia, nil return albumMedia, nil
} }
func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData) ([]*models.MediaURL, error) { func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, newModTime time.Time) ([]*models.MediaURL, error) {
// Make sure media cache directory exists // Make sure media cache directory exists
mediaCachePath, err := mediaData.Media.CachePath() mediaCachePath, err := mediaData.Media.CachePath()
@ -213,5 +226,5 @@ func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.Encode
return []*models.MediaURL{}, errors.Wrap(err, "cache directory error") return []*models.MediaURL{}, errors.Wrap(err, "cache directory error")
} }
return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, mediaCachePath) return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, mediaCachePath, newModTime)
} }

3
api/scanner/scanner_media.go
View File

@ -5,6 +5,7 @@ import (
"log" "log"
"os" "os"
"path" "path"
"time"
"github.com/photoview/photoview/api/graphql/models" "github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner/media_encoding" "github.com/photoview/photoview/api/scanner/media_encoding"
@ -92,7 +93,7 @@ func ProcessSingleMedia(db *gorm.DB, media *models.Media) error {
return err return err
} }
updated_urls, err := scanner_tasks.Tasks.ProcessMedia(new_ctx, &media_data, mediaCachePath) updated_urls, err := scanner_tasks.Tasks.ProcessMedia(new_ctx, &media_data, mediaCachePath, time.Now())
if err != nil { if err != nil {
return err return err
} }

2
api/scanner/scanner_task/scanner_task.go
View File

@ -32,7 +32,7 @@ type ScannerTask interface {
AfterMediaFound(ctx TaskContext, media *models.Media, newMedia bool, newModTime time.Time) error AfterMediaFound(ctx TaskContext, media *models.Media, newMedia bool, newModTime time.Time) error
BeforeProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData) (TaskContext, error) BeforeProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData) (TaskContext, error)
ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error)
AfterProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error AfterProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error
} }

2
api/scanner/scanner_task/scanner_task_base.go
View File

@ -31,7 +31,7 @@ func (t ScannerTaskBase) BeforeProcessMedia(ctx TaskContext, mediaData *media_en
return ctx, nil return ctx, nil
} }
func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) { func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error) {
return []*models.MediaURL{}, nil return []*models.MediaURL{}, nil
} }

12
api/scanner/scanner_tasks/processing_tasks/process_photo_task.go
View File

@ -5,6 +5,7 @@ import (
"log" "log"
"os" "os"
"path" "path"
"time"
"github.com/photoview/photoview/api/graphql/models" "github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner/media_encoding" "github.com/photoview/photoview/api/scanner/media_encoding"
@ -25,7 +26,7 @@ type ProcessPhotoTask struct {
scanner_task.ScannerTaskBase scanner_task.ScannerTaskBase
} }
func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) { func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
if mediaData.Media.Type != models.MediaTypePhoto { if mediaData.Media.Type != models.MediaTypePhoto {
return []*models.MediaURL{}, nil return []*models.MediaURL{}, nil
} }
@ -66,7 +67,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
return []*models.MediaURL{}, err return []*models.MediaURL{}, err
} }
if !contentType.IsWebCompatible() { if !contentType.IsWebCompatible() || mediaData.Media.UpdatedAt.Before(newModTime) {
highresName := generateUniqueMediaNamePrefixed("highres", photo.Path, ".jpg") highresName := generateUniqueMediaNamePrefixed("highres", photo.Path, ".jpg")
baseImagePath = path.Join(mediaCachePath, highresName) baseImagePath = path.Join(mediaCachePath, highresName)
@ -81,7 +82,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
// Verify that highres photo still exists in cache // Verify that highres photo still exists in cache
baseImagePath = path.Join(mediaCachePath, highResURL.MediaName) baseImagePath = path.Join(mediaCachePath, highResURL.MediaName)
if _, err := os.Stat(baseImagePath); os.IsNotExist(err) { if _, err := os.Stat(baseImagePath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName) fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName)
updatedURLs = append(updatedURLs, highResURL) updatedURLs = append(updatedURLs, highResURL)
@ -93,7 +94,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
} }
// Save original photo to database // Save original photo to database
if origURL == nil { if origURL == nil || mediaData.Media.UpdatedAt.Before(newModTime) {
// Make sure photo dimensions is set // Make sure photo dimensions is set
if photoDimensions == nil { if photoDimensions == nil {
@ -124,7 +125,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
// Verify that thumbnail photo still exists in cache // Verify that thumbnail photo still exists in cache
thumbPath := path.Join(mediaCachePath, thumbURL.MediaName) thumbPath := path.Join(mediaCachePath, thumbURL.MediaName)
if _, err := os.Stat(thumbPath); os.IsNotExist(err) { if _, err := os.Stat(thumbPath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
updatedURLs = append(updatedURLs, thumbURL) updatedURLs = append(updatedURLs, thumbURL)
fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.MediaName) fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.MediaName)
@ -135,6 +136,5 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
} }
} }
ctx.GetDB().Save(mediaData.Media)
return updatedURLs, nil return updatedURLs, nil
} }

8
api/scanner/scanner_tasks/processing_tasks/process_video_task.go
View File

@ -23,7 +23,7 @@ type ProcessVideoTask struct {
scanner_task.ScannerTaskBase scanner_task.ScannerTaskBase
} }
func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) { func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
if mediaData.Media.Type != models.MediaTypeVideo { if mediaData.Media.Type != models.MediaTypeVideo {
return []*models.MediaURL{}, nil return []*models.MediaURL{}, nil
} }
@ -55,7 +55,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
return []*models.MediaURL{}, errors.Wrap(err, "error getting video content type") return []*models.MediaURL{}, errors.Wrap(err, "error getting video content type")
} }
if videoOriginalURL == nil && videoType.IsWebCompatible() { if videoType.IsWebCompatible() && (videoOriginalURL == nil || mediaData.Media.UpdatedAt.Before(newModTime)) {
origVideoPath := video.Path origVideoPath := video.Path
videoMediaName := generateUniqueMediaName(video.Path) videoMediaName := generateUniqueMediaName(video.Path)
@ -86,7 +86,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
updatedURLs = append(updatedURLs, &mediaURL) updatedURLs = append(updatedURLs, &mediaURL)
} }
if videoWebURL == nil && !videoType.IsWebCompatible() { if !videoType.IsWebCompatible() && (videoWebURL == nil || mediaData.Media.UpdatedAt.Before(newModTime)) {
web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken()) web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
web_video_name = strings.ReplaceAll(web_video_name, ".", "_") web_video_name = strings.ReplaceAll(web_video_name, ".", "_")
web_video_name = strings.ReplaceAll(web_video_name, " ", "_") web_video_name = strings.ReplaceAll(web_video_name, " ", "_")
@ -173,7 +173,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
// Verify that video thumbnail still exists in cache // Verify that video thumbnail still exists in cache
thumbImagePath := path.Join(mediaCachePath, videoThumbnailURL.MediaName) thumbImagePath := path.Join(mediaCachePath, videoThumbnailURL.MediaName)
if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) { if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName) fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
updatedURLs = append(updatedURLs, videoThumbnailURL) updatedURLs = append(updatedURLs, videoThumbnailURL)

2
api/scanner/scanner_tasks/processing_tasks/sidecar_task.go
View File

@ -53,7 +53,7 @@ func (t SidecarTask) AfterMediaFound(ctx scanner_task.TaskContext, media *models
return nil return nil
} }
func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) { func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error) {
mediaType, err := mediaData.ContentType() mediaType, err := mediaData.ContentType()
if err != nil { if err != nil {
return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, process media") return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, process media")

4
api/scanner/scanner_tasks/scanner_tasks.go
View File

@ -116,7 +116,7 @@ func (t scannerTasks) BeforeProcessMedia(ctx scanner_task.TaskContext, mediaData
return ctx, nil return ctx, nil
} }
func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) { func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
allNewMedia := make([]*models.MediaURL, 0) allNewMedia := make([]*models.MediaURL, 0)
for _, task := range allTasks { for _, task := range allTasks {
@ -126,7 +126,7 @@ func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *medi
default: default:
} }
newMedia, err := task.ProcessMedia(ctx, mediaData, mediaCachePath) newMedia, err := task.ProcessMedia(ctx, mediaData, mediaCachePath, newModTime)
if err != nil { if err != nil {
return []*models.MediaURL{}, err return []*models.MediaURL{}, err
} }