From 880ef195fbda7150dca8654480e47876270f99ea Mon Sep 17 00:00:00 2001
From: amit handa <amit.handa@gmail.com>
Date: Tue, 24 Jan 2023 18:25:15 -0800
Subject: [PATCH] make the processing idempotent

---
 api/scanner/scanner_album.go                  | 37 +++++++++++++------
 api/scanner/scanner_media.go                  |  3 +-
 api/scanner/scanner_task/scanner_task.go      |  2 +-
 api/scanner/scanner_task/scanner_task_base.go |  2 +-
 .../processing_tasks/process_photo_task.go    | 12 +++---
 .../processing_tasks/process_video_task.go    |  8 ++--
 .../processing_tasks/sidecar_task.go          |  2 +-
 api/scanner/scanner_tasks/scanner_tasks.go    |  4 +-
 8 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/api/scanner/scanner_album.go b/api/scanner/scanner_album.go
index 0005e4a..304d889 100644
--- a/api/scanner/scanner_album.go
+++ b/api/scanner/scanner_album.go
@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 	"path"
+	"time"
 
 	"github.com/photoview/photoview/api/graphql/models"
 	"github.com/photoview/photoview/api/scanner/media_encoding"
@@ -85,6 +86,11 @@ func ValidRootPath(rootPath string) bool {
 	return true
 }
 
+type ChangedMedia struct {
+	media      *models.Media
+	newModTime time.Time
+}
+
 func ScanAlbum(ctx scanner_task.TaskContext) error {
 
 	newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx)
@@ -94,16 +100,16 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
 	ctx = newCtx
 
 	// Scan for photos
-	albumMedia, err := findMediaForAlbum(ctx)
+	albumUpdatedMedia, err := findMediaForAlbum(ctx)
 	if err != nil {
 		return errors.Wrapf(err, "find media for album (%s): %s", ctx.GetAlbum().Path, err)
 	}
 
 	changedMedia := make([]*models.Media, 0)
-	for i, media := range albumMedia {
+	for i, media := range albumUpdatedMedia {
 		updatedURLs := []*models.MediaURL{}
 
-		mediaData := media_encoding.NewEncodeMediaData(media)
+		mediaData := media_encoding.NewEncodeMediaData(media.media)
 
 		// define new ctx for scope of for-loop
 		ctx, err := scanner_tasks.Tasks.BeforeProcessMedia(ctx, &mediaData)
@@ -112,15 +118,18 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
 		}
 
 		transactionError := ctx.DatabaseTransaction(func(ctx scanner_task.TaskContext) error {
-			updatedURLs, err = processMedia(ctx, &mediaData)
+			updatedURLs, err = processMedia(ctx, &mediaData, media.newModTime)
 			if err != nil {
-				return errors.Wrapf(err, "process media (%s)", media.Path)
+				return errors.Wrapf(err, "process media (%s)", media.media.Path)
 			}
 
 			if len(updatedURLs) > 0 {
-				changedMedia = append(changedMedia, media)
+				changedMedia = append(changedMedia, media.media)
 			}
 
+			if media.media.UpdatedAt.Before(media.newModTime) {
+				ctx.GetDB().Save(mediaData.Media)
+			}
 			return nil
 		})
 
@@ -128,11 +137,15 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
 			return errors.Wrap(err, "process media database transaction")
 		}
 
-		if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumMedia)); err != nil {
+		if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumUpdatedMedia)); err != nil {
 			return errors.Wrap(err, "after process media")
 		}
 	}
 
+	albumMedia := make([]*models.Media, 0)
+	for _, media := range albumUpdatedMedia {
+		albumMedia = append(albumMedia, media.media)
+	}
 	if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, changedMedia, albumMedia); err != nil {
 		return errors.Wrap(err, "after scan album")
 	}
@@ -140,9 +153,9 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
 	return nil
 }
 
-func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
+func findMediaForAlbum(ctx scanner_task.TaskContext) ([]ChangedMedia, error) {
 
-	albumMedia := make([]*models.Media, 0)
+	albumMedia := make([]ChangedMedia, 0)
 
 	dirContent, err := ioutil.ReadDir(ctx.GetAlbum().Path)
 	if err != nil {
@@ -189,7 +202,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
 					return err
 				}
 
-				albumMedia = append(albumMedia, media)
+				albumMedia = append(albumMedia, ChangedMedia{media, item.ModTime()})
 
 				return nil
 			})
@@ -205,7 +218,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
 	return albumMedia, nil
 }
 
-func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData) ([]*models.MediaURL, error) {
+func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, newModTime time.Time) ([]*models.MediaURL, error) {
 
 	// Make sure media cache directory exists
 	mediaCachePath, err := mediaData.Media.CachePath()
@@ -213,5 +226,5 @@ func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.Encode
 		return []*models.MediaURL{}, errors.Wrap(err, "cache directory error")
 	}
 
-	return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, mediaCachePath)
+	return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, mediaCachePath, newModTime)
 }
diff --git a/api/scanner/scanner_media.go b/api/scanner/scanner_media.go
index f76b1eb..e7f6f0e 100644
--- a/api/scanner/scanner_media.go
+++ b/api/scanner/scanner_media.go
@@ -5,6 +5,7 @@ import (
 	"log"
 	"os"
 	"path"
+	"time"
 
 	"github.com/photoview/photoview/api/graphql/models"
 	"github.com/photoview/photoview/api/scanner/media_encoding"
@@ -92,7 +93,7 @@ func ProcessSingleMedia(db *gorm.DB, media *models.Media) error {
 		return err
 	}
 
-	updated_urls, err := scanner_tasks.Tasks.ProcessMedia(new_ctx, &media_data, mediaCachePath)
+	updated_urls, err := scanner_tasks.Tasks.ProcessMedia(new_ctx, &media_data, mediaCachePath, time.Now())
 	if err != nil {
 		return err
 	}
diff --git a/api/scanner/scanner_task/scanner_task.go b/api/scanner/scanner_task/scanner_task.go
index c374b0d..fc4db02 100644
--- a/api/scanner/scanner_task/scanner_task.go
+++ b/api/scanner/scanner_task/scanner_task.go
@@ -32,7 +32,7 @@ type ScannerTask interface {
 	AfterMediaFound(ctx TaskContext, media *models.Media, newMedia bool, newModTime time.Time) error
 
 	BeforeProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData) (TaskContext, error)
-	ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error)
+	ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error)
 	AfterProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error
 }
 
diff --git a/api/scanner/scanner_task/scanner_task_base.go b/api/scanner/scanner_task/scanner_task_base.go
index 78187d1..2e51904 100644
--- a/api/scanner/scanner_task/scanner_task_base.go
+++ b/api/scanner/scanner_task/scanner_task_base.go
@@ -31,7 +31,7 @@ func (t ScannerTaskBase) BeforeProcessMedia(ctx TaskContext, mediaData *media_en
 	return ctx, nil
 }
 
-func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) {
+func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error) {
 	return []*models.MediaURL{}, nil
 }
 
diff --git a/api/scanner/scanner_tasks/processing_tasks/process_photo_task.go b/api/scanner/scanner_tasks/processing_tasks/process_photo_task.go
index 702d078..27e3e2f 100644
--- a/api/scanner/scanner_tasks/processing_tasks/process_photo_task.go
+++ b/api/scanner/scanner_tasks/processing_tasks/process_photo_task.go
@@ -5,6 +5,7 @@ import (
 	"log"
 	"os"
 	"path"
+	"time"
 
 	"github.com/photoview/photoview/api/graphql/models"
 	"github.com/photoview/photoview/api/scanner/media_encoding"
@@ -25,7 +26,7 @@ type ProcessPhotoTask struct {
 	scanner_task.ScannerTaskBase
 }
 
-func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
+func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
 	if mediaData.Media.Type != models.MediaTypePhoto {
 		return []*models.MediaURL{}, nil
 	}
@@ -66,7 +67,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 			return []*models.MediaURL{}, err
 		}
 
-		if !contentType.IsWebCompatible() {
+		if !contentType.IsWebCompatible() || mediaData.Media.UpdatedAt.Before(newModTime) {
 			highresName := generateUniqueMediaNamePrefixed("highres", photo.Path, ".jpg")
 			baseImagePath = path.Join(mediaCachePath, highresName)
 
@@ -81,7 +82,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 		// Verify that highres photo still exists in cache
 		baseImagePath = path.Join(mediaCachePath, highResURL.MediaName)
 
-		if _, err := os.Stat(baseImagePath); os.IsNotExist(err) {
+		if _, err := os.Stat(baseImagePath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
 			fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName)
 			updatedURLs = append(updatedURLs, highResURL)
 
@@ -93,7 +94,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 	}
 
 	// Save original photo to database
-	if origURL == nil {
+	if origURL == nil || mediaData.Media.UpdatedAt.Before(newModTime) {
 
 		// Make sure photo dimensions is set
 		if photoDimensions == nil {
@@ -124,7 +125,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 		// Verify that thumbnail photo still exists in cache
 		thumbPath := path.Join(mediaCachePath, thumbURL.MediaName)
 
-		if _, err := os.Stat(thumbPath); os.IsNotExist(err) {
+		if _, err := os.Stat(thumbPath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
 			updatedURLs = append(updatedURLs, thumbURL)
 			fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.MediaName)
 
@@ -135,6 +136,5 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 		}
 	}
 
-	ctx.GetDB().Save(mediaData.Media)
 	return updatedURLs, nil
 }
diff --git a/api/scanner/scanner_tasks/processing_tasks/process_video_task.go b/api/scanner/scanner_tasks/processing_tasks/process_video_task.go
index b29f482..6aee242 100644
--- a/api/scanner/scanner_tasks/processing_tasks/process_video_task.go
+++ b/api/scanner/scanner_tasks/processing_tasks/process_video_task.go
@@ -23,7 +23,7 @@ type ProcessVideoTask struct {
 	scanner_task.ScannerTaskBase
 }
 
-func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
+func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
 	if mediaData.Media.Type != models.MediaTypeVideo {
 		return []*models.MediaURL{}, nil
 	}
@@ -55,7 +55,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 		return []*models.MediaURL{}, errors.Wrap(err, "error getting video content type")
 	}
 
-	if videoOriginalURL == nil && videoType.IsWebCompatible() {
+	if videoType.IsWebCompatible() && (videoOriginalURL == nil || mediaData.Media.UpdatedAt.Before(newModTime)) {
 		origVideoPath := video.Path
 		videoMediaName := generateUniqueMediaName(video.Path)
 
@@ -86,7 +86,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 		updatedURLs = append(updatedURLs, &mediaURL)
 	}
 
-	if videoWebURL == nil && !videoType.IsWebCompatible() {
+	if !videoType.IsWebCompatible() && (videoWebURL == nil || mediaData.Media.UpdatedAt.Before(newModTime)) {
 		web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
 		web_video_name = strings.ReplaceAll(web_video_name, ".", "_")
 		web_video_name = strings.ReplaceAll(web_video_name, " ", "_")
@@ -173,7 +173,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
 		// Verify that video thumbnail still exists in cache
 		thumbImagePath := path.Join(mediaCachePath, videoThumbnailURL.MediaName)
 
-		if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) {
+		if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
 			fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
 			updatedURLs = append(updatedURLs, videoThumbnailURL)
 
diff --git a/api/scanner/scanner_tasks/processing_tasks/sidecar_task.go b/api/scanner/scanner_tasks/processing_tasks/sidecar_task.go
index 9cd64af..c81c8e9 100644
--- a/api/scanner/scanner_tasks/processing_tasks/sidecar_task.go
+++ b/api/scanner/scanner_tasks/processing_tasks/sidecar_task.go
@@ -53,7 +53,7 @@ func (t SidecarTask) AfterMediaFound(ctx scanner_task.TaskContext, media *models
 	return nil
 }
 
-func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) {
+func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error) {
 	mediaType, err := mediaData.ContentType()
 	if err != nil {
 		return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, process media")
diff --git a/api/scanner/scanner_tasks/scanner_tasks.go b/api/scanner/scanner_tasks/scanner_tasks.go
index 510c46e..7dc2c23 100644
--- a/api/scanner/scanner_tasks/scanner_tasks.go
+++ b/api/scanner/scanner_tasks/scanner_tasks.go
@@ -116,7 +116,7 @@ func (t scannerTasks) BeforeProcessMedia(ctx scanner_task.TaskContext, mediaData
 	return ctx, nil
 }
 
-func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
+func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
 	allNewMedia := make([]*models.MediaURL, 0)
 
 	for _, task := range allTasks {
@@ -126,7 +126,7 @@ func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *medi
 		default:
 		}
 
-		newMedia, err := task.ProcessMedia(ctx, mediaData, mediaCachePath)
+		newMedia, err := task.ProcessMedia(ctx, mediaData, mediaCachePath, newModTime)
 		if err != nil {
 			return []*models.MediaURL{}, err
 		}
