mirror
/
photoview
mirror of https://github.com/photoview/photoview.git
1
Fork 0
Code

make the processing idempotent

This commit is contained in:
amit handa 2023-01-24 18:25:15 -08:00
parent 3048c41c3e
commit 880ef195fb
8 changed files with 42 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
api/scanner/scanner_album.go
View File

@ -6,6 +6,7 @@ import (
"log"
"os"
"path"
"time"
"github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner/media_encoding"
@ -85,6 +86,11 @@ func ValidRootPath(rootPath string) bool {
return true
}
type ChangedMedia struct {
media *models.Media
newModTime time.Time
}
func ScanAlbum(ctx scanner_task.TaskContext) error {
newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx)
@ -94,16 +100,16 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
ctx = newCtx
// Scan for photos
albumMedia, err := findMediaForAlbum(ctx)
albumUpdatedMedia, err := findMediaForAlbum(ctx)
if err != nil {
return errors.Wrapf(err, "find media for album (%s): %s", ctx.GetAlbum().Path, err)
}
changedMedia := make([]*models.Media, 0)
for i, media := range albumMedia {
for i, media := range albumUpdatedMedia {
updatedURLs := []*models.MediaURL{}
mediaData := media_encoding.NewEncodeMediaData(media)
mediaData := media_encoding.NewEncodeMediaData(media.media)
// define new ctx for scope of for-loop
ctx, err := scanner_tasks.Tasks.BeforeProcessMedia(ctx, &mediaData)
@ -112,15 +118,18 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
}
transactionError := ctx.DatabaseTransaction(func(ctx scanner_task.TaskContext) error {
updatedURLs, err = processMedia(ctx, &mediaData)
updatedURLs, err = processMedia(ctx, &mediaData, media.newModTime)
if err != nil {
return errors.Wrapf(err, "process media (%s)", media.Path)
return errors.Wrapf(err, "process media (%s)", media.media.Path)
}
if len(updatedURLs) > 0 {
changedMedia = append(changedMedia, media)
changedMedia = append(changedMedia, media.media)
}
if media.media.UpdatedAt.Before(media.newModTime) {
ctx.GetDB().Save(mediaData.Media)
}
return nil
})
@ -128,11 +137,15 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
return errors.Wrap(err, "process media database transaction")
}
if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumMedia)); err != nil {
if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumUpdatedMedia)); err != nil {
return errors.Wrap(err, "after process media")
}
}
albumMedia := make([]*models.Media, 0)
for _, media := range albumUpdatedMedia {
albumMedia = append(albumMedia, media.media)
}
if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, changedMedia, albumMedia); err != nil {
return errors.Wrap(err, "after scan album")
}
@ -140,9 +153,9 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
return nil
}
func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
func findMediaForAlbum(ctx scanner_task.TaskContext) ([]ChangedMedia, error) {
albumMedia := make([]*models.Media, 0)
albumMedia := make([]ChangedMedia, 0)
dirContent, err := ioutil.ReadDir(ctx.GetAlbum().Path)
if err != nil {
@ -189,7 +202,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
return err
}
albumMedia = append(albumMedia, media)
albumMedia = append(albumMedia, ChangedMedia{media, item.ModTime()})
return nil
})
@ -205,7 +218,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
return albumMedia, nil
}
func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData) ([]*models.MediaURL, error) {
func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, newModTime time.Time) ([]*models.MediaURL, error) {
// Make sure media cache directory exists
mediaCachePath, err := mediaData.Media.CachePath()
@ -213,5 +226,5 @@ func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.Encode
return []*models.MediaURL{}, errors.Wrap(err, "cache directory error")
}
return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, mediaCachePath)
return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, mediaCachePath, newModTime)
}

3
api/scanner/scanner_media.go
View File

@ -5,6 +5,7 @@ import (
"log"
"os"
"path"
"time"
"github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner/media_encoding"
@ -92,7 +93,7 @@ func ProcessSingleMedia(db *gorm.DB, media *models.Media) error {
return err
}
updated_urls, err := scanner_tasks.Tasks.ProcessMedia(new_ctx, &media_data, mediaCachePath)
updated_urls, err := scanner_tasks.Tasks.ProcessMedia(new_ctx, &media_data, mediaCachePath, time.Now())
if err != nil {
return err
}

2
api/scanner/scanner_task/scanner_task.go
View File

@ -32,7 +32,7 @@ type ScannerTask interface {
AfterMediaFound(ctx TaskContext, media *models.Media, newMedia bool, newModTime time.Time) error
BeforeProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData) (TaskContext, error)
ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error)
ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error)
AfterProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error
}

2
api/scanner/scanner_task/scanner_task_base.go
View File

@ -31,7 +31,7 @@ func (t ScannerTaskBase) BeforeProcessMedia(ctx TaskContext, mediaData *media_en
return ctx, nil
}
func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) {
func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error) {
return []*models.MediaURL{}, nil
}

12
api/scanner/scanner_tasks/processing_tasks/process_photo_task.go
View File

@ -5,6 +5,7 @@ import (
"log"
"os"
"path"
"time"
"github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner/media_encoding"
@ -25,7 +26,7 @@ type ProcessPhotoTask struct {
scanner_task.ScannerTaskBase
}
func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
if mediaData.Media.Type != models.MediaTypePhoto {
return []*models.MediaURL{}, nil
}
@ -66,7 +67,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
return []*models.MediaURL{}, err
}
if !contentType.IsWebCompatible() {
if !contentType.IsWebCompatible() || mediaData.Media.UpdatedAt.Before(newModTime) {
highresName := generateUniqueMediaNamePrefixed("highres", photo.Path, ".jpg")
baseImagePath = path.Join(mediaCachePath, highresName)
@ -81,7 +82,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
// Verify that highres photo still exists in cache
baseImagePath = path.Join(mediaCachePath, highResURL.MediaName)
if _, err := os.Stat(baseImagePath); os.IsNotExist(err) {
if _, err := os.Stat(baseImagePath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName)
updatedURLs = append(updatedURLs, highResURL)
@ -93,7 +94,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
}
// Save original photo to database
if origURL == nil {
if origURL == nil || mediaData.Media.UpdatedAt.Before(newModTime) {
// Make sure photo dimensions is set
if photoDimensions == nil {
@ -124,7 +125,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
// Verify that thumbnail photo still exists in cache
thumbPath := path.Join(mediaCachePath, thumbURL.MediaName)
if _, err := os.Stat(thumbPath); os.IsNotExist(err) {
if _, err := os.Stat(thumbPath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
updatedURLs = append(updatedURLs, thumbURL)
fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.MediaName)
@ -135,6 +136,5 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
}
}
ctx.GetDB().Save(mediaData.Media)
return updatedURLs, nil
}

8
api/scanner/scanner_tasks/processing_tasks/process_video_task.go
View File

@ -23,7 +23,7 @@ type ProcessVideoTask struct {
scanner_task.ScannerTaskBase
}
func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
if mediaData.Media.Type != models.MediaTypeVideo {
return []*models.MediaURL{}, nil
}
@ -55,7 +55,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
return []*models.MediaURL{}, errors.Wrap(err, "error getting video content type")
}
if videoOriginalURL == nil && videoType.IsWebCompatible() {
if videoType.IsWebCompatible() && (videoOriginalURL == nil || mediaData.Media.UpdatedAt.Before(newModTime)) {
origVideoPath := video.Path
videoMediaName := generateUniqueMediaName(video.Path)
@ -86,7 +86,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
updatedURLs = append(updatedURLs, &mediaURL)
}
if videoWebURL == nil && !videoType.IsWebCompatible() {
if !videoType.IsWebCompatible() && (videoWebURL == nil || mediaData.Media.UpdatedAt.Before(newModTime)) {
web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
web_video_name = strings.ReplaceAll(web_video_name, ".", "_")
web_video_name = strings.ReplaceAll(web_video_name, " ", "_")
@ -173,7 +173,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
// Verify that video thumbnail still exists in cache
thumbImagePath := path.Join(mediaCachePath, videoThumbnailURL.MediaName)
if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) {
if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
updatedURLs = append(updatedURLs, videoThumbnailURL)

2
api/scanner/scanner_tasks/processing_tasks/sidecar_task.go
View File

@ -53,7 +53,7 @@ func (t SidecarTask) AfterMediaFound(ctx scanner_task.TaskContext, media *models
return nil
}
func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) {
func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error) {
mediaType, err := mediaData.ContentType()
if err != nil {
return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, process media")

4
api/scanner/scanner_tasks/scanner_tasks.go
View File

@ -116,7 +116,7 @@ func (t scannerTasks) BeforeProcessMedia(ctx scanner_task.TaskContext, mediaData
return ctx, nil
}
func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
allNewMedia := make([]*models.MediaURL, 0)
for _, task := range allTasks {
@ -126,7 +126,7 @@ func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *medi
default:
}
newMedia, err := task.ProcessMedia(ctx, mediaData, mediaCachePath)
newMedia, err := task.ProcessMedia(ctx, mediaData, mediaCachePath, newModTime)
if err != nil {
return []*models.MediaURL{}, err
}