make the processing idempotent
This commit is contained in:
parent
3048c41c3e
commit
880ef195fb
|
@ -6,6 +6,7 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding"
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
@ -85,6 +86,11 @@ func ValidRootPath(rootPath string) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ChangedMedia struct {
|
||||||
|
media *models.Media
|
||||||
|
newModTime time.Time
|
||||||
|
}
|
||||||
|
|
||||||
func ScanAlbum(ctx scanner_task.TaskContext) error {
|
func ScanAlbum(ctx scanner_task.TaskContext) error {
|
||||||
|
|
||||||
newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx)
|
newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx)
|
||||||
|
@ -94,16 +100,16 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
|
||||||
ctx = newCtx
|
ctx = newCtx
|
||||||
|
|
||||||
// Scan for photos
|
// Scan for photos
|
||||||
albumMedia, err := findMediaForAlbum(ctx)
|
albumUpdatedMedia, err := findMediaForAlbum(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "find media for album (%s): %s", ctx.GetAlbum().Path, err)
|
return errors.Wrapf(err, "find media for album (%s): %s", ctx.GetAlbum().Path, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
changedMedia := make([]*models.Media, 0)
|
changedMedia := make([]*models.Media, 0)
|
||||||
for i, media := range albumMedia {
|
for i, media := range albumUpdatedMedia {
|
||||||
updatedURLs := []*models.MediaURL{}
|
updatedURLs := []*models.MediaURL{}
|
||||||
|
|
||||||
mediaData := media_encoding.NewEncodeMediaData(media)
|
mediaData := media_encoding.NewEncodeMediaData(media.media)
|
||||||
|
|
||||||
// define new ctx for scope of for-loop
|
// define new ctx for scope of for-loop
|
||||||
ctx, err := scanner_tasks.Tasks.BeforeProcessMedia(ctx, &mediaData)
|
ctx, err := scanner_tasks.Tasks.BeforeProcessMedia(ctx, &mediaData)
|
||||||
|
@ -112,15 +118,18 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
transactionError := ctx.DatabaseTransaction(func(ctx scanner_task.TaskContext) error {
|
transactionError := ctx.DatabaseTransaction(func(ctx scanner_task.TaskContext) error {
|
||||||
updatedURLs, err = processMedia(ctx, &mediaData)
|
updatedURLs, err = processMedia(ctx, &mediaData, media.newModTime)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "process media (%s)", media.Path)
|
return errors.Wrapf(err, "process media (%s)", media.media.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(updatedURLs) > 0 {
|
if len(updatedURLs) > 0 {
|
||||||
changedMedia = append(changedMedia, media)
|
changedMedia = append(changedMedia, media.media)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if media.media.UpdatedAt.Before(media.newModTime) {
|
||||||
|
ctx.GetDB().Save(mediaData.Media)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -128,11 +137,15 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
|
||||||
return errors.Wrap(err, "process media database transaction")
|
return errors.Wrap(err, "process media database transaction")
|
||||||
}
|
}
|
||||||
|
|
||||||
if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumMedia)); err != nil {
|
if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumUpdatedMedia)); err != nil {
|
||||||
return errors.Wrap(err, "after process media")
|
return errors.Wrap(err, "after process media")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
albumMedia := make([]*models.Media, 0)
|
||||||
|
for _, media := range albumUpdatedMedia {
|
||||||
|
albumMedia = append(albumMedia, media.media)
|
||||||
|
}
|
||||||
if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, changedMedia, albumMedia); err != nil {
|
if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, changedMedia, albumMedia); err != nil {
|
||||||
return errors.Wrap(err, "after scan album")
|
return errors.Wrap(err, "after scan album")
|
||||||
}
|
}
|
||||||
|
@ -140,9 +153,9 @@ func ScanAlbum(ctx scanner_task.TaskContext) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
|
func findMediaForAlbum(ctx scanner_task.TaskContext) ([]ChangedMedia, error) {
|
||||||
|
|
||||||
albumMedia := make([]*models.Media, 0)
|
albumMedia := make([]ChangedMedia, 0)
|
||||||
|
|
||||||
dirContent, err := ioutil.ReadDir(ctx.GetAlbum().Path)
|
dirContent, err := ioutil.ReadDir(ctx.GetAlbum().Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -189,7 +202,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
albumMedia = append(albumMedia, media)
|
albumMedia = append(albumMedia, ChangedMedia{media, item.ModTime()})
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
@ -205,7 +218,7 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
|
||||||
return albumMedia, nil
|
return albumMedia, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData) ([]*models.MediaURL, error) {
|
func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, newModTime time.Time) ([]*models.MediaURL, error) {
|
||||||
|
|
||||||
// Make sure media cache directory exists
|
// Make sure media cache directory exists
|
||||||
mediaCachePath, err := mediaData.Media.CachePath()
|
mediaCachePath, err := mediaData.Media.CachePath()
|
||||||
|
@ -213,5 +226,5 @@ func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.Encode
|
||||||
return []*models.MediaURL{}, errors.Wrap(err, "cache directory error")
|
return []*models.MediaURL{}, errors.Wrap(err, "cache directory error")
|
||||||
}
|
}
|
||||||
|
|
||||||
return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, mediaCachePath)
|
return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, mediaCachePath, newModTime)
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding"
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
@ -92,7 +93,7 @@ func ProcessSingleMedia(db *gorm.DB, media *models.Media) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
updated_urls, err := scanner_tasks.Tasks.ProcessMedia(new_ctx, &media_data, mediaCachePath)
|
updated_urls, err := scanner_tasks.Tasks.ProcessMedia(new_ctx, &media_data, mediaCachePath, time.Now())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -32,7 +32,7 @@ type ScannerTask interface {
|
||||||
AfterMediaFound(ctx TaskContext, media *models.Media, newMedia bool, newModTime time.Time) error
|
AfterMediaFound(ctx TaskContext, media *models.Media, newMedia bool, newModTime time.Time) error
|
||||||
|
|
||||||
BeforeProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData) (TaskContext, error)
|
BeforeProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData) (TaskContext, error)
|
||||||
ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error)
|
ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error)
|
||||||
AfterProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error
|
AfterProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -31,7 +31,7 @@ func (t ScannerTaskBase) BeforeProcessMedia(ctx TaskContext, mediaData *media_en
|
||||||
return ctx, nil
|
return ctx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) {
|
func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error) {
|
||||||
return []*models.MediaURL{}, nil
|
return []*models.MediaURL{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -5,6 +5,7 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding"
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
@ -25,7 +26,7 @@ type ProcessPhotoTask struct {
|
||||||
scanner_task.ScannerTaskBase
|
scanner_task.ScannerTaskBase
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
|
func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
|
||||||
if mediaData.Media.Type != models.MediaTypePhoto {
|
if mediaData.Media.Type != models.MediaTypePhoto {
|
||||||
return []*models.MediaURL{}, nil
|
return []*models.MediaURL{}, nil
|
||||||
}
|
}
|
||||||
|
@ -66,7 +67,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
|
||||||
return []*models.MediaURL{}, err
|
return []*models.MediaURL{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if !contentType.IsWebCompatible() {
|
if !contentType.IsWebCompatible() || mediaData.Media.UpdatedAt.Before(newModTime) {
|
||||||
highresName := generateUniqueMediaNamePrefixed("highres", photo.Path, ".jpg")
|
highresName := generateUniqueMediaNamePrefixed("highres", photo.Path, ".jpg")
|
||||||
baseImagePath = path.Join(mediaCachePath, highresName)
|
baseImagePath = path.Join(mediaCachePath, highresName)
|
||||||
|
|
||||||
|
@ -81,7 +82,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
|
||||||
// Verify that highres photo still exists in cache
|
// Verify that highres photo still exists in cache
|
||||||
baseImagePath = path.Join(mediaCachePath, highResURL.MediaName)
|
baseImagePath = path.Join(mediaCachePath, highResURL.MediaName)
|
||||||
|
|
||||||
if _, err := os.Stat(baseImagePath); os.IsNotExist(err) {
|
if _, err := os.Stat(baseImagePath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
|
||||||
fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName)
|
fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName)
|
||||||
updatedURLs = append(updatedURLs, highResURL)
|
updatedURLs = append(updatedURLs, highResURL)
|
||||||
|
|
||||||
|
@ -93,7 +94,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
|
||||||
}
|
}
|
||||||
|
|
||||||
// Save original photo to database
|
// Save original photo to database
|
||||||
if origURL == nil {
|
if origURL == nil || mediaData.Media.UpdatedAt.Before(newModTime) {
|
||||||
|
|
||||||
// Make sure photo dimensions is set
|
// Make sure photo dimensions is set
|
||||||
if photoDimensions == nil {
|
if photoDimensions == nil {
|
||||||
|
@ -124,7 +125,7 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
|
||||||
// Verify that thumbnail photo still exists in cache
|
// Verify that thumbnail photo still exists in cache
|
||||||
thumbPath := path.Join(mediaCachePath, thumbURL.MediaName)
|
thumbPath := path.Join(mediaCachePath, thumbURL.MediaName)
|
||||||
|
|
||||||
if _, err := os.Stat(thumbPath); os.IsNotExist(err) {
|
if _, err := os.Stat(thumbPath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
|
||||||
updatedURLs = append(updatedURLs, thumbURL)
|
updatedURLs = append(updatedURLs, thumbURL)
|
||||||
fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.MediaName)
|
fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.MediaName)
|
||||||
|
|
||||||
|
@ -135,6 +136,5 @@ func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ctx.GetDB().Save(mediaData.Media)
|
|
||||||
return updatedURLs, nil
|
return updatedURLs, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -23,7 +23,7 @@ type ProcessVideoTask struct {
|
||||||
scanner_task.ScannerTaskBase
|
scanner_task.ScannerTaskBase
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
|
func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
|
||||||
if mediaData.Media.Type != models.MediaTypeVideo {
|
if mediaData.Media.Type != models.MediaTypeVideo {
|
||||||
return []*models.MediaURL{}, nil
|
return []*models.MediaURL{}, nil
|
||||||
}
|
}
|
||||||
|
@ -55,7 +55,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
|
||||||
return []*models.MediaURL{}, errors.Wrap(err, "error getting video content type")
|
return []*models.MediaURL{}, errors.Wrap(err, "error getting video content type")
|
||||||
}
|
}
|
||||||
|
|
||||||
if videoOriginalURL == nil && videoType.IsWebCompatible() {
|
if videoType.IsWebCompatible() && (videoOriginalURL == nil || mediaData.Media.UpdatedAt.Before(newModTime)) {
|
||||||
origVideoPath := video.Path
|
origVideoPath := video.Path
|
||||||
videoMediaName := generateUniqueMediaName(video.Path)
|
videoMediaName := generateUniqueMediaName(video.Path)
|
||||||
|
|
||||||
|
@ -86,7 +86,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
|
||||||
updatedURLs = append(updatedURLs, &mediaURL)
|
updatedURLs = append(updatedURLs, &mediaURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
if videoWebURL == nil && !videoType.IsWebCompatible() {
|
if !videoType.IsWebCompatible() && (videoWebURL == nil || mediaData.Media.UpdatedAt.Before(newModTime)) {
|
||||||
web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
|
web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
|
||||||
web_video_name = strings.ReplaceAll(web_video_name, ".", "_")
|
web_video_name = strings.ReplaceAll(web_video_name, ".", "_")
|
||||||
web_video_name = strings.ReplaceAll(web_video_name, " ", "_")
|
web_video_name = strings.ReplaceAll(web_video_name, " ", "_")
|
||||||
|
@ -173,7 +173,7 @@ func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *
|
||||||
// Verify that video thumbnail still exists in cache
|
// Verify that video thumbnail still exists in cache
|
||||||
thumbImagePath := path.Join(mediaCachePath, videoThumbnailURL.MediaName)
|
thumbImagePath := path.Join(mediaCachePath, videoThumbnailURL.MediaName)
|
||||||
|
|
||||||
if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) {
|
if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) || mediaData.Media.UpdatedAt.Before(newModTime) {
|
||||||
fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
|
fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
|
||||||
updatedURLs = append(updatedURLs, videoThumbnailURL)
|
updatedURLs = append(updatedURLs, videoThumbnailURL)
|
||||||
|
|
||||||
|
|
|
@ -53,7 +53,7 @@ func (t SidecarTask) AfterMediaFound(ctx scanner_task.TaskContext, media *models
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) {
|
func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) (updatedURLs []*models.MediaURL, err error) {
|
||||||
mediaType, err := mediaData.ContentType()
|
mediaType, err := mediaData.ContentType()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, process media")
|
return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, process media")
|
||||||
|
|
|
@ -116,7 +116,7 @@ func (t scannerTasks) BeforeProcessMedia(ctx scanner_task.TaskContext, mediaData
|
||||||
return ctx, nil
|
return ctx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
|
func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string, newModTime time.Time) ([]*models.MediaURL, error) {
|
||||||
allNewMedia := make([]*models.MediaURL, 0)
|
allNewMedia := make([]*models.MediaURL, 0)
|
||||||
|
|
||||||
for _, task := range allTasks {
|
for _, task := range allTasks {
|
||||||
|
@ -126,7 +126,7 @@ func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *medi
|
||||||
default:
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
newMedia, err := task.ProcessMedia(ctx, mediaData, mediaCachePath)
|
newMedia, err := task.ProcessMedia(ctx, mediaData, mediaCachePath, newModTime)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return []*models.MediaURL{}, err
|
return []*models.MediaURL{}, err
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue