WIP: split up scanner into separate tasks
This commit is contained in:
parent
6361df1793
commit
6e2a64bc77
|
@ -30,9 +30,6 @@ type Media struct {
|
||||||
SideCarHash *string `gorm:"unique"`
|
SideCarHash *string `gorm:"unique"`
|
||||||
Faces []*ImageFace `gorm:"constraint:OnDelete:CASCADE;"`
|
Faces []*ImageFace `gorm:"constraint:OnDelete:CASCADE;"`
|
||||||
Blurhash *string `gorm:""`
|
Blurhash *string `gorm:""`
|
||||||
|
|
||||||
// Only used internally
|
|
||||||
CounterpartPath *string `gorm:"-"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (Media) TableName() string {
|
func (Media) TableName() string {
|
||||||
|
@ -65,6 +62,21 @@ func (m *Media) GetThumbnail() (*MediaURL, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (m *Media) GetHighRes() (*MediaURL, error) {
|
||||||
|
if len(m.MediaURL) == 0 {
|
||||||
|
return nil, errors.New("media.MediaURL is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, url := range m.MediaURL {
|
||||||
|
if url.Purpose == PhotoHighRes {
|
||||||
|
url.Media = m
|
||||||
|
return &url, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
type MediaType string
|
type MediaType string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
|
@ -54,10 +54,11 @@ func encodeImageJPEG(image image.Image, outputPath string, jpegQuality int) erro
|
||||||
|
|
||||||
// EncodeMediaData is used to easily decode media data, with a cache so expensive operations are not repeated
|
// EncodeMediaData is used to easily decode media data, with a cache so expensive operations are not repeated
|
||||||
type EncodeMediaData struct {
|
type EncodeMediaData struct {
|
||||||
Media *models.Media
|
Media *models.Media
|
||||||
_photoImage image.Image
|
CounterpartPath *string
|
||||||
_contentType *media_type.MediaType
|
_photoImage image.Image
|
||||||
_videoMetadata *ffprobe.ProbeData
|
_contentType *media_type.MediaType
|
||||||
|
_videoMetadata *ffprobe.ProbeData
|
||||||
}
|
}
|
||||||
|
|
||||||
// ContentType reads the image to determine its content type
|
// ContentType reads the image to determine its content type
|
||||||
|
@ -86,7 +87,7 @@ func (img *EncodeMediaData) EncodeHighRes(outputPath string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Use darktable if there is no counterpart JPEG file to use instead
|
// Use darktable if there is no counterpart JPEG file to use instead
|
||||||
if contentType.IsRaw() && img.Media.CounterpartPath == nil {
|
if contentType.IsRaw() && img.CounterpartPath == nil {
|
||||||
if executable_worker.DarktableCli.IsInstalled() {
|
if executable_worker.DarktableCli.IsInstalled() {
|
||||||
err := executable_worker.DarktableCli.EncodeJpeg(img.Media.Path, outputPath, 70)
|
err := executable_worker.DarktableCli.EncodeJpeg(img.Media.Path, outputPath, 70)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -114,8 +115,8 @@ func (img *EncodeMediaData) photoImage() (image.Image, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
var photoPath string
|
var photoPath string
|
||||||
if img.Media.CounterpartPath != nil {
|
if img.CounterpartPath != nil {
|
||||||
photoPath = *img.Media.CounterpartPath
|
photoPath = *img.CounterpartPath
|
||||||
} else {
|
} else {
|
||||||
photoPath = img.Media.Path
|
photoPath = img.Media.Path
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,411 +0,0 @@
|
||||||
package scanner
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"path"
|
|
||||||
"strconv"
|
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding"
|
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding/media_utils"
|
|
||||||
"github.com/photoview/photoview/api/scanner/media_type"
|
|
||||||
"github.com/photoview/photoview/api/utils"
|
|
||||||
"github.com/pkg/errors"
|
|
||||||
"gorm.io/gorm"
|
|
||||||
|
|
||||||
// Image decoders
|
|
||||||
_ "image/gif"
|
|
||||||
_ "image/png"
|
|
||||||
|
|
||||||
_ "golang.org/x/image/bmp"
|
|
||||||
_ "golang.org/x/image/tiff"
|
|
||||||
_ "golang.org/x/image/webp"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Higher order function used to check if MediaURL for a given MediaPurpose exists
|
|
||||||
func makePhotoURLChecker(tx *gorm.DB, mediaID int) func(purpose models.MediaPurpose) (*models.MediaURL, error) {
|
|
||||||
return func(purpose models.MediaPurpose) (*models.MediaURL, error) {
|
|
||||||
var mediaURL []*models.MediaURL
|
|
||||||
|
|
||||||
result := tx.Where("purpose = ?", purpose).Where("media_id = ?", mediaID).Find(&mediaURL)
|
|
||||||
|
|
||||||
if result.Error != nil {
|
|
||||||
return nil, result.Error
|
|
||||||
}
|
|
||||||
|
|
||||||
if result.RowsAffected > 0 {
|
|
||||||
return mediaURL[0], nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateUniqueMediaNamePrefixed(prefix string, mediaPath string, extension string) string {
|
|
||||||
mediaName := fmt.Sprintf("%s_%s_%s", prefix, path.Base(mediaPath), utils.GenerateToken())
|
|
||||||
mediaName = models.SanitizeMediaName(mediaName)
|
|
||||||
mediaName = mediaName + extension
|
|
||||||
return mediaName
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateUniqueMediaName(mediaPath string) string {
|
|
||||||
|
|
||||||
filename := path.Base(mediaPath)
|
|
||||||
baseName := filename[0 : len(filename)-len(path.Ext(filename))]
|
|
||||||
baseExt := path.Ext(filename)
|
|
||||||
|
|
||||||
mediaName := fmt.Sprintf("%s_%s", baseName, utils.GenerateToken())
|
|
||||||
mediaName = models.SanitizeMediaName(mediaName) + baseExt
|
|
||||||
|
|
||||||
return mediaName
|
|
||||||
}
|
|
||||||
|
|
||||||
func ProcessMedia(tx *gorm.DB, media *models.Media) (bool, error) {
|
|
||||||
imageData := media_encoding.EncodeMediaData{
|
|
||||||
Media: media,
|
|
||||||
}
|
|
||||||
|
|
||||||
contentType, err := imageData.ContentType()
|
|
||||||
if err != nil {
|
|
||||||
return false, errors.Wrapf(err, "get content-type of media (%s)", media.Path)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Make sure media cache directory exists
|
|
||||||
mediaCachePath, err := makeMediaCacheDir(media)
|
|
||||||
if err != nil {
|
|
||||||
return false, errors.Wrap(err, "cache directory error")
|
|
||||||
}
|
|
||||||
|
|
||||||
if contentType.IsVideo() {
|
|
||||||
return processVideo(tx, &imageData, mediaCachePath)
|
|
||||||
} else {
|
|
||||||
return processPhoto(tx, &imageData, mediaCachePath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func processPhoto(tx *gorm.DB, imageData *media_encoding.EncodeMediaData, photoCachePath *string) (bool, error) {
|
|
||||||
|
|
||||||
photo := imageData.Media
|
|
||||||
|
|
||||||
log.Printf("Processing photo: %s\n", photo.Path)
|
|
||||||
|
|
||||||
didProcess := false
|
|
||||||
|
|
||||||
photoURLFromDB := makePhotoURLChecker(tx, photo.ID)
|
|
||||||
|
|
||||||
// original photo url
|
|
||||||
origURL, err := photoURLFromDB(models.MediaOriginal)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Thumbnail
|
|
||||||
thumbURL, err := photoURLFromDB(models.PhotoThumbnail)
|
|
||||||
if err != nil {
|
|
||||||
return false, errors.Wrap(err, "error processing photo thumbnail")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Highres
|
|
||||||
highResURL, err := photoURLFromDB(models.PhotoHighRes)
|
|
||||||
if err != nil {
|
|
||||||
return false, errors.Wrap(err, "error processing photo highres")
|
|
||||||
}
|
|
||||||
|
|
||||||
var photoDimensions *media_utils.PhotoDimensions
|
|
||||||
var baseImagePath string = photo.Path
|
|
||||||
|
|
||||||
mediaType, err := media_type.GetMediaType(photo.Path)
|
|
||||||
if err != nil {
|
|
||||||
return false, errors.Wrap(err, "could determine if media was photo or video")
|
|
||||||
}
|
|
||||||
|
|
||||||
if mediaType.IsRaw() {
|
|
||||||
err = processRawSideCar(tx, imageData, highResURL, thumbURL, photoCachePath)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
counterpartFile := scanForCompressedCounterpartFile(photo.Path)
|
|
||||||
if counterpartFile != nil {
|
|
||||||
imageData.Media.CounterpartPath = counterpartFile
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate high res jpeg
|
|
||||||
if highResURL == nil {
|
|
||||||
|
|
||||||
contentType, err := imageData.ContentType()
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !contentType.IsWebCompatible() {
|
|
||||||
didProcess = true
|
|
||||||
|
|
||||||
highresName := generateUniqueMediaNamePrefixed("highres", photo.Path, ".jpg")
|
|
||||||
|
|
||||||
baseImagePath = path.Join(*photoCachePath, highresName)
|
|
||||||
|
|
||||||
_, err := generateSaveHighResJPEG(tx, photo, imageData, highresName, baseImagePath, nil)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Verify that highres photo still exists in cache
|
|
||||||
baseImagePath = path.Join(*photoCachePath, highResURL.MediaName)
|
|
||||||
|
|
||||||
if _, err := os.Stat(baseImagePath); os.IsNotExist(err) {
|
|
||||||
fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName)
|
|
||||||
didProcess = true
|
|
||||||
|
|
||||||
err = imageData.EncodeHighRes(baseImagePath)
|
|
||||||
if err != nil {
|
|
||||||
return false, errors.Wrap(err, "creating high-res cached image")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save original photo to database
|
|
||||||
if origURL == nil {
|
|
||||||
didProcess = true
|
|
||||||
|
|
||||||
// Make sure photo dimensions is set
|
|
||||||
if photoDimensions == nil {
|
|
||||||
photoDimensions, err = media_utils.GetPhotoDimensions(baseImagePath)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = saveOriginalPhotoToDB(tx, photo, imageData, photoDimensions); err != nil {
|
|
||||||
return false, errors.Wrap(err, "saving original photo to database")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Save thumbnail to cache
|
|
||||||
if thumbURL == nil {
|
|
||||||
didProcess = true
|
|
||||||
|
|
||||||
thumbnailName := generateUniqueMediaNamePrefixed("thumbnail", photo.Path, ".jpg")
|
|
||||||
|
|
||||||
_, err := generateSaveThumbnailJPEG(tx, photo, thumbnailName, photoCachePath, baseImagePath, nil)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
} else {
|
|
||||||
// Verify that thumbnail photo still exists in cache
|
|
||||||
thumbPath := path.Join(*photoCachePath, thumbURL.MediaName)
|
|
||||||
|
|
||||||
if _, err := os.Stat(thumbPath); os.IsNotExist(err) {
|
|
||||||
didProcess = true
|
|
||||||
fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.MediaName)
|
|
||||||
|
|
||||||
_, err := media_encoding.EncodeThumbnail(baseImagePath, thumbPath)
|
|
||||||
if err != nil {
|
|
||||||
return false, errors.Wrap(err, "could not create thumbnail cached image")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return didProcess, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func makeMediaCacheDir(media *models.Media) (*string, error) {
|
|
||||||
|
|
||||||
// Make root cache dir if not exists
|
|
||||||
if _, err := os.Stat(utils.MediaCachePath()); os.IsNotExist(err) {
|
|
||||||
if err := os.Mkdir(utils.MediaCachePath(), os.ModePerm); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "could not make root image cache directory")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Make album cache dir if not exists
|
|
||||||
albumCachePath := path.Join(utils.MediaCachePath(), strconv.Itoa(int(media.AlbumID)))
|
|
||||||
if _, err := os.Stat(albumCachePath); os.IsNotExist(err) {
|
|
||||||
if err := os.Mkdir(albumCachePath, os.ModePerm); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "could not make album image cache directory")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Make photo cache dir if not exists
|
|
||||||
photoCachePath := path.Join(albumCachePath, strconv.Itoa(int(media.ID)))
|
|
||||||
if _, err := os.Stat(photoCachePath); os.IsNotExist(err) {
|
|
||||||
if err := os.Mkdir(photoCachePath, os.ModePerm); err != nil {
|
|
||||||
return nil, errors.Wrap(err, "could not make photo image cache directory")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return &photoCachePath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func saveOriginalPhotoToDB(tx *gorm.DB, photo *models.Media, imageData *media_encoding.EncodeMediaData, photoDimensions *media_utils.PhotoDimensions) error {
|
|
||||||
originalImageName := generateUniqueMediaName(photo.Path)
|
|
||||||
|
|
||||||
contentType, err := imageData.ContentType()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
fileStats, err := os.Stat(photo.Path)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Wrap(err, "reading file stats of original photo")
|
|
||||||
}
|
|
||||||
|
|
||||||
mediaURL := models.MediaURL{
|
|
||||||
Media: photo,
|
|
||||||
MediaName: originalImageName,
|
|
||||||
Width: photoDimensions.Width,
|
|
||||||
Height: photoDimensions.Height,
|
|
||||||
Purpose: models.MediaOriginal,
|
|
||||||
ContentType: string(*contentType),
|
|
||||||
FileSize: fileStats.Size(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := tx.Create(&mediaURL).Error; err != nil {
|
|
||||||
return errors.Wrapf(err, "inserting original photo url: %d, %s", photo.ID, photo.Title)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *media_encoding.EncodeMediaData, highres_name string, imagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) {
|
|
||||||
|
|
||||||
err := imageData.EncodeHighRes(imagePath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "creating high-res cached image")
|
|
||||||
}
|
|
||||||
|
|
||||||
photoDimensions, err := media_utils.GetPhotoDimensions(imagePath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
fileStats, err := os.Stat(imagePath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "reading file stats of highres photo")
|
|
||||||
}
|
|
||||||
|
|
||||||
if mediaURL == nil {
|
|
||||||
|
|
||||||
mediaURL = &models.MediaURL{
|
|
||||||
MediaID: media.ID,
|
|
||||||
MediaName: highres_name,
|
|
||||||
Width: photoDimensions.Width,
|
|
||||||
Height: photoDimensions.Height,
|
|
||||||
Purpose: models.PhotoHighRes,
|
|
||||||
ContentType: "image/jpeg",
|
|
||||||
FileSize: fileStats.Size(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := tx.Create(&mediaURL).Error; err != nil {
|
|
||||||
return nil, errors.Wrapf(err, "could not insert highres media url (%d, %s)", media.ID, highres_name)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
mediaURL.Width = photoDimensions.Width
|
|
||||||
mediaURL.Height = photoDimensions.Height
|
|
||||||
mediaURL.FileSize = fileStats.Size()
|
|
||||||
|
|
||||||
if err := tx.Save(&mediaURL).Error; err != nil {
|
|
||||||
return nil, errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, highres_name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return mediaURL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateSaveThumbnailJPEG(tx *gorm.DB, media *models.Media, thumbnail_name string, photoCachePath *string, baseImagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) {
|
|
||||||
thumbOutputPath := path.Join(*photoCachePath, thumbnail_name)
|
|
||||||
|
|
||||||
thumbSize, err := media_encoding.EncodeThumbnail(baseImagePath, thumbOutputPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "could not create thumbnail cached image")
|
|
||||||
}
|
|
||||||
|
|
||||||
fileStats, err := os.Stat(thumbOutputPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, errors.Wrap(err, "reading file stats of thumbnail photo")
|
|
||||||
}
|
|
||||||
|
|
||||||
if mediaURL == nil {
|
|
||||||
|
|
||||||
mediaURL = &models.MediaURL{
|
|
||||||
MediaID: media.ID,
|
|
||||||
MediaName: thumbnail_name,
|
|
||||||
Width: thumbSize.Width,
|
|
||||||
Height: thumbSize.Height,
|
|
||||||
Purpose: models.PhotoThumbnail,
|
|
||||||
ContentType: "image/jpeg",
|
|
||||||
FileSize: fileStats.Size(),
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := tx.Create(&mediaURL).Error; err != nil {
|
|
||||||
return nil, errors.Wrapf(err, "could not insert thumbnail media url (%d, %s)", media.ID, thumbnail_name)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
mediaURL.Width = thumbSize.Width
|
|
||||||
mediaURL.Height = thumbSize.Height
|
|
||||||
mediaURL.FileSize = fileStats.Size()
|
|
||||||
|
|
||||||
if err := tx.Save(&mediaURL).Error; err != nil {
|
|
||||||
return nil, errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, thumbnail_name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return mediaURL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func processRawSideCar(tx *gorm.DB, imageData *media_encoding.EncodeMediaData, highResURL *models.MediaURL, thumbURL *models.MediaURL, photoCachePath *string) error {
|
|
||||||
photo := imageData.Media
|
|
||||||
sideCarFileHasChanged := false
|
|
||||||
var currentFileHash *string
|
|
||||||
currentSideCarPath := scanForSideCarFile(photo.Path)
|
|
||||||
|
|
||||||
if currentSideCarPath != nil {
|
|
||||||
currentFileHash = hashSideCarFile(currentSideCarPath)
|
|
||||||
if photo.SideCarHash == nil || *photo.SideCarHash != *currentFileHash {
|
|
||||||
sideCarFileHasChanged = true
|
|
||||||
}
|
|
||||||
} else if photo.SideCarPath != nil { // sidecar has been deleted since last scan
|
|
||||||
sideCarFileHasChanged = true
|
|
||||||
}
|
|
||||||
|
|
||||||
if sideCarFileHasChanged {
|
|
||||||
fmt.Printf("Detected changed sidecar file for %s recreating JPG's to reflect changes\n", photo.Path)
|
|
||||||
|
|
||||||
// update high res image may be cropped so dimentions and file size can change
|
|
||||||
baseImagePath := path.Join(*photoCachePath, highResURL.MediaName) // update base image path for thumbnail
|
|
||||||
tempHighResPath := baseImagePath + ".hold"
|
|
||||||
os.Rename(baseImagePath, tempHighResPath)
|
|
||||||
_, err := generateSaveHighResJPEG(tx, photo, imageData, highResURL.MediaName, baseImagePath, highResURL)
|
|
||||||
if err != nil {
|
|
||||||
os.Rename(tempHighResPath, baseImagePath)
|
|
||||||
return errors.Wrap(err, "recreating high-res cached image")
|
|
||||||
}
|
|
||||||
os.Remove(tempHighResPath)
|
|
||||||
|
|
||||||
// update thumbnail image may be cropped so dimentions and file size can change
|
|
||||||
thumbPath := path.Join(*photoCachePath, thumbURL.MediaName)
|
|
||||||
tempThumbPath := thumbPath + ".hold" // hold onto the original image incase for some reason we fail to recreate one with the new settings
|
|
||||||
os.Rename(thumbPath, tempThumbPath)
|
|
||||||
_, err = generateSaveThumbnailJPEG(tx, photo, thumbURL.MediaName, photoCachePath, baseImagePath, thumbURL)
|
|
||||||
if err != nil {
|
|
||||||
os.Rename(tempThumbPath, thumbPath)
|
|
||||||
return errors.Wrap(err, "recreating thumbnail cached image")
|
|
||||||
}
|
|
||||||
os.Remove(tempThumbPath)
|
|
||||||
|
|
||||||
photo.SideCarHash = currentFileHash
|
|
||||||
photo.SideCarPath = currentSideCarPath
|
|
||||||
|
|
||||||
// save new side car hash
|
|
||||||
if err := tx.Save(&photo).Error; err != nil {
|
|
||||||
return errors.Wrapf(err, "could not update side car hash for media: %s", photo.Path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
|
@ -6,9 +6,9 @@ import (
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
"github.com/photoview/photoview/api/scanner/face_detection"
|
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding"
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
"github.com/photoview/photoview/api/scanner/scanner_task"
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
"github.com/photoview/photoview/api/scanner/scanner_tasks"
|
"github.com/photoview/photoview/api/scanner/scanner_tasks"
|
||||||
|
@ -86,68 +86,61 @@ func ValidRootPath(rootPath string) bool {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func ScanAlbum(ctx scanner_task.TaskContext) {
|
func ScanAlbum(ctx scanner_task.TaskContext) error {
|
||||||
|
|
||||||
newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx)
|
newCtx, err := scanner_tasks.Tasks.BeforeScanAlbum(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
scanner_utils.ScannerError("before scan album (%s): %s", ctx.GetAlbum().Path, err)
|
return errors.Wrapf(err, "before scan album (%s)", ctx.GetAlbum().Path)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
ctx = newCtx
|
ctx = newCtx
|
||||||
|
|
||||||
// Scan for photos
|
// Scan for photos
|
||||||
albumMedia, err := findMediaForAlbum(ctx)
|
albumMedia, err := findMediaForAlbum(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
scanner_utils.ScannerError("find media for album (%s): %s", ctx.GetAlbum().Path, err)
|
return errors.Wrapf(err, "find media for album (%s): %s", ctx.GetAlbum().Path)
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
albumHasChanges := false
|
changedMedia := make([]*models.Media, 0)
|
||||||
for count, media := range albumMedia {
|
for i, media := range albumMedia {
|
||||||
didProcess := false
|
updatedURLs := []*models.MediaURL{}
|
||||||
|
|
||||||
transactionError := ctx.GetDB().Transaction(func(tx *gorm.DB) error {
|
mediaData := media_encoding.EncodeMediaData{
|
||||||
// processing_was_needed, err = ProcessMedia(tx, media)
|
Media: media,
|
||||||
didProcess, err = processMedia(ctx, media)
|
}
|
||||||
|
|
||||||
|
// define new ctx for scope of for-loop
|
||||||
|
ctx, err := scanner_tasks.Tasks.BeforeProcessMedia(ctx, &mediaData)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
transactionError := ctx.DatabaseTransaction(func(ctx scanner_task.TaskContext) error {
|
||||||
|
updatedURLs, err = processMedia(ctx, &mediaData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "process media (%s)", media.Path)
|
return errors.Wrapf(err, "process media (%s)", media.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
if didProcess {
|
if len(updatedURLs) > 0 {
|
||||||
albumHasChanges = true
|
changedMedia = append(changedMedia, media)
|
||||||
}
|
|
||||||
|
|
||||||
if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, media, didProcess, count, len(albumMedia)); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
if transactionError != nil {
|
if transactionError != nil {
|
||||||
scanner_utils.ScannerError("begin database transaction: %s", transactionError)
|
return errors.Wrap(err, "process media database transaction")
|
||||||
}
|
}
|
||||||
|
|
||||||
if didProcess && media.Type == models.MediaTypePhoto {
|
if err = scanner_tasks.Tasks.AfterProcessMedia(ctx, &mediaData, updatedURLs, i, len(albumMedia)); err != nil {
|
||||||
go func(media *models.Media) {
|
return errors.Wrap(err, "after process media")
|
||||||
if face_detection.GlobalFaceDetector == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err := face_detection.GlobalFaceDetector.DetectFaces(ctx.GetDB(), media); err != nil {
|
|
||||||
scanner_utils.ScannerError("Error detecting faces in image (%s): %s", media.Path, err)
|
|
||||||
}
|
|
||||||
}(media)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanup_errors := CleanupMedia(ctx.GetDB(), ctx.GetAlbum().ID, albumMedia)
|
if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, changedMedia, albumMedia); err != nil {
|
||||||
for _, err := range cleanup_errors {
|
return errors.Wrap(err, "after scan album")
|
||||||
scanner_utils.ScannerError("delete old media: %s", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := scanner_tasks.Tasks.AfterScanAlbum(ctx, albumHasChanges); err != nil {
|
return nil
|
||||||
scanner_utils.ScannerError("after scan album: %s", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
|
func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
|
||||||
|
@ -178,14 +171,9 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Skip the JPEGs that are compressed version of raw files
|
err = ctx.DatabaseTransaction(func(ctx scanner_task.TaskContext) error {
|
||||||
counterpartFile := scanForRawCounterpartFile(mediaPath)
|
|
||||||
if counterpartFile != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
err = ctx.GetDB().Transaction(func(tx *gorm.DB) error {
|
media, isNewMedia, err := ScanMedia(ctx.GetDB(), mediaPath, ctx.GetAlbum().ID, ctx.GetCache())
|
||||||
media, isNewMedia, err := ScanMedia(tx, mediaPath, ctx.GetAlbum().ID, ctx.GetCache())
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "scanning media error (%s)", mediaPath)
|
return errors.Wrapf(err, "scanning media error (%s)", mediaPath)
|
||||||
}
|
}
|
||||||
|
@ -210,21 +198,46 @@ func findMediaForAlbum(ctx scanner_task.TaskContext) ([]*models.Media, error) {
|
||||||
return albumMedia, nil
|
return albumMedia, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func processMedia(ctx scanner_task.TaskContext, media *models.Media) (bool, error) {
|
func processMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData) ([]*models.MediaURL, error) {
|
||||||
mediaData := media_encoding.EncodeMediaData{
|
|
||||||
Media: media,
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err := mediaData.ContentType()
|
_, err := mediaData.ContentType()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrapf(err, "get content-type of media (%s)", media.Path)
|
return []*models.MediaURL{}, errors.Wrapf(err, "get content-type of media (%s)", mediaData.Media.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Make sure media cache directory exists
|
// Make sure media cache directory exists
|
||||||
mediaCachePath, err := makeMediaCacheDir(media)
|
mediaCachePath, err := makeMediaCacheDir(mediaData.Media)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "cache directory error")
|
return []*models.MediaURL{}, errors.Wrap(err, "cache directory error")
|
||||||
}
|
}
|
||||||
|
|
||||||
return scanner_tasks.Tasks.ProcessMedia(ctx, &mediaData, *mediaCachePath)
|
return scanner_tasks.Tasks.ProcessMedia(ctx, mediaData, *mediaCachePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeMediaCacheDir(media *models.Media) (*string, error) {
|
||||||
|
|
||||||
|
// Make root cache dir if not exists
|
||||||
|
if _, err := os.Stat(utils.MediaCachePath()); os.IsNotExist(err) {
|
||||||
|
if err := os.Mkdir(utils.MediaCachePath(), os.ModePerm); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "could not make root image cache directory")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make album cache dir if not exists
|
||||||
|
albumCachePath := path.Join(utils.MediaCachePath(), strconv.Itoa(int(media.AlbumID)))
|
||||||
|
if _, err := os.Stat(albumCachePath); os.IsNotExist(err) {
|
||||||
|
if err := os.Mkdir(albumCachePath, os.ModePerm); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "could not make album image cache directory")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make photo cache dir if not exists
|
||||||
|
photoCachePath := path.Join(albumCachePath, strconv.Itoa(int(media.ID)))
|
||||||
|
if _, err := os.Stat(photoCachePath); os.IsNotExist(err) {
|
||||||
|
if err := os.Mkdir(photoCachePath, os.ModePerm); err != nil {
|
||||||
|
return nil, errors.Wrap(err, "could not make photo image cache directory")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return &photoCachePath, nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,92 +1,16 @@
|
||||||
package scanner
|
package scanner
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/md5"
|
|
||||||
"encoding/hex"
|
|
||||||
"io"
|
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
"github.com/photoview/photoview/api/scanner/exif"
|
|
||||||
"github.com/photoview/photoview/api/scanner/media_type"
|
|
||||||
"github.com/photoview/photoview/api/scanner/scanner_cache"
|
"github.com/photoview/photoview/api/scanner/scanner_cache"
|
||||||
"github.com/photoview/photoview/api/scanner/scanner_utils"
|
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
func scanForSideCarFile(path string) *string {
|
|
||||||
testPath := path + ".xmp"
|
|
||||||
|
|
||||||
if scanner_utils.FileExists(testPath) {
|
|
||||||
return &testPath
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func scanForRawCounterpartFile(imagePath string) *string {
|
|
||||||
ext := filepath.Ext(imagePath)
|
|
||||||
fileExtType, found := media_type.GetExtensionMediaType(ext)
|
|
||||||
|
|
||||||
if found {
|
|
||||||
if !fileExtType.IsBasicTypeSupported() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
rawPath := media_type.RawCounterpart(imagePath)
|
|
||||||
if rawPath != nil {
|
|
||||||
return rawPath
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func scanForCompressedCounterpartFile(imagePath string) *string {
|
|
||||||
ext := filepath.Ext(imagePath)
|
|
||||||
fileExtType, found := media_type.GetExtensionMediaType(ext)
|
|
||||||
|
|
||||||
if found {
|
|
||||||
if fileExtType.IsBasicTypeSupported() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pathWithoutExt := strings.TrimSuffix(imagePath, path.Ext(imagePath))
|
|
||||||
for _, ext := range media_type.TypeJpeg.FileExtensions() {
|
|
||||||
testPath := pathWithoutExt + ext
|
|
||||||
if scanner_utils.FileExists(testPath) {
|
|
||||||
return &testPath
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func hashSideCarFile(path *string) *string {
|
|
||||||
if path == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
f, err := os.Open(*path)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("ERROR: %s", err)
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
h := md5.New()
|
|
||||||
if _, err := io.Copy(h, f); err != nil {
|
|
||||||
log.Printf("ERROR: %s", err)
|
|
||||||
}
|
|
||||||
hash := hex.EncodeToString(h.Sum(nil))
|
|
||||||
return &hash
|
|
||||||
}
|
|
||||||
|
|
||||||
func ScanMedia(tx *gorm.DB, mediaPath string, albumId int, cache *scanner_cache.AlbumScannerCache) (*models.Media, bool, error) {
|
func ScanMedia(tx *gorm.DB, mediaPath string, albumId int, cache *scanner_cache.AlbumScannerCache) (*models.Media, bool, error) {
|
||||||
mediaName := path.Base(mediaPath)
|
mediaName := path.Base(mediaPath)
|
||||||
|
|
||||||
|
@ -115,20 +39,10 @@ func ScanMedia(tx *gorm.DB, mediaPath string, albumId int, cache *scanner_cache.
|
||||||
|
|
||||||
var mediaTypeText models.MediaType
|
var mediaTypeText models.MediaType
|
||||||
|
|
||||||
var sideCarPath *string = nil
|
|
||||||
var sideCarHash *string = nil
|
|
||||||
|
|
||||||
if mediaType.IsVideo() {
|
if mediaType.IsVideo() {
|
||||||
mediaTypeText = models.MediaTypeVideo
|
mediaTypeText = models.MediaTypeVideo
|
||||||
} else {
|
} else {
|
||||||
mediaTypeText = models.MediaTypePhoto
|
mediaTypeText = models.MediaTypePhoto
|
||||||
// search for sidecar files
|
|
||||||
if mediaType.IsRaw() {
|
|
||||||
sideCarPath = scanForSideCarFile(mediaPath)
|
|
||||||
if sideCarPath != nil {
|
|
||||||
sideCarHash = hashSideCarFile(sideCarPath)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
stat, err := os.Stat(mediaPath)
|
stat, err := os.Stat(mediaPath)
|
||||||
|
@ -137,24 +51,17 @@ func ScanMedia(tx *gorm.DB, mediaPath string, albumId int, cache *scanner_cache.
|
||||||
}
|
}
|
||||||
|
|
||||||
media := models.Media{
|
media := models.Media{
|
||||||
Title: mediaName,
|
Title: mediaName,
|
||||||
Path: mediaPath,
|
Path: mediaPath,
|
||||||
SideCarPath: sideCarPath,
|
AlbumID: albumId,
|
||||||
SideCarHash: sideCarHash,
|
Type: mediaTypeText,
|
||||||
AlbumID: albumId,
|
DateShot: stat.ModTime(),
|
||||||
Type: mediaTypeText,
|
|
||||||
DateShot: stat.ModTime(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := tx.Create(&media).Error; err != nil {
|
if err := tx.Create(&media).Error; err != nil {
|
||||||
return nil, false, errors.Wrap(err, "could not insert media into database")
|
return nil, false, errors.Wrap(err, "could not insert media into database")
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = exif.SaveEXIF(tx, &media)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("WARN: SaveEXIF for %s failed: %s\n", mediaName, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if media.Type == models.MediaTypeVideo {
|
if media.Type == models.MediaTypeVideo {
|
||||||
if err = ScanVideoMetadata(tx, &media); err != nil {
|
if err = ScanVideoMetadata(tx, &media); err != nil {
|
||||||
log.Printf("WARN: ScanVideoMetadata for %s failed: %s\n", mediaName, err)
|
log.Printf("WARN: ScanVideoMetadata for %s failed: %s\n", mediaName, err)
|
||||||
|
|
|
@ -2,6 +2,7 @@ package scanner_task
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"database/sql"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
|
@ -15,14 +16,22 @@ type ScannerTask interface {
|
||||||
// BeforeScanAlbum will run at the beginning of the scan task.
|
// BeforeScanAlbum will run at the beginning of the scan task.
|
||||||
// New values can be stored in the returned TaskContext that will live throughout the lifetime of the task.
|
// New values can be stored in the returned TaskContext that will live throughout the lifetime of the task.
|
||||||
BeforeScanAlbum(ctx TaskContext) (TaskContext, error)
|
BeforeScanAlbum(ctx TaskContext) (TaskContext, error)
|
||||||
AfterScanAlbum(ctx TaskContext, albumHadChanges bool) error
|
|
||||||
|
|
||||||
|
// AfterScanAlbum will run at the end of the scan task.
|
||||||
|
AfterScanAlbum(ctx TaskContext, changedMedia []*models.Media, albumMedia []*models.Media) error
|
||||||
|
|
||||||
|
// MediaFound will run for each media file found on the filesystem.
|
||||||
|
// It will run even when the media is already present in the database.
|
||||||
|
// If the returned skip value is true, the media will be skipped and further steps will not be executed for the given file.
|
||||||
MediaFound(ctx TaskContext, fileInfo fs.FileInfo, mediaPath string) (skip bool, err error)
|
MediaFound(ctx TaskContext, fileInfo fs.FileInfo, mediaPath string) (skip bool, err error)
|
||||||
|
|
||||||
|
// AfterMediaFound will run each media file after is has been saved to the database, but not processed yet.
|
||||||
|
// It will run even when the media is already present in the database, in that case `newMedia` will be true.
|
||||||
AfterMediaFound(ctx TaskContext, media *models.Media, newMedia bool) error
|
AfterMediaFound(ctx TaskContext, media *models.Media, newMedia bool) error
|
||||||
|
|
||||||
BeforeProcessMedia(ctx TaskContext, media *models.Media) (TaskContext, error)
|
BeforeProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData) (TaskContext, error)
|
||||||
ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (didProcess bool, err error)
|
ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error)
|
||||||
AfterProcessMedia(ctx TaskContext, media *models.Media, didProcess bool, mediaIndex int, mediaTotal int) error
|
AfterProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error
|
||||||
}
|
}
|
||||||
|
|
||||||
type TaskContext struct {
|
type TaskContext struct {
|
||||||
|
@ -30,14 +39,12 @@ type TaskContext struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewTaskContext(parent context.Context, db *gorm.DB, album *models.Album, cache *scanner_cache.AlbumScannerCache) TaskContext {
|
func NewTaskContext(parent context.Context, db *gorm.DB, album *models.Album, cache *scanner_cache.AlbumScannerCache) TaskContext {
|
||||||
ctx := parent
|
ctx := TaskContext{ctx: parent}
|
||||||
ctx = context.WithValue(ctx, taskCtxKeyAlbum, album)
|
ctx = ctx.WithValue(taskCtxKeyAlbum, album)
|
||||||
ctx = context.WithValue(ctx, taskCtxKeyAlbumCache, cache)
|
ctx = ctx.WithValue(taskCtxKeyAlbumCache, cache)
|
||||||
ctx = context.WithValue(ctx, taskCtxKeyDatabase, db.WithContext(ctx))
|
ctx = ctx.WithDB(db)
|
||||||
|
|
||||||
return TaskContext{
|
return ctx
|
||||||
ctx: ctx,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type taskCtxKeyType string
|
type taskCtxKeyType string
|
||||||
|
@ -60,6 +67,12 @@ func (c TaskContext) GetDB() *gorm.DB {
|
||||||
return c.ctx.Value(taskCtxKeyDatabase).(*gorm.DB)
|
return c.ctx.Value(taskCtxKeyDatabase).(*gorm.DB)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c TaskContext) DatabaseTransaction(transFunc func(ctx TaskContext) error, opts ...*sql.TxOptions) error {
|
||||||
|
return c.GetDB().Transaction(func(tx *gorm.DB) error {
|
||||||
|
return transFunc(c.WithDB(tx))
|
||||||
|
}, opts...)
|
||||||
|
}
|
||||||
|
|
||||||
func (c TaskContext) WithValue(key, val interface{}) TaskContext {
|
func (c TaskContext) WithValue(key, val interface{}) TaskContext {
|
||||||
return TaskContext{
|
return TaskContext{
|
||||||
ctx: context.WithValue(c.ctx, key, val),
|
ctx: context.WithValue(c.ctx, key, val),
|
||||||
|
@ -69,3 +82,15 @@ func (c TaskContext) WithValue(key, val interface{}) TaskContext {
|
||||||
func (c TaskContext) Value(key interface{}) interface{} {
|
func (c TaskContext) Value(key interface{}) interface{} {
|
||||||
return c.ctx.Value(key)
|
return c.ctx.Value(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c TaskContext) WithDB(db *gorm.DB) TaskContext {
|
||||||
|
return c.WithValue(taskCtxKeyDatabase, db.WithContext(c.ctx))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c TaskContext) Done() <-chan struct{} {
|
||||||
|
return c.ctx.Done()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c TaskContext) Err() error {
|
||||||
|
return c.ctx.Err()
|
||||||
|
}
|
||||||
|
|
|
@ -14,7 +14,7 @@ func (t ScannerTaskBase) BeforeScanAlbum(ctx TaskContext) (TaskContext, error) {
|
||||||
return ctx, nil
|
return ctx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t ScannerTaskBase) AfterScanAlbum(ctx TaskContext, albumHadChanges bool) error {
|
func (t ScannerTaskBase) AfterScanAlbum(ctx TaskContext, changedMedia []*models.Media, albumMedia []*models.Media) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -26,14 +26,14 @@ func (t ScannerTaskBase) AfterMediaFound(ctx TaskContext, media *models.Media, n
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t ScannerTaskBase) BeforeProcessMedia(ctx TaskContext, media *models.Media) (TaskContext, error) {
|
func (t ScannerTaskBase) BeforeProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData) (TaskContext, error) {
|
||||||
return ctx, nil
|
return ctx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (bool, error) {
|
func (t ScannerTaskBase) ProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) {
|
||||||
return false, nil
|
return []*models.MediaURL{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t ScannerTaskBase) AfterProcessMedia(ctx TaskContext, media *models.Media, didProcess bool, mediaIndex int, mediaTotal int) error {
|
func (t ScannerTaskBase) AfterProcessMedia(ctx TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
package scanner
|
package cleanup_tasks
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
|
@ -13,6 +13,7 @@ import (
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// CleanupMedia removes media entries from the database that are no longer present on the filesystem
|
||||||
func CleanupMedia(db *gorm.DB, albumId int, albumMedia []*models.Media) []error {
|
func CleanupMedia(db *gorm.DB, albumId int, albumMedia []*models.Media) []error {
|
||||||
albumMediaIds := make([]int, len(albumMedia))
|
albumMediaIds := make([]int, len(albumMedia))
|
||||||
for i, media := range albumMedia {
|
for i, media := range albumMedia {
|
||||||
|
@ -63,8 +64,8 @@ func CleanupMedia(db *gorm.DB, albumId int, albumMedia []*models.Media) []error
|
||||||
return deleteErrors
|
return deleteErrors
|
||||||
}
|
}
|
||||||
|
|
||||||
// Find and delete old albums in the database and cache that does not exist on the filesystem anymore.
|
// DeleteOldUserAlbums finds and deletes old albums in the database and cache that does not exist on the filesystem anymore.
|
||||||
func deleteOldUserAlbums(db *gorm.DB, scannedAlbums []*models.Album, user *models.User) []error {
|
func DeleteOldUserAlbums(db *gorm.DB, scannedAlbums []*models.Album, user *models.User) []error {
|
||||||
if len(scannedAlbums) == 0 {
|
if len(scannedAlbums) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
package scanner_test
|
package cleanup_tasks_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
|
@ -0,0 +1,21 @@
|
||||||
|
package cleanup_tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
type MediaCleanupTask struct {
|
||||||
|
scanner_task.ScannerTaskBase
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t MediaCleanupTask) AfterScanAlbum(ctx scanner_task.TaskContext, changedMedia []*models.Media, albumMedia []*models.Media) error {
|
||||||
|
|
||||||
|
cleanup_errors := CleanupMedia(ctx.GetDB(), ctx.GetAlbum().ID, albumMedia)
|
||||||
|
for _, err := range cleanup_errors {
|
||||||
|
scanner_utils.ScannerError("delete old media: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -0,0 +1,24 @@
|
||||||
|
package scanner_tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
|
"github.com/photoview/photoview/api/scanner/exif"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ExifTask struct {
|
||||||
|
scanner_task.ScannerTaskBase
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t ExifTask) AfterProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error {
|
||||||
|
|
||||||
|
_, err := exif.SaveEXIF(ctx.GetDB(), mediaData.Media)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("WARN: SaveEXIF for %s failed: %s\n", mediaData.Media.Title, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -0,0 +1,30 @@
|
||||||
|
package scanner_tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
|
"github.com/photoview/photoview/api/scanner/face_detection"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_utils"
|
||||||
|
)
|
||||||
|
|
||||||
|
type FaceDetectionTask struct {
|
||||||
|
scanner_task.ScannerTaskBase
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t FaceDetectionTask) AfterProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error {
|
||||||
|
didProcess := len(updatedURLs) > 0
|
||||||
|
|
||||||
|
if didProcess && mediaData.Media.Type == models.MediaTypePhoto {
|
||||||
|
go func(media *models.Media) {
|
||||||
|
if face_detection.GlobalFaceDetector == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := face_detection.GlobalFaceDetector.DetectFaces(ctx.GetDB(), media); err != nil {
|
||||||
|
scanner_utils.ScannerError("Error detecting faces in image (%s): %s", media.Path, err)
|
||||||
|
}
|
||||||
|
}(mediaData.Media)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -6,6 +6,7 @@ import (
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
"github.com/photoview/photoview/api/graphql/notification"
|
"github.com/photoview/photoview/api/graphql/notification"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
"github.com/photoview/photoview/api/scanner/scanner_task"
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
"github.com/photoview/photoview/api/utils"
|
"github.com/photoview/photoview/api/utils"
|
||||||
)
|
)
|
||||||
|
@ -41,14 +42,14 @@ func (t NotificationTask) AfterMediaFound(ctx scanner_task.TaskContext, media *m
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t NotificationTask) AfterProcessMedia(ctx scanner_task.TaskContext, media *models.Media, didProcess bool, mediaIndex int, mediaTotal int) error {
|
func (t NotificationTask) AfterProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error {
|
||||||
if didProcess {
|
if len(updatedURLs) > 0 {
|
||||||
progress := float64(mediaIndex) / float64(mediaTotal) * 100.0
|
progress := float64(mediaIndex) / float64(mediaTotal) * 100.0
|
||||||
notification.BroadcastNotification(&models.Notification{
|
notification.BroadcastNotification(&models.Notification{
|
||||||
Key: t.albumKey,
|
Key: t.albumKey,
|
||||||
Type: models.NotificationTypeProgress,
|
Type: models.NotificationTypeProgress,
|
||||||
Header: fmt.Sprintf("Processing media for album '%s'", ctx.GetAlbum().Title),
|
Header: fmt.Sprintf("Processing media for album '%s'", ctx.GetAlbum().Title),
|
||||||
Content: fmt.Sprintf("Processed media at %s", media.Path),
|
Content: fmt.Sprintf("Processed media at %s", mediaData.Media.Path),
|
||||||
Progress: &progress,
|
Progress: &progress,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
@ -56,8 +57,8 @@ func (t NotificationTask) AfterProcessMedia(ctx scanner_task.TaskContext, media
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t NotificationTask) AfterScanAlbum(ctx scanner_task.TaskContext, albumHadChanges bool) error {
|
func (t NotificationTask) AfterScanAlbum(ctx scanner_task.TaskContext, changedMedia []*models.Media, albumMedia []*models.Media) error {
|
||||||
if albumHadChanges {
|
if len(changedMedia) > 0 {
|
||||||
timeoutDelay := 2000
|
timeoutDelay := 2000
|
||||||
notification.BroadcastNotification(&models.Notification{
|
notification.BroadcastNotification(&models.Notification{
|
||||||
Key: t.albumKey,
|
Key: t.albumKey,
|
||||||
|
|
|
@ -0,0 +1,87 @@
|
||||||
|
package processing_tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_type"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_utils"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CounterpartFilesTask struct {
|
||||||
|
scanner_task.ScannerTaskBase
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t CounterpartFilesTask) MediaFound(ctx scanner_task.TaskContext, fileInfo fs.FileInfo, mediaPath string) (skip bool, err error) {
|
||||||
|
|
||||||
|
// Skip the JPEGs that are compressed version of raw files
|
||||||
|
counterpartFile := scanForRawCounterpartFile(mediaPath)
|
||||||
|
if counterpartFile != nil {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t CounterpartFilesTask) BeforeProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData) (scanner_task.TaskContext, error) {
|
||||||
|
|
||||||
|
mediaType, err := ctx.GetCache().GetMediaType(mediaData.Media.Path)
|
||||||
|
if err != nil {
|
||||||
|
return ctx, errors.Wrap(err, "scan for counterpart file")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !mediaType.IsRaw() {
|
||||||
|
return ctx, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
counterpartFile := scanForCompressedCounterpartFile(mediaData.Media.Path)
|
||||||
|
if counterpartFile != nil {
|
||||||
|
mediaData.CounterpartPath = counterpartFile
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanForCompressedCounterpartFile(imagePath string) *string {
|
||||||
|
ext := filepath.Ext(imagePath)
|
||||||
|
fileExtType, found := media_type.GetExtensionMediaType(ext)
|
||||||
|
|
||||||
|
if found {
|
||||||
|
if fileExtType.IsBasicTypeSupported() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pathWithoutExt := strings.TrimSuffix(imagePath, path.Ext(imagePath))
|
||||||
|
for _, ext := range media_type.TypeJpeg.FileExtensions() {
|
||||||
|
testPath := pathWithoutExt + ext
|
||||||
|
if scanner_utils.FileExists(testPath) {
|
||||||
|
return &testPath
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanForRawCounterpartFile(imagePath string) *string {
|
||||||
|
ext := filepath.Ext(imagePath)
|
||||||
|
fileExtType, found := media_type.GetExtensionMediaType(ext)
|
||||||
|
|
||||||
|
if found {
|
||||||
|
if !fileExtType.IsBasicTypeSupported() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rawPath := media_type.RawCounterpart(imagePath)
|
||||||
|
if rawPath != nil {
|
||||||
|
return rawPath
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -0,0 +1,139 @@
|
||||||
|
package processing_tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
|
||||||
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding/media_utils"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
|
||||||
|
// Image decoders
|
||||||
|
_ "image/gif"
|
||||||
|
_ "image/png"
|
||||||
|
|
||||||
|
_ "golang.org/x/image/bmp"
|
||||||
|
_ "golang.org/x/image/tiff"
|
||||||
|
_ "golang.org/x/image/webp"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ProcessPhotoTask struct {
|
||||||
|
scanner_task.ScannerTaskBase
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t ProcessPhotoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
|
||||||
|
if mediaData.Media.Type != models.MediaTypePhoto {
|
||||||
|
return []*models.MediaURL{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
updatedURLs := make([]*models.MediaURL, 0)
|
||||||
|
photo := mediaData.Media
|
||||||
|
|
||||||
|
log.Printf("Processing photo: %s\n", photo.Path)
|
||||||
|
|
||||||
|
photoURLFromDB := makePhotoURLChecker(ctx.GetDB(), photo.ID)
|
||||||
|
|
||||||
|
// original photo url
|
||||||
|
origURL, err := photoURLFromDB(models.MediaOriginal)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Thumbnail
|
||||||
|
thumbURL, err := photoURLFromDB(models.PhotoThumbnail)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "error processing photo thumbnail")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Highres
|
||||||
|
highResURL, err := photoURLFromDB(models.PhotoHighRes)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "error processing photo highres")
|
||||||
|
}
|
||||||
|
|
||||||
|
var photoDimensions *media_utils.PhotoDimensions
|
||||||
|
var baseImagePath string = photo.Path
|
||||||
|
|
||||||
|
// Generate high res jpeg
|
||||||
|
if highResURL == nil {
|
||||||
|
|
||||||
|
contentType, err := mediaData.ContentType()
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !contentType.IsWebCompatible() {
|
||||||
|
highresName := generateUniqueMediaNamePrefixed("highres", photo.Path, ".jpg")
|
||||||
|
baseImagePath = path.Join(mediaCachePath, highresName)
|
||||||
|
|
||||||
|
highRes, err := generateSaveHighResJPEG(ctx.GetDB(), photo, mediaData, highresName, baseImagePath, nil)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
updatedURLs = append(updatedURLs, highRes)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Verify that highres photo still exists in cache
|
||||||
|
baseImagePath = path.Join(mediaCachePath, highResURL.MediaName)
|
||||||
|
|
||||||
|
if _, err := os.Stat(baseImagePath); os.IsNotExist(err) {
|
||||||
|
fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName)
|
||||||
|
updatedURLs = append(updatedURLs, highResURL)
|
||||||
|
|
||||||
|
err = mediaData.EncodeHighRes(baseImagePath)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "creating high-res cached image")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save original photo to database
|
||||||
|
if origURL == nil {
|
||||||
|
|
||||||
|
// Make sure photo dimensions is set
|
||||||
|
if photoDimensions == nil {
|
||||||
|
photoDimensions, err = media_utils.GetPhotoDimensions(baseImagePath)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
original, err := saveOriginalPhotoToDB(ctx.GetDB(), photo, mediaData, photoDimensions)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "saving original photo to database")
|
||||||
|
}
|
||||||
|
|
||||||
|
updatedURLs = append(updatedURLs, original)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save thumbnail to cache
|
||||||
|
if thumbURL == nil {
|
||||||
|
thumbnailName := generateUniqueMediaNamePrefixed("thumbnail", photo.Path, ".jpg")
|
||||||
|
thumbnail, err := generateSaveThumbnailJPEG(ctx.GetDB(), photo, thumbnailName, mediaCachePath, baseImagePath, nil)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
updatedURLs = append(updatedURLs, thumbnail)
|
||||||
|
} else {
|
||||||
|
// Verify that thumbnail photo still exists in cache
|
||||||
|
thumbPath := path.Join(mediaCachePath, thumbURL.MediaName)
|
||||||
|
|
||||||
|
if _, err := os.Stat(thumbPath); os.IsNotExist(err) {
|
||||||
|
updatedURLs = append(updatedURLs, thumbURL)
|
||||||
|
fmt.Printf("Thumbnail photo found in database but not in cache, re-encoding photo to cache: %s\n", thumbURL.MediaName)
|
||||||
|
|
||||||
|
_, err := media_encoding.EncodeThumbnail(baseImagePath, thumbPath)
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "could not create thumbnail cached image")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return updatedURLs, nil
|
||||||
|
}
|
|
@ -1,4 +1,4 @@
|
||||||
package scanner
|
package processing_tasks
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
@ -13,54 +13,60 @@ import (
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding"
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
|
"github.com/photoview/photoview/api/scanner/media_encoding/executable_worker"
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding/media_utils"
|
"github.com/photoview/photoview/api/scanner/media_encoding/media_utils"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
"github.com/photoview/photoview/api/utils"
|
"github.com/photoview/photoview/api/utils"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
"gopkg.in/vansante/go-ffprobe.v2"
|
"gopkg.in/vansante/go-ffprobe.v2"
|
||||||
"gorm.io/gorm"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func processVideo(tx *gorm.DB, mediaData *media_encoding.EncodeMediaData, videoCachePath *string) (bool, error) {
|
type ProcessVideoTask struct {
|
||||||
|
scanner_task.ScannerTaskBase
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t ProcessVideoTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
|
||||||
|
if mediaData.Media.Type != models.MediaTypeVideo {
|
||||||
|
return []*models.MediaURL{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
updatedURLs := make([]*models.MediaURL, 0)
|
||||||
video := mediaData.Media
|
video := mediaData.Media
|
||||||
didProcess := false
|
|
||||||
|
|
||||||
log.Printf("Processing video: %s", video.Path)
|
log.Printf("Processing video: %s", video.Path)
|
||||||
|
|
||||||
mediaURLFromDB := makePhotoURLChecker(tx, video.ID)
|
mediaURLFromDB := makePhotoURLChecker(ctx.GetDB(), video.ID)
|
||||||
|
|
||||||
videoOriginalURL, err := mediaURLFromDB(models.MediaOriginal)
|
videoOriginalURL, err := mediaURLFromDB(models.MediaOriginal)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "error processing video original format")
|
return []*models.MediaURL{}, errors.Wrap(err, "error processing video original format")
|
||||||
}
|
}
|
||||||
|
|
||||||
videoWebURL, err := mediaURLFromDB(models.VideoWeb)
|
videoWebURL, err := mediaURLFromDB(models.VideoWeb)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "error processing video web-format")
|
return []*models.MediaURL{}, errors.Wrap(err, "error processing video web-format")
|
||||||
}
|
}
|
||||||
|
|
||||||
videoThumbnailURL, err := mediaURLFromDB(models.VideoThumbnail)
|
videoThumbnailURL, err := mediaURLFromDB(models.VideoThumbnail)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "error processing video thumbnail")
|
return []*models.MediaURL{}, errors.Wrap(err, "error processing video thumbnail")
|
||||||
}
|
}
|
||||||
|
|
||||||
videoType, err := mediaData.ContentType()
|
videoType, err := mediaData.ContentType()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "error getting video content type")
|
return []*models.MediaURL{}, errors.Wrap(err, "error getting video content type")
|
||||||
}
|
}
|
||||||
|
|
||||||
if videoOriginalURL == nil && videoType.IsWebCompatible() {
|
if videoOriginalURL == nil && videoType.IsWebCompatible() {
|
||||||
didProcess = true
|
|
||||||
|
|
||||||
origVideoPath := video.Path
|
origVideoPath := video.Path
|
||||||
videoMediaName := generateUniqueMediaName(video.Path)
|
videoMediaName := generateUniqueMediaName(video.Path)
|
||||||
|
|
||||||
webMetadata, err := readVideoStreamMetadata(origVideoPath)
|
webMetadata, err := readVideoStreamMetadata(origVideoPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrapf(err, "failed to read metadata for original video (%s)", video.Title)
|
return []*models.MediaURL{}, errors.Wrapf(err, "failed to read metadata for original video (%s)", video.Title)
|
||||||
}
|
}
|
||||||
|
|
||||||
fileStats, err := os.Stat(origVideoPath)
|
fileStats, err := os.Stat(origVideoPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "reading file stats of original video")
|
return []*models.MediaURL{}, errors.Wrap(err, "reading file stats of original video")
|
||||||
}
|
}
|
||||||
|
|
||||||
mediaURL := models.MediaURL{
|
mediaURL := models.MediaURL{
|
||||||
|
@ -73,35 +79,34 @@ func processVideo(tx *gorm.DB, mediaData *media_encoding.EncodeMediaData, videoC
|
||||||
FileSize: fileStats.Size(),
|
FileSize: fileStats.Size(),
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := tx.Create(&mediaURL).Error; err != nil {
|
if err := ctx.GetDB().Create(&mediaURL).Error; err != nil {
|
||||||
return false, errors.Wrapf(err, "failed to insert original video into database (%s)", video.Title)
|
return []*models.MediaURL{}, errors.Wrapf(err, "insert original video into database (%s)", video.Title)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
updatedURLs = append(updatedURLs, &mediaURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
if videoWebURL == nil && !videoType.IsWebCompatible() {
|
if videoWebURL == nil && !videoType.IsWebCompatible() {
|
||||||
didProcess = true
|
|
||||||
|
|
||||||
web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
|
web_video_name := fmt.Sprintf("web_video_%s_%s", path.Base(video.Path), utils.GenerateToken())
|
||||||
web_video_name = strings.ReplaceAll(web_video_name, ".", "_")
|
web_video_name = strings.ReplaceAll(web_video_name, ".", "_")
|
||||||
web_video_name = strings.ReplaceAll(web_video_name, " ", "_")
|
web_video_name = strings.ReplaceAll(web_video_name, " ", "_")
|
||||||
web_video_name = web_video_name + ".mp4"
|
web_video_name = web_video_name + ".mp4"
|
||||||
|
|
||||||
webVideoPath := path.Join(*videoCachePath, web_video_name)
|
webVideoPath := path.Join(mediaCachePath, web_video_name)
|
||||||
|
|
||||||
err = executable_worker.FfmpegCli.EncodeMp4(video.Path, webVideoPath)
|
err = executable_worker.FfmpegCli.EncodeMp4(video.Path, webVideoPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrapf(err, "could not encode mp4 video (%s)", video.Path)
|
return []*models.MediaURL{}, errors.Wrapf(err, "could not encode mp4 video (%s)", video.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
webMetadata, err := readVideoStreamMetadata(webVideoPath)
|
webMetadata, err := readVideoStreamMetadata(webVideoPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrapf(err, "failed to read metadata for encoded web-video (%s)", video.Title)
|
return []*models.MediaURL{}, errors.Wrapf(err, "failed to read metadata for encoded web-video (%s)", video.Title)
|
||||||
}
|
}
|
||||||
|
|
||||||
fileStats, err := os.Stat(webVideoPath)
|
fileStats, err := os.Stat(webVideoPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "reading file stats of web-optimized video")
|
return []*models.MediaURL{}, errors.Wrap(err, "reading file stats of web-optimized video")
|
||||||
}
|
}
|
||||||
|
|
||||||
mediaURL := models.MediaURL{
|
mediaURL := models.MediaURL{
|
||||||
|
@ -114,39 +119,39 @@ func processVideo(tx *gorm.DB, mediaData *media_encoding.EncodeMediaData, videoC
|
||||||
FileSize: fileStats.Size(),
|
FileSize: fileStats.Size(),
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := tx.Create(&mediaURL).Error; err != nil {
|
if err := ctx.GetDB().Create(&mediaURL).Error; err != nil {
|
||||||
return false, errors.Wrapf(err, "failed to insert encoded web-video into database (%s)", video.Title)
|
return []*models.MediaURL{}, errors.Wrapf(err, "failed to insert encoded web-video into database (%s)", video.Title)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
updatedURLs = append(updatedURLs, &mediaURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
probeData, err := mediaData.VideoMetadata()
|
probeData, err := mediaData.VideoMetadata()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return []*models.MediaURL{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if videoThumbnailURL == nil {
|
if videoThumbnailURL == nil {
|
||||||
didProcess = true
|
|
||||||
|
|
||||||
video_thumb_name := fmt.Sprintf("video_thumb_%s_%s", path.Base(video.Path), utils.GenerateToken())
|
video_thumb_name := fmt.Sprintf("video_thumb_%s_%s", path.Base(video.Path), utils.GenerateToken())
|
||||||
video_thumb_name = strings.ReplaceAll(video_thumb_name, ".", "_")
|
video_thumb_name = strings.ReplaceAll(video_thumb_name, ".", "_")
|
||||||
video_thumb_name = strings.ReplaceAll(video_thumb_name, " ", "_")
|
video_thumb_name = strings.ReplaceAll(video_thumb_name, " ", "_")
|
||||||
video_thumb_name = video_thumb_name + ".jpg"
|
video_thumb_name = video_thumb_name + ".jpg"
|
||||||
|
|
||||||
thumbImagePath := path.Join(*videoCachePath, video_thumb_name)
|
thumbImagePath := path.Join(mediaCachePath, video_thumb_name)
|
||||||
|
|
||||||
err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
|
err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title)
|
return []*models.MediaURL{}, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title)
|
||||||
}
|
}
|
||||||
|
|
||||||
thumbDimensions, err := media_utils.GetPhotoDimensions(thumbImagePath)
|
thumbDimensions, err := media_utils.GetPhotoDimensions(thumbImagePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "get dimensions of video thumbnail image")
|
return []*models.MediaURL{}, errors.Wrap(err, "get dimensions of video thumbnail image")
|
||||||
}
|
}
|
||||||
|
|
||||||
fileStats, err := os.Stat(thumbImagePath)
|
fileStats, err := os.Stat(thumbImagePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "reading file stats of video thumbnail")
|
return []*models.MediaURL{}, errors.Wrap(err, "reading file stats of video thumbnail")
|
||||||
}
|
}
|
||||||
|
|
||||||
thumbMediaURL := models.MediaURL{
|
thumbMediaURL := models.MediaURL{
|
||||||
|
@ -159,43 +164,45 @@ func processVideo(tx *gorm.DB, mediaData *media_encoding.EncodeMediaData, videoC
|
||||||
FileSize: fileStats.Size(),
|
FileSize: fileStats.Size(),
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := tx.Create(&thumbMediaURL).Error; err != nil {
|
if err := ctx.GetDB().Create(&thumbMediaURL).Error; err != nil {
|
||||||
return false, errors.Wrapf(err, "failed to insert video thumbnail image into database (%s)", video.Title)
|
return []*models.MediaURL{}, errors.Wrapf(err, "failed to insert video thumbnail image into database (%s)", video.Title)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
updatedURLs = append(updatedURLs, &thumbMediaURL)
|
||||||
} else {
|
} else {
|
||||||
// Verify that video thumbnail still exists in cache
|
// Verify that video thumbnail still exists in cache
|
||||||
thumbImagePath := path.Join(*videoCachePath, videoThumbnailURL.MediaName)
|
thumbImagePath := path.Join(mediaCachePath, videoThumbnailURL.MediaName)
|
||||||
|
|
||||||
if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) {
|
if _, err := os.Stat(thumbImagePath); os.IsNotExist(err) {
|
||||||
fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
|
fmt.Printf("Video thumbnail found in database but not in cache, re-encoding photo to cache: %s\n", videoThumbnailURL.MediaName)
|
||||||
didProcess = true
|
updatedURLs = append(updatedURLs, videoThumbnailURL)
|
||||||
|
|
||||||
err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
|
err = executable_worker.FfmpegCli.EncodeVideoThumbnail(video.Path, thumbImagePath, probeData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title)
|
return []*models.MediaURL{}, errors.Wrapf(err, "failed to generate thumbnail for video (%s)", video.Title)
|
||||||
}
|
}
|
||||||
|
|
||||||
thumbDimensions, err := media_utils.GetPhotoDimensions(thumbImagePath)
|
thumbDimensions, err := media_utils.GetPhotoDimensions(thumbImagePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "get dimensions of video thumbnail image")
|
return []*models.MediaURL{}, errors.Wrap(err, "get dimensions of video thumbnail image")
|
||||||
}
|
}
|
||||||
|
|
||||||
fileStats, err := os.Stat(thumbImagePath)
|
fileStats, err := os.Stat(thumbImagePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.Wrap(err, "reading file stats of video thumbnail")
|
return []*models.MediaURL{}, errors.Wrap(err, "reading file stats of video thumbnail")
|
||||||
}
|
}
|
||||||
|
|
||||||
videoThumbnailURL.Width = thumbDimensions.Width
|
videoThumbnailURL.Width = thumbDimensions.Width
|
||||||
videoThumbnailURL.Height = thumbDimensions.Height
|
videoThumbnailURL.Height = thumbDimensions.Height
|
||||||
videoThumbnailURL.FileSize = fileStats.Size()
|
videoThumbnailURL.FileSize = fileStats.Size()
|
||||||
|
|
||||||
if err := tx.Save(videoThumbnailURL).Error; err != nil {
|
if err := ctx.GetDB().Save(videoThumbnailURL).Error; err != nil {
|
||||||
return false, errors.Wrap(err, "updating video thumbnail url in database after re-encoding")
|
return []*models.MediaURL{}, errors.Wrap(err, "updating video thumbnail url in database after re-encoding")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return didProcess, nil
|
return updatedURLs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func readVideoMetadata(videoPath string) (*ffprobe.ProbeData, error) {
|
func readVideoMetadata(videoPath string) (*ffprobe.ProbeData, error) {
|
|
@ -0,0 +1,98 @@
|
||||||
|
package processing_tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
|
||||||
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding/media_utils"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *media_encoding.EncodeMediaData, highres_name string, imagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) {
|
||||||
|
|
||||||
|
err := imageData.EncodeHighRes(imagePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "creating high-res cached image")
|
||||||
|
}
|
||||||
|
|
||||||
|
photoDimensions, err := media_utils.GetPhotoDimensions(imagePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
fileStats, err := os.Stat(imagePath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "reading file stats of highres photo")
|
||||||
|
}
|
||||||
|
|
||||||
|
if mediaURL == nil {
|
||||||
|
|
||||||
|
mediaURL = &models.MediaURL{
|
||||||
|
MediaID: media.ID,
|
||||||
|
MediaName: highres_name,
|
||||||
|
Width: photoDimensions.Width,
|
||||||
|
Height: photoDimensions.Height,
|
||||||
|
Purpose: models.PhotoHighRes,
|
||||||
|
ContentType: "image/jpeg",
|
||||||
|
FileSize: fileStats.Size(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Create(&mediaURL).Error; err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "could not insert highres media url (%d, %s)", media.ID, highres_name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
mediaURL.Width = photoDimensions.Width
|
||||||
|
mediaURL.Height = photoDimensions.Height
|
||||||
|
mediaURL.FileSize = fileStats.Size()
|
||||||
|
|
||||||
|
if err := tx.Save(&mediaURL).Error; err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, highres_name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mediaURL, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateSaveThumbnailJPEG(tx *gorm.DB, media *models.Media, thumbnail_name string, photoCachePath string, baseImagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) {
|
||||||
|
thumbOutputPath := path.Join(photoCachePath, thumbnail_name)
|
||||||
|
|
||||||
|
thumbSize, err := media_encoding.EncodeThumbnail(baseImagePath, thumbOutputPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "could not create thumbnail cached image")
|
||||||
|
}
|
||||||
|
|
||||||
|
fileStats, err := os.Stat(thumbOutputPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "reading file stats of thumbnail photo")
|
||||||
|
}
|
||||||
|
|
||||||
|
if mediaURL == nil {
|
||||||
|
|
||||||
|
mediaURL = &models.MediaURL{
|
||||||
|
MediaID: media.ID,
|
||||||
|
MediaName: thumbnail_name,
|
||||||
|
Width: thumbSize.Width,
|
||||||
|
Height: thumbSize.Height,
|
||||||
|
Purpose: models.PhotoThumbnail,
|
||||||
|
ContentType: "image/jpeg",
|
||||||
|
FileSize: fileStats.Size(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Create(&mediaURL).Error; err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "could not insert thumbnail media url (%d, %s)", media.ID, thumbnail_name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
mediaURL.Width = thumbSize.Width
|
||||||
|
mediaURL.Height = thumbSize.Height
|
||||||
|
mediaURL.FileSize = fileStats.Size()
|
||||||
|
|
||||||
|
if err := tx.Save(&mediaURL).Error; err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "could not update media url after side car changes (%d, %s)", media.ID, thumbnail_name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return mediaURL, nil
|
||||||
|
}
|
|
@ -0,0 +1,82 @@
|
||||||
|
package processing_tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
|
||||||
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding/media_utils"
|
||||||
|
"github.com/photoview/photoview/api/utils"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Higher order function used to check if MediaURL for a given MediaPurpose exists
|
||||||
|
func makePhotoURLChecker(tx *gorm.DB, mediaID int) func(purpose models.MediaPurpose) (*models.MediaURL, error) {
|
||||||
|
return func(purpose models.MediaPurpose) (*models.MediaURL, error) {
|
||||||
|
var mediaURL []*models.MediaURL
|
||||||
|
|
||||||
|
result := tx.Where("purpose = ?", purpose).Where("media_id = ?", mediaID).Find(&mediaURL)
|
||||||
|
|
||||||
|
if result.Error != nil {
|
||||||
|
return nil, result.Error
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.RowsAffected > 0 {
|
||||||
|
return mediaURL[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateUniqueMediaNamePrefixed(prefix string, mediaPath string, extension string) string {
|
||||||
|
mediaName := fmt.Sprintf("%s_%s_%s", prefix, path.Base(mediaPath), utils.GenerateToken())
|
||||||
|
mediaName = models.SanitizeMediaName(mediaName)
|
||||||
|
mediaName = mediaName + extension
|
||||||
|
return mediaName
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateUniqueMediaName(mediaPath string) string {
|
||||||
|
|
||||||
|
filename := path.Base(mediaPath)
|
||||||
|
baseName := filename[0 : len(filename)-len(path.Ext(filename))]
|
||||||
|
baseExt := path.Ext(filename)
|
||||||
|
|
||||||
|
mediaName := fmt.Sprintf("%s_%s", baseName, utils.GenerateToken())
|
||||||
|
mediaName = models.SanitizeMediaName(mediaName) + baseExt
|
||||||
|
|
||||||
|
return mediaName
|
||||||
|
}
|
||||||
|
|
||||||
|
func saveOriginalPhotoToDB(tx *gorm.DB, photo *models.Media, imageData *media_encoding.EncodeMediaData, photoDimensions *media_utils.PhotoDimensions) (*models.MediaURL, error) {
|
||||||
|
originalImageName := generateUniqueMediaName(photo.Path)
|
||||||
|
|
||||||
|
contentType, err := imageData.ContentType()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
fileStats, err := os.Stat(photo.Path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "reading file stats of original photo")
|
||||||
|
}
|
||||||
|
|
||||||
|
mediaURL := models.MediaURL{
|
||||||
|
Media: photo,
|
||||||
|
MediaName: originalImageName,
|
||||||
|
Width: photoDimensions.Width,
|
||||||
|
Height: photoDimensions.Height,
|
||||||
|
Purpose: models.MediaOriginal,
|
||||||
|
ContentType: string(*contentType),
|
||||||
|
FileSize: fileStats.Size(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Create(&mediaURL).Error; err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "inserting original photo url: %d, %s", photo.ID, photo.Title)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &mediaURL, nil
|
||||||
|
}
|
|
@ -0,0 +1,159 @@
|
||||||
|
package processing_tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
|
||||||
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_utils"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
type SidecarTask struct {
|
||||||
|
scanner_task.ScannerTaskBase
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t SidecarTask) AfterMediaFound(ctx scanner_task.TaskContext, media *models.Media, newMedia bool) error {
|
||||||
|
if media.Type != models.MediaTypePhoto || !newMedia {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
mediaType, err := ctx.GetCache().GetMediaType(media.Path)
|
||||||
|
if err != nil {
|
||||||
|
return errors.Wrap(err, "scan for sidecar file")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !mediaType.IsRaw() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var sideCarPath *string = nil
|
||||||
|
var sideCarHash *string = nil
|
||||||
|
|
||||||
|
sideCarPath = scanForSideCarFile(media.Path)
|
||||||
|
if sideCarPath != nil {
|
||||||
|
sideCarHash = hashSideCarFile(sideCarPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add sidecar data to media
|
||||||
|
media.SideCarPath = sideCarPath
|
||||||
|
media.SideCarHash = sideCarHash
|
||||||
|
if err := ctx.GetDB().Save(media).Error; err != nil {
|
||||||
|
return errors.Wrapf(err, "update media sidecar info (%s)", *sideCarPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t SidecarTask) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (updatedURLs []*models.MediaURL, err error) {
|
||||||
|
mediaType, err := mediaData.ContentType()
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, process media")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !mediaType.IsRaw() {
|
||||||
|
return []*models.MediaURL{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
photo := mediaData.Media
|
||||||
|
|
||||||
|
sideCarFileHasChanged := false
|
||||||
|
var currentFileHash *string
|
||||||
|
currentSideCarPath := scanForSideCarFile(photo.Path)
|
||||||
|
|
||||||
|
if currentSideCarPath != nil {
|
||||||
|
currentFileHash = hashSideCarFile(currentSideCarPath)
|
||||||
|
if photo.SideCarHash == nil || *photo.SideCarHash != *currentFileHash {
|
||||||
|
sideCarFileHasChanged = true
|
||||||
|
}
|
||||||
|
} else if photo.SideCarPath != nil { // sidecar has been deleted since last scan
|
||||||
|
sideCarFileHasChanged = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if !sideCarFileHasChanged {
|
||||||
|
return []*models.MediaURL{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Detected changed sidecar file for %s recreating JPG's to reflect changes\n", photo.Path)
|
||||||
|
|
||||||
|
highResURL, err := photo.GetHighRes()
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, get high-res media_url")
|
||||||
|
}
|
||||||
|
|
||||||
|
thumbURL, err := photo.GetThumbnail()
|
||||||
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, get high-res media_url")
|
||||||
|
}
|
||||||
|
|
||||||
|
// update high res image may be cropped so dimentions and file size can change
|
||||||
|
baseImagePath := path.Join(mediaCachePath, highResURL.MediaName) // update base image path for thumbnail
|
||||||
|
tempHighResPath := baseImagePath + ".hold"
|
||||||
|
os.Rename(baseImagePath, tempHighResPath)
|
||||||
|
updatedHighRes, err := generateSaveHighResJPEG(ctx.GetDB(), photo, mediaData, highResURL.MediaName, baseImagePath, highResURL)
|
||||||
|
if err != nil {
|
||||||
|
os.Rename(tempHighResPath, baseImagePath)
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "sidecar task, recreating high-res cached image")
|
||||||
|
}
|
||||||
|
os.Remove(tempHighResPath)
|
||||||
|
|
||||||
|
// update thumbnail image may be cropped so dimentions and file size can change
|
||||||
|
thumbPath := path.Join(mediaCachePath, thumbURL.MediaName)
|
||||||
|
tempThumbPath := thumbPath + ".hold" // hold onto the original image incase for some reason we fail to recreate one with the new settings
|
||||||
|
os.Rename(thumbPath, tempThumbPath)
|
||||||
|
updatedThumbnail, err := generateSaveThumbnailJPEG(ctx.GetDB(), photo, thumbURL.MediaName, mediaCachePath, baseImagePath, thumbURL)
|
||||||
|
if err != nil {
|
||||||
|
os.Rename(tempThumbPath, thumbPath)
|
||||||
|
return []*models.MediaURL{}, errors.Wrap(err, "recreating thumbnail cached image")
|
||||||
|
}
|
||||||
|
os.Remove(tempThumbPath)
|
||||||
|
|
||||||
|
photo.SideCarHash = currentFileHash
|
||||||
|
photo.SideCarPath = currentSideCarPath
|
||||||
|
|
||||||
|
// save new side car hash
|
||||||
|
if err := ctx.GetDB().Save(&photo).Error; err != nil {
|
||||||
|
return []*models.MediaURL{}, errors.Wrapf(err, "could not update side car hash for media: %s", photo.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
return []*models.MediaURL{
|
||||||
|
updatedThumbnail,
|
||||||
|
updatedHighRes,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func scanForSideCarFile(path string) *string {
|
||||||
|
testPath := path + ".xmp"
|
||||||
|
|
||||||
|
if scanner_utils.FileExists(testPath) {
|
||||||
|
return &testPath
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hashSideCarFile(path *string) *string {
|
||||||
|
if path == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
f, err := os.Open(*path)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("ERROR: %s", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
h := md5.New()
|
||||||
|
if _, err := io.Copy(h, f); err != nil {
|
||||||
|
log.Printf("ERROR: %s", err)
|
||||||
|
}
|
||||||
|
hash := hex.EncodeToString(h.Sum(nil))
|
||||||
|
return &hash
|
||||||
|
}
|
|
@ -6,11 +6,20 @@ import (
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
"github.com/photoview/photoview/api/scanner/media_encoding"
|
"github.com/photoview/photoview/api/scanner/media_encoding"
|
||||||
"github.com/photoview/photoview/api/scanner/scanner_task"
|
"github.com/photoview/photoview/api/scanner/scanner_task"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_tasks/cleanup_tasks"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_tasks/processing_tasks"
|
||||||
)
|
)
|
||||||
|
|
||||||
var allTasks []scanner_task.ScannerTask = []scanner_task.ScannerTask{
|
var allTasks []scanner_task.ScannerTask = []scanner_task.ScannerTask{
|
||||||
NotificationTask{},
|
NotificationTask{},
|
||||||
IgnorefileTask{},
|
IgnorefileTask{},
|
||||||
|
processing_tasks.CounterpartFilesTask{},
|
||||||
|
processing_tasks.SidecarTask{},
|
||||||
|
processing_tasks.ProcessPhotoTask{},
|
||||||
|
processing_tasks.ProcessVideoTask{},
|
||||||
|
FaceDetectionTask{},
|
||||||
|
ExifTask{},
|
||||||
|
cleanup_tasks.MediaCleanupTask{},
|
||||||
}
|
}
|
||||||
|
|
||||||
type scannerTasks struct {
|
type scannerTasks struct {
|
||||||
|
@ -34,6 +43,23 @@ func getSubContextsProcessing(ctx scanner_task.TaskContext) []scanner_task.TaskC
|
||||||
return ctx.Value(tasksSubContextsGlobal).([]scanner_task.TaskContext)
|
return ctx.Value(tasksSubContextsGlobal).([]scanner_task.TaskContext)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func simpleCombinedTasks(subContexts []scanner_task.TaskContext, doTask func(ctx scanner_task.TaskContext, task scanner_task.ScannerTask) error) error {
|
||||||
|
for i, task := range allTasks {
|
||||||
|
subCtx := subContexts[i]
|
||||||
|
select {
|
||||||
|
case <-subCtx.Done():
|
||||||
|
continue
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
err := doTask(subCtx, task)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (t scannerTasks) BeforeScanAlbum(ctx scanner_task.TaskContext) (scanner_task.TaskContext, error) {
|
func (t scannerTasks) BeforeScanAlbum(ctx scanner_task.TaskContext) (scanner_task.TaskContext, error) {
|
||||||
subContexts := make([]scanner_task.TaskContext, len(allTasks))
|
subContexts := make([]scanner_task.TaskContext, len(allTasks))
|
||||||
|
|
||||||
|
@ -43,6 +69,12 @@ func (t scannerTasks) BeforeScanAlbum(ctx scanner_task.TaskContext) (scanner_tas
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ctx, err
|
return ctx, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx, ctx.Err()
|
||||||
|
default:
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return ctx.WithValue(tasksSubContextsGlobal, subContexts), nil
|
return ctx.WithValue(tasksSubContextsGlobal, subContexts), nil
|
||||||
|
@ -53,7 +85,14 @@ func (t scannerTasks) MediaFound(ctx scanner_task.TaskContext, fileInfo fs.FileI
|
||||||
subContexts := getSubContextsGlobal(ctx)
|
subContexts := getSubContextsGlobal(ctx)
|
||||||
|
|
||||||
for i, task := range allTasks {
|
for i, task := range allTasks {
|
||||||
skip, err := task.MediaFound(subContexts[i], fileInfo, mediaPath)
|
subCtx := subContexts[i]
|
||||||
|
select {
|
||||||
|
case <-subCtx.Done():
|
||||||
|
continue
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
skip, err := task.MediaFound(subCtx, fileInfo, mediaPath)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
|
@ -67,65 +106,63 @@ func (t scannerTasks) MediaFound(ctx scanner_task.TaskContext, fileInfo fs.FileI
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t scannerTasks) AfterScanAlbum(ctx scanner_task.TaskContext, albumHadChanges bool) error {
|
func (t scannerTasks) AfterScanAlbum(ctx scanner_task.TaskContext, changedMedia []*models.Media, albumMedia []*models.Media) error {
|
||||||
subContexts := getSubContextsGlobal(ctx)
|
return simpleCombinedTasks(getSubContextsGlobal(ctx), func(ctx scanner_task.TaskContext, task scanner_task.ScannerTask) error {
|
||||||
for i, task := range allTasks {
|
return task.AfterScanAlbum(ctx, changedMedia, albumMedia)
|
||||||
err := task.AfterScanAlbum(subContexts[i], albumHadChanges)
|
})
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t scannerTasks) AfterMediaFound(ctx scanner_task.TaskContext, media *models.Media, newMedia bool) error {
|
func (t scannerTasks) AfterMediaFound(ctx scanner_task.TaskContext, media *models.Media, newMedia bool) error {
|
||||||
subContexts := getSubContextsGlobal(ctx)
|
return simpleCombinedTasks(getSubContextsGlobal(ctx), func(ctx scanner_task.TaskContext, task scanner_task.ScannerTask) error {
|
||||||
for i, task := range allTasks {
|
return task.AfterMediaFound(ctx, media, newMedia)
|
||||||
err := task.AfterMediaFound(subContexts[i], media, newMedia)
|
})
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t scannerTasks) BeforeProcessMedia(ctx scanner_task.TaskContext, media *models.Media) (scanner_task.TaskContext, error) {
|
func (t scannerTasks) BeforeProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData) (scanner_task.TaskContext, error) {
|
||||||
subContexts := make([]scanner_task.TaskContext, len(allTasks))
|
globalSubContexts := getSubContextsGlobal(ctx)
|
||||||
|
processSubContexts := make([]scanner_task.TaskContext, len(allTasks))
|
||||||
|
|
||||||
for i, task := range allTasks {
|
for i, task := range allTasks {
|
||||||
|
select {
|
||||||
|
case <-globalSubContexts[i].Done():
|
||||||
|
continue
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
subContexts[i], err = task.BeforeProcessMedia(ctx, media)
|
processSubContexts[i], err = task.BeforeProcessMedia(ctx, mediaData)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ctx, err
|
return ctx, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return ctx.WithValue(tasksSubContextsProcessing, subContexts), nil
|
return ctx.WithValue(tasksSubContextsProcessing, processSubContexts), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) (bool, error) {
|
func (t scannerTasks) ProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, mediaCachePath string) ([]*models.MediaURL, error) {
|
||||||
subContexts := getSubContextsProcessing(ctx)
|
subContexts := getSubContextsProcessing(ctx)
|
||||||
didProcess := false
|
allNewMedia := make([]*models.MediaURL, 0)
|
||||||
|
|
||||||
for i, task := range allTasks {
|
for i, task := range allTasks {
|
||||||
singleDidProcess, err := task.ProcessMedia(subContexts[i], mediaData, mediaCachePath)
|
select {
|
||||||
if err != nil {
|
case <-subContexts[i].Done():
|
||||||
return false, err
|
continue
|
||||||
|
default:
|
||||||
}
|
}
|
||||||
|
|
||||||
if singleDidProcess {
|
newMedia, err := task.ProcessMedia(subContexts[i], mediaData, mediaCachePath)
|
||||||
didProcess = true
|
if err != nil {
|
||||||
|
return []*models.MediaURL{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
allNewMedia = append(allNewMedia, newMedia...)
|
||||||
}
|
}
|
||||||
return didProcess, nil
|
|
||||||
|
return allNewMedia, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t scannerTasks) AfterProcessMedia(ctx scanner_task.TaskContext, media *models.Media, didProcess bool, mediaIndex int, mediaTotal int) error {
|
func (t scannerTasks) AfterProcessMedia(ctx scanner_task.TaskContext, mediaData *media_encoding.EncodeMediaData, updatedURLs []*models.MediaURL, mediaIndex int, mediaTotal int) error {
|
||||||
subContexts := getSubContextsProcessing(ctx)
|
return simpleCombinedTasks(getSubContextsProcessing(ctx), func(ctx scanner_task.TaskContext, task scanner_task.ScannerTask) error {
|
||||||
for i, task := range allTasks {
|
return task.AfterProcessMedia(ctx, mediaData, updatedURLs, mediaIndex, mediaTotal)
|
||||||
err := task.AfterProcessMedia(subContexts[i], media, didProcess, mediaIndex, mediaTotal)
|
})
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,6 +10,7 @@ import (
|
||||||
|
|
||||||
"github.com/photoview/photoview/api/graphql/models"
|
"github.com/photoview/photoview/api/graphql/models"
|
||||||
"github.com/photoview/photoview/api/scanner/scanner_cache"
|
"github.com/photoview/photoview/api/scanner/scanner_cache"
|
||||||
|
"github.com/photoview/photoview/api/scanner/scanner_tasks/cleanup_tasks"
|
||||||
"github.com/photoview/photoview/api/scanner/scanner_utils"
|
"github.com/photoview/photoview/api/scanner/scanner_utils"
|
||||||
"github.com/photoview/photoview/api/utils"
|
"github.com/photoview/photoview/api/utils"
|
||||||
"github.com/pkg/errors"
|
"github.com/pkg/errors"
|
||||||
|
@ -215,7 +216,7 @@ func FindAlbumsForUser(db *gorm.DB, user *models.User, album_cache *scanner_cach
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
deleteErrors := deleteOldUserAlbums(db, userAlbums, user)
|
deleteErrors := cleanup_tasks.DeleteOldUserAlbums(db, userAlbums, user)
|
||||||
scanErrors = append(scanErrors, deleteErrors...)
|
scanErrors = append(scanErrors, deleteErrors...)
|
||||||
|
|
||||||
return userAlbums, scanErrors
|
return userAlbums, scanErrors
|
||||||
|
|
Loading…
Reference in New Issue