Photo duplication detection (#148)

* Fixes viktorstrate/photoview#8
- Added new property CounterpartPath to Media struct to hold the path to the counterpart JPEG file (if any)
- Added new MediaType method isBasicSupportedisBasicTypeSupported()
- Added new function isFileExists() to minimize the code duplication

* Fixes viktorstrate/photoview#8
- Chaned CounterpartPath definition from string to *string
- Added new helper method FileExtensions()
- Simplified the logic inside scanForRawCounterpartFile() and scanForCompressedCounterpartFile() functions, reducing the code duplication

* Fixes viktorstrate/photoview#8
- Added debug to fileExists() function

* Cleanup fileExists logging

Co-authored-by: viktorstrate <viktorstrate@gmail.com>

api/graphql/models/media.go

@@ -23,6 +23,7 @@ type Media struct {
 	VideoMetadataId *int
 	SideCarPath     *string
 	SideCarHash     *string
+	CounterpartPath *string
 }
 
 func (p *Media) ID() int {

api/scanner/encode_photo.go

@@ -126,7 +126,8 @@ func (img *EncodeMediaData) EncodeHighRes(tx *sql.Tx, outputPath string) error {
 		return errors.New("could not convert photo as file format is not supported")
 	}
 
-	if contentType.isRaw() {
+	// Use darktable if there is no counterpart JPEG file to use instead
+	if contentType.isRaw() && img.media.CounterpartPath == nil {
 		if DarktableCli.IsInstalled() {
 			err := DarktableCli.EncodeJpeg(img.media.Path, outputPath, 70)
 			if err != nil {
@@ -170,21 +171,16 @@ func (img *EncodeMediaData) photoImage(tx *sql.Tx) (image.Image, error) {
 		return img._photoImage, nil
 	}
 
-	photoImg, err := DecodeImage(img.media.Path)
-	if err != nil {
-		return nil, utils.HandleError("image decoding", err)
+	var photoPath string
+	if img.media.CounterpartPath != nil {
+		photoPath = *img.media.CounterpartPath
+	} else {
+		photoPath = img.media.Path
 	}
 
-	// Get orientation from exif data
-	row := tx.QueryRow("SELECT media_exif.orientation FROM media JOIN media_exif WHERE media.exif_id = media_exif.exif_id AND media.media_id = ?", img.media.MediaID)
-	var orientation *int
-	if err = row.Scan(&orientation); err != nil {
-		// If not found use default orientation (not rotate)
-		if err == sql.ErrNoRows {
-			orientation = nil
-		} else {
-			return nil, err
-		}
+	photoImg, err := DecodeImage(photoPath)
+	if err != nil {
+		return nil, utils.HandleError("image decoding", err)
 	}
 
 	img._photoImage = photoImg

api/scanner/media_type.go

@@ -209,14 +209,22 @@ func (imgType *MediaType) isVideo() bool {
 	return false
 }
 
-// isSupported determines if the given type can be processed
-func (imgType *MediaType) isSupported() bool {
-	for _, supported_mime := range SupportedMimetypes {
-		if supported_mime == *imgType {
+func (imgType *MediaType) isBasicTypeSupported() bool {
+	for _, img_mime := range SupportedMimetypes {
+		if img_mime == *imgType {
 			return true
 		}
 	}
 
+	return false
+}
+
+// isSupported determines if the given type can be processed
+func (imgType *MediaType) isSupported() bool {
+	if imgType.isBasicTypeSupported() {
+		return true
+	}
+
 	if DarktableCli.IsInstalled() && imgType.isRaw() {
 		return true
 	}
@@ -296,3 +304,16 @@ func isPathMedia(mediaPath string, cache *AlbumScannerCache) bool {
 	log.Printf("File is not a supported media %s\n", mediaPath)
 	return false
 }
+
+func (mediaType MediaType) FileExtensions() []string {
+	var extensions []string
+
+	for ext, extType := range fileExtensions {
+		if extType == mediaType {
+			extensions = append(extensions, ext)
+			extensions = append(extensions, strings.ToUpper(ext))
+		}
+	}
+
+	return extensions
+}

api/scanner/process_photo.go

@@ -109,6 +109,11 @@ func processPhoto(tx *sql.Tx, imageData *EncodeMediaData, photoCachePath *string
 		if err != nil {
 			return false, err
 		}
+
+		counterpartFile := scanForCompressedCounterpartFile(photo.Path)
+		if counterpartFile != nil {
+			photo.CounterpartPath = counterpartFile
+		}
 	}
 
 	// Generate high res jpeg
@@ -173,11 +178,6 @@ func processPhoto(tx *sql.Tx, imageData *EncodeMediaData, photoCachePath *string
 		thumbnail_name = models.SanitizeMediaName(thumbnail_name)
 		thumbnail_name = thumbnail_name + ".jpg"
 
-		// thumbnailImage, err := imageData.ThumbnailImage(tx)
-		// if err != nil {
-		// 	return err
-		// }
-
 		err = generateSaveThumbnailJPEG(tx, photo.MediaID, thumbnail_name, photoCachePath, baseImagePath, -1)
 		if err != nil {
 			return false, err

api/scanner/scanner_album.go

@@ -98,6 +98,12 @@ func findMediaForAlbum(album *models.Album, cache *AlbumScannerCache, db *sql.DB
 		photoPath := path.Join(album.Path, item.Name())
 
 		if !item.IsDir() && isPathMedia(photoPath, cache) {
+			// Skip the JPEGs that are compressed version of raw files
+			counterpartFile := scanForRawCounterpartFile(photoPath)
+			if counterpartFile != nil {
+				continue
+			}
+
 			tx, err := db.Begin()
 			if err != nil {
 				ScannerError("Could not begin database transaction for image %s: %s\n", photoPath, err)

api/scanner/scanner_media.go

@@ -8,24 +8,79 @@ import (
 	"log"
 	"os"
 	"path"
+	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/viktorstrate/photoview/api/graphql/models"
 )
 
-func scanForSideCarFile(path string) *string {
-	testPath := path + ".xmp"
+func fileExists(testPath string) bool {
 	_, err := os.Stat(testPath)
 
 	if os.IsNotExist(err) {
-		return nil
+		return false
 	} else if err != nil {
 		// unexpected error logging
-		log.Printf("ERROR: %s", err)
-		return nil
+		log.Printf("Error: checking for file existence (%s): %s", testPath, err)
+		return false
 	}
-	return &testPath
+	return true
+}
 
+func scanForSideCarFile(path string) *string {
+	testPath := path + ".xmp"
+
+	if fileExists(testPath) {
+		return &testPath
+	}
+
+	return nil
+}
+
+func scanForRawCounterpartFile(imagePath string) *string {
+	ext := filepath.Ext(imagePath)
+	fileExtType, found := fileExtensions[strings.ToLower(ext)]
+
+	if found {
+		if !fileExtType.isBasicTypeSupported() {
+			return nil
+		}
+	}
+
+	pathWithoutExt := strings.TrimSuffix(imagePath, path.Ext(imagePath))
+
+	for _, rawType := range RawMimeTypes {
+		for _, ext := range rawType.FileExtensions() {
+			testPath := pathWithoutExt + ext
+			if fileExists(testPath) {
+				return &testPath
+			}
+		}
+	}
+
+	return nil
+}
+
+func scanForCompressedCounterpartFile(imagePath string) *string {
+	ext := filepath.Ext(imagePath)
+	fileExtType, found := fileExtensions[strings.ToLower(ext)]
+
+	if found {
+		if fileExtType.isBasicTypeSupported() {
+			return nil
+		}
+	}
+
+	pathWithoutExt := strings.TrimSuffix(imagePath, path.Ext(imagePath))
+	for _, ext := range TypeJpeg.FileExtensions() {
+		testPath := pathWithoutExt + ext
+		if fileExists(testPath) {
+			return &testPath
+		}
+	}
+
+	return nil
 }
 
 func hashSideCarFile(path *string) *string {