Start on face detection

api/database/database.go

@@ -164,6 +164,10 @@ func MigrateDatabase(db *gorm.DB) error {
 		&models.VideoMetadata{},
 		&models.ShareToken{},
 		&models.UserMediaData{},
+
+		// Face detection
+		&models.FaceGroup{},
+		&models.ImageFace{},
 	)
 
 	// v2.1.0 - Replaced by Media.CreatedAt

api/go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/99designs/gqlgen v0.13.0
+	github.com/Kagami/go-face v0.0.0-20200825065730-3dd2d74dccfb
 	github.com/agnivade/levenshtein v1.1.0 // indirect
 	github.com/disintegration/imaging v1.6.2
 	github.com/gorilla/handlers v1.5.1

api/go.sum

@@ -1,6 +1,8 @@
 github.com/99designs/gqlgen v0.13.0 h1:haLTcUp3Vwp80xMVEg5KRNwzfUrgFdRmtBY8fuB8scA=
 github.com/99designs/gqlgen v0.13.0/go.mod h1:NV130r6f4tpRWuAI+zsrSdooO/eWUv+Gyyoi3rEfXIk=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Kagami/go-face v0.0.0-20200825065730-3dd2d74dccfb h1:DXwA1Te9paM+nsdTGc7uve37lq7WEbQO+gwGBPVwQuQ=
+github.com/Kagami/go-face v0.0.0-20200825065730-3dd2d74dccfb/go.mod h1:9wdDJkRgo3SGTcFwbQ7elVIQhIr2bbBjecuY7VoqmPU=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
 github.com/agnivade/levenshtein v1.0.3/go.mod h1:4SFRZbbXWLF4MU1T9Qg0pGgH3Pjs+t6ie5efyrwRJXs=
 github.com/agnivade/levenshtein v1.1.0 h1:n6qGwyHG61v3ABce1rPVZklEYRT8NFpCMrpZdBUbYGM=

api/graphql/models/face_detection.go

@@ -0,0 +1,48 @@
+package models
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/binary"
+
+	"github.com/Kagami/go-face"
+)
+
+type FaceGroup struct {
+	Model
+	Label      *string
+	ImageFaces []ImageFace `gorm:"constraint:OnDelete:CASCADE;"`
+}
+
+type ImageFace struct {
+	Model
+	FaceGroupID int            `gorm:"not null;index"`
+	MediaID     int            `gorm:"not null;index"`
+	Media       Media          `gorm:"constraint:OnDelete:CASCADE;"`
+	Descriptor  FaceDescriptor `gorm:"not null"`
+}
+
+type FaceDescriptor face.Descriptor
+
+// GormDataType datatype used in database
+func (fd FaceDescriptor) GormDataType() string {
+	return "BLOB"
+}
+
+// Scan tells GORM how to convert database data to Go format
+func (fd *FaceDescriptor) Scan(value interface{}) error {
+	byteValue := value.([]byte)
+	reader := bytes.NewReader(byteValue)
+	binary.Read(reader, binary.LittleEndian, fd)
+	return nil
+}
+
+// Value tells GORM how to save into the database
+func (fd FaceDescriptor) Value() (driver.Value, error) {
+	buf := new(bytes.Buffer)
+	if err := binary.Write(buf, binary.LittleEndian, fd); err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}

api/graphql/models/media.go

@@ -1,11 +1,14 @@
 package models
 
 import (
+	"fmt"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/photoview/photoview/api/utils"
+	"github.com/pkg/errors"
 	"gorm.io/gorm"
 )
 
@@ -59,7 +62,7 @@ const (
 type MediaURL struct {
 	Model
 	MediaID     int          `gorm:"not null;index"`
-	Media       Media        `gorm:"constraint:OnDelete:CASCADE;"`
+	Media       *Media       `gorm:"constraint:OnDelete:CASCADE;"`
 	MediaName   string       `gorm:"not null"`
 	Width       int          `gorm:"not null"`
 	Height      int          `gorm:"not null"`
@@ -80,6 +83,24 @@ func (p *MediaURL) URL() string {
 	return imageUrl.String()
 }
 
+func (p *MediaURL) CachedPath() (string, error) {
+	var cachedPath string
+
+	if p.Media == nil {
+		return "", errors.New("mediaURL.Media is nil")
+	}
+
+	if p.Purpose == PhotoThumbnail || p.Purpose == PhotoHighRes || p.Purpose == VideoThumbnail {
+		cachedPath = path.Join(utils.MediaCachePath(), strconv.Itoa(int(p.Media.AlbumID)), strconv.Itoa(int(p.MediaID)), p.MediaName)
+	} else if p.Purpose == MediaOriginal {
+		cachedPath = p.Media.Path
+	} else {
+		return "", errors.New(fmt.Sprintf("cannot determine cache path for purpose (%s)", p.Purpose))
+	}
+
+	return cachedPath, nil
+}
+
 func SanitizeMediaName(mediaName string) string {
 	result := mediaName
 	result = strings.ReplaceAll(result, "/", "")

api/graphql/resolvers/user.go

@@ -12,6 +12,7 @@ import (
 	"github.com/photoview/photoview/api/graphql/auth"
 	"github.com/photoview/photoview/api/graphql/models"
 	"github.com/photoview/photoview/api/scanner"
+	"github.com/photoview/photoview/api/utils"
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/bcrypt"
 	"gorm.io/gorm"
@@ -252,7 +253,7 @@ func (r *mutationResolver) DeleteUser(ctx context.Context, id int) (*models.User
 
 	// If there is only one associated user, clean up the cache folder and delete the album row
 	for _, deletedAlbumID := range deletedAlbumIDs {
-		cachePath := path.Join(scanner.MediaCachePath(), strconv.Itoa(int(deletedAlbumID)))
+		cachePath := path.Join(utils.MediaCachePath(), strconv.Itoa(int(deletedAlbumID)))
 		if err := os.RemoveAll(cachePath); err != nil {
 			return &user, err
 		}
@@ -370,7 +371,7 @@ func (r *mutationResolver) UserRemoveRootAlbum(ctx context.Context, userID int,
 	if deletedAlbumIDs != nil {
 		// Delete albums from cache
 		for _, id := range deletedAlbumIDs {
-			cacheAlbumPath := path.Join(scanner.MediaCachePath(), strconv.Itoa(id))
+			cacheAlbumPath := path.Join(utils.MediaCachePath(), strconv.Itoa(id))
 
 			if err := os.RemoveAll(cacheAlbumPath); err != nil {
 				return nil, err

api/routes/photos.go

@@ -4,8 +4,6 @@ import (
 	"log"
 	"net/http"
 	"os"
-	"path"
-	"strconv"
 
 	"github.com/gorilla/mux"
 	"gorm.io/gorm"
@@ -27,7 +25,7 @@ func RegisterPhotoRoutes(db *gorm.DB, router *mux.Router) {
 			return
 		}
 
-		media := &mediaURL.Media
+		media := mediaURL.Media
 
 		if success, response, status, err := authenticateMedia(media, db, r); !success {
 			if err != nil {
@@ -38,14 +36,9 @@ func RegisterPhotoRoutes(db *gorm.DB, router *mux.Router) {
 			return
 		}
 
-		var cachedPath string
+		cachedPath, err := mediaURL.CachedPath()
-
+		if err != nil {
-		if mediaURL.Purpose == models.PhotoThumbnail || mediaURL.Purpose == models.PhotoHighRes || mediaURL.Purpose == models.VideoThumbnail {
+			log.Printf("ERROR: %s\n", err)
-			cachedPath = path.Join(scanner.MediaCachePath(), strconv.Itoa(int(media.AlbumID)), strconv.Itoa(int(mediaURL.MediaID)), mediaURL.MediaName)
-		} else if mediaURL.Purpose == models.MediaOriginal {
-			cachedPath = media.Path
-		} else {
-			log.Printf("ERROR: Can not handle media_purpose for photo: %s\n", mediaURL.Purpose)
 			w.WriteHeader(http.StatusInternalServerError)
 			w.Write([]byte("internal server error"))
 			return

api/routes/videos.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/photoview/photoview/api/graphql/models"
 	"github.com/photoview/photoview/api/scanner"
+	"github.com/photoview/photoview/api/utils"
 	"gorm.io/gorm"
 )
 
@@ -26,7 +27,7 @@ func RegisterVideoRoutes(db *gorm.DB, router *mux.Router) {
 			return
 		}
 
-		var media = &mediaURL.Media
+		var media = mediaURL.Media
 
 		if success, response, status, err := authenticateMedia(media, db, r); !success {
 			if err != nil {
@@ -40,7 +41,7 @@ func RegisterVideoRoutes(db *gorm.DB, router *mux.Router) {
 		var cachedPath string
 
 		if mediaURL.Purpose == models.VideoWeb {
-			cachedPath = path.Join(scanner.MediaCachePath(), strconv.Itoa(int(media.AlbumID)), strconv.Itoa(int(mediaURL.MediaID)), mediaURL.MediaName)
+			cachedPath = path.Join(utils.MediaCachePath(), strconv.Itoa(int(media.AlbumID)), strconv.Itoa(int(mediaURL.MediaID)), mediaURL.MediaName)
 		} else {
 			log.Printf("ERROR: Can not handle media_purpose for video: %s\n", mediaURL.Purpose)
 			w.WriteHeader(http.StatusInternalServerError)

api/scanner/cleanup_media.go

@@ -6,6 +6,7 @@ import (
 	"strconv"
 
 	"github.com/photoview/photoview/api/graphql/models"
+	"github.com/photoview/photoview/api/utils"
 	"github.com/pkg/errors"
 	"gorm.io/gorm"
 )
@@ -36,7 +37,7 @@ func CleanupMedia(db *gorm.DB, albumId int, albumMedia []*models.Media) []error
 	for _, media := range mediaList {
 
 		mediaIDs = append(mediaIDs, media.ID)
-		cachePath := path.Join(MediaCachePath(), strconv.Itoa(int(albumId)), strconv.Itoa(int(media.ID)))
+		cachePath := path.Join(utils.MediaCachePath(), strconv.Itoa(int(albumId)), strconv.Itoa(int(media.ID)))
 		err := os.RemoveAll(cachePath)
 		if err != nil {
 			deleteErrors = append(deleteErrors, errors.Wrapf(err, "delete unused cache folder (%s)", cachePath))
@@ -90,7 +91,7 @@ func deleteOldUserAlbums(db *gorm.DB, scannedAlbums []*models.Album, user *model
 	deleteAlbumIDs := make([]int, len(deleteAlbums))
 	for i, album := range deleteAlbums {
 		deleteAlbumIDs[i] = album.ID
-		cachePath := path.Join(MediaCachePath(), strconv.Itoa(int(album.ID)))
+		cachePath := path.Join(utils.MediaCachePath(), strconv.Itoa(int(album.ID)))
 		err := os.RemoveAll(cachePath)
 		if err != nil {
 			deleteErrors = append(deleteErrors, errors.Wrapf(err, "delete unused cache folder (%s)", cachePath))

api/scanner/face_detection/face_detector.go

@@ -0,0 +1,135 @@
+package face_detection
+
+import (
+	"log"
+	"path/filepath"
+	"sync"
+
+	"github.com/Kagami/go-face"
+	"github.com/photoview/photoview/api/graphql/models"
+	"github.com/pkg/errors"
+	"gorm.io/gorm"
+)
+
+type FaceDetector struct {
+	mutex   sync.Mutex
+	db      *gorm.DB
+	rec     *face.Recognizer
+	samples []face.Descriptor
+	cats    []int32
+}
+
+var GlobalFaceDetector FaceDetector
+
+func InitializeFaceDetector(db *gorm.DB) error {
+
+	log.Println("Initializing face detector")
+
+	rec, err := face.NewRecognizer(filepath.Join("data", "models"))
+	if err != nil {
+		return errors.Wrap(err, "initialize facedetect recognizer")
+	}
+
+	samples, cats, err := getSamplesFromDatabase(db)
+	if err != nil {
+		return errors.Wrap(err, "get face detection samples from database")
+	}
+
+	GlobalFaceDetector = FaceDetector{
+		db:      db,
+		rec:     rec,
+		samples: samples,
+		cats:    cats,
+	}
+
+	return nil
+}
+
+func getSamplesFromDatabase(db *gorm.DB) (samples []face.Descriptor, cats []int32, err error) {
+	samples = make([]face.Descriptor, 0)
+	cats = make([]int32, 0)
+
+	return
+}
+
+func (fd *FaceDetector) DetectFaces(media *models.Media) error {
+	if err := fd.db.Model(media).Preload("MediaURL").First(&media).Error; err != nil {
+		return err
+	}
+
+	var thumbnailURL *models.MediaURL
+	for _, url := range media.MediaURL {
+		if url.Purpose == models.PhotoThumbnail {
+			thumbnailURL = &url
+			thumbnailURL.Media = media
+			break
+		}
+	}
+
+	if thumbnailURL == nil {
+		return errors.New("thumbnail url is missing")
+	}
+
+	thumbnailPath, err := thumbnailURL.CachedPath()
+	if err != nil {
+		return err
+	}
+
+	fd.mutex.Lock()
+	faces, err := fd.rec.RecognizeFile(thumbnailPath)
+	fd.mutex.Unlock()
+
+	if err != nil {
+		return errors.Wrap(err, "error read faces")
+	}
+
+	for _, face := range faces {
+		fd.classifyFace(&face, media)
+	}
+
+	return nil
+}
+
+func (fd *FaceDetector) classifyFace(face *face.Face, media *models.Media) error {
+	fd.mutex.Lock()
+	defer fd.mutex.Unlock()
+
+	match := fd.rec.ClassifyThreshold(face.Descriptor, 0.2)
+
+	imageFace := models.ImageFace{
+		MediaID:    media.ID,
+		Descriptor: models.FaceDescriptor(face.Descriptor),
+	}
+
+	var faceGroup models.FaceGroup
+
+	// If no match add it new to samples
+	if match < 0 {
+		log.Println("No match, assigning new face")
+
+		faceGroup = models.FaceGroup{
+			ImageFaces: []models.ImageFace{imageFace},
+		}
+
+		if err := fd.db.Create(&faceGroup).Error; err != nil {
+			return err
+		}
+
+	} else {
+		log.Println("Found match")
+
+		if err := fd.db.First(&faceGroup, int(match)).Error; err != nil {
+			return err
+		}
+
+		if err := fd.db.Model(&faceGroup).Association("ImageFaces").Append(&imageFace); err != nil {
+			return err
+		}
+	}
+
+	fd.samples = append(fd.samples, face.Descriptor)
+	fd.cats = append(fd.cats, int32(faceGroup.ID))
+
+	fd.rec.SetSamples(fd.samples, fd.cats)
+	return nil
+}

api/scanner/process_photo.go

@@ -217,14 +217,14 @@ func processPhoto(tx *gorm.DB, imageData *EncodeMediaData, photoCachePath *strin
 func makeMediaCacheDir(media *models.Media) (*string, error) {
 
 	// Make root cache dir if not exists
-	if _, err := os.Stat(MediaCachePath()); os.IsNotExist(err) {
+	if _, err := os.Stat(utils.MediaCachePath()); os.IsNotExist(err) {
-		if err := os.Mkdir(MediaCachePath(), os.ModePerm); err != nil {
+		if err := os.Mkdir(utils.MediaCachePath(), os.ModePerm); err != nil {
 			return nil, errors.Wrap(err, "could not make root image cache directory")
 		}
 	}
 
 	// Make album cache dir if not exists
-	albumCachePath := path.Join(MediaCachePath(), strconv.Itoa(int(media.AlbumID)))
+	albumCachePath := path.Join(utils.MediaCachePath(), strconv.Itoa(int(media.AlbumID)))
 	if _, err := os.Stat(albumCachePath); os.IsNotExist(err) {
 		if err := os.Mkdir(albumCachePath, os.ModePerm); err != nil {
 			return nil, errors.Wrap(err, "could not make album image cache directory")
@@ -256,7 +256,7 @@ func saveOriginalPhotoToDB(tx *gorm.DB, photo *models.Media, imageData *EncodeMe
 	}
 
 	mediaURL := models.MediaURL{
-		Media:       *photo,
+		Media:       photo,
 		MediaName:   originalImageName,
 		Width:       photoDimensions.Width,
 		Height:      photoDimensions.Height,

api/scanner/scanner_album.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/photoview/photoview/api/graphql/models"
 	"github.com/photoview/photoview/api/graphql/notification"
+	"github.com/photoview/photoview/api/scanner/face_detection"
 	"github.com/photoview/photoview/api/utils"
 	"github.com/pkg/errors"
 	"gorm.io/gorm"
@@ -54,7 +55,7 @@ func scanAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB) {
 	notifyThrottle.Trigger(nil)
 
 	// Scan for photos
-	albumPhotos, err := findMediaForAlbum(album, cache, db, func(photo *models.Media, newPhoto bool) {
+	albumMedia, err := findMediaForAlbum(album, cache, db, func(photo *models.Media, newPhoto bool) {
 		if newPhoto {
 			notifyThrottle.Trigger(func() {
 				notification.BroadcastNotification(&models.Notification{
@@ -71,25 +72,33 @@ func scanAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB) {
 	}
 
 	album_has_changes := false
-	for count, photo := range albumPhotos {
+	for count, media := range albumMedia {
 		// tx, err := db.Begin()
 
 		transactionError := db.Transaction(func(tx *gorm.DB) error {
-			processing_was_needed, err := ProcessMedia(tx, photo)
+			processing_was_needed, err := ProcessMedia(tx, media)
 			if err != nil {
-				return errors.Wrapf(err, "failed to process photo (%s)", photo.Path)
+				return errors.Wrapf(err, "failed to process photo (%s)", media.Path)
 			}
 
 			if processing_was_needed {
 				album_has_changes = true
-				progress := float64(count) / float64(len(albumPhotos)) * 100.0
+				progress := float64(count) / float64(len(albumMedia)) * 100.0
 				notification.BroadcastNotification(&models.Notification{
 					Key:      album_notify_key,
 					Type:     models.NotificationTypeProgress,
 					Header:   fmt.Sprintf("Processing media for album '%s'", album.Title),
-					Content:  fmt.Sprintf("Processed media at %s", photo.Path),
+					Content:  fmt.Sprintf("Processed media at %s", media.Path),
 					Progress: &progress,
 				})
+
+				if media.Type == models.MediaTypePhoto {
+					go func() {
+						if err := face_detection.GlobalFaceDetector.DetectFaces(media); err != nil {
+							ScannerError("Error detecting faces in image (%s): %s", media.Path, err)
+						}
+					}()
+				}
 			}
 
 			return nil
@@ -100,7 +109,7 @@ func scanAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB) {
 		}
 	}
 
-	cleanup_errors := CleanupMedia(db, album.ID, albumPhotos)
+	cleanup_errors := CleanupMedia(db, album.ID, albumMedia)
 	for _, err := range cleanup_errors {
 		ScannerError("Failed to delete old media: %s", err)
 	}
@@ -138,14 +147,14 @@ func findMediaForAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.D
 			}
 
 			err := db.Transaction(func(tx *gorm.DB) error {
-				photo, isNewPhoto, err := ScanMedia(tx, photoPath, album.ID, cache)
+				media, isNewMedia, err := ScanMedia(tx, photoPath, album.ID, cache)
 				if err != nil {
 					return errors.Wrapf(err, "Scanning media error (%s)", photoPath)
 				}
 
-				onScanPhoto(photo, isNewPhoto)
+				onScanPhoto(media, isNewMedia)
 
-				albumPhotos = append(albumPhotos, photo)
+				albumPhotos = append(albumPhotos, media)
 
 				return nil
 			})

api/scanner/scanner_user.go

@@ -220,13 +220,3 @@ func ScannerError(format string, args ...interface{}) {
 		Negative: true,
 	})
 }
-
-// MediaCachePath returns the path for where the media cache is located on the file system
-func MediaCachePath() string {
-	photoCache := utils.EnvMediaCachePath.GetValue()
-	if photoCache == "" {
-		photoCache = "./media_cache"
-	}
-
-	return photoCache
-}

api/server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/photoview/photoview/api/graphql/dataloader"
 	"github.com/photoview/photoview/api/routes"
 	"github.com/photoview/photoview/api/scanner"
+	"github.com/photoview/photoview/api/scanner/face_detection"
 	"github.com/photoview/photoview/api/server"
 	"github.com/photoview/photoview/api/utils"
 
@@ -51,6 +52,10 @@ func main() {
 
 	scanner.InitializeExecutableWorkers()
 
+	if err := face_detection.InitializeFaceDetector(db); err != nil {
+		log.Panicf("Could not initialize face detector: %s\n", err)
+	}
+
 	rootRouter := mux.NewRouter()
 
 	rootRouter.Use(dataloader.Middleware(db))

api/utils/utils.go

@@ -41,3 +41,13 @@ func HandleError(message string, err error) PhotoviewError {
 		original: err,
 	}
 }
+
+// MediaCachePath returns the path for where the media cache is located on the file system
+func MediaCachePath() string {
+	photoCache := EnvMediaCachePath.GetValue()
+	if photoCache == "" {
+		photoCache = "./media_cache"
+	}
+
+	return photoCache
+}