mirror
/
photoview
mirror of https://github.com/photoview/photoview.git
1
Fork 0
Code

Merge pull request #586 from photoview/scanner-cleanup-tests 

Fix database cleanup error when multiple users have overlapping paths
This commit is contained in:
Viktor Strate Kløvedal 2021-11-06 13:36:35 +01:00 committed by GitHub
parent 91a208b1b9 6545a04225
commit 18b83dbbea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 181 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/database/database.go
View File

@ -103,6 +103,11 @@ func ConfigureDatabase(config *gorm.Config) (*gorm.DB, error) {
return nil, err
}
// Manually enable foreign keys for sqlite, as this isn't done by default
if drivers.DatabaseDriver() == drivers.DatabaseDriverSqlite {
db.Exec("PRAGMA foreign_keys = ON")
}
return db, nil
}

1
api/go.mod
View File

@ -19,6 +19,7 @@ require (
github.com/joho/godotenv v1.3.0
github.com/mattn/go-sqlite3 v1.14.6 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/otiai10/copy v1.6.0 // indirect
github.com/pkg/errors v0.9.1
github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f
github.com/stretchr/objx v0.3.0 // indirect

6
api/go.sum
View File

@ -437,6 +437,12 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/openzipkin/zipkin-go v0.2.2 h1:nY8Hti+WKaP0cRsSeQ026wU03QsM762XBeCXBb9NAWI=
github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ=
github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E=
github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE=
github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
github.com/pact-foundation/pact-go v1.0.4 h1:OYkFijGHoZAYbOIb1LWXrwKQbMMRUv1oQ89blD2Mh2Q=
github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs=

2
api/graphql/resolvers/user.go
View File

@ -343,7 +343,7 @@ func (r *mutationResolver) UserRemoveRootAlbum(ctx context.Context, userID int,
}
// Reload faces as media might have been deleted
if face_detection.GlobalFaceDetector == nil {
if face_detection.GlobalFaceDetector != nil {
if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(r.Database); err != nil {
return nil, err
}

16
api/routes/authenticate_routes_test.go
View File

@ -88,8 +88,8 @@ func TestAuthenticateRoute(t *testing.T) {
assert.NoError(t, err)
assert.True(t, success)
assert.Equal(t, responseMessage, "success")
assert.Equal(t, responseStatus, http.StatusAccepted)
assert.Equal(t, "success", responseMessage)
assert.Equal(t, http.StatusAccepted, responseStatus)
})
})
@ -103,8 +103,8 @@ func TestAuthenticateRoute(t *testing.T) {
assert.NoError(t, err)
assert.True(t, success)
assert.Equal(t, responseMessage, "success")
assert.Equal(t, responseStatus, http.StatusAccepted)
assert.Equal(t, "success", responseMessage)
assert.Equal(t, http.StatusAccepted, responseStatus)
})
t.Run("Request without access token", func(t *testing.T) {
@ -114,8 +114,8 @@ func TestAuthenticateRoute(t *testing.T) {
assert.Error(t, err)
assert.False(t, success)
assert.Equal(t, responseMessage, "unauthorized")
assert.Equal(t, responseStatus, http.StatusForbidden)
assert.Equal(t, "unauthorized", responseMessage)
assert.Equal(t, http.StatusForbidden, responseStatus)
})
expire := time.Now().Add(time.Hour * 24 * 30)
@ -139,8 +139,8 @@ func TestAuthenticateRoute(t *testing.T) {
assert.NoError(t, err)
assert.True(t, success)
assert.Equal(t, responseMessage, "success")
assert.Equal(t, responseStatus, http.StatusAccepted)
assert.Equal(t, "success", responseMessage)
assert.Equal(t, http.StatusAccepted, responseStatus)
})
})

7
api/scanner/cleanup_media.go
View File

@ -83,7 +83,6 @@ func deleteOldUserAlbums(db *gorm.DB, scannedAlbums []*models.Album, user *model
Table("user_albums").
Joins("JOIN albums ON user_albums.album_id = albums.id").
Where("user_id = ?", user.ID).
// Where("album_id IN (?)", userAlbumIDs).
Where("album_id NOT IN (?)", scannedAlbumIDs)
if err := query.Find(&deleteAlbums).Error; err != nil {
@ -109,11 +108,11 @@ func deleteOldUserAlbums(db *gorm.DB, scannedAlbums []*models.Album, user *model
// Delete old albums from database
err := db.Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&user).Association("Albums").Delete(deleteAlbums); err != nil {
if err := tx.Where("album_id IN (?)", deleteAlbumIDs).Delete(&models.UserAlbums{}).Error; err != nil {
return err
}
if err := tx.Where("id IN ?", deleteAlbumIDs).Delete(models.Album{}).Error; err != nil {
if err := tx.Where("id IN (?)", deleteAlbumIDs).Delete(models.Album{}).Error; err != nil {
return err
}
@ -126,7 +125,7 @@ func deleteOldUserAlbums(db *gorm.DB, scannedAlbums []*models.Album, user *model
}
// Reload faces after deleting albums
if face_detection.GlobalFaceDetector == nil {
if face_detection.GlobalFaceDetector != nil {
if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(db); err != nil {
deleteErrors = append(deleteErrors, err)
}

101
api/scanner/cleanup_media_test.go Normal file
View File

@ -0,0 +1,101 @@
package scanner_test
import (
"os"
"path"
"testing"
"github.com/otiai10/copy"
"github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner/face_detection"
"github.com/photoview/photoview/api/test_utils"
"github.com/stretchr/testify/assert"
)
func TestCleanupMedia(t *testing.T) {
test_utils.FilesystemTest(t)
db := test_utils.DatabaseTest(t)
if !assert.NoError(t, face_detection.InitializeFaceDetector(db)) {
return
}
test_dir := t.TempDir()
copy.Copy("./test_data", test_dir)
countAllMedia := func() int {
var all_media []*models.Media
if !assert.NoError(t, db.Find(&all_media).Error) {
return -1
}
return len(all_media)
}
countAllMediaURLs := func() int {
var all_media_urls []*models.MediaURL
if !assert.NoError(t, db.Find(&all_media_urls).Error) {
return -1
}
return len(all_media_urls)
}
pass := "1234"
user1, err := models.RegisterUser(db, "user1", &pass, true)
if !assert.NoError(t, err) {
return
}
user2, err := models.RegisterUser(db, "user2", &pass, true)
if !assert.NoError(t, err) {
return
}
root_album := models.Album{
Title: "root album",
Path: test_dir,
}
if !assert.NoError(t, db.Save(&root_album).Error) {
return
}
err = db.Model(user1).Association("Albums").Append(&root_album)
if !assert.NoError(t, err) {
return
}
err = db.Model(user2).Association("Albums").Append(&root_album)
if !assert.NoError(t, err) {
return
}
t.Run("Modify albums", func(t *testing.T) {
test_utils.RunScannerOnUser(t, db, user1)
assert.Equal(t, 9, countAllMedia())
assert.Equal(t, 18, countAllMediaURLs())
// move faces directory
assert.NoError(t, os.Rename(path.Join(test_dir, "faces"), path.Join(test_dir, "faces_moved")))
test_utils.RunScannerAll(t, db)
assert.Equal(t, 9, countAllMedia())
assert.Equal(t, 18, countAllMediaURLs())
// remove faces_moved directory
assert.NoError(t, os.RemoveAll(path.Join(test_dir, "faces_moved")))
test_utils.RunScannerAll(t, db)
assert.Equal(t, 3, countAllMedia())
assert.Equal(t, 6, countAllMediaURLs())
})
t.Run("Modify images", func(t *testing.T) {
assert.NoError(t, os.Rename(path.Join(test_dir, "buttercup_close_summer_yellow.jpg"), path.Join(test_dir, "yellow-flower.jpg")))
test_utils.RunScannerAll(t, db)
assert.Equal(t, 3, countAllMedia())
assert.Equal(t, 6, countAllMediaURLs())
assert.NoError(t, os.Remove(path.Join(test_dir, "lilac_lilac_bush_lilac.jpg")))
test_utils.RunScannerAll(t, db)
assert.Equal(t, 2, countAllMedia())
assert.Equal(t, 4, countAllMediaURLs())
})
}

8
api/scanner/exif/exif.go
View File

@ -9,19 +9,19 @@ import (
"github.com/photoview/photoview/api/graphql/models"
)
type exifParser interface {
type ExifParser interface {
ParseExif(media_path string) (*models.MediaEXIF, error)
}
var globalExifParser exifParser
var globalExifParser ExifParser
func InitializeEXIFParser() {
// Decide between internal or external Exif parser
exiftoolParser, err := newExiftoolParser()
exiftoolParser, err := NewExiftoolParser()
if err != nil {
log.Printf("Failed to get exiftool, using internal exif parser instead: %v\n", err)
globalExifParser = &internalExifParser{}
globalExifParser = NewInternalExifParser()
} else {
log.Println("Found exiftool")
globalExifParser = exiftoolParser

2
api/scanner/exif/exif_parser_external.go
View File

@ -15,7 +15,7 @@ type externalExifParser struct {
dataLoader *dataloader.ExiftoolLoader
}
func newExiftoolParser() (*externalExifParser, error) {
func NewExiftoolParser() (ExifParser, error) {
et, err := exiftool.NewExiftool(exiftool.NoPrintConversion())
if err != nil {

6
api/scanner/exif/exif_parser_internal.go
View File

@ -16,7 +16,11 @@ import (
// internalExifParser is an exif parser that parses the media without the use of external tools
type internalExifParser struct{}
func (p *internalExifParser) ParseExif(media_path string) (returnExif *models.MediaEXIF, returnErr error) {
func NewInternalExifParser() ExifParser {
return internalExifParser{}
}
func (p internalExifParser) ParseExif(media_path string) (returnExif *models.MediaEXIF, returnErr error) {
photoFile, err := os.Open(media_path)
if err != nil {
return nil, err

11
api/scanner/exif/exif_test.go
View File

@ -1,4 +1,4 @@
package exif
package exif_test
import (
"fmt"
@ -9,6 +9,7 @@ import (
"github.com/barasher/go-exiftool"
"github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner/exif"
"github.com/photoview/photoview/api/test_utils"
"github.com/stretchr/testify/assert"
)
@ -22,18 +23,18 @@ func TestExifParsers(t *testing.T) {
parsers := []struct {
name string
parser exifParser
parser exif.ExifParser
}{
{
name: "internal",
parser: &internalExifParser{},
parser: exif.NewInternalExifParser(),
},
}
if externalParser, err := newExiftoolParser(); err == nil {
if externalParser, err := exif.NewExiftoolParser(); err == nil {
parsers = append(parsers, struct {
name string
parser exifParser
parser exif.ExifParser
}{
name: "external",
parser: externalParser,

12
api/scanner/scanner_test.go
View File

@ -6,7 +6,6 @@ import (
"time"
"github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner"
"github.com/photoview/photoview/api/scanner/face_detection"
"github.com/photoview/photoview/api/test_utils"
"github.com/stretchr/testify/assert"
@ -40,20 +39,11 @@ func TestFullScan(t *testing.T) {
return
}
if !assert.NoError(t, scanner.InitializeScannerQueue(db)) {
return
}
if !assert.NoError(t, face_detection.InitializeFaceDetector(db)) {
return
}
if !assert.NoError(t, scanner.AddUserToQueue(user)) {
return
}
// wait for all jobs to finish
scanner.CloseScannerQueue()
test_utils.RunScannerOnUser(t, db, user)
var all_media []*models.Media
if !assert.NoError(t, db.Find(&all_media).Error) {

2
api/scanner/scanner_user.go
View File

@ -23,7 +23,7 @@ func getPhotoviewIgnore(ignorePath string) ([]string, error) {
// Open .photoviewignore file, if exists
photoviewIgnoreFile, err := os.Open(path.Join(ignorePath, ".photoviewignore"))
if err != nil {
if err == os.ErrNotExist {
if os.IsNotExist(err) {
return photoviewIgnore, nil
}
return photoviewIgnore, err

36
api/test_utils/scanner_helpers.go Normal file
View File

@ -0,0 +1,36 @@
package test_utils
import (
"testing"
"github.com/photoview/photoview/api/graphql/models"
"github.com/photoview/photoview/api/scanner"
"github.com/stretchr/testify/assert"
"gorm.io/gorm"
)
func RunScannerOnUser(t *testing.T, db *gorm.DB, user *models.User) {
if !assert.NoError(t, scanner.InitializeScannerQueue(db)) {
return
}
if !assert.NoError(t, scanner.AddUserToQueue(user)) {
return
}
// wait for all jobs to finish
scanner.CloseScannerQueue()
}
func RunScannerAll(t *testing.T, db *gorm.DB) {
if !assert.NoError(t, scanner.InitializeScannerQueue(db)) {
return
}
if !assert.NoError(t, scanner.AddAllToQueue()) {
return
}
// wait for all jobs to finish
scanner.CloseScannerQueue()
}

4
api/utils/environment_variables.go
View File

@ -66,13 +66,13 @@ func (v EnvironmentVariable) GetBool() bool {
// ShouldServeUI whether or not the "serve ui" option is enabled
func ShouldServeUI() bool {
return EnvServeUI.GetValue() == "1"
return EnvServeUI.GetBool()
}
// DevelopmentMode describes whether or not the server is running in development mode,
// and should thus print debug informations and enable other features related to developing.
func DevelopmentMode() bool {
return EnvDevelopmentMode.GetValue() == "1"
return EnvDevelopmentMode.GetBool()
}
// UIPath returns the value from where the static UI files are located if SERVE_UI=1