mirror
/
photoview
mirror of https://github.com/photoview/photoview.git
1
Fork 0
Code

InnoDB has a maximum index length of 767 bytes for tables that use COMPACT or REDUNDANT row format, so for utf8mb3 or utf8mb4 columns, you can index a maximum of 255 or 191 characters, respectively. 

That's why I created a new column storing a MD5 hash of the path and made it unique. The MD5 hash has only 32 characters and can be generated natively in MySQL and MariaDB. It helps us to avoid maximum key length and in the same time enforce unique photo and album paths.

- Added path_hash column to photo and album tables
- Added down migration file for 005_utf8_migration (just for consistency)
- Added PathHash field to Album and Photo struct
- album_scanner.go and photo_scanner.goo perform checks by MD5 hash now
This commit is contained in:
stz184 2020-06-24 11:43:20 +03:00
parent abfcea9072
commit f88fa33e5f
6 changed files with 32 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/database/migrations/002_photo.up.sql
View File

@ -20,7 +20,8 @@ CREATE TABLE IF NOT EXISTS album (
title varchar(256) NOT NULL,
parent_album int,
owner_id int NOT NULL,
path varchar(512) NOT NULL UNIQUE,
path varchar(512) NOT NULL,
path_hash varchar(32) NOT NULL UNIQUE,
PRIMARY KEY (album_id),
FOREIGN KEY (parent_album) REFERENCES album(album_id) ON DELETE CASCADE,
@ -30,7 +31,8 @@ CREATE TABLE IF NOT EXISTS album (
CREATE TABLE IF NOT EXISTS photo (
photo_id int NOT NULL AUTO_INCREMENT,
title varchar(256) NOT NULL,
path varchar(1024) NOT NULL UNIQUE,
path varchar(1024) NOT NULL,
path_hash varchar(32) NOT NULL UNIQUE,
album_id int NOT NULL,
exif_id int,

9
api/database/migrations/005_utf8_migration.down.sql Normal file
View File

@ -0,0 +1,9 @@
-- Migrate all tables in database to use utf8 for better language support
ALTER TABLE access_token CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
ALTER TABLE album CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
ALTER TABLE photo CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
ALTER TABLE photo_exif CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
ALTER TABLE photo_url CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
ALTER TABLE share_token CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
ALTER TABLE site_info CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
ALTER TABLE user CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;

5
api/graphql/models/album.go
View File

@ -10,6 +10,7 @@ type Album struct {
ParentAlbum *int
OwnerID int
Path string
PathHash string
}
func (a *Album) ID() int {
@ -23,7 +24,7 @@ func (a *Album) FilePath() string {
func NewAlbumFromRow(row *sql.Row) (*Album, error) {
album := Album{}
if err := row.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path); err != nil {
if err := row.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path, &album.PathHash); err != nil {
return nil, err
}
@ -35,7 +36,7 @@ func NewAlbumsFromRows(rows *sql.Rows) ([]*Album, error) {
for rows.Next() {
var album Album
if err := rows.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path); err != nil {
if err := rows.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path, &album.PathHash); err != nil {
return nil, err
}
albums = append(albums, &album)

15
api/graphql/models/photo.go
View File

@ -8,11 +8,12 @@ import (
)
type Photo struct {
PhotoID int
Title string
Path string
AlbumId int
ExifId *int
PhotoID int
Title string
Path string
PathHash string
AlbumId int
ExifId *int
}
func (p *Photo) ID() int {
@ -40,7 +41,7 @@ type PhotoURL struct {
func NewPhotoFromRow(row *sql.Row) (*Photo, error) {
photo := Photo{}
if err := row.Scan(&photo.PhotoID, &photo.Title, &photo.Path, &photo.AlbumId, &photo.ExifId); err != nil {
if err := row.Scan(&photo.PhotoID, &photo.Title, &photo.Path, &photo.PathHash, &photo.AlbumId, &photo.ExifId); err != nil {
return nil, err
}
@ -52,7 +53,7 @@ func NewPhotosFromRows(rows *sql.Rows) ([]*Photo, error) {
for rows.Next() {
var photo Photo
if err := rows.Scan(&photo.PhotoID, &photo.Title, &photo.Path, &photo.AlbumId, &photo.ExifId); err != nil {
if err := rows.Scan(&photo.PhotoID, &photo.Title, &photo.Path, &photo.PathHash, &photo.AlbumId, &photo.ExifId); err != nil {
return nil, err
}
photos = append(photos, &photo)

12
api/scanner/album_scanner.go
View File

@ -166,14 +166,14 @@ func scan(database *sql.DB, user *models.User) {
// Make album if not exists
albumTitle := path.Base(albumPath)
_, err = tx.Exec("INSERT IGNORE INTO album (title, parent_album, owner_id, path) VALUES (?, ?, ?, ?)", albumTitle, albumParentId, user.UserID, albumPath)
_, err = tx.Exec("INSERT IGNORE INTO album (title, parent_album, owner_id, path, path_hash) VALUES (?, ?, ?, ?, MD5(path))", albumTitle, albumParentId, user.UserID, albumPath)
if err != nil {
ScannerError("Could not insert album into database: %s\n", err)
tx.Rollback()
continue
}
row := tx.QueryRow("SELECT album_id FROM album WHERE path = ?", albumPath)
row := tx.QueryRow("SELECT album_id FROM album WHERE path_hash = MD5(?)", albumPath)
var albumId int
if err := row.Scan(&albumId); err != nil {
ScannerError("Could not get id of album: %s\n", err)
@ -400,8 +400,8 @@ func cleanupCache(database *sql.DB, scanned_albums []interface{}, scanned_photos
album_args = append(album_args, user.UserID)
album_args = append(album_args, scanned_albums...)
albums_questions := strings.Repeat("?,", len(scanned_albums))[:len(scanned_albums)*2-1]
rows, err := database.Query("SELECT album_id FROM album WHERE album.owner_id = ? AND path NOT IN ("+albums_questions+")", album_args...)
albums_questions := strings.Repeat("MD5(?),", len(scanned_albums))[:len(scanned_albums)*7-1]
rows, err := database.Query("SELECT album_id FROM album WHERE album.owner_id = ? AND path_hash NOT IN ("+albums_questions+")", album_args...)
if err != nil {
ScannerError("Could not get albums from database: %s\n", err)
return
@ -436,11 +436,11 @@ func cleanupCache(database *sql.DB, scanned_albums []interface{}, scanned_photos
photo_args = append(photo_args, user.UserID)
photo_args = append(photo_args, scanned_photos...)
photo_questions := strings.Repeat("?,", len(scanned_photos))[:len(scanned_photos)*2-1]
photo_questions := strings.Repeat("MD5(?),", len(scanned_photos))[:len(scanned_photos)*7-1]
rows, err = database.Query(`
SELECT photo.photo_id as photo_id, album.album_id as album_id FROM photo JOIN album ON photo.album_id = album.album_id
WHERE album.owner_id = ? AND photo.path NOT IN (`+photo_questions+`)
WHERE album.owner_id = ? AND photo.path_hash NOT IN (`+photo_questions+`)
`, photo_args...)
if err != nil {
ScannerError("Could not get deleted photos from database: %s\n", err)

4
api/scanner/photo_scanner.go
View File

@ -16,7 +16,7 @@ func ScanPhoto(tx *sql.Tx, photoPath string, albumId int, notificationKey string
// Check if image already exists
{
row := tx.QueryRow("SELECT * FROM photo WHERE path = ?", photoPath)
row := tx.QueryRow("SELECT * FROM photo WHERE path_hash = MD5(?)", photoPath)
photo, err := models.NewPhotoFromRow(row)
if err != sql.ErrNoRows {
if err == nil {
@ -28,7 +28,7 @@ func ScanPhoto(tx *sql.Tx, photoPath string, albumId int, notificationKey string
}
}
result, err := tx.Exec("INSERT INTO photo (title, path, album_id) VALUES (?, ?, ?)", photoName, photoPath, albumId)
result, err := tx.Exec("INSERT INTO photo (title, path, path_hash, album_id) VALUES (?, ?, MD5(path), ?)", photoName, photoPath, albumId)
if err != nil {
log.Printf("ERROR: Could not insert photo into database")
return nil, false, err