InnoDB has a maximum index length of 767 bytes for tables that use COMPACT or REDUNDANT row format, so for utf8mb3 or utf8mb4 columns, you can index a maximum of 255 or 191 characters, respectively.
That's why I created a new column storing a MD5 hash of the path and made it unique. The MD5 hash has only 32 characters and can be generated natively in MySQL and MariaDB. It helps us to avoid maximum key length and in the same time enforce unique photo and album paths. - Added path_hash column to photo and album tables - Added down migration file for 005_utf8_migration (just for consistency) - Added PathHash field to Album and Photo struct - album_scanner.go and photo_scanner.goo perform checks by MD5 hash now
This commit is contained in:
parent
abfcea9072
commit
f88fa33e5f
|
@ -20,7 +20,8 @@ CREATE TABLE IF NOT EXISTS album (
|
|||
title varchar(256) NOT NULL,
|
||||
parent_album int,
|
||||
owner_id int NOT NULL,
|
||||
path varchar(512) NOT NULL UNIQUE,
|
||||
path varchar(512) NOT NULL,
|
||||
path_hash varchar(32) NOT NULL UNIQUE,
|
||||
|
||||
PRIMARY KEY (album_id),
|
||||
FOREIGN KEY (parent_album) REFERENCES album(album_id) ON DELETE CASCADE,
|
||||
|
@ -30,7 +31,8 @@ CREATE TABLE IF NOT EXISTS album (
|
|||
CREATE TABLE IF NOT EXISTS photo (
|
||||
photo_id int NOT NULL AUTO_INCREMENT,
|
||||
title varchar(256) NOT NULL,
|
||||
path varchar(1024) NOT NULL UNIQUE,
|
||||
path varchar(1024) NOT NULL,
|
||||
path_hash varchar(32) NOT NULL UNIQUE,
|
||||
album_id int NOT NULL,
|
||||
exif_id int,
|
||||
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
-- Migrate all tables in database to use utf8 for better language support
|
||||
ALTER TABLE access_token CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
||||
ALTER TABLE album CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
||||
ALTER TABLE photo CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
||||
ALTER TABLE photo_exif CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
||||
ALTER TABLE photo_url CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
||||
ALTER TABLE share_token CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
||||
ALTER TABLE site_info CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
||||
ALTER TABLE user CONVERT TO CHARACTER SET utf8 COLLATE utf8_unicode_ci;
|
|
@ -10,6 +10,7 @@ type Album struct {
|
|||
ParentAlbum *int
|
||||
OwnerID int
|
||||
Path string
|
||||
PathHash string
|
||||
}
|
||||
|
||||
func (a *Album) ID() int {
|
||||
|
@ -23,7 +24,7 @@ func (a *Album) FilePath() string {
|
|||
func NewAlbumFromRow(row *sql.Row) (*Album, error) {
|
||||
album := Album{}
|
||||
|
||||
if err := row.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path); err != nil {
|
||||
if err := row.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path, &album.PathHash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
@ -35,7 +36,7 @@ func NewAlbumsFromRows(rows *sql.Rows) ([]*Album, error) {
|
|||
|
||||
for rows.Next() {
|
||||
var album Album
|
||||
if err := rows.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path); err != nil {
|
||||
if err := rows.Scan(&album.AlbumID, &album.Title, &album.ParentAlbum, &album.OwnerID, &album.Path, &album.PathHash); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
albums = append(albums, &album)
|
||||
|
|
|
@ -8,11 +8,12 @@ import (
|
|||
)
|
||||
|
||||
type Photo struct {
|
||||
PhotoID int
|
||||
Title string
|
||||
Path string
|
||||
AlbumId int
|
||||
ExifId *int
|
||||
PhotoID int
|
||||
Title string
|
||||
Path string
|
||||
PathHash string
|
||||
AlbumId int
|
||||
ExifId *int
|
||||
}
|
||||
|
||||
func (p *Photo) ID() int {
|
||||
|
@ -40,7 +41,7 @@ type PhotoURL struct {
|
|||
func NewPhotoFromRow(row *sql.Row) (*Photo, error) {
|
||||
photo := Photo{}
|
||||
|
||||
if err := row.Scan(&photo.PhotoID, &photo.Title, &photo.Path, &photo.AlbumId, &photo.ExifId); err != nil {
|
||||
if err := row.Scan(&photo.PhotoID, &photo.Title, &photo.Path, &photo.PathHash, &photo.AlbumId, &photo.ExifId); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
@ -52,7 +53,7 @@ func NewPhotosFromRows(rows *sql.Rows) ([]*Photo, error) {
|
|||
|
||||
for rows.Next() {
|
||||
var photo Photo
|
||||
if err := rows.Scan(&photo.PhotoID, &photo.Title, &photo.Path, &photo.AlbumId, &photo.ExifId); err != nil {
|
||||
if err := rows.Scan(&photo.PhotoID, &photo.Title, &photo.Path, &photo.PathHash, &photo.AlbumId, &photo.ExifId); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
photos = append(photos, &photo)
|
||||
|
|
|
@ -166,14 +166,14 @@ func scan(database *sql.DB, user *models.User) {
|
|||
|
||||
// Make album if not exists
|
||||
albumTitle := path.Base(albumPath)
|
||||
_, err = tx.Exec("INSERT IGNORE INTO album (title, parent_album, owner_id, path) VALUES (?, ?, ?, ?)", albumTitle, albumParentId, user.UserID, albumPath)
|
||||
_, err = tx.Exec("INSERT IGNORE INTO album (title, parent_album, owner_id, path, path_hash) VALUES (?, ?, ?, ?, MD5(path))", albumTitle, albumParentId, user.UserID, albumPath)
|
||||
if err != nil {
|
||||
ScannerError("Could not insert album into database: %s\n", err)
|
||||
tx.Rollback()
|
||||
continue
|
||||
}
|
||||
|
||||
row := tx.QueryRow("SELECT album_id FROM album WHERE path = ?", albumPath)
|
||||
row := tx.QueryRow("SELECT album_id FROM album WHERE path_hash = MD5(?)", albumPath)
|
||||
var albumId int
|
||||
if err := row.Scan(&albumId); err != nil {
|
||||
ScannerError("Could not get id of album: %s\n", err)
|
||||
|
@ -400,8 +400,8 @@ func cleanupCache(database *sql.DB, scanned_albums []interface{}, scanned_photos
|
|||
album_args = append(album_args, user.UserID)
|
||||
album_args = append(album_args, scanned_albums...)
|
||||
|
||||
albums_questions := strings.Repeat("?,", len(scanned_albums))[:len(scanned_albums)*2-1]
|
||||
rows, err := database.Query("SELECT album_id FROM album WHERE album.owner_id = ? AND path NOT IN ("+albums_questions+")", album_args...)
|
||||
albums_questions := strings.Repeat("MD5(?),", len(scanned_albums))[:len(scanned_albums)*7-1]
|
||||
rows, err := database.Query("SELECT album_id FROM album WHERE album.owner_id = ? AND path_hash NOT IN ("+albums_questions+")", album_args...)
|
||||
if err != nil {
|
||||
ScannerError("Could not get albums from database: %s\n", err)
|
||||
return
|
||||
|
@ -436,11 +436,11 @@ func cleanupCache(database *sql.DB, scanned_albums []interface{}, scanned_photos
|
|||
photo_args = append(photo_args, user.UserID)
|
||||
photo_args = append(photo_args, scanned_photos...)
|
||||
|
||||
photo_questions := strings.Repeat("?,", len(scanned_photos))[:len(scanned_photos)*2-1]
|
||||
photo_questions := strings.Repeat("MD5(?),", len(scanned_photos))[:len(scanned_photos)*7-1]
|
||||
|
||||
rows, err = database.Query(`
|
||||
SELECT photo.photo_id as photo_id, album.album_id as album_id FROM photo JOIN album ON photo.album_id = album.album_id
|
||||
WHERE album.owner_id = ? AND photo.path NOT IN (`+photo_questions+`)
|
||||
WHERE album.owner_id = ? AND photo.path_hash NOT IN (`+photo_questions+`)
|
||||
`, photo_args...)
|
||||
if err != nil {
|
||||
ScannerError("Could not get deleted photos from database: %s\n", err)
|
||||
|
|
|
@ -16,7 +16,7 @@ func ScanPhoto(tx *sql.Tx, photoPath string, albumId int, notificationKey string
|
|||
|
||||
// Check if image already exists
|
||||
{
|
||||
row := tx.QueryRow("SELECT * FROM photo WHERE path = ?", photoPath)
|
||||
row := tx.QueryRow("SELECT * FROM photo WHERE path_hash = MD5(?)", photoPath)
|
||||
photo, err := models.NewPhotoFromRow(row)
|
||||
if err != sql.ErrNoRows {
|
||||
if err == nil {
|
||||
|
@ -28,7 +28,7 @@ func ScanPhoto(tx *sql.Tx, photoPath string, albumId int, notificationKey string
|
|||
}
|
||||
}
|
||||
|
||||
result, err := tx.Exec("INSERT INTO photo (title, path, album_id) VALUES (?, ?, ?)", photoName, photoPath, albumId)
|
||||
result, err := tx.Exec("INSERT INTO photo (title, path, path_hash, album_id) VALUES (?, ?, MD5(path), ?)", photoName, photoPath, albumId)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Could not insert photo into database")
|
||||
return nil, false, err
|
||||
|
|
Loading…
Reference in New Issue