From 3ae92086cd40549cee7f5cef7aa06536a388fc72 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Tue, 16 Mar 2021 18:26:51 +0100 Subject: [PATCH 01/22] Pass database as argument to individual face_detector functions This allows the face detector to use transactions, such that faces can be detected on media that has not been fully commited yet. This solves #214 --- api/graphql/resolvers/user.go | 2 +- api/scanner/cleanup_media.go | 4 ++-- api/scanner/face_detection/face_detector.go | 20 +++++++++----------- api/scanner/scanner_album.go | 8 ++++---- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/api/graphql/resolvers/user.go b/api/graphql/resolvers/user.go index 11e6bc8..cd0cbee 100644 --- a/api/graphql/resolvers/user.go +++ b/api/graphql/resolvers/user.go @@ -361,7 +361,7 @@ func (r *mutationResolver) UserRemoveRootAlbum(ctx context.Context, userID int, return err } - if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(); err != nil { + if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(tx); err != nil { return err } } diff --git a/api/scanner/cleanup_media.go b/api/scanner/cleanup_media.go index 068891a..9031122 100644 --- a/api/scanner/cleanup_media.go +++ b/api/scanner/cleanup_media.go @@ -52,7 +52,7 @@ func CleanupMedia(db *gorm.DB, albumId int, albumMedia []*models.Media) []error } // Reload faces after deleting media - if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(); err != nil { + if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(db); err != nil { deleteErrors = append(deleteErrors, errors.Wrap(err, "reload faces from database")) } } @@ -123,7 +123,7 @@ func deleteOldUserAlbums(db *gorm.DB, scannedAlbums []*models.Album, user *model } // Reload faces after deleting albums - if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(); err != nil { + if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(db); err != nil { deleteErrors = append(deleteErrors, err) } diff --git a/api/scanner/face_detection/face_detector.go b/api/scanner/face_detection/face_detector.go index c9ee45a..9fdce00 100644 --- a/api/scanner/face_detection/face_detector.go +++ b/api/scanner/face_detection/face_detector.go @@ -13,7 +13,6 @@ import ( type FaceDetector struct { mutex sync.Mutex - db *gorm.DB rec *face.Recognizer faceDescriptors []face.Descriptor faceGroupIDs []int32 @@ -37,7 +36,6 @@ func InitializeFaceDetector(db *gorm.DB) error { } GlobalFaceDetector = FaceDetector{ - db: db, rec: rec, faceDescriptors: faceDescriptors, faceGroupIDs: faceGroupIDs, @@ -69,8 +67,8 @@ func getSamplesFromDatabase(db *gorm.DB) (samples []face.Descriptor, faceGroupID } // ReloadFacesFromDatabase replaces the in-memory face descriptors with the ones in the database -func (fd *FaceDetector) ReloadFacesFromDatabase() error { - faceDescriptors, faceGroupIDs, imageFaceIDs, err := getSamplesFromDatabase(fd.db) +func (fd *FaceDetector) ReloadFacesFromDatabase(db *gorm.DB) error { + faceDescriptors, faceGroupIDs, imageFaceIDs, err := getSamplesFromDatabase(db) if err != nil { return err } @@ -86,8 +84,8 @@ func (fd *FaceDetector) ReloadFacesFromDatabase() error { } // DetectFaces finds the faces in the given image and saves them to the database -func (fd *FaceDetector) DetectFaces(media *models.Media) error { - if err := fd.db.Model(media).Preload("MediaURL").First(&media).Error; err != nil { +func (fd *FaceDetector) DetectFaces(tx *gorm.DB, media *models.Media) error { + if err := tx.Model(media).Preload("MediaURL").First(&media).Error; err != nil { return err } @@ -118,7 +116,7 @@ func (fd *FaceDetector) DetectFaces(media *models.Media) error { } for _, face := range faces { - fd.classifyFace(&face, media, thumbnailPath) + fd.classifyFace(tx, &face, media, thumbnailPath) } return nil @@ -128,7 +126,7 @@ func (fd *FaceDetector) classifyDescriptor(descriptor face.Descriptor) int32 { return int32(fd.rec.ClassifyThreshold(descriptor, 0.2)) } -func (fd *FaceDetector) classifyFace(face *face.Face, media *models.Media, imagePath string) error { +func (fd *FaceDetector) classifyFace(tx *gorm.DB, face *face.Face, media *models.Media, imagePath string) error { fd.mutex.Lock() defer fd.mutex.Unlock() @@ -155,18 +153,18 @@ func (fd *FaceDetector) classifyFace(face *face.Face, media *models.Media, image ImageFaces: []models.ImageFace{imageFace}, } - if err := fd.db.Create(&faceGroup).Error; err != nil { + if err := tx.Create(&faceGroup).Error; err != nil { return err } } else { log.Println("Found match") - if err := fd.db.First(&faceGroup, int(match)).Error; err != nil { + if err := tx.First(&faceGroup, int(match)).Error; err != nil { return err } - if err := fd.db.Model(&faceGroup).Association("ImageFaces").Append(&imageFace); err != nil { + if err := tx.Model(&faceGroup).Association("ImageFaces").Append(&imageFace); err != nil { return err } } diff --git a/api/scanner/scanner_album.go b/api/scanner/scanner_album.go index aa1fd64..4686181 100644 --- a/api/scanner/scanner_album.go +++ b/api/scanner/scanner_album.go @@ -12,7 +12,7 @@ import ( "github.com/photoview/photoview/api/scanner/face_detection" "github.com/photoview/photoview/api/utils" "github.com/pkg/errors" - "github.com/sabhiram/go-gitignore" + ignore "github.com/sabhiram/go-gitignore" "gorm.io/gorm" ) @@ -96,7 +96,7 @@ func scanAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB) { if media.Type == models.MediaTypePhoto { go func() { - if err := face_detection.GlobalFaceDetector.DetectFaces(media); err != nil { + if err := face_detection.GlobalFaceDetector.DetectFaces(tx, media); err != nil { ScannerError("Error detecting faces in image (%s): %s", media.Path, err) } }() @@ -139,14 +139,14 @@ func findMediaForAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.D } // Get ignore data - albumIgnore := ignore.CompileIgnoreLines(*cache.GetAlbumIgnore(album.Path)...) + albumIgnore := ignore.CompileIgnoreLines(*cache.GetAlbumIgnore(album.Path)...) for _, item := range dirContent { photoPath := path.Join(album.Path, item.Name()) if !item.IsDir() && isPathMedia(photoPath, cache) { // Match file against ignore data - if (albumIgnore.MatchesPath(item.Name())) { + if albumIgnore.MatchesPath(item.Name()) { log.Printf("File %s ignored\n", item.Name()) continue } From a14b12b8d49abc20e4308ceb0ba92815db671701 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Tue, 16 Mar 2021 22:27:27 +0100 Subject: [PATCH 02/22] Fix bug where DetectFaces would be called with the wrong media. This happend because the go routine did not capture the media variable, and so the it would change before the go routine could start and call the DetectFaces function. --- api/graphql/resolvers/user.go | 21 ++++++++++---------- api/scanner/face_detection/face_detector.go | 14 ++++++------- api/scanner/scanner_album.go | 22 +++++++++++---------- 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/api/graphql/resolvers/user.go b/api/graphql/resolvers/user.go index cd0cbee..a48bbb8 100644 --- a/api/graphql/resolvers/user.go +++ b/api/graphql/resolvers/user.go @@ -321,7 +321,7 @@ func (r *mutationResolver) UserRemoveRootAlbum(ctx context.Context, userID int, var deletedAlbumIDs []int = nil - err := r.Database.Transaction(func(tx *gorm.DB) error { + transactionError := r.Database.Transaction(func(tx *gorm.DB) error { if err := tx.Raw("DELETE FROM user_albums WHERE user_id = ? AND album_id = ?", userID, albumID).Error; err != nil { return err } @@ -346,12 +346,12 @@ func (r *mutationResolver) UserRemoveRootAlbum(ctx context.Context, userID int, } // Cleanup if no user owns the album anymore - var count int - if err := tx.Raw("SELECT COUNT(user_id) FROM user_albums WHERE album_id = ?", albumID).Scan(&count).Error; err != nil { + var userAlbumCount int + if err := tx.Raw("SELECT COUNT(user_id) FROM user_albums WHERE album_id = ?", albumID).Scan(&userAlbumCount).Error; err != nil { return err } - if count == 0 { + if userAlbumCount == 0 { deletedAlbumIDs = append(childAlbumIDs, albumID) childAlbumIDs = nil @@ -360,17 +360,13 @@ func (r *mutationResolver) UserRemoveRootAlbum(ctx context.Context, userID int, deletedAlbumIDs = nil return err } - - if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(tx); err != nil { - return err - } } return nil }) - if err != nil { - return nil, err + if transactionError != nil { + return nil, transactionError } if deletedAlbumIDs != nil { @@ -382,6 +378,11 @@ func (r *mutationResolver) UserRemoveRootAlbum(ctx context.Context, userID int, return nil, err } } + + // Reload faces as media might have been deleted + if err := face_detection.GlobalFaceDetector.ReloadFacesFromDatabase(r.Database); err != nil { + return nil, err + } } return &album, nil diff --git a/api/scanner/face_detection/face_detector.go b/api/scanner/face_detection/face_detector.go index 9fdce00..09e44a1 100644 --- a/api/scanner/face_detection/face_detector.go +++ b/api/scanner/face_detection/face_detector.go @@ -84,8 +84,8 @@ func (fd *FaceDetector) ReloadFacesFromDatabase(db *gorm.DB) error { } // DetectFaces finds the faces in the given image and saves them to the database -func (fd *FaceDetector) DetectFaces(tx *gorm.DB, media *models.Media) error { - if err := tx.Model(media).Preload("MediaURL").First(&media).Error; err != nil { +func (fd *FaceDetector) DetectFaces(db *gorm.DB, media *models.Media) error { + if err := db.Model(media).Preload("MediaURL").First(&media).Error; err != nil { return err } @@ -116,7 +116,7 @@ func (fd *FaceDetector) DetectFaces(tx *gorm.DB, media *models.Media) error { } for _, face := range faces { - fd.classifyFace(tx, &face, media, thumbnailPath) + fd.classifyFace(db, &face, media, thumbnailPath) } return nil @@ -126,7 +126,7 @@ func (fd *FaceDetector) classifyDescriptor(descriptor face.Descriptor) int32 { return int32(fd.rec.ClassifyThreshold(descriptor, 0.2)) } -func (fd *FaceDetector) classifyFace(tx *gorm.DB, face *face.Face, media *models.Media, imagePath string) error { +func (fd *FaceDetector) classifyFace(db *gorm.DB, face *face.Face, media *models.Media, imagePath string) error { fd.mutex.Lock() defer fd.mutex.Unlock() @@ -153,18 +153,18 @@ func (fd *FaceDetector) classifyFace(tx *gorm.DB, face *face.Face, media *models ImageFaces: []models.ImageFace{imageFace}, } - if err := tx.Create(&faceGroup).Error; err != nil { + if err := db.Create(&faceGroup).Error; err != nil { return err } } else { log.Println("Found match") - if err := tx.First(&faceGroup, int(match)).Error; err != nil { + if err := db.First(&faceGroup, int(match)).Error; err != nil { return err } - if err := tx.Model(&faceGroup).Association("ImageFaces").Append(&imageFace); err != nil { + if err := db.Model(&faceGroup).Association("ImageFaces").Append(&imageFace); err != nil { return err } } diff --git a/api/scanner/scanner_album.go b/api/scanner/scanner_album.go index 4686181..9e89b3a 100644 --- a/api/scanner/scanner_album.go +++ b/api/scanner/scanner_album.go @@ -75,10 +75,11 @@ func scanAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB) { album_has_changes := false for count, media := range albumMedia { - // tx, err := db.Begin() + processing_was_needed := false transactionError := db.Transaction(func(tx *gorm.DB) error { - processing_was_needed, err := ProcessMedia(tx, media) + log.Printf("Process media transaction enter (%s)", media.Path) + processing_was_needed, err = ProcessMedia(tx, media) if err != nil { return errors.Wrapf(err, "failed to process photo (%s)", media.Path) } @@ -93,22 +94,23 @@ func scanAlbum(album *models.Album, cache *AlbumScannerCache, db *gorm.DB) { Content: fmt.Sprintf("Processed media at %s", media.Path), Progress: &progress, }) - - if media.Type == models.MediaTypePhoto { - go func() { - if err := face_detection.GlobalFaceDetector.DetectFaces(tx, media); err != nil { - ScannerError("Error detecting faces in image (%s): %s", media.Path, err) - } - }() - } } + log.Printf("Process media transaction exit (%s)", media.Path) return nil }) if transactionError != nil { ScannerError("Failed to begin database transaction: %s", transactionError) } + + if processing_was_needed && media.Type == models.MediaTypePhoto { + go func(media *models.Media) { + if err := face_detection.GlobalFaceDetector.DetectFaces(db, media); err != nil { + ScannerError("Error detecting faces in image (%s): %s", media.Path, err) + } + }(media) + } } cleanup_errors := CleanupMedia(db, album.ID, albumMedia) From b590413e7672228b7b2312581c851c969963471d Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Tue, 23 Mar 2021 14:04:44 +0100 Subject: [PATCH 03/22] Fix share media not working with postgres This closes #265 --- api/graphql/resolvers/share_token.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/graphql/resolvers/share_token.go b/api/graphql/resolvers/share_token.go index 6817d01..524f7b2 100644 --- a/api/graphql/resolvers/share_token.go +++ b/api/graphql/resolvers/share_token.go @@ -147,8 +147,15 @@ func (r *mutationResolver) ShareMedia(ctx context.Context, mediaID int, expire * var media models.Media + var query string + if r.Database.Dialector.Name() == "postgres" { + query = "EXISTS (SELECT * FROM user_albums WHERE user_albums.album_id = \"Album\".id AND user_albums.user_id = ?)" + } else { + query = "EXISTS (SELECT * FROM user_albums WHERE user_albums.album_id = Album.id AND user_albums.user_id = ?)" + } + err := r.Database.Joins("Album"). - Where("EXISTS (SELECT * FROM user_albums WHERE user_albums.album_id = Album.id AND user_albums.user_id = ?)", user.ID). + Where(query, user.ID). First(&media, mediaID). Error From bb03cb1631f3dcb6f4241bd09b1d281598757246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Fri, 26 Mar 2021 11:20:41 +0100 Subject: [PATCH 04/22] Add sponsor section to readme (#270) --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 83cfcf1..eabc322 100644 --- a/README.md +++ b/README.md @@ -139,3 +139,10 @@ cd ./ui && npm start The site can now be accessed at [localhost:1234](http://localhost:1234). And the graphql playground at [localhost:4001](http://localhost:4001) + +# Sponsors + + +
+ @ericerkz +
From 4344e895d18cb637eb94dcf739c7976ce96529da Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Sat, 27 Mar 2021 21:14:42 +0100 Subject: [PATCH 05/22] Support .heic images + refactor EncodeMediaData --- api/go.mod | 1 + api/go.sum | 2 + api/scanner/encode_photo.go | 101 ++++++++++++++++++++--------------- api/scanner/media_type.go | 3 ++ api/scanner/process_photo.go | 4 +- 5 files changed, 66 insertions(+), 45 deletions(-) diff --git a/api/go.mod b/api/go.mod index 6aa9063..32895c1 100644 --- a/api/go.mod +++ b/api/go.mod @@ -14,6 +14,7 @@ require ( github.com/joho/godotenv v1.3.0 github.com/pkg/errors v0.9.1 github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f + github.com/strukturag/libheif v1.11.0 // indirect github.com/vektah/gqlparser/v2 v2.1.0 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 github.com/xor-gate/goexif2 v1.1.0 diff --git a/api/go.sum b/api/go.sum index 6bc5e98..f7bfa58 100644 --- a/api/go.sum +++ b/api/go.sum @@ -198,6 +198,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/strukturag/libheif v1.11.0 h1:HaWu5re98INSXNq7C8o5AwLcv2qD8+U7a+jVCpGWemI= +github.com/strukturag/libheif v1.11.0/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e h1:+w0Zm/9gaWpEAyDlU1eKOuk5twTjAjuevXqcJJw8hrg= diff --git a/api/scanner/encode_photo.go b/api/scanner/encode_photo.go index faf9e0b..6b1f945 100644 --- a/api/scanner/encode_photo.go +++ b/api/scanner/encode_photo.go @@ -11,34 +11,29 @@ import ( "github.com/photoview/photoview/api/utils" "github.com/pkg/errors" "gopkg.in/vansante/go-ffprobe.v2" - "gorm.io/gorm" + + _ "github.com/strukturag/libheif/go/heif" ) -func DecodeImage(imagePath string) (image.Image, error) { - file, err := os.Open(imagePath) - if err != nil { - return nil, errors.Wrapf(err, "failed to open file to decode image (%s)", imagePath) - } - defer file.Close() +func EncodeThumbnail(inputPath string, outputPath string) (*image_helpers.PhotoDimensions, error) { - image, err := imaging.Decode(file, imaging.AutoOrientation(true)) + inputImage, err := imaging.Open(inputPath, imaging.AutoOrientation(true)) if err != nil { - return nil, errors.Wrapf(err, "failed to decode image (%s)", imagePath) + return nil, err } - return image, nil + dimensions := image_helpers.PhotoDimensionsFromRect(inputImage.Bounds()) + dimensions = dimensions.ThumbnailScale() + + thumbImage := imaging.Resize(inputImage, dimensions.Width, dimensions.Height, imaging.NearestNeighbor) + if err = encodeImageJPEG(thumbImage, outputPath, 60); err != nil { + return nil, err + } + + return &dimensions, nil } -// EncodeMediaData is used to easily decode media data, with a cache so expensive operations are not repeated -type EncodeMediaData struct { - media *models.Media - _photoImage image.Image - _thumbnailImage image.Image - _contentType *MediaType - _videoMetadata *ffprobe.ProbeData -} - -func EncodeImageJPEG(image image.Image, outputPath string, jpegQuality int) error { +func encodeImageJPEG(image image.Image, outputPath string, jpegQuality int) error { photo_file, err := os.Create(outputPath) if err != nil { return errors.Wrapf(err, "could not create file: %s", outputPath) @@ -53,6 +48,14 @@ func EncodeImageJPEG(image image.Image, outputPath string, jpegQuality int) erro return nil } +// EncodeMediaData is used to easily decode media data, with a cache so expensive operations are not repeated +type EncodeMediaData struct { + media *models.Media + _photoImage image.Image + _contentType *MediaType + _videoMetadata *ffprobe.ProbeData +} + // ContentType reads the image to determine its content type func (img *EncodeMediaData) ContentType() (*MediaType, error) { if img._contentType != nil { @@ -68,7 +71,7 @@ func (img *EncodeMediaData) ContentType() (*MediaType, error) { return imgType, nil } -func (img *EncodeMediaData) EncodeHighRes(tx *gorm.DB, outputPath string) error { +func (img *EncodeMediaData) EncodeHighRes(outputPath string) error { contentType, err := img.ContentType() if err != nil { return err @@ -89,36 +92,19 @@ func (img *EncodeMediaData) EncodeHighRes(tx *gorm.DB, outputPath string) error return errors.New("could not convert photo as no RAW converter was found") } } else { - image, err := img.photoImage(tx) + image, err := img.photoImage() if err != nil { return err } - EncodeImageJPEG(image, outputPath, 70) + encodeImageJPEG(image, outputPath, 70) } return nil } -func EncodeThumbnail(inputPath string, outputPath string) (*image_helpers.PhotoDimensions, error) { - inputImage, err := DecodeImage(inputPath) - if err != nil { - return nil, err - } - - dimensions := image_helpers.PhotoDimensionsFromRect(inputImage.Bounds()) - dimensions = dimensions.ThumbnailScale() - - thumbImage := imaging.Resize(inputImage, dimensions.Width, dimensions.Height, imaging.NearestNeighbor) - if err = EncodeImageJPEG(thumbImage, outputPath, 60); err != nil { - return nil, err - } - - return &dimensions, nil -} - -// PhotoImage reads and decodes the image file and saves it in a cache so the photo in only decoded once -func (img *EncodeMediaData) photoImage(tx *gorm.DB) (image.Image, error) { +// photoImage reads and decodes the image file and saves it in a cache so the photo in only decoded once +func (img *EncodeMediaData) photoImage() (image.Image, error) { if img._photoImage != nil { return img._photoImage, nil } @@ -130,7 +116,7 @@ func (img *EncodeMediaData) photoImage(tx *gorm.DB) (image.Image, error) { photoPath = img.media.Path } - photoImg, err := DecodeImage(photoPath) + photoImg, err := img.decodeImage(photoPath) if err != nil { return nil, utils.HandleError("image decoding", err) } @@ -138,3 +124,32 @@ func (img *EncodeMediaData) photoImage(tx *gorm.DB) (image.Image, error) { img._photoImage = photoImg return img._photoImage, nil } + +func (img *EncodeMediaData) decodeImage(imagePath string) (image.Image, error) { + file, err := os.Open(imagePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to open file to decode image (%s)", imagePath) + } + defer file.Close() + + mediaType, err := img.ContentType() + if err != nil { + return nil, errors.Wrapf(err, "failed to get media content type needed to decode it (%s)", imagePath) + } + + var decodedImage image.Image + + if *mediaType == TypeHeic { + decodedImage, _, err = image.Decode(file) + if err != nil { + return nil, errors.Wrapf(err, "failed to decode HEIF image (%s)", imagePath) + } + } else { + decodedImage, err = imaging.Decode(file, imaging.AutoOrientation(true)) + if err != nil { + return nil, errors.Wrapf(err, "failed to decode image (%s)", imagePath) + } + } + + return decodedImage, nil +} diff --git a/api/scanner/media_type.go b/api/scanner/media_type.go index a1dfce6..60d0e4e 100644 --- a/api/scanner/media_type.go +++ b/api/scanner/media_type.go @@ -20,6 +20,7 @@ const ( TypeTiff MediaType = "image/tiff" TypeWebp MediaType = "image/webp" TypeBmp MediaType = "image/bmp" + TypeHeic MediaType = "image/heic" // Raw formats TypeDNG MediaType = "image/x-adobe-dng" @@ -73,6 +74,7 @@ var SupportedMimetypes = [...]MediaType{ TypeTiff, TypeWebp, TypeBmp, + TypeHeic, } var WebMimetypes = [...]MediaType{ @@ -144,6 +146,7 @@ var fileExtensions = map[string]MediaType{ ".tif": TypeTiff, ".tiff": TypeTiff, ".bmp": TypeBmp, + ".heic": TypeHeic, // RAW formats ".dng": TypeDNG, diff --git a/api/scanner/process_photo.go b/api/scanner/process_photo.go index 389d25c..47baa03 100644 --- a/api/scanner/process_photo.go +++ b/api/scanner/process_photo.go @@ -149,7 +149,7 @@ func processPhoto(tx *gorm.DB, imageData *EncodeMediaData, photoCachePath *strin fmt.Printf("High-res photo found in database but not in cache, re-encoding photo to cache: %s\n", highResURL.MediaName) didProcess = true - err = imageData.EncodeHighRes(tx, baseImagePath) + err = imageData.EncodeHighRes(baseImagePath) if err != nil { return false, errors.Wrap(err, "creating high-res cached image") } @@ -275,7 +275,7 @@ func saveOriginalPhotoToDB(tx *gorm.DB, photo *models.Media, imageData *EncodeMe func generateSaveHighResJPEG(tx *gorm.DB, media *models.Media, imageData *EncodeMediaData, highres_name string, imagePath string, mediaURL *models.MediaURL) (*models.MediaURL, error) { - err := imageData.EncodeHighRes(tx, imagePath) + err := imageData.EncodeHighRes(imagePath) if err != nil { return nil, errors.Wrap(err, "creating high-res cached image") } From a039941dda3548eec28e5be111acfea374a4c9bf Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Sat, 27 Mar 2021 22:30:26 +0100 Subject: [PATCH 06/22] Add libheif to Dockerfile --- Dockerfile | 13 +++++++++++-- README.md | 3 ++- api/go.mod | 2 +- api/go.sum | 52 ---------------------------------------------------- 4 files changed, 14 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb0d2a5..d87bc66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,9 +22,12 @@ RUN npm run build -- --public-url $UI_PUBLIC_URL ### Build API ### FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.15-buster AS api +# Use latest unstable packages for libheif +RUN echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list.d/sources.list + # Install G++/GCC cross compilers RUN dpkg --add-architecture arm64 && dpkg --add-architecture armhf -RUN apt-get update && apt-get install -y \ +RUN apt-get update && apt-get install -t buster -y \ g++-aarch64-linux-gnu \ libc6-dev-arm64-cross \ g++-arm-linux-gnueabihf \ @@ -34,6 +37,8 @@ RUN apt-get update && apt-get install -y \ libblas-dev libblas-dev:arm64 libblas-dev:armhf \ liblapack-dev liblapack-dev:arm64 liblapack-dev:armhf \ libjpeg62-turbo-dev libjpeg62-turbo-dev:arm64 libjpeg62-turbo-dev:armhf \ + # Install libheif for HEIF media decoding + && apt-get install -t bullseye -y libheif-dev/bullseye libheif-dev:arm64/bullseye libheif-dev:armhf/bullseye \ # Cleanup && apt-get clean && rm -rf /var/lib/apt/lists/* @@ -71,9 +76,13 @@ WORKDIR /app COPY api/data /app/data +# Use latest unstable packages for libheif +RUN echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list.d/sources.list + RUN apt-get update \ # Required dependencies - && apt-get install -y curl gpg libdlib19 ffmpeg + && apt-get install -t buster -y curl gpg libdlib19 ffmpeg \ + && apt-get install -t bullseye -y libheif1 # Install Darktable if building for a supported architecture RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ] || [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ diff --git a/README.md b/README.md index eabc322..1c6a545 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Password: **demo** - [Why yet another self-hosted photo gallery](#why-yet-another-self-hosted-photo-gallery) - [Getting started - Setup with Docker](#getting-started---setup-with-docker) - [Setup development environment](#setup-development-environment) +- [Sponsors](#sponsors) ## Main features @@ -140,7 +141,7 @@ cd ./ui && npm start The site can now be accessed at [localhost:1234](http://localhost:1234). And the graphql playground at [localhost:4001](http://localhost:4001) -# Sponsors +## Sponsors
diff --git a/api/go.mod b/api/go.mod index 32895c1..af9dd63 100644 --- a/api/go.mod +++ b/api/go.mod @@ -14,7 +14,7 @@ require ( github.com/joho/godotenv v1.3.0 github.com/pkg/errors v0.9.1 github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f - github.com/strukturag/libheif v1.11.0 // indirect + github.com/strukturag/libheif v1.11.0 github.com/vektah/gqlparser/v2 v2.1.0 github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 github.com/xor-gate/goexif2 v1.1.0 diff --git a/api/go.sum b/api/go.sum index f7bfa58..dd36b09 100644 --- a/api/go.sum +++ b/api/go.sum @@ -1,6 +1,5 @@ github.com/99designs/gqlgen v0.13.0 h1:haLTcUp3Vwp80xMVEg5KRNwzfUrgFdRmtBY8fuB8scA= github.com/99designs/gqlgen v0.13.0/go.mod h1:NV130r6f4tpRWuAI+zsrSdooO/eWUv+Gyyoi3rEfXIk= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kagami/go-face v0.0.0-20200825065730-3dd2d74dccfb h1:DXwA1Te9paM+nsdTGc7uve37lq7WEbQO+gwGBPVwQuQ= github.com/Kagami/go-face v0.0.0-20200825065730-3dd2d74dccfb/go.mod h1:9wdDJkRgo3SGTcFwbQ7elVIQhIr2bbBjecuY7VoqmPU= @@ -14,11 +13,8 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -29,19 +25,14 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/go-chi/chi v3.3.2+incompatible h1:uQNcQN3NsV1j4ANsPh42P4ew4t6rnRbJb8frvpp31qQ= github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/protobuf v1.0.0 h1:2jyBKDKU/8v3v2xVR2PtiWQviFUyiaGk2rpfyFT8rTM= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f h1:9oNbS1z4rVpbnkHBdPZU4jo9bSmrLpII768arSyMFgk= github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= @@ -105,7 +96,6 @@ github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= @@ -113,83 +103,61 @@ github.com/jinzhu/now v1.1.1 h1:g39TucaRWyV3dwDO++eEc6qf8TVIQ/Da48WmqjZ3i7E= github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007 h1:reVOUXwnhsYv/8UqjvhrMOu5CNT9UapHFLbQ2JcXsmg= github.com/matryer/moq v0.0.0-20200106131100-75d0ddfc0007/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047 h1:zCoDWFD5nrJJVjbXiDZcVhOBSzKn3o9LgRLLMRNuru8= github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rs/cors v1.6.0 h1:G9tHG9lebljV9mfp9SNPDL36nCDxmo3zTlAf1YgvzmI= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= -github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f h1:8P2MkG70G76gnZBOPGwmMIgwBb/rESQuwsJ7K8ds4NE= github.com/sabhiram/go-gitignore v0.0.0-20201211210132-54b8a0bf510f/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc h1:jUIKcSPO9MoMJBbEoyE/RJoE8vz7Mb8AjvifMMwSyvY= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371 h1:SWV2fHctRpRrp49VXJ6UZja7gU9QLHwRpIPBN89SKEo= github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/vfsgen v0.0.0-20180121065927-ffb13db8def0 h1:JJV9CsgM9EC9w2iVkwuz+sMx8yRFe89PJRUrv6hPCIA= github.com/shurcooL/vfsgen v0.0.0-20180121065927-ffb13db8def0/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -200,9 +168,7 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/strukturag/libheif v1.11.0 h1:HaWu5re98INSXNq7C8o5AwLcv2qD8+U7a+jVCpGWemI= github.com/strukturag/libheif v1.11.0/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= -github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e h1:+w0Zm/9gaWpEAyDlU1eKOuk5twTjAjuevXqcJJw8hrg= github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U= github.com/vektah/gqlparser/v2 v2.1.0 h1:uiKJ+T5HMGGQM2kRKQ8Pxw8+Zq9qhhZhz/lieYvCMns= github.com/vektah/gqlparser/v2 v2.1.0/go.mod h1:SyUiHgLATUR8BiYURfTirrTcGpcE+4XkV2se04Px1Ms= @@ -210,19 +176,14 @@ github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlV github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= github.com/xor-gate/goexif2 v1.1.0 h1:OvTZ5iEvsDhRWFjV5xY3wT7uHFna28nSSP7ucau+cXQ= github.com/xor-gate/goexif2 v1.1.0/go.mod h1:eRjn3VSkAwpNpxEx/CGmd0zg0JFGL3akrSMxnJ581AY= -github.com/zenazn/goji v0.9.0 h1:RSQQAbXGArQ0dIDEq+PI6WqN6if+5KHu6x2Cx/GXLTQ= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= @@ -237,17 +198,13 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk= golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee h1:WG0RUwxtNT4qqaXX3DPA8zHFNm/D9xaBpxzHt1WcA/E= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -259,9 +216,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -276,7 +231,6 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589 h1:rjUrONFu4kLchcZTfp3/96bR8bW8dIa8uz3cR5n0cgM= golang.org/x/tools v0.0.0-20200114235610-7ae403b6b589/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -286,11 +240,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec h1:RlWgLqCMMIYYEVcAR5MDsuHlVkaIPDAF+5Dehzg8L5A= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/vansante/go-ffprobe.v2 v2.0.2 h1:DdxSfFnlqeawPIVbIQEI6LR6OQHQNR7tNgWb2mWuC4w= gopkg.in/vansante/go-ffprobe.v2 v2.0.2/go.mod h1:qF0AlAjk7Nqzqf3y333Ly+KxN3cKF2JqA3JT5ZheUGE= @@ -308,9 +259,6 @@ gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9D gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.20.12 h1:ebZ5KrSHzet+sqOCVdH9mTjW91L298nX3v5lVxAzSUY= gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755 h1:d2maSb13hr/ArmfK3rW+wNUKKfytCol7W1/vDHxMPiE= sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 h1:e1sMhtVq9AfcEy8AXNb8eSg6gbzfdpYhoNqnPJa+GzI= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= From 1cd94eab5f9f76fce4109dc59c7f06e249960bb2 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Sat, 27 Mar 2021 22:36:21 +0100 Subject: [PATCH 07/22] Add libheif-dev dependenc to gh-actions api tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 295cf40..3e00829 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: - name: Get C dependencies run: | sudo apt-get update - sudo apt-get install -y libdlib-dev libblas-dev liblapack-dev libjpeg-turbo8-dev + sudo apt-get install -y libdlib-dev libblas-dev liblapack-dev libjpeg-turbo8-dev libheif-dev - name: Get GO dependencies run: | From 93d6962a3568289ecf398f78c77a12cc7d066c08 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Sat, 27 Mar 2021 23:15:14 +0100 Subject: [PATCH 08/22] Add libheif ppa to ghactions API tests --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e00829..2d199d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,6 +27,8 @@ jobs: - name: Get C dependencies run: | + sudo add-apt-repository ppa:strukturag/libheif + sudo add-apt-repository ppa:strukturag/libde265 sudo apt-get update sudo apt-get install -y libdlib-dev libblas-dev liblapack-dev libjpeg-turbo8-dev libheif-dev From b59ba20bd1813c514975cc2c1e2e217cf7c9f460 Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Sun, 28 Mar 2021 13:57:57 +0200 Subject: [PATCH 09/22] Refactor Dockerfile - Use debian:bullseye instad for newer packages - Only install necessary dependencies for targeted arch - Make it work with libheif --- Dockerfile | 41 ++++++---------------------- api/go.sum | 2 ++ docker/go_wrapper.sh | 2 +- docker/install_build_dependencies.sh | 38 ++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 34 deletions(-) create mode 100644 docker/install_build_dependencies.sh diff --git a/Dockerfile b/Dockerfile index d87bc66..126880b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,34 +20,17 @@ COPY ui /app RUN npm run build -- --public-url $UI_PUBLIC_URL ### Build API ### -FROM --platform=${BUILDPLATFORM:-linux/amd64} golang:1.15-buster AS api +FROM --platform=${BUILDPLATFORM:-linux/amd64} debian:bullseye-slim AS api +ARG TARGETPLATFORM -# Use latest unstable packages for libheif -RUN echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list.d/sources.list - -# Install G++/GCC cross compilers -RUN dpkg --add-architecture arm64 && dpkg --add-architecture armhf -RUN apt-get update && apt-get install -t buster -y \ - g++-aarch64-linux-gnu \ - libc6-dev-arm64-cross \ - g++-arm-linux-gnueabihf \ - libc6-dev-armhf-cross \ - # Install go-face dependencies - libdlib-dev libdlib-dev:arm64 libdlib-dev:armhf \ - libblas-dev libblas-dev:arm64 libblas-dev:armhf \ - liblapack-dev liblapack-dev:arm64 liblapack-dev:armhf \ - libjpeg62-turbo-dev libjpeg62-turbo-dev:arm64 libjpeg62-turbo-dev:armhf \ - # Install libheif for HEIF media decoding - && apt-get install -t bullseye -y libheif-dev/bullseye libheif-dev:arm64/bullseye libheif-dev:armhf/bullseye \ - # Cleanup - && apt-get clean && rm -rf /var/lib/apt/lists/* +COPY docker/install_build_dependencies.sh /tmp/ +RUN chmod +x /tmp/install_build_dependencies.sh && /tmp/install_build_dependencies.sh COPY docker/go_wrapper.sh /go/bin/go RUN chmod +x /go/bin/go ENV CGO_ENABLED 1 -ARG TARGETPLATFORM RUN go env RUN mkdir -p /app @@ -58,7 +41,7 @@ COPY api/go.mod api/go.sum /app/ RUN go mod download # Patch go-face -RUN sed -i 's/-march=native//g' /go/pkg/mod/github.com/!kagami/go-face*/face.go +RUN sed -i 's/-march=native//g' /root/go/pkg/mod/github.com/!kagami/go-face*/face.go # Build dependencies that use CGO RUN go install \ @@ -70,27 +53,19 @@ COPY api /app RUN go build -v -o photoview . ### Copy api and ui to production environment ### -FROM debian:buster +FROM debian:bullseye-slim ARG TARGETPLATFORM WORKDIR /app COPY api/data /app/data -# Use latest unstable packages for libheif -RUN echo 'deb http://deb.debian.org/debian bullseye main' >> /etc/apt/sources.list.d/sources.list - RUN apt-get update \ # Required dependencies - && apt-get install -t buster -y curl gpg libdlib19 ffmpeg \ - && apt-get install -t bullseye -y libheif1 + && apt-get install -y curl gpg libdlib19 ffmpeg libheif1 # Install Darktable if building for a supported architecture RUN if [ "${TARGETPLATFORM}" = "linux/amd64" ] || [ "${TARGETPLATFORM}" = "linux/arm64" ]; then \ - echo 'deb http://download.opensuse.org/repositories/graphics:/darktable/Debian_10/ /' | tee /etc/apt/sources.list.d/graphics:darktable.list \ - && curl -fsSL https://download.opensuse.org/repositories/graphics:darktable/Debian_10/Release.key | gpg --dearmor | tee /etc/apt/trusted.gpg.d/graphics_darktable.gpg > /dev/null \ - && apt-get update \ - && apt-get install -y darktable; \ - fi + apt-get install -y darktable; fi # Remove build dependencies and cleanup RUN apt-get purge -y curl gpg \ diff --git a/api/go.sum b/api/go.sum index dd36b09..0c317b7 100644 --- a/api/go.sum +++ b/api/go.sum @@ -166,6 +166,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/strukturag/libheif v1.3.2 h1:NXft6czPSJiBeGc6hchNgczIY9aRPxsIcNIlVlDm2IY= +github.com/strukturag/libheif v1.3.2/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= github.com/strukturag/libheif v1.11.0 h1:HaWu5re98INSXNq7C8o5AwLcv2qD8+U7a+jVCpGWemI= github.com/strukturag/libheif v1.11.0/go.mod h1:E/PNRlmVtrtj9j2AvBZlrO4dsBDu6KfwDZn7X1Ce8Ks= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= diff --git a/docker/go_wrapper.sh b/docker/go_wrapper.sh index 4526ecc..e388755 100644 --- a/docker/go_wrapper.sh +++ b/docker/go_wrapper.sh @@ -104,4 +104,4 @@ if [ -z "$GOBIN" ] && [ -n "$GOPATH" ] && [ -n "$GOARCH" ] && [ -n "$GOOS" ]; th export PATH=${GOPATH}/bin/${GOOS}_${GOARCH}:${PATH} fi -exec /usr/local/go/bin/go "$@" +exec /usr/bin/go "$@" diff --git a/docker/install_build_dependencies.sh b/docker/install_build_dependencies.sh new file mode 100644 index 0000000..0b71d39 --- /dev/null +++ b/docker/install_build_dependencies.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +if [ "$TARGETPLATFORM" == "linux/arm64" ]; then + dpkg --add-architecture arm64 + DEBIAN_ARCH='arm64' +elif [ "$TARGETPLATFORM" == "linux/arm/v6" ] || [ "$TARGETPLATFORM" == "linux/arm/v7" ]; then + dpkg --add-architecture armhf + DEBIAN_ARCH='armhf' +else + DEBIAN_ARCH='amd64' +fi + +apt-get update + +# Install Golang +apt-get install -y ca-certificates golang + +# Install G++/GCC cross compilers +apt-get install -y \ + g++-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + g++-arm-linux-gnueabihf \ + libc6-dev-armhf-cross + +# Install go-face dependencies +apt-get install -y \ + libdlib-dev:$DEBIAN_ARCH \ + libblas-dev:$DEBIAN_ARCH \ + liblapack-dev:$DEBIAN_ARCH \ + libjpeg62-turbo-dev:$DEBIAN_ARCH \ + +# Install libheif for HEIF media decoding +apt-get install -y \ + libheif-dev:$DEBIAN_ARCH + +# Cleanup +apt-get clean +rm -rf /var/lib/apt/lists/* From ec2a6e4dedd30778f0e3327b4f90c507f617da6c Mon Sep 17 00:00:00 2001 From: viktorstrate Date: Sun, 28 Mar 2021 18:43:58 +0200 Subject: [PATCH 10/22] Remove cache restore key --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 754d18b..f80b38d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,8 +27,6 @@ jobs: with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - name: Set up QEMU uses: docker/setup-qemu-action@v1 From e29beea1ffe718f8218b4def0bafbeb88c8381be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 19:27:25 +0200 Subject: [PATCH 11/22] Add secret to cache key To make it possible to invalidate cache if needed --- .github/workflows/build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f80b38d..474d9a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,9 @@ jobs: id: cache with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} + key: ${{ runner.os }}-buildx-${{ secrets.CACHE_KEY }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-${{ secrets.CACHE_KEY }}- - name: Set up QEMU uses: docker/setup-qemu-action@v1 From 3224cc8ce794b7d3cc7cc9464019c8887bb1b95a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 19:39:22 +0200 Subject: [PATCH 12/22] Use a ghaction matrix in docker build to build each architecture individually --- .github/workflows/build.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 474d9a3..ffea465 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,8 +1,5 @@ name: Docker builds -env: - TARGET_PLATFORMS: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64 - on: schedule: - cron: "0 10 * * *" # everyday at 10am @@ -17,6 +14,15 @@ jobs: build: name: Build and deploy docker images runs-on: ubuntu-20.04 + + strategy: + matrix: + target_platform: + - "linux/amd64" + - "linux/arm/v6" + - "linux/arm/v7" + - "linux/arm64" + steps: - name: Checkout uses: actions/checkout@v2 @@ -26,14 +32,14 @@ jobs: id: cache with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ secrets.CACHE_KEY }}-${{ github.sha }} + key: ${{ runner.os }}-buildx-${{ secrets.CACHE_KEY }}-${{ matrix.target_platform }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-buildx-${{ secrets.CACHE_KEY }}- + ${{ runner.os }}-buildx-${{ secrets.CACHE_KEY }}-${{ matrix.target_platform }}- - name: Set up QEMU uses: docker/setup-qemu-action@v1 with: - platforms: ${{ env.TARGET_PLATFORMS }} + platforms: ${{ matrix.target_platform }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 @@ -43,7 +49,7 @@ jobs: run: | DOCKER_USERNAME=viktorstrate DOCKER_IMAGE=viktorstrate/photoview - DOCKER_PLATFORMS=${TARGET_PLATFORMS} + DOCKER_PLATFORMS=${{ matrix.target_platform }} VERSION=edge if [[ $GITHUB_REF == refs/tags/* ]]; then From 5ec05e54ea9c27c58f980afb092c43cdc8cd3913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 21:24:32 +0200 Subject: [PATCH 13/22] Ghactions save docker images in artifacts --- .github/workflows/build.yml | 47 ++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ffea465..7cfb9f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ on: jobs: build: - name: Build and deploy docker images + name: Build Docker Image runs-on: ubuntu-20.04 strategy: @@ -78,21 +78,41 @@ jobs: --build-arg VCS_REF=${GITHUB_SHA::8} \ ${TAGS} --file Dockerfile . - - name: Build Docker images + - name: Docker Build run: | - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --output "type=oci,dest=/tmp/docker-image-${{ matrix.target_platform }}-${{ github.sha }}.oci" ${{ steps.prepare.outputs.buildx_args }} + + - name: Save Image + uses: actions/upload-artifact@v2 + with: + name: docker-image-${{ matrix.target_platform }} + path: /tmp/docker-image-${{ matrix.target_platform }}-${{ github.sha }}.oci - - name: Docker Login - if: success() && github.event_name != 'pull_request' && github.repository == 'photoview/photoview' - env: - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - run: | - echo "${DOCKER_PASSWORD}" | docker login --username "${{ steps.prepare.outputs.docker_username }}" --password-stdin + push: + name: Push Docker Images + runs-on: ubuntu-20.04 + if: github.event_name != 'pull_request' && github.repository == 'photoview/photoview' + + needs: [build] - - name: Push to Docker Hub - if: success() && github.event_name != 'pull_request' && github.repository == 'photoview/photoview' - run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + steps: + - name: Load Images + uses: actions/download-artifact@v2 + with: + path: /docker-images + + - name: Print Artifacts + run: ls -lah /docker-images + + #- name: Docker Login + # env: + # DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + # run: | + # echo "${DOCKER_PASSWORD}" | docker login --username "${{ steps.prepare.outputs.docker_username }}" --password-stdin + + #- name: Push to Docker Hub + # run: | + # docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} # - name: Docker Check Manifest # if: always() && github.event_name != 'pull_request' && github.repository == 'photoview/photoview' @@ -100,6 +120,5 @@ jobs: # docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} - name: Clear - if: always() && github.event_name != 'pull_request' run: | rm -f ${HOME}/.docker/config.json From d7bdefa42b46a0abc9f321e40dbf09141f4772a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 21:30:20 +0200 Subject: [PATCH 14/22] Add missing docker build output directory --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7cfb9f3..28db343 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,8 @@ jobs: - name: Docker Build run: | - docker buildx build --output "type=oci,dest=/tmp/docker-image-${{ matrix.target_platform }}-${{ github.sha }}.oci" ${{ steps.prepare.outputs.buildx_args }} + mkdir -p /tmp/docker-image-${{ matrix.target_platform }} + docker buildx build --output "type=oci,dest=/tmp/docker-image-${{ matrix.target_platform }}/${{ github.sha }}.oci" ${{ steps.prepare.outputs.buildx_args }} - name: Save Image uses: actions/upload-artifact@v2 From 9d4310412697880ab568ecc19fc1710742ee1c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 21:32:22 +0200 Subject: [PATCH 15/22] Fix path to artifact --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 28db343..8703eb8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,7 +87,7 @@ jobs: uses: actions/upload-artifact@v2 with: name: docker-image-${{ matrix.target_platform }} - path: /tmp/docker-image-${{ matrix.target_platform }}-${{ github.sha }}.oci + path: /tmp/docker-image-${{ matrix.target_platform }}/${{ github.sha }}.oci push: name: Push Docker Images From 41520f912a2e472d3b2428a1e4e56f7e796dbddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 21:46:50 +0200 Subject: [PATCH 16/22] Format artifact name to not contain '/' --- .github/workflows/build.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8703eb8..4ef843f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,17 +77,20 @@ jobs: --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ ${TAGS} --file Dockerfile . + + OUTPUT_PLATFORM=$(echo ${{ matrix.target_platform }} | sed 's/\//-/g') + echo ::set-output name=output_platform::${OUTPUT_PLATFORM} - name: Docker Build run: | mkdir -p /tmp/docker-image-${{ matrix.target_platform }} - docker buildx build --output "type=oci,dest=/tmp/docker-image-${{ matrix.target_platform }}/${{ github.sha }}.oci" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --output "type=oci,dest=/tmp/docker-image-${{ steps.prepare.outputs.output_platform }}-${{ github.sha }}.oci" ${{ steps.prepare.outputs.buildx_args }} - name: Save Image uses: actions/upload-artifact@v2 with: - name: docker-image-${{ matrix.target_platform }} - path: /tmp/docker-image-${{ matrix.target_platform }}/${{ github.sha }}.oci + name: docker-image-${{ steps.prepare.outputs.output_platform }} + path: /tmp/docker-image-${{ steps.prepare.outputs.output_platform }}-${{ github.sha }}.oci push: name: Push Docker Images From 1543f74f6ccd996e612b376971d064adcc5af00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 21:52:26 +0200 Subject: [PATCH 17/22] Update load images step --- .github/workflows/build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4ef843f..e418b7b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -102,11 +102,9 @@ jobs: steps: - name: Load Images uses: actions/download-artifact@v2 - with: - path: /docker-images - name: Print Artifacts - run: ls -lah /docker-images + run: ls -R #- name: Docker Login # env: From cf36b9ec8d38445789d90a4bfd1993d725bdecdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 23:03:42 +0200 Subject: [PATCH 18/22] Push individual images, combine afterwards --- .github/workflows/build.yml | 88 +++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 47 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e418b7b..a291344 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,6 +9,11 @@ on: branches: master tags: - v* + +env: + DOCKER_USERNAME: viktorstrate + DOCKER_IMAGE: viktorstrate/photoview + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} jobs: build: @@ -19,9 +24,9 @@ jobs: matrix: target_platform: - "linux/amd64" - - "linux/arm/v6" - - "linux/arm/v7" - "linux/arm64" + - "linux/arm/v7" + - "linux/arm/v6" steps: - name: Checkout @@ -47,8 +52,6 @@ jobs: - name: Prepare id: prepare run: | - DOCKER_USERNAME=viktorstrate - DOCKER_IMAGE=viktorstrate/photoview DOCKER_PLATFORMS=${{ matrix.target_platform }} VERSION=edge @@ -58,13 +61,11 @@ jobs: if [ "${{ github.event_name }}" = "schedule" ]; then VERSION=nightly fi - - TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION =~ ^(([0-9]{1,3})\.[0-9]{1,3})\.[0-9]{1,3}$ ]]; then - VERSION_MINOR=${BASH_REMATCH[1]} - VERSION_MAJOR=${BASH_REMATCH[2]} - TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest --tag ${DOCKER_IMAGE}:${VERSION_MINOR} --tag ${DOCKER_IMAGE}:${VERSION_MAJOR}" - fi + + OUTPUT_PLATFORM=$(echo ${{ matrix.target_platform }} | sed 's/\//-/g') + echo ::set-output name=output_platform::${OUTPUT_PLATFORM} + + TAG="--tag ${OUTPUT_PLATFORM}-${GITHUB_SHA::8}" echo ::set-output name=docker_username::${DOCKER_USERNAME} echo ::set-output name=docker_image::${DOCKER_IMAGE} @@ -76,51 +77,44 @@ jobs: --build-arg VERSION=${VERSION} \ --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ --build-arg VCS_REF=${GITHUB_SHA::8} \ - ${TAGS} --file Dockerfile . - - OUTPUT_PLATFORM=$(echo ${{ matrix.target_platform }} | sed 's/\//-/g') - echo ::set-output name=output_platform::${OUTPUT_PLATFORM} + ${TAG} --file Dockerfile . - name: Docker Build run: | mkdir -p /tmp/docker-image-${{ matrix.target_platform }} - docker buildx build --output "type=oci,dest=/tmp/docker-image-${{ steps.prepare.outputs.output_platform }}-${{ github.sha }}.oci" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args - - name: Save Image - uses: actions/upload-artifact@v2 - with: - name: docker-image-${{ steps.prepare.outputs.output_platform }} - path: /tmp/docker-image-${{ steps.prepare.outputs.output_platform }}-${{ github.sha }}.oci + - name: Docker Login + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${{ steps.prepare.outputs.docker_username }}" --password-stdin - push: - name: Push Docker Images + - name: Push to Docker Hub + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + + - name: Clear + run: | + rm -f ${HOME}/.docker/config.json + + docker-manifest: + name: Combine Docker Images runs-on: ubuntu-20.04 if: github.event_name != 'pull_request' && github.repository == 'photoview/photoview' needs: [build] - - steps: - - name: Load Images - uses: actions/download-artifact@v2 - - - name: Print Artifacts - run: ls -R - #- name: Docker Login - # env: - # DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - # run: | - # echo "${DOCKER_PASSWORD}" | docker login --username "${{ steps.prepare.outputs.docker_username }}" --password-stdin - - #- name: Push to Docker Hub - # run: | - # docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} - - # - name: Docker Check Manifest - # if: always() && github.event_name != 'pull_request' && github.repository == 'photoview/photoview' - # run: | - # docker run --rm mplatform/mquery ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} - - - name: Clear + steps: + - name: Docker Login run: | - rm -f ${HOME}/.docker/config.json + echo "${DOCKER_PASSWORD}" | docker login --username "${{ steps.prepare.outputs.docker_username }}" --password-stdin + + - name: Create Manifest + run: | + DOCKER_IMAGES="viktorstrate/photoview:linux-amd64-${GITHUB_SHA::8}" + DOCKER_IMAGES="${DOCKER_IMAGES} viktorstrate/photoview:linux-arm64-${GITHUB_SHA::8}" + DOCKER_IMAGES="${DOCKER_IMAGES} viktorstrate/photoview:linux-arm-v7-${GITHUB_SHA::8}" + DOCKER_IMAGES="${DOCKER_IMAGES} viktorstrate/photoview:linux-arm-v6-${GITHUB_SHA::8}" + + docker manifest create viktorstrate/photoview:test ${DOCKER_IMAGES} + docker manifest push viktorstrate/photoview:test + From c432036a7fd715df59c894b06f91299bd142ea87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 23:07:06 +0200 Subject: [PATCH 19/22] Fix syntax error --- .github/workflows/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a291344..85dc46a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,8 +81,7 @@ jobs: - name: Docker Build run: | - mkdir -p /tmp/docker-image-${{ matrix.target_platform }} - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} - name: Docker Login run: | From e2fd7a7a7085be0a88303d9b8f965310b037295b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 23:11:00 +0200 Subject: [PATCH 20/22] Fix typo --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 85dc46a..908bff3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -65,7 +65,7 @@ jobs: OUTPUT_PLATFORM=$(echo ${{ matrix.target_platform }} | sed 's/\//-/g') echo ::set-output name=output_platform::${OUTPUT_PLATFORM} - TAG="--tag ${OUTPUT_PLATFORM}-${GITHUB_SHA::8}" + TAG="--tag ${DOCKER_IMAGE}:${OUTPUT_PLATFORM}-${GITHUB_SHA::8}" echo ::set-output name=docker_username::${DOCKER_USERNAME} echo ::set-output name=docker_image::${DOCKER_IMAGE} From aa7588145fc0825a15024e50140a666d12a0aad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 23:15:42 +0200 Subject: [PATCH 21/22] Fix docker username variable --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 908bff3..6389829 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -105,15 +105,15 @@ jobs: steps: - name: Docker Login run: | - echo "${DOCKER_PASSWORD}" | docker login --username "${{ steps.prepare.outputs.docker_username }}" --password-stdin + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin - name: Create Manifest run: | - DOCKER_IMAGES="viktorstrate/photoview:linux-amd64-${GITHUB_SHA::8}" - DOCKER_IMAGES="${DOCKER_IMAGES} viktorstrate/photoview:linux-arm64-${GITHUB_SHA::8}" - DOCKER_IMAGES="${DOCKER_IMAGES} viktorstrate/photoview:linux-arm-v7-${GITHUB_SHA::8}" - DOCKER_IMAGES="${DOCKER_IMAGES} viktorstrate/photoview:linux-arm-v6-${GITHUB_SHA::8}" + DOCKER_IMAGES="${DOCKER_IMAGE}:linux-amd64-${GITHUB_SHA::8}" + DOCKER_IMAGES="${DOCKER_IMAGES} ${DOCKER_IMAGE}:linux-arm64-${GITHUB_SHA::8}" + DOCKER_IMAGES="${DOCKER_IMAGES} ${DOCKER_IMAGE}:linux-arm-v7-${GITHUB_SHA::8}" + DOCKER_IMAGES="${DOCKER_IMAGES} ${DOCKER_IMAGE}:linux-arm-v6-${GITHUB_SHA::8}" - docker manifest create viktorstrate/photoview:test ${DOCKER_IMAGES} - docker manifest push viktorstrate/photoview:test + docker manifest create ${DOCKER_IMAGE}:test ${DOCKER_IMAGES} + docker manifest push ${DOCKER_IMAGE}:test From 804b4576ec242bb9337e8476ddf4abcfaea5a073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 28 Mar 2021 23:32:41 +0200 Subject: [PATCH 22/22] Push correct tags to dockerhub --- .github/workflows/build.yml | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6389829..30f417c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,8 +1,6 @@ name: Docker builds on: - schedule: - - cron: "0 10 * * *" # everyday at 10am pull_request: branches: master push: @@ -53,14 +51,11 @@ jobs: id: prepare run: | DOCKER_PLATFORMS=${{ matrix.target_platform }} + VERSION=edge - if [[ $GITHUB_REF == refs/tags/* ]]; then VERSION=${GITHUB_REF#refs/tags/v} fi - if [ "${{ github.event_name }}" = "schedule" ]; then - VERSION=nightly - fi OUTPUT_PLATFORM=$(echo ${{ matrix.target_platform }} | sed 's/\//-/g') echo ::set-output name=output_platform::${OUTPUT_PLATFORM} @@ -107,13 +102,28 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin - - name: Create Manifest + - name: Create Manifests run: | DOCKER_IMAGES="${DOCKER_IMAGE}:linux-amd64-${GITHUB_SHA::8}" DOCKER_IMAGES="${DOCKER_IMAGES} ${DOCKER_IMAGE}:linux-arm64-${GITHUB_SHA::8}" DOCKER_IMAGES="${DOCKER_IMAGES} ${DOCKER_IMAGE}:linux-arm-v7-${GITHUB_SHA::8}" DOCKER_IMAGES="${DOCKER_IMAGES} ${DOCKER_IMAGE}:linux-arm-v6-${GITHUB_SHA::8}" - docker manifest create ${DOCKER_IMAGE}:test ${DOCKER_IMAGES} - docker manifest push ${DOCKER_IMAGE}:test - + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + + TAGS=("${VERSION}") + + if [[ $VERSION =~ ^(([0-9]{1,3})\.[0-9]{1,3})\.[0-9]{1,3}$ ]]; then + VERSION_MINOR=${BASH_REMATCH[1]} + VERSION_MAJOR=${BASH_REMATCH[2]} + TAGS+=("${VERSION_MAJOR}", "${VERSION_MINOR}") + fi + + for TAG in ${TAGS[*]}; do + echo "Creating tag: ${TAG}" + docker manifest create ${DOCKER_IMAGE}:${TAG} ${DOCKER_IMAGES} + docker manifest push ${DOCKER_IMAGE}:${TAG} + done