Refactor scanner
This commit is contained in:
parent
56800b14a8
commit
6aa5200f95
|
@ -18,7 +18,7 @@ const isImage = async path => {
|
||||||
return type != null
|
return type != null
|
||||||
}
|
}
|
||||||
|
|
||||||
const isRawImage = async path => {
|
export const isRawImage = async path => {
|
||||||
const buffer = await readChunk(path, 0, 12)
|
const buffer = await readChunk(path, 0, 12)
|
||||||
const { ext } = imageType(buffer)
|
const { ext } = imageType(buffer)
|
||||||
|
|
||||||
|
@ -37,6 +37,9 @@ class PhotoScanner {
|
||||||
this.scanAlbum = this.scanAlbum.bind(this)
|
this.scanAlbum = this.scanAlbum.bind(this)
|
||||||
this.scanUser = this.scanUser.bind(this)
|
this.scanUser = this.scanUser.bind(this)
|
||||||
this.processImage = this.processImage.bind(this)
|
this.processImage = this.processImage.bind(this)
|
||||||
|
|
||||||
|
this.imagesToProgress = 0
|
||||||
|
this.finishedImages = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
async scanAll() {
|
async scanAll() {
|
||||||
|
@ -53,6 +56,8 @@ class PhotoScanner {
|
||||||
|
|
||||||
let session = this.driver.session()
|
let session = this.driver.session()
|
||||||
|
|
||||||
|
let allUserScans = []
|
||||||
|
|
||||||
session.run('MATCH (u:User) return u').subscribe({
|
session.run('MATCH (u:User) return u').subscribe({
|
||||||
onNext: record => {
|
onNext: record => {
|
||||||
const user = record.toObject().u.properties
|
const user = record.toObject().u.properties
|
||||||
|
@ -64,15 +69,19 @@ class PhotoScanner {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
this.scanUser(user)
|
allUserScans.push(this.scanUser(user))
|
||||||
|
|
||||||
console.log(`Scanning ${user.username}...`)
|
console.log(`Scanning ${user.username}...`)
|
||||||
},
|
},
|
||||||
onCompleted: () => {
|
onCompleted: () => {
|
||||||
session.close()
|
session.close()
|
||||||
this.isRunning = false
|
this.isRunning = false
|
||||||
console.log('Done scanning')
|
|
||||||
|
|
||||||
|
Promise.all(allUserScans)
|
||||||
|
.then(() => {
|
||||||
|
console.log(
|
||||||
|
`Done scanning ${this.finishedImages} of ${this.imagesToProgress}`
|
||||||
|
)
|
||||||
this.pubsub.publish(EVENT_SCANNER_PROGRESS, {
|
this.pubsub.publish(EVENT_SCANNER_PROGRESS, {
|
||||||
scannerStatusUpdate: {
|
scannerStatusUpdate: {
|
||||||
progress: 100,
|
progress: 100,
|
||||||
|
@ -81,6 +90,17 @@ class PhotoScanner {
|
||||||
errorMessage: '',
|
errorMessage: '',
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
})
|
||||||
|
.catch(error => {
|
||||||
|
this.pubsub.publish(EVENT_SCANNER_PROGRESS, {
|
||||||
|
scannerStatusUpdate: {
|
||||||
|
progress: 0,
|
||||||
|
finished: false,
|
||||||
|
error: true,
|
||||||
|
errorMessage: error.message,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
})
|
||||||
},
|
},
|
||||||
onError: error => {
|
onError: error => {
|
||||||
console.error(error)
|
console.error(error)
|
||||||
|
@ -187,6 +207,8 @@ class PhotoScanner {
|
||||||
if (await isImage(itemPath)) {
|
if (await isImage(itemPath)) {
|
||||||
const session = this.driver.session()
|
const session = this.driver.session()
|
||||||
|
|
||||||
|
this.imagesToProgress++
|
||||||
|
|
||||||
const photoResult = await session.run(
|
const photoResult = await session.run(
|
||||||
`MATCH (p:Photo {path: {imgPath} })<--(a:Album {id: {albumId}}) RETURN p`,
|
`MATCH (p:Photo {path: {imgPath} })<--(a:Album {id: {albumId}}) RETURN p`,
|
||||||
{
|
{
|
||||||
|
@ -197,6 +219,20 @@ class PhotoScanner {
|
||||||
|
|
||||||
if (photoResult.records.length != 0) {
|
if (photoResult.records.length != 0) {
|
||||||
console.log(`Photo already exists ${item}`)
|
console.log(`Photo already exists ${item}`)
|
||||||
|
|
||||||
|
const id = photoResult.records[0].get('p').properties.id
|
||||||
|
|
||||||
|
const thumbnailPath = pathResolve(
|
||||||
|
config.cachePath,
|
||||||
|
id,
|
||||||
|
'thumbnail.jpg'
|
||||||
|
)
|
||||||
|
|
||||||
|
if (!(await fs.exists(thumbnailPath))) {
|
||||||
|
this.processImage(id)
|
||||||
|
} else {
|
||||||
|
this.finishedImages++
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
console.log(`Found new image at ${itemPath}`)
|
console.log(`Found new image at ${itemPath}`)
|
||||||
const imageId = uuid()
|
const imageId = uuid()
|
||||||
|
@ -248,16 +284,23 @@ class PhotoScanner {
|
||||||
// Resize image
|
// Resize image
|
||||||
console.log('Resizing image', resizeBaseImg)
|
console.log('Resizing image', resizeBaseImg)
|
||||||
await sharp(resizeBaseImg)
|
await sharp(resizeBaseImg)
|
||||||
|
.jpeg({ quality: 80 })
|
||||||
.resize(1440, 1080, { fit: 'inside', withoutEnlargement: true })
|
.resize(1440, 1080, { fit: 'inside', withoutEnlargement: true })
|
||||||
.toFile(path.resolve(imagePath, 'thumbnail.jpg'))
|
.toFile(path.resolve(imagePath, 'thumbnail.jpg'))
|
||||||
|
|
||||||
await sharp(resizeBaseImg)
|
|
||||||
.jpeg({ quality: 85 })
|
|
||||||
.toFile(path.resolve(imagePath, 'original.jpg'))
|
|
||||||
|
|
||||||
session.close()
|
session.close()
|
||||||
|
|
||||||
console.log('Processing done')
|
console.log('Processing done')
|
||||||
|
this.finishedImages++
|
||||||
|
|
||||||
|
this.pubsub.publish(EVENT_SCANNER_PROGRESS, {
|
||||||
|
scannerStatusUpdate: {
|
||||||
|
progress: this.finishedImages / this.imagesToProgress,
|
||||||
|
finished: false,
|
||||||
|
error: false,
|
||||||
|
errorMessage: '',
|
||||||
|
},
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -72,6 +72,8 @@ const driver = neo4j.driver(
|
||||||
|
|
||||||
const scanner = new PhotoScanner(driver)
|
const scanner = new PhotoScanner(driver)
|
||||||
|
|
||||||
|
setInterval(scanner.scanAll, 1000 * 60 * 60 * 4)
|
||||||
|
|
||||||
// Specify port and path for GraphQL endpoint
|
// Specify port and path for GraphQL endpoint
|
||||||
const port = process.env.GRAPHQL_LISTEN_PORT || 4001
|
const port = process.env.GRAPHQL_LISTEN_PORT || 4001
|
||||||
const graphPath = '/graphql'
|
const graphPath = '/graphql'
|
||||||
|
|
|
@ -3,6 +3,7 @@ import { promisify } from 'util'
|
||||||
import fs from 'fs-extra'
|
import fs from 'fs-extra'
|
||||||
import path from 'path'
|
import path from 'path'
|
||||||
import config from '../config'
|
import config from '../config'
|
||||||
|
import { isRawImage } from '../Scanner'
|
||||||
|
|
||||||
const imageSize = promisify(require('image-size'))
|
const imageSize = promisify(require('image-size'))
|
||||||
|
|
||||||
|
@ -147,7 +148,53 @@ function photoResolver(image) {
|
||||||
}
|
}
|
||||||
|
|
||||||
const Photo = {
|
const Photo = {
|
||||||
original: photoResolver('original.jpg'),
|
// TODO: Make original point to the right path
|
||||||
|
original: async (root, args, ctx, info) => {
|
||||||
|
async function getPath(retryAfterScan = false) {
|
||||||
|
let imgPath = path.resolve(
|
||||||
|
config.cachePath,
|
||||||
|
'images',
|
||||||
|
root.id,
|
||||||
|
'extracted.jpg'
|
||||||
|
)
|
||||||
|
|
||||||
|
if (!(await fs.exists(imgPath))) {
|
||||||
|
imgPath = root.path
|
||||||
|
|
||||||
|
if (!imgPath) {
|
||||||
|
const session = ctx.driver.session()
|
||||||
|
|
||||||
|
const result = await session.run(
|
||||||
|
'MATCH (p:Photo { id: {id} }) return p.path as path',
|
||||||
|
{
|
||||||
|
id: root.id,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
imgPath = result.get('path')
|
||||||
|
session.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!(await fs.exists(imgPath)) || (await isRawImage(imgPath))) {
|
||||||
|
if (retryAfterScan)
|
||||||
|
throw new Error('Could not find image after rescan')
|
||||||
|
await ctx.scanner.processImage(root.id)
|
||||||
|
return getPath(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return imgPath
|
||||||
|
}
|
||||||
|
|
||||||
|
const imgPath = await getPath()
|
||||||
|
|
||||||
|
const { width, height } = await imageSize(imgPath)
|
||||||
|
return {
|
||||||
|
path: `${ctx.endpoint}/images/${root.id}/${path.basename(imgPath)}`,
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
}
|
||||||
|
},
|
||||||
thumbnail: photoResolver('thumbnail.jpg'),
|
thumbnail: photoResolver('thumbnail.jpg'),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -6,8 +6,8 @@ enum Role {
|
||||||
type User @isAuthenticated {
|
type User @isAuthenticated {
|
||||||
id: ID!
|
id: ID!
|
||||||
username: String!
|
username: String!
|
||||||
#: password: String
|
|
||||||
albums: [Album] @relation(name: "OWNS", direction: "OUT")
|
albums: [Album] @relation(name: "OWNS", direction: "OUT")
|
||||||
|
# Local filepath for the user's photos
|
||||||
rootPath: String! @hasRole(roles: [Admin])
|
rootPath: String! @hasRole(roles: [Admin])
|
||||||
admin: Boolean
|
admin: Boolean
|
||||||
}
|
}
|
||||||
|
@ -20,22 +20,25 @@ type Album {
|
||||||
path: String
|
path: String
|
||||||
}
|
}
|
||||||
|
|
||||||
type UnimportedAlbum {
|
|
||||||
id: ID!
|
|
||||||
relativePath: String
|
|
||||||
}
|
|
||||||
|
|
||||||
type PhotoURL {
|
type PhotoURL {
|
||||||
|
# URL for the image
|
||||||
path: String
|
path: String
|
||||||
|
# Width of the image in pixels
|
||||||
width: Int
|
width: Int
|
||||||
|
# Height of the image in pixels
|
||||||
height: Int
|
height: Int
|
||||||
}
|
}
|
||||||
|
|
||||||
type Photo {
|
type Photo {
|
||||||
id: ID!
|
id: ID!
|
||||||
title: String
|
title: String
|
||||||
|
# Local filepath for the photo
|
||||||
|
path: String
|
||||||
|
# URL to display the photo in full resolution
|
||||||
original: PhotoURL @neo4j_ignore
|
original: PhotoURL @neo4j_ignore
|
||||||
|
# URL to display the photo in a smaller resolution
|
||||||
thumbnail: PhotoURL @neo4j_ignore
|
thumbnail: PhotoURL @neo4j_ignore
|
||||||
|
# The album that holds the photo
|
||||||
album: Album! @relation(name: "CONTAINS", direction: "IN")
|
album: Album! @relation(name: "CONTAINS", direction: "IN")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import React, { Component } from 'react'
|
import React, { Component, useState } from 'react'
|
||||||
import styled from 'styled-components'
|
import styled from 'styled-components'
|
||||||
import { Message, Progress } from 'semantic-ui-react'
|
import { Message, Progress } from 'semantic-ui-react'
|
||||||
import gql from 'graphql-tag'
|
import gql from 'graphql-tag'
|
||||||
|
@ -47,10 +47,24 @@ const MessageProgress = ({ header, content, percent, ...props }) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
class Messages extends Component {
|
class Messages extends Component {
|
||||||
|
constructor(props) {
|
||||||
|
super(props)
|
||||||
|
|
||||||
|
this.state = {
|
||||||
|
showSyncMessage: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
render() {
|
render() {
|
||||||
return (
|
return (
|
||||||
<Container>
|
<Container>
|
||||||
<Subscription subscription={syncSubscription} shouldResubscribe>
|
<Subscription
|
||||||
|
subscription={syncSubscription}
|
||||||
|
shouldResubscribe
|
||||||
|
onSubscriptionData={() => {
|
||||||
|
this.setState({ showSyncMessage: true })
|
||||||
|
}}
|
||||||
|
>
|
||||||
{({ loading, error, data }) => {
|
{({ loading, error, data }) => {
|
||||||
if (error) return <div>error {error.message}</div>
|
if (error) return <div>error {error.message}</div>
|
||||||
if (loading) return null
|
if (loading) return null
|
||||||
|
@ -60,6 +74,10 @@ class Messages extends Component {
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<MessageProgress
|
<MessageProgress
|
||||||
|
hidden={!this.state.showSyncMessage}
|
||||||
|
onDismiss={() => {
|
||||||
|
this.setState({ showSyncMessage: false })
|
||||||
|
}}
|
||||||
header={update.finished ? 'Synced' : 'Syncing'}
|
header={update.finished ? 'Synced' : 'Syncing'}
|
||||||
content={
|
content={
|
||||||
update.finished ? 'Finished syncing' : 'Syncing in progress'
|
update.finished ? 'Finished syncing' : 'Syncing in progress'
|
||||||
|
|
Loading…
Reference in New Issue