mirror
/
photoview
mirror of https://github.com/photoview/photoview.git
1
Fork 0
Code

Scanner work + image processing

This commit is contained in:
viktorstrate 2019-07-16 18:31:48 +02:00
parent 286cfa992b
commit f0bf1c2dc4
13 changed files with 420 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,5 +1,7 @@
# See https://help.github.com/ignore-files/ for more about ignoring files.
/api/src/cache/
# dependencies
node_modules/

7
api/package.json
View File

@ -23,14 +23,21 @@
"apollo-server": "^2.6.2",
"babel-runtime": "^6.26.0",
"body-parser": "^1.19.0",
"cookie-parser": "^1.4.4",
"core-js": "^3.1.4",
"dotenv": "^8.0.0",
"exiftool-vendored": "^8.15.0",
"fs-extra": "^8.1.0",
"graphql-tag": "^2.10.1",
"image-size": "^0.7.4",
"img-type": "^0.1.13",
"jsonwebtoken": "^8.5.1",
"lodash": "^4.17.14",
"neo4j-driver": "^1.7.3",
"neo4j-graphql-js": "^2.6.3",
"node-fetch": "^2.3.0",
"regenerator-runtime": "^0.13.2",
"sharp": "^0.22.1",
"uuid": "^3.3.2"
},
"devDependencies": {

258
api/src/Scanner.js
View File

@ -1,4 +1,12 @@
import fs from 'fs-extra'
import path from 'path'
import { resolve as pathResolve, basename as pathBasename } from 'path'
import { PubSub } from 'apollo-server'
import imgType from 'img-type'
import uuid from 'uuid'
import { exiftool } from 'exiftool-vendored'
import sharp from 'sharp'
import config from './config'
export const EVENT_SCANNER_PROGRESS = 'SCANNER_PROGRESS'
@ -7,6 +15,11 @@ class PhotoScanner {
this.driver = driver
this.isRunning = false
this.pubsub = new PubSub()
this.scanAll = this.scanAll.bind(this)
this.scanAlbum = this.scanAlbum.bind(this)
this.scanUser = this.scanUser.bind(this)
this.processImage = this.processImage.bind(this)
}
async scanAll() {
@ -23,55 +36,216 @@ class PhotoScanner {
let session = this.driver.session()
session
.run(
'MATCH (u:User) return u.id AS id, u.username AS username, u.rootPath as path'
)
.subscribe({
onNext: function(record) {
const username = record.get('username')
const id = record.get('id')
const path = record.get('path')
session.run('MATCH (u:User) return u').subscribe({
onNext: record => {
const user = record.toObject().u.properties
if (!path) {
console.log(`User ${username}, has no root path, skipping`)
return
}
console.log('USER', user)
this.scanUser(id)
if (!user.rootPath) {
console.log(`User ${user.username}, has no root path, skipping`)
return
}
console.log(`Scanning ${username}...`)
},
onCompleted: () => {
session.close()
this.isRunning = false
console.log('Done scanning')
this.scanUser(user)
this.pubsub.publish(EVENT_SCANNER_PROGRESS, {
scannerStatusUpdate: {
progress: 100,
finished: true,
error: false,
errorMessage: '',
},
})
},
onError: error => {
console.error(error)
console.log(`Scanning ${user.username}...`)
},
onCompleted: () => {
session.close()
this.isRunning = false
console.log('Done scanning')
this.pubsub.publish(EVENT_SCANNER_PROGRESS, {
scannerStatusUpdate: {
progress: 0,
finished: false,
error: true,
errorMessage: error.message,
},
})
},
})
this.pubsub.publish(EVENT_SCANNER_PROGRESS, {
scannerStatusUpdate: {
progress: 100,
finished: true,
error: false,
errorMessage: '',
},
})
},
onError: error => {
console.error(error)
this.pubsub.publish(EVENT_SCANNER_PROGRESS, {
scannerStatusUpdate: {
progress: 0,
finished: false,
error: true,
errorMessage: error.message,
},
})
},
})
}
async scanUser(id) {}
async scanUser(user) {
console.log('Scanning path', user.rootPath)
const driver = this.driver
const scanAlbum = this.scanAlbum
async function scanPath(path) {
const list = fs.readdirSync(path)
let foundImage = false
for (const item of list) {
const itemPath = pathResolve(path, item)
// console.log(`Scanning item ${itemPath}...`)
const stat = fs.statSync(itemPath)
if (stat.isDirectory()) {
// console.log(`Entering directory ${itemPath}`)
const imagesInDirectory = await scanPath(itemPath)
if (imagesInDirectory) {
console.log(`Found album at ${itemPath}`)
const session = driver.session()
const findAlbumResult = await session.run(
'MATCH (a:Album { path: {path} }) RETURN a',
{
path: itemPath,
}
)
console.log('FIND ALBUM RESULT', findAlbumResult.records)
if (findAlbumResult.records.length != 0) {
console.log('Album already exists')
const album = findAlbumResult.records[0].toObject().a.properties
scanAlbum(album)
continue
}
console.log('Adding album')
const albumId = uuid()
const albumResult = await session.run(
`MATCH (u:User { id: {userId} })
CREATE (a:Album { id: {id}, title: {title}, path: {path} })
CREATE (u)-[own:OWNS]->(a)
RETURN a`,
{
id: albumId,
userId: user.id,
title: item,
path: itemPath,
}
)
const album = albumResult.records[0].toObject().a.properties
scanAlbum(album)
session.close()
}
continue
}
if (!foundImage && (await imgType.isImg(itemPath))) {
foundImage = true
}
}
return foundImage
}
await scanPath(user.rootPath)
console.log('User scan complete')
}
async scanAlbum(album) {
const { title, path, id } = album
console.log('Scanning album', title)
const list = fs.readdirSync(path)
for (const item of list) {
const itemPath = pathResolve(path, item)
if (await imgType.isImg(itemPath)) {
const session = this.driver.session()
const photoResult = await session.run(
`MATCH (p:Photo {path: {imgPath} })<--(a:Album {id: {albumId}}) RETURN p`,
{
imgPath: itemPath,
albumId: id,
}
)
if (photoResult.records.length != 0) {
console.log(`Photo already exists ${item}`)
} else {
console.log(`Found new image at ${itemPath}`)
const imageId = uuid()
await session.run(
`MATCH (a:Album { id: {albumId} })
CREATE (p:Photo {id: {id}, path: {path}, title: {title} })
CREATE (a)-[:CONTAINS]->(p)`,
{
id: imageId,
path: itemPath,
title: item,
albumId: id,
}
)
this.processImage(imageId)
}
}
}
}
async processImage(id) {
console.log('Processing image')
const session = this.driver.session()
const result = await session.run('MATCH (p:Photo { id: {id} }) return p', {
id,
})
const photo = result.records[0].get('p').properties
console.log('PHOTO', photo.path)
const imagePath = path.resolve(config.cachePath, 'images', id)
await fs.remove(imagePath)
await fs.mkdirp(imagePath)
const type = await imgType.getType(photo.path)
let resizeBaseImg = photo.path
const rawTypes = ['cr2', 'arw', 'crw', 'dng']
if (rawTypes.includes(type)) {
console.log('Processing RAW image')
const extractedPath = path.resolve(imagePath, 'extracted.jpg')
await exiftool.extractPreview(photo.path, extractedPath)
resizeBaseImg = extractedPath
}
// Resize image
console.log('Resizing image', resizeBaseImg)
await sharp(resizeBaseImg)
.resize(1440, 1080, { fit: 'inside', withoutEnlargement: true })
.toFile(path.resolve(imagePath, 'thumbnail.jpg'))
await sharp(resizeBaseImg)
.jpeg({ quality: 85 })
.toFile(path.resolve(imagePath, 'original.jpg'))
session.close()
console.log('Processing done')
}
}
export default PhotoScanner

5
api/src/config.js Normal file
View File

@ -0,0 +1,5 @@
import path from 'path'
export default {
cachePath: path.resolve(__dirname, 'cache'),
}

101
api/src/index.js
View File

@ -1,13 +1,16 @@
import fs from 'fs'
import fs from 'fs-extra'
import path from 'path'
import { ApolloServer } from 'apollo-server-express'
import express from 'express'
import bodyParser from 'body-parser'
import cookieParser from 'cookie-parser'
import { v1 as neo4j } from 'neo4j-driver'
import { makeAugmentedSchema } from 'neo4j-graphql-js'
import dotenv from 'dotenv'
import http from 'http'
import PhotoScanner from './Scanner'
import _ from 'lodash'
import config from './config'
import { getUserFromToken, getTokenFromBearer } from './token'
@ -16,6 +19,7 @@ dotenv.config()
const app = express()
app.use(bodyParser.json())
app.use(cookieParser())
/*
* Create an executable GraphQL schema object from GraphQL type definitions
@ -33,6 +37,9 @@ const typeDefs = fs
import usersResolver from './resolvers/users'
import scannerResolver from './resolvers/scanner'
import photosResolver from './resolvers/photos'
const resolvers = [usersResolver, scannerResolver, photosResolver]
const schema = makeAugmentedSchema({
typeDefs,
@ -42,19 +49,12 @@ const schema = makeAugmentedSchema({
hasRole: true,
},
mutation: false,
query: {
exclude: ['ScannerResult', 'AuthorizeResult', 'Subscription'],
},
},
resolvers: {
Mutation: {
...usersResolver.mutation,
...scannerResolver.mutation,
},
Subscription: {
...scannerResolver.subscription,
},
query: false,
// query: {
// exclude: ['ScannerResult', 'AuthorizeResult', 'Subscription'],
// },
},
resolvers: resolvers.reduce((prev, curr) => _.merge(prev, curr), {}),
})
/*
@ -72,6 +72,10 @@ const driver = neo4j.driver(
const scanner = new PhotoScanner(driver)
// Specify port and path for GraphQL endpoint
const port = process.env.GRAPHQL_LISTEN_PORT || 4001
const graphPath = '/graphql'
/*
* Create a new ApolloServer instance, serving the GraphQL schema
* created using makeAugmentedSchema above and injecting the Neo4j driver
@ -81,13 +85,21 @@ const scanner = new PhotoScanner(driver)
const server = new ApolloServer({
context: async function({ req }) {
let user = null
let token = null
if (req && req.headers.authorization) {
const token = getTokenFromBearer(req.headers.authorization)
token = getTokenFromBearer(req.headers.authorization)
user = await getUserFromToken(token, driver)
}
return { ...req, driver, scanner, user }
return {
...req,
driver,
scanner,
user,
token,
endpoint: `http://localhost:${port}`,
}
},
schema: schema,
introspection: true,
@ -105,16 +117,59 @@ const server = new ApolloServer({
},
})
// Specify port and path for GraphQL endpoint
const port = process.env.GRAPHQL_LISTEN_PORT || 4001
const graphPath = '/graphql'
/*
* Optionally, apply Express middleware for authentication, etc
* This also also allows us to specify a path for the GraphQL endpoint
*/
server.applyMiddleware({ app, graphPath })
app.use('/images/:id/:image', async function(req, res) {
const { id, image } = req.params
console.log('image', image)
if (image != 'original.jpg' && image != 'thumbnail.jpg') {
return res.status(404).send('Image not found')
}
let user = null
try {
const token = req.cookies.token
user = await getUserFromToken(token, driver)
} catch (err) {
return res.status(401).send(err.message)
}
const session = driver.session()
const result = await session.run(
'MATCH (p:Photo { id: {id} }) MATCH (p)<-[:CONTAINS]-(:Album)<-[:OWNS]-(u:User) RETURN p as photo, u.id as userId',
{
id,
}
)
if (result.records.length == 0) {
return res.status(404).send(`Image not found`)
}
const userId = result.records[0].get('userId')
const photo = result.records[0].get('photo')
if (userId != user.id) {
return res.status(401).send(`Image not owned by you`)
}
session.close()
const imagePath = path.resolve(config.cachePath, 'images', id, image)
const imageFound = await fs.exists(imagePath)
if (!imageFound) {
console.log('Image not found', imagePath)
await scanner.processImage(id)
}
res.sendFile(imagePath)
})
const httpServer = http.createServer(app)
server.installSubscriptionHandlers(httpServer)

74
api/src/resolvers/photos.js Normal file
View File

@ -0,0 +1,74 @@
import { cypherQuery } from 'neo4j-graphql-js'
import { promisify } from 'util'
import fs from 'fs-extra'
import path from 'path'
import config from '../config'
const imageSize = promisify(require('image-size'))
function injectAt(query, index, injection) {
return query.substr(0, index) + injection + query.substr(index)
}
const Query = {
myAlbums: async function(root, args, ctx, info) {
const query = cypherQuery(args, ctx, info)
const whereSplit = query[0].indexOf('RETURN')
query[0] = injectAt(
query[0],
whereSplit,
`MATCH (u:User { id: {userid} }) WHERE (u)-[:OWNS]->(album) `
)
query[1].userid = ctx.user.id
console.log(query)
const addIDSplit = query[0].indexOf('album_photos {') + 14
console.log('ID SPLIT', query[0].substr(0, addIDSplit))
query[0] = injectAt(query[0], addIDSplit, `.id,`)
const session = ctx.driver.session()
const result = await session.run(...query)
session.close()
return result.records.map(record => record.get('album'))
},
}
const Photo = {
async original(root, args, ctx, info) {
const imgPath = path.resolve(
config.cachePath,
'images',
root.id,
'original.jpg'
)
if (!(await fs.exists(imgPath))) {
await ctx.scanner.processImage(root.id)
}
const { width, height } = await imageSize(imgPath)
return {
path: `${ctx.endpoint}/images/${root.id}/original.jpg`,
width,
height,
}
},
async thumbnail(root, args, ctx, info) {
return {
path: `${ctx.endpoint}/images/${root.id}/thumbnail.jpg`,
width: 120,
height: 240,
}
},
}
export default {
Query,
Photo,
}

8
api/src/resolvers/scanner.js
View File

@ -1,6 +1,6 @@
import { EVENT_SCANNER_PROGRESS } from '../scanner'
const mutation = {
const Mutation = {
async scanAll(root, args, ctx, info) {
if (ctx.scanner.isRunning) {
return {
@ -20,7 +20,7 @@ const mutation = {
},
}
const subscription = {
const Subscription = {
scannerStatusUpdate: {
subscribe(root, args, ctx, info) {
return ctx.scanner.pubsub.asyncIterator([EVENT_SCANNER_PROGRESS])
@ -29,6 +29,6 @@ const subscription = {
}
export default {
mutation,
subscription,
Mutation,
Subscription,
}

5
api/src/resolvers/users.js
View File

@ -1,8 +1,9 @@
import jwt from 'jsonwebtoken'
import uuid from 'uuid'
const mutation = {
const Mutation = {
async authorizeUser(root, args, ctx, info) {
console.log('Authorize user')
let { username, password } = args
let session = ctx.driver.session()
@ -69,5 +70,5 @@ const mutation = {
}
export default {
mutation,
Mutation,
}

35
api/src/schema.graphql
View File

@ -6,28 +6,42 @@ enum Role {
type User @isAuthenticated {
id: ID!
username: String!
#: password: String
albums: [Album] @relation(name: "OWNS", direction: "OUT")
rootPath: String! @hasRole(roles: [Admin])
admin: Boolean
}
type Album @isAuthenticated {
type Album {
id: ID!
title: String
photos: [Photo] @relation(name: "CONTAINS", direction: "OUT")
owner: User! @relation(name: "OWNS", direction: "IN")
path: String
}
type Photo @isAuthenticated {
type UnimportedAlbum {
id: ID!
relativePath: String
}
type PhotoURL {
path: String
width: Int
height: Int
}
type Photo {
id: ID!
title: String
path: String!
original: PhotoURL @neo4j_ignore
thumbnail: PhotoURL @neo4j_ignore
album: Album! @relation(name: "CONTAINS", direction: "IN")
}
type SiteInfo {
signupEnabled: Boolean!
firstSignup: Boolean!
initialSetup: Boolean!
}
type AuthorizeResult {
@ -49,18 +63,15 @@ type Subscription {
type Mutation {
authorizeUser(username: String!, password: String!): AuthorizeResult!
@neo4j_ignore
registerUser(username: String!, password: String!): AuthorizeResult!
@neo4j_ignore
scanAll: ScannerResult! @isAuthenticated
scanAll: ScannerResult! @isAuthenticated @neo4j_ignore
}
type Query {
siteInfo: SiteInfo
}
# type Query {
# usersBySubstring(substring: String): [User]
# @cypher(
# statement: "MATCH (u:User) WHERE u.name CONTAINS $substring RETURN u"
# )
# }
myAlbums: [Album] @isAuthenticated
}

1
api/src/test.js
View File

@ -1 +0,0 @@
const neo4j = require('neo4j-graphql-js')

4
api/src/token.js
View File

@ -14,10 +14,10 @@ export const getUserFromToken = async function(token, driver) {
)
if (userResult.records.length == 0) {
throw new Error(`User doesn't exist anymore`)
throw new Error(`User was not found`)
}
const user = userResult.records[0].toObject()
let user = userResult.records[0].toObject().u.properties
session.close()

1
ui/package.json
View File

@ -11,6 +11,7 @@
"apollo-link-error": "^1.1.11",
"apollo-link-http": "^1.5.15",
"babel-plugin-styled-components": "^1.10.6",
"cookie": "^0.4.0",
"graphql": "^14.2.1",
"graphql-tag": "^2.10.1",
"parcel-bundler": "^1.12.3",

5
ui/yarn.lock
View File

@ -1814,6 +1814,11 @@ convert-source-map@^1.1.0, convert-source-map@^1.5.1:
dependencies:
safe-buffer "~5.1.1"
cookie@^0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.0.tgz#beb437e7022b3b6d49019d088665303ebe9c14ba"
integrity sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==
copy-descriptor@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d"