From e71e5d42076d434d9d9fbd287c38f6068864f979 Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 6 Mar 2017 23:27:44 +1100 Subject: [PATCH] Replace custom "make" DSL with the output task list (#60) --- package.json | 2 +- src/index.js | 148 +++++++++++----------------------- src/input/database.js | 2 +- src/input/file.js | 6 +- src/input/output.js | 52 ++++++------ src/output-media/resize.js | 82 +++++++++++++++++++ src/output-media/tasks.js | 51 ++++++++++++ src/output-media/thumbs.js | 79 ------------------ src/output-website/website.js | 1 - src/utils/files.js | 16 ---- src/utils/make.js | 60 -------------- src/utils/progress.js | 30 ++++--- test/input/file.spec.js | 3 +- 13 files changed, 234 insertions(+), 298 deletions(-) create mode 100644 src/output-media/resize.js create mode 100644 src/output-media/tasks.js delete mode 100644 src/output-media/thumbs.js delete mode 100644 src/utils/files.js delete mode 100644 src/utils/make.js diff --git a/package.json b/package.json index a791add..4adf442 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "async": "^2.1.2", "debug": "^2.6.1", "exiftool-json-db": "~1.0.0", - "fs-extra": "^1.0.0", + "fs-extra": "^2.0.0", "gm": "^1.23.0", "handlebars": "~4.0.5", "less": "^2.7.1", diff --git a/src/index.js b/src/index.js index 302929b..1e845e8 100644 --- a/src/index.js +++ b/src/index.js @@ -1,136 +1,84 @@ const async = require('async') const fs = require('fs-extra') -const make = require('./utils/make') const pad = require('pad') const path = require('path') const database = require('./input/database') +const progress = require('./utils/progress') const File = require('./input/file') const Media = require('./model/media') const hierarchy = require('./model/hierarchy.js') -const thumbs = require('./output-media/thumbs') +const resize = require('./output-media/resize') const website = require('./output-website/website') exports.build = function (opts) { - thumbs.sizes.thumb = opts.thumbSize; - thumbs.sizes.large = opts.largeSize; + resize.sizes.thumb = opts.thumbSize + resize.sizes.large = opts.largeSize - fs.mkdirpSync(opts.output); - var media = path.join(opts.output, 'media'); - var databaseFile = path.join(opts.output, 'metadata.json'); + fs.mkdirpSync(opts.output) + const media = path.join(opts.output, 'media') + const databaseFile = path.join(opts.output, 'metadata.json') - // --------------------- - // These variables are set later during the async phase - // --------------------- var album = null // root album with nested albums var collection = null // all files in the database - function buildStep (options) { - return function(callback) { - if (options.condition !== false) { - make.exec(opts.input, media, collection, options, callback); - } else { - callback(); - } - } - } - - function callbackStep (name, fn) { - return function(next) { - process.stdout.write(pad(name, 20)); - fn(function(err) { - if (err) { - console.log('[====================] error'); - next(err); - } else { - console.log('[====================] done'); - next(); - } - }); - } - } - - function copyFile(task, callback) { - fs.copy(task.src, task.dest, callback); - } - async.series([ - function updateDatabase(callback) { + function updateDatabase (callback) { database.update(opts.input, databaseFile, (err, dbFiles) => { collection = dbFiles.map(f => new File(f, opts)) callback(err) }) }, - buildStep({ - condition: opts.originalPhotos, - message: 'Photos: original', - ext: 'jpg|jpeg|png|gif', - dest: '/original/$path/$name.$ext', - func: copyFile - }), - - buildStep({ - message: 'Photos: large', - ext: 'jpg|jpeg|png|gif', - dest: '/large/$path/$name.$ext', - func: thumbs.photoLarge - }), - - buildStep({ - message: 'Photos: thumbnails', - ext: 'jpg|jpeg|png|gif', - dest: '/thumbs/$path/$name.$ext', - func: thumbs.photoSquare - }), - - buildStep({ - condition: opts.originalVideos, - message: 'Videos: original', - ext: 'mp4|mov|mts|m2ts', - dest: '/original/$path/$name.$ext', - func: copyFile - }), - - buildStep({ - message: 'Videos: resized', - ext: 'mp4|mov|mts|m2ts', - dest: '/large/$path/$name.mp4', - func: thumbs.videoWeb - }), - - buildStep({ - message: 'Videos: poster', - ext: 'mp4|mov|mts|m2ts', - dest: '/large/$path/$name.jpg', - func: thumbs.videoLarge - }), + function processPhotos (callback) { + const tasks = require('./output-media/tasks') + const imageTasks = tasks.create(opts, collection, 'image') + const imageBar = progress.create('Processing photos', imageTasks.length) + async.parallelLimit(imageTasks.map(asyncProgress(imageBar)), 2, callback) + }, - buildStep({ - message: 'Videos: thumbnails', - ext: 'mp4|mov|mts|m2ts', - dest: '/thumbs/$path/$name.jpg', - func: thumbs.videoSquare - }), + function processVideos (callback) { + const tasks = require('./output-media/tasks') + const videoTasks = tasks.create(opts, collection, 'video') + const videoBar = progress.create('Processing videos', videoTasks.length) + async.parallelLimit(videoTasks.map(asyncProgress(videoBar)), 2, callback) + }, - callbackStep('Album hierarchy', function(next) { + function createAlbums (callback) { + const bar = progress.create('Creating albums') const mediaCollection = collection.map(f => new Media(f)) - albums = hierarchy.createAlbums(mediaCollection, opts) - next() - }), + album = hierarchy.createAlbums(mediaCollection, opts) + bar.tick(1) + callback() + }, + + function createWebsite (callback) { + const bar = progress.create('Building website') + website.build(album, opts, (err) => { + bar.tick(1) + callback(err) + }) + } - callbackStep('Static website', function(next) { - website.build(albums, opts, next) - }) + ], finish) - ], finish); +} +function asyncProgress (bar) { + return fn => { + return done => { + fn(err => { + bar.tick(1) + done(err) + }) + } + } } function finish (err) { - console.log(); - console.log(err || 'Gallery generated successfully'); - console.log(); + console.log(err ? 'Unexpected error' : '') + console.log(err || 'Gallery generated successfully') + console.log() process.exit(err ? 1 : 0) } diff --git a/src/input/database.js b/src/input/database.js index d4c6cca..dd36a50 100644 --- a/src/input/database.js +++ b/src/input/database.js @@ -18,7 +18,7 @@ exports.update = function(media, databasePath, callback) { emitter.on('stats', (stats) => { debug(`Database stats: total=${stats.total}`) - const totalBar = progress.create('Finding media files', stats.total) + const totalBar = progress.create('Finding media', stats.total) totalBar.tick(stats.total) updateBar = progress.create('Updating database', stats.added + stats.modified) }) diff --git a/src/input/file.js b/src/input/file.js index 846433c..83050fd 100644 --- a/src/input/file.js +++ b/src/input/file.js @@ -13,7 +13,7 @@ function File (dbEntry, opts) { this.path = dbEntry.SourceFile this.date = fileDate(dbEntry) this.type = mediaType(dbEntry) - this.output = output.paths(this.path, this.mediaType, opts) + this.output = output.paths(this.path, this.type, opts) } function fileDate (dbEntry) { @@ -22,8 +22,8 @@ function fileDate (dbEntry) { function mediaType (dbEntry) { const match = MIME_REGEX.exec(dbEntry.File.MIMEType) - // "image" or "video" - if (match) return match[1] + if (match && match[1] === 'image') return 'image' + if (match && match[1] === 'video') return 'video' return 'unknown' } diff --git a/src/input/output.js b/src/input/output.js index 9704477..1955880 100644 --- a/src/input/output.js +++ b/src/input/output.js @@ -1,27 +1,27 @@ +const debug = require('debug')('thumbsup') exports.paths = function (filepath, mediaType, config) { - if (mediaType === 'video') { + if (mediaType === 'image') { + var originals = config ? config.originalPhotos : false + return imageOutput(filepath, originals) + } else if (mediaType === 'video') { var originals = config ? config.originalVideos : false - return videoOutput(filepath, original) + return videoOutput(filepath, originals) } else { - var originals = config ? config.originalPhotos : false - return imageOutput(filepath) + debug(`Unsupported file type: ${mediaType}`) + return {} } } -function videoOutput (filepath, originals) { - var output = { +function imageOutput (filepath, originals) { + const output = { thumbnail: { - path: 'media/thumbs/' + ext(filepath, 'jpg'), - rel: 'video:thumbnail' + path: 'media/thumbs/' + filepath, + rel: 'photo:thumbnail' }, large: { - path: 'media/large/' + ext(filepath, 'jpg'), - rel: 'video:poster' - }, - video: { - path: 'media/large/' + ext(filepath, 'mp4'), - rel: 'video:resized' + path: 'media/large/' + filepath, + rel: 'photo:large' } } if (originals) { @@ -30,20 +30,24 @@ function videoOutput (filepath, originals) { rel: 'original' } } else { - output.download = output.video + output.download = output.large } - return output; + return output } -function imageOutput (filepath, originals) { - const output = { +function videoOutput (filepath, originals) { + var output = { thumbnail: { - path: 'media/thumbs/' + filepath, - rel: 'photo:thumbnail' + path: 'media/thumbs/' + ext(filepath, 'jpg'), + rel: 'video:thumbnail' }, large: { - path: 'media/large/' + filepath, - rel: 'photo:large' + path: 'media/large/' + ext(filepath, 'jpg'), + rel: 'video:poster' + }, + video: { + path: 'media/large/' + ext(filepath, 'mp4'), + rel: 'video:resized' } } if (originals) { @@ -52,9 +56,9 @@ function imageOutput (filepath, originals) { rel: 'original' } } else { - output.download = output.large + output.download = output.video } - return output + return output; } function ext(file, ext) { diff --git a/src/output-media/resize.js b/src/output-media/resize.js new file mode 100644 index 0000000..7985f4f --- /dev/null +++ b/src/output-media/resize.js @@ -0,0 +1,82 @@ +const async = require('async') +const exec = require('child_process').exec +const fs = require('fs') +const gm = require('gm') +const path = require('path') + +exports.sizes = { + thumb: 120, + large: 1000, +} + +exports.copy = function (task, callback) { + fs.copy(task.src, task.dest, callback) +} + +// Small square photo thumbnail +exports.photoSquare = function (task, callback) { + gm(task.src) + .autoOrient() + .coalesce() + .resize(exports.sizes.thumb, exports.sizes.thumb, '^') + .gravity('Center') + .crop(exports.sizes.thumb, exports.sizes.thumb) + .quality(90) + .write(task.dest, callback) +} + +// Large photo +exports.photoLarge = function (task, callback) { + gm(task.src) + .autoOrient() + .resize(null, exports.sizes.large, '>') + .quality(90) + .write(task.dest, callback) +} + +// Web-streaming friendly video +exports.videoWeb = function (task, callback) { + var ffmpeg = 'ffmpeg -i "' + task.src + '" -y "'+ task.dest +'" -f mp4 -vcodec libx264 -ab 96k' + // AVCHD/MTS videos need a full-frame export to avoid interlacing artefacts + if (path.extname(task.src).toLowerCase() === '.mts') { + ffmpeg += ' -vf yadif=1 -qscale:v 4' + } else { + ffmpeg += ' -vb 1200k' + } + exec(ffmpeg, callback) +} + +// Large video preview (before you click play) +exports.videoLarge = function (task, callback) { + async.series([ + function(next) { + extractFrame(task, next) + }, + function(next) { + exports.photoLarge({ + src: task.dest, + dest: task.dest + }, next) + } + ], callback) +} + +// Small square video preview +exports.videoSquare = function (task, callback) { + async.series([ + function(next) { + extractFrame(task, next) + }, + function(next) { + exports.photoSquare({ + src: task.dest, + dest: task.dest + }, next) + } + ], callback) +} + +function extractFrame (task, callback) { + const ffmpeg = 'ffmpeg -itsoffset -1 -i "' + task.src + '" -ss 0.1 -vframes 1 -y "' + task.dest + '"' + exec(ffmpeg, callback) +} diff --git a/src/output-media/tasks.js b/src/output-media/tasks.js new file mode 100644 index 0000000..3922aa1 --- /dev/null +++ b/src/output-media/tasks.js @@ -0,0 +1,51 @@ +const debug = require('debug')('thumbsup') +const fs = require('fs-extra') +const path = require('path') +const resize = require('./resize') + +const ACTION_MAP = { + 'original': resize.copy, + 'photo:thumbnail': resize.photoSquare, + 'photo:large': resize.photoLarge, + 'video:thumbnail': resize.videoSquare, + 'video:poster': resize.videoLarge, + 'video:resized': resize.videoWeb +} + +/* + Return a list of task to build all required outputs (new or updated) + Can be filtered by type (image/video) to give more accurate ETAs +*/ +exports.create = function (opts, files, filterType) { + var tasks = {} + // accumulate all tasks into an object + // to remove duplicate destinations + files.filter(f => f.type === filterType).forEach(f => { + debug(`Tasks for ${f.path}, ${JSON.stringify(f.output)}`) + Object.keys(f.output).forEach(out => { + var src = path.join(opts.input, f.path) + var dest = path.join(opts.output, f.output[out].path) + var destDate = modifiedDate(dest) + if (f.date > destDate) { + var action = ACTION_MAP[f.output[out].rel] + tasks[dest] = (done) => { + fs.mkdirsSync(path.dirname(dest)) + debug(`${f.output[out].rel} from ${src} to ${dest}`) + action({src: src, dest: dest}, done) + } + } + }) + }) + // back into an array + const list = Object.keys(tasks).map(t => tasks[t]) + debug(`Created ${list.length} ${filterType} tasks`) + return list +} + +function modifiedDate (filepath) { + try { + return fs.statSync(filepath).mtime.getTime() + } catch (ex) { + return 0 + } +} diff --git a/src/output-media/thumbs.js b/src/output-media/thumbs.js deleted file mode 100644 index 1c5f73e..0000000 --- a/src/output-media/thumbs.js +++ /dev/null @@ -1,79 +0,0 @@ -var exec = require('child_process').exec; -var path = require('path'); -var gm = require('gm'); -var async = require('async'); - -exports.sizes = { - thumb: 120, - large: 1000, -}; - -// Small square photo thumbnail -exports.photoSquare = function(task, callback) { - gm(task.src) - .autoOrient() - .coalesce() - .resize(exports.sizes.thumb, exports.sizes.thumb, '^') - .gravity('Center') - .crop(exports.sizes.thumb, exports.sizes.thumb) - .quality(90) - .write(task.dest, callback); -}; - -// Large photo -exports.photoLarge = function(task, callback) { - gm(task.src) - .autoOrient() - .resize(null, exports.sizes.large, '>') - .quality(90) - .write(task.dest, callback); -}; - -// Web-streaming friendly video -exports.videoWeb = function(task, callback) { - var ffmpeg = 'ffmpeg -i "' + task.src + '" -y "'+ task.dest +'" -f mp4 -vcodec libx264 -ab 96k'; - // AVCHD/MTS videos need a full-frame export to avoid interlacing artefacts - if (path.extname(task.src).toLowerCase() === '.mts') { - ffmpeg += ' -vf yadif=1 -qscale:v 4'; - } else { - ffmpeg += ' -vb 1200k'; - } - exec(ffmpeg, callback); -}; - -// Large video preview (before you click play) -exports.videoLarge = function(task, callback) { - async.series([ - function(next) { - extractFrame(task, next); - }, - function(next) { - exports.photoLarge({ - src: task.dest, - dest: task.dest, - metadata: task.metadata - }, next); - } - ], callback); -}; - -// Small square video preview -exports.videoSquare = function(task, callback) { - async.series([ - function(next) { - extractFrame(task, next); - }, - function(next) { - exports.photoSquare({ - src: task.dest, - dest: task.dest, - metadata: task.metadata - }, next); - } - ], callback); -}; - -function extractFrame(task, callback) { - var ffmpeg = 'ffmpeg -itsoffset -1 -i "' + task.src + '" -ss 0.1 -vframes 1 -y "' + task.dest + '"'; - exec(ffmpeg, callback); -} diff --git a/src/output-website/website.js b/src/output-website/website.js index 30d453c..1721cd4 100644 --- a/src/output-website/website.js +++ b/src/output-website/website.js @@ -4,7 +4,6 @@ var path = require('path'); var async = require('async'); var pad = require('pad'); var less = require('less'); -var files = require('../utils/files'); var Album = require('../model/album'); var byFolder = require('../model//by-folder'); var byDate = require('../model//by-date'); diff --git a/src/utils/files.js b/src/utils/files.js deleted file mode 100644 index d182baf..0000000 --- a/src/utils/files.js +++ /dev/null @@ -1,16 +0,0 @@ -var fs = require('fs'); - -exports.newer = function(src, dest) { - var srcTime = 0; - try { - var srcTime = fs.statSync(src).mtime.getTime(); - } catch (ex) { - return false; - } - try { - var destTime = fs.statSync(dest).mtime.getTime(); - return srcTime > destTime; - } catch (ex) { - return true; - } -}; diff --git a/src/utils/make.js b/src/utils/make.js deleted file mode 100644 index 7ec20e7..0000000 --- a/src/utils/make.js +++ /dev/null @@ -1,60 +0,0 @@ -const debug = require('debug')('thumbsup') -const fs = require('fs-extra') -const path = require('path') -const pad = require('pad') -const async = require('async') -const progress = require('./progress') - -exports.exec = function(input, output, collection, options, callback) { - var message = pad(options.message, 20) - // create {src, dest} tasks - var tasks = collection.filter(extension(options.ext)).map(file => { - return { - src: path.join(input, file.path), - dest: path.join(output, transform(file.path, options.dest)), - metadata: file - }; - }); - // only keep the ones where dest is out of date - var process = tasks.filter(function(task) { - try { - var destDate = fs.statSync(task.dest).mtime.getTime(); - return task.metadata.fileDate > destDate; - } catch (ex) { - return true; - } - }); - // run all in sequence - var bar = progress.create(options.message, process.length); - if (process.length > 0) { - var ops = process.map(function(task) { - return function(next) { - debug(`Transforming ${task.src} into ${task.dest}`) - fs.mkdirpSync(path.dirname(task.dest)); - options.func(task, function(err) { - bar.tick(); - next(err); - }); - }; - }); - async.parallelLimit(ops, 2, callback); - } else { - callback(); - } -} - -function extension(regex) { - return function(file) { - return file.path.match(new RegExp('\.(' + regex + ')$', 'i')); - } -} - -function transform(file, pattern) { - var absolutePrefix = (pattern[0] === '/') ? '/' : ''; - var parts = pattern.split('/'); - var full = path.join.apply(this, parts); - return absolutePrefix + - full.replace('$path', path.dirname(file)) - .replace('$name', path.basename(file, path.extname(file))) - .replace('$ext', path.extname(file).substr(1)); -} diff --git a/src/utils/progress.js b/src/utils/progress.js index f884834..9d6bf61 100644 --- a/src/utils/progress.js +++ b/src/utils/progress.js @@ -2,18 +2,28 @@ const pad = require('pad') const ProgressBar = require('progress') const util = require('util') -function BetterProgressBar (message, count) { +exports.create = function(message, count) { + if (typeof count === 'undefined') { + var format = pad(message, 20) + '[:bar] :eta' + return new BetterProgressBar(format, 1) + } + if (Array.isArray(count)) count = count.length if (count > 0) { - var format = pad(message, 20) + '[:bar] :current/:total files :eta' - ProgressBar.call(this, format, { total: count, width: 20 }) - this.tick(0) + var format = pad(message, 20) + '[:bar] :current/:total :eta' + return new BetterProgressBar(format, count) } else { var format = pad(message, 20) + '[:bar] up to date' - ProgressBar.call(this, format, { total: 1, width: 20, incomplete: '=' }) - this.tick(1) + var bar = new BetterProgressBar(format, 1) + bar.tick(1) + return bar } } +function BetterProgressBar (format, count) { + ProgressBar.call(this, format, { total: count, width: 25 }) + this.tick(0) +} + util.inherits(BetterProgressBar, ProgressBar) BetterProgressBar.prototype.eta = function () { @@ -34,7 +44,7 @@ BetterProgressBar.prototype.render = function (tokens) { } function formatEta (ms) { - if (isNaN(ms) || !isFinite(ms)) return '(calculating...)' + if (isNaN(ms) || !isFinite(ms)) return '' if (ms > 60 * 1000) { var min = Math.floor(ms / 60 / 1000) return `(${min.toFixed(0)}min left)` @@ -45,10 +55,6 @@ function formatEta (ms) { var sec = ms / 1000 return `(a few seconds left)` } else { - return '' + return 'done' } } - -exports.create = function(message, count) { - return new BetterProgressBar(message, count) -} diff --git a/test/input/file.spec.js b/test/input/file.spec.js index 2db6fed..2d37307 100644 --- a/test/input/file.spec.js +++ b/test/input/file.spec.js @@ -40,7 +40,8 @@ describe('Input file', function () { }) function dbFile (data) { - // needs at least a file date + // some required data + if (!data.SourceFile) data.SourceFile = 'photo.jpg' if (!data.File) data.File = {} if (!data.File.FileModifyDate) data.File.FileModifyDate = '1999:12:31 23:59:59+00:00' return data