mirror of https://github.com/thumbsup/thumbsup
Replace custom "make" DSL with the output task list (#60)
Romain
7 years ago
committed by
GitHub
parent
e719bf15b1
commit
e71e5d4207
@ -1,136 +1,84 @@
|
||||
const async = require('async')
|
||||
const fs = require('fs-extra')
|
||||
const make = require('./utils/make')
|
||||
const pad = require('pad')
|
||||
const path = require('path')
|
||||
const database = require('./input/database')
|
||||
const progress = require('./utils/progress')
|
||||
const File = require('./input/file')
|
||||
const Media = require('./model/media')
|
||||
const hierarchy = require('./model/hierarchy.js')
|
||||
const thumbs = require('./output-media/thumbs')
|
||||
const resize = require('./output-media/resize')
|
||||
const website = require('./output-website/website')
|
||||
|
||||
exports.build = function (opts) {
|
||||
|
||||
thumbs.sizes.thumb = opts.thumbSize;
|
||||
thumbs.sizes.large = opts.largeSize;
|
||||
resize.sizes.thumb = opts.thumbSize
|
||||
resize.sizes.large = opts.largeSize
|
||||
|
||||
fs.mkdirpSync(opts.output);
|
||||
var media = path.join(opts.output, 'media');
|
||||
var databaseFile = path.join(opts.output, 'metadata.json');
|
||||
fs.mkdirpSync(opts.output)
|
||||
const media = path.join(opts.output, 'media')
|
||||
const databaseFile = path.join(opts.output, 'metadata.json')
|
||||
|
||||
// ---------------------
|
||||
// These variables are set later during the async phase
|
||||
// ---------------------
|
||||
var album = null // root album with nested albums
|
||||
var collection = null // all files in the database
|
||||
|
||||
function buildStep (options) {
|
||||
return function(callback) {
|
||||
if (options.condition !== false) {
|
||||
make.exec(opts.input, media, collection, options, callback);
|
||||
} else {
|
||||
callback();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function callbackStep (name, fn) {
|
||||
return function(next) {
|
||||
process.stdout.write(pad(name, 20));
|
||||
fn(function(err) {
|
||||
if (err) {
|
||||
console.log('[====================] error');
|
||||
next(err);
|
||||
} else {
|
||||
console.log('[====================] done');
|
||||
next();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function copyFile(task, callback) {
|
||||
fs.copy(task.src, task.dest, callback);
|
||||
}
|
||||
|
||||
async.series([
|
||||
|
||||
function updateDatabase(callback) {
|
||||
function updateDatabase (callback) {
|
||||
database.update(opts.input, databaseFile, (err, dbFiles) => {
|
||||
collection = dbFiles.map(f => new File(f, opts))
|
||||
callback(err)
|
||||
})
|
||||
},
|
||||
|
||||
buildStep({
|
||||
condition: opts.originalPhotos,
|
||||
message: 'Photos: original',
|
||||
ext: 'jpg|jpeg|png|gif',
|
||||
dest: '/original/$path/$name.$ext',
|
||||
func: copyFile
|
||||
}),
|
||||
|
||||
buildStep({
|
||||
message: 'Photos: large',
|
||||
ext: 'jpg|jpeg|png|gif',
|
||||
dest: '/large/$path/$name.$ext',
|
||||
func: thumbs.photoLarge
|
||||
}),
|
||||
|
||||
buildStep({
|
||||
message: 'Photos: thumbnails',
|
||||
ext: 'jpg|jpeg|png|gif',
|
||||
dest: '/thumbs/$path/$name.$ext',
|
||||
func: thumbs.photoSquare
|
||||
}),
|
||||
|
||||
buildStep({
|
||||
condition: opts.originalVideos,
|
||||
message: 'Videos: original',
|
||||
ext: 'mp4|mov|mts|m2ts',
|
||||
dest: '/original/$path/$name.$ext',
|
||||
func: copyFile
|
||||
}),
|
||||
|
||||
buildStep({
|
||||
message: 'Videos: resized',
|
||||
ext: 'mp4|mov|mts|m2ts',
|
||||
dest: '/large/$path/$name.mp4',
|
||||
func: thumbs.videoWeb
|
||||
}),
|
||||
|
||||
buildStep({
|
||||
message: 'Videos: poster',
|
||||
ext: 'mp4|mov|mts|m2ts',
|
||||
dest: '/large/$path/$name.jpg',
|
||||
func: thumbs.videoLarge
|
||||
}),
|
||||
function processPhotos (callback) {
|
||||
const tasks = require('./output-media/tasks')
|
||||
const imageTasks = tasks.create(opts, collection, 'image')
|
||||
const imageBar = progress.create('Processing photos', imageTasks.length)
|
||||
async.parallelLimit(imageTasks.map(asyncProgress(imageBar)), 2, callback)
|
||||
},
|
||||
|
||||
buildStep({
|
||||
message: 'Videos: thumbnails',
|
||||
ext: 'mp4|mov|mts|m2ts',
|
||||
dest: '/thumbs/$path/$name.jpg',
|
||||
func: thumbs.videoSquare
|
||||
}),
|
||||
function processVideos (callback) {
|
||||
const tasks = require('./output-media/tasks')
|
||||
const videoTasks = tasks.create(opts, collection, 'video')
|
||||
const videoBar = progress.create('Processing videos', videoTasks.length)
|
||||
async.parallelLimit(videoTasks.map(asyncProgress(videoBar)), 2, callback)
|
||||
},
|
||||
|
||||
callbackStep('Album hierarchy', function(next) {
|
||||
function createAlbums (callback) {
|
||||
const bar = progress.create('Creating albums')
|
||||
const mediaCollection = collection.map(f => new Media(f))
|
||||
albums = hierarchy.createAlbums(mediaCollection, opts)
|
||||
next()
|
||||
}),
|
||||
album = hierarchy.createAlbums(mediaCollection, opts)
|
||||
bar.tick(1)
|
||||
callback()
|
||||
},
|
||||
|
||||
function createWebsite (callback) {
|
||||
const bar = progress.create('Building website')
|
||||
website.build(album, opts, (err) => {
|
||||
bar.tick(1)
|
||||
callback(err)
|
||||
})
|
||||
}
|
||||
|
||||
callbackStep('Static website', function(next) {
|
||||
website.build(albums, opts, next)
|
||||
})
|
||||
], finish)
|
||||
|
||||
], finish);
|
||||
}
|
||||
|
||||
function asyncProgress (bar) {
|
||||
return fn => {
|
||||
return done => {
|
||||
fn(err => {
|
||||
bar.tick(1)
|
||||
done(err)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function finish (err) {
|
||||
console.log();
|
||||
console.log(err || 'Gallery generated successfully');
|
||||
console.log();
|
||||
console.log(err ? 'Unexpected error' : '')
|
||||
console.log(err || 'Gallery generated successfully')
|
||||
console.log()
|
||||
process.exit(err ? 1 : 0)
|
||||
}
|
||||
|
||||
@ -0,0 +1,82 @@
|
||||
const async = require('async')
|
||||
const exec = require('child_process').exec
|
||||
const fs = require('fs')
|
||||
const gm = require('gm')
|
||||
const path = require('path')
|
||||
|
||||
exports.sizes = {
|
||||
thumb: 120,
|
||||
large: 1000,
|
||||
}
|
||||
|
||||
exports.copy = function (task, callback) {
|
||||
fs.copy(task.src, task.dest, callback)
|
||||
}
|
||||
|
||||
// Small square photo thumbnail
|
||||
exports.photoSquare = function (task, callback) {
|
||||
gm(task.src)
|
||||
.autoOrient()
|
||||
.coalesce()
|
||||
.resize(exports.sizes.thumb, exports.sizes.thumb, '^')
|
||||
.gravity('Center')
|
||||
.crop(exports.sizes.thumb, exports.sizes.thumb)
|
||||
.quality(90)
|
||||
.write(task.dest, callback)
|
||||
}
|
||||
|
||||
// Large photo
|
||||
exports.photoLarge = function (task, callback) {
|
||||
gm(task.src)
|
||||
.autoOrient()
|
||||
.resize(null, exports.sizes.large, '>')
|
||||
.quality(90)
|
||||
.write(task.dest, callback)
|
||||
}
|
||||
|
||||
// Web-streaming friendly video
|
||||
exports.videoWeb = function (task, callback) {
|
||||
var ffmpeg = 'ffmpeg -i "' + task.src + '" -y "'+ task.dest +'" -f mp4 -vcodec libx264 -ab 96k'
|
||||
// AVCHD/MTS videos need a full-frame export to avoid interlacing artefacts
|
||||
if (path.extname(task.src).toLowerCase() === '.mts') {
|
||||
ffmpeg += ' -vf yadif=1 -qscale:v 4'
|
||||
} else {
|
||||
ffmpeg += ' -vb 1200k'
|
||||
}
|
||||
exec(ffmpeg, callback)
|
||||
}
|
||||
|
||||
// Large video preview (before you click play)
|
||||
exports.videoLarge = function (task, callback) {
|
||||
async.series([
|
||||
function(next) {
|
||||
extractFrame(task, next)
|
||||
},
|
||||
function(next) {
|
||||
exports.photoLarge({
|
||||
src: task.dest,
|
||||
dest: task.dest
|
||||
}, next)
|
||||
}
|
||||
], callback)
|
||||
}
|
||||
|
||||
// Small square video preview
|
||||
exports.videoSquare = function (task, callback) {
|
||||
async.series([
|
||||
function(next) {
|
||||
extractFrame(task, next)
|
||||
},
|
||||
function(next) {
|
||||
exports.photoSquare({
|
||||
src: task.dest,
|
||||
dest: task.dest
|
||||
}, next)
|
||||
}
|
||||
], callback)
|
||||
}
|
||||
|
||||
function extractFrame (task, callback) {
|
||||
const ffmpeg = 'ffmpeg -itsoffset -1 -i "' + task.src + '" -ss 0.1 -vframes 1 -y "' + task.dest + '"'
|
||||
exec(ffmpeg, callback)
|
||||
}
|
||||
@ -0,0 +1,51 @@
|
||||
const debug = require('debug')('thumbsup')
|
||||
const fs = require('fs-extra')
|
||||
const path = require('path')
|
||||
const resize = require('./resize')
|
||||
|
||||
const ACTION_MAP = {
|
||||
'original': resize.copy,
|
||||
'photo:thumbnail': resize.photoSquare,
|
||||
'photo:large': resize.photoLarge,
|
||||
'video:thumbnail': resize.videoSquare,
|
||||
'video:poster': resize.videoLarge,
|
||||
'video:resized': resize.videoWeb
|
||||
}
|
||||
|
||||
/*
|
||||
Return a list of task to build all required outputs (new or updated)
|
||||
Can be filtered by type (image/video) to give more accurate ETAs
|
||||
*/
|
||||
exports.create = function (opts, files, filterType) {
|
||||
var tasks = {}
|
||||
// accumulate all tasks into an object
|
||||
// to remove duplicate destinations
|
||||
files.filter(f => f.type === filterType).forEach(f => {
|
||||
debug(`Tasks for ${f.path}, ${JSON.stringify(f.output)}`)
|
||||
Object.keys(f.output).forEach(out => {
|
||||
var src = path.join(opts.input, f.path)
|
||||
var dest = path.join(opts.output, f.output[out].path)
|
||||
var destDate = modifiedDate(dest)
|
||||
if (f.date > destDate) {
|
||||
var action = ACTION_MAP[f.output[out].rel]
|
||||
tasks[dest] = (done) => {
|
||||
fs.mkdirsSync(path.dirname(dest))
|
||||
debug(`${f.output[out].rel} from ${src} to ${dest}`)
|
||||
action({src: src, dest: dest}, done)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
// back into an array
|
||||
const list = Object.keys(tasks).map(t => tasks[t])
|
||||
debug(`Created ${list.length} ${filterType} tasks`)
|
||||
return list
|
||||
}
|
||||
|
||||
function modifiedDate (filepath) {
|
||||
try {
|
||||
return fs.statSync(filepath).mtime.getTime()
|
||||
} catch (ex) {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
@ -1,79 +0,0 @@
|
||||
var exec = require('child_process').exec;
|
||||
var path = require('path');
|
||||
var gm = require('gm');
|
||||
var async = require('async');
|
||||
|
||||
exports.sizes = {
|
||||
thumb: 120,
|
||||
large: 1000,
|
||||
};
|
||||
|
||||
// Small square photo thumbnail
|
||||
exports.photoSquare = function(task, callback) {
|
||||
gm(task.src)
|
||||
.autoOrient()
|
||||
.coalesce()
|
||||
.resize(exports.sizes.thumb, exports.sizes.thumb, '^')
|
||||
.gravity('Center')
|
||||
.crop(exports.sizes.thumb, exports.sizes.thumb)
|
||||
.quality(90)
|
||||
.write(task.dest, callback);
|
||||
};
|
||||
|
||||
// Large photo
|
||||
exports.photoLarge = function(task, callback) {
|
||||
gm(task.src)
|
||||
.autoOrient()
|
||||
.resize(null, exports.sizes.large, '>')
|
||||
.quality(90)
|
||||
.write(task.dest, callback);
|
||||
};
|
||||
|
||||
// Web-streaming friendly video
|
||||
exports.videoWeb = function(task, callback) {
|
||||
var ffmpeg = 'ffmpeg -i "' + task.src + '" -y "'+ task.dest +'" -f mp4 -vcodec libx264 -ab 96k';
|
||||
// AVCHD/MTS videos need a full-frame export to avoid interlacing artefacts
|
||||
if (path.extname(task.src).toLowerCase() === '.mts') {
|
||||
ffmpeg += ' -vf yadif=1 -qscale:v 4';
|
||||
} else {
|
||||
ffmpeg += ' -vb 1200k';
|
||||
}
|
||||
exec(ffmpeg, callback);
|
||||
};
|
||||
|
||||
// Large video preview (before you click play)
|
||||
exports.videoLarge = function(task, callback) {
|
||||
async.series([
|
||||
function(next) {
|
||||
extractFrame(task, next);
|
||||
},
|
||||
function(next) {
|
||||
exports.photoLarge({
|
||||
src: task.dest,
|
||||
dest: task.dest,
|
||||
metadata: task.metadata
|
||||
}, next);
|
||||
}
|
||||
], callback);
|
||||
};
|
||||
|
||||
// Small square video preview
|
||||
exports.videoSquare = function(task, callback) {
|
||||
async.series([
|
||||
function(next) {
|
||||
extractFrame(task, next);
|
||||
},
|
||||
function(next) {
|
||||
exports.photoSquare({
|
||||
src: task.dest,
|
||||
dest: task.dest,
|
||||
metadata: task.metadata
|
||||
}, next);
|
||||
}
|
||||
], callback);
|
||||
};
|
||||
|
||||
function extractFrame(task, callback) {
|
||||
var ffmpeg = 'ffmpeg -itsoffset -1 -i "' + task.src + '" -ss 0.1 -vframes 1 -y "' + task.dest + '"';
|
||||
exec(ffmpeg, callback);
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
var fs = require('fs');
|
||||
|
||||
exports.newer = function(src, dest) {
|
||||
var srcTime = 0;
|
||||
try {
|
||||
var srcTime = fs.statSync(src).mtime.getTime();
|
||||
} catch (ex) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
var destTime = fs.statSync(dest).mtime.getTime();
|
||||
return srcTime > destTime;
|
||||
} catch (ex) {
|
||||
return true;
|
||||
}
|
||||
};
|
||||
@ -1,60 +0,0 @@
|
||||
const debug = require('debug')('thumbsup')
|
||||
const fs = require('fs-extra')
|
||||
const path = require('path')
|
||||
const pad = require('pad')
|
||||
const async = require('async')
|
||||
const progress = require('./progress')
|
||||
|
||||
exports.exec = function(input, output, collection, options, callback) {
|
||||
var message = pad(options.message, 20)
|
||||
// create {src, dest} tasks
|
||||
var tasks = collection.filter(extension(options.ext)).map(file => {
|
||||
return {
|
||||
src: path.join(input, file.path),
|
||||
dest: path.join(output, transform(file.path, options.dest)),
|
||||
metadata: file
|
||||
};
|
||||
});
|
||||
// only keep the ones where dest is out of date
|
||||
var process = tasks.filter(function(task) {
|
||||
try {
|
||||
var destDate = fs.statSync(task.dest).mtime.getTime();
|
||||
return task.metadata.fileDate > destDate;
|
||||
} catch (ex) {
|
||||
return true;
|
||||
}
|
||||
});
|
||||
// run all in sequence
|
||||
var bar = progress.create(options.message, process.length);
|
||||
if (process.length > 0) {
|
||||
var ops = process.map(function(task) {
|
||||
return function(next) {
|
||||
debug(`Transforming ${task.src} into ${task.dest}`)
|
||||
fs.mkdirpSync(path.dirname(task.dest));
|
||||
options.func(task, function(err) {
|
||||
bar.tick();
|
||||
next(err);
|
||||
});
|
||||
};
|
||||
});
|
||||
async.parallelLimit(ops, 2, callback);
|
||||
} else {
|
||||
callback();
|
||||
}
|
||||
}
|
||||
|
||||
function extension(regex) {
|
||||
return function(file) {
|
||||
return file.path.match(new RegExp('\.(' + regex + ')$', 'i'));
|
||||
}
|
||||
}
|
||||
|
||||
function transform(file, pattern) {
|
||||
var absolutePrefix = (pattern[0] === '/') ? '/' : '';
|
||||
var parts = pattern.split('/');
|
||||
var full = path.join.apply(this, parts);
|
||||
return absolutePrefix +
|
||||
full.replace('$path', path.dirname(file))
|
||||
.replace('$name', path.basename(file, path.extname(file)))
|
||||
.replace('$ext', path.extname(file).substr(1));
|
||||
}
|
||||
Loading…
Reference in New Issue