mirror of
https://github.com/thumbsup/thumbsup
synced 2024-11-05 12:01:04 +00:00
24b2f9bd7c
1. Move from a JSON index to a SQLite database. - This allows the indexing to be interrupted & resumed - Updating the index consumes less RAM than loading / saving an entire JSON object - Loading the index consumes less RAM since it can be streamed, only exacting the properties we need every time (instead of loading all EXIF data in memory, only to discard most of it later) - These make a big difference when processing 10,000+ photos 2. Switch from <glob> to a manual <readdir> - Glob would take several hundred or GB of RAM when asked to find several thousand files - Manual approach with <micromatch> library does the same thing in a fraction of the time / memory usage 3. Exiftool optimisations - Run 1 exiftool process per CPU, still in batch mode (divide all files to be read into 1 bucket per CPU) - Stream the exiftool output instead of buffering it in memory
115 lines
2.6 KiB
JavaScript
115 lines
2.6 KiB
JavaScript
const delta = require('../../../src/components/index/delta')
|
|
const should = require('should/as-function')
|
|
|
|
describe('Index: delta', () => {
|
|
it('no changes', () => {
|
|
const database = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000000000
|
|
}
|
|
const disk = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000000000
|
|
}
|
|
const res = delta.calculate(database, disk)
|
|
should(res).eql({
|
|
unchanged: ['IMG_0001', 'IMG_0002'],
|
|
added: [],
|
|
modified: [],
|
|
deleted: []
|
|
})
|
|
})
|
|
|
|
it('no changes within a second', () => {
|
|
const database = {
|
|
'IMG_0001': 1410000001000,
|
|
'IMG_0002': 1420000001000
|
|
}
|
|
const disk = {
|
|
'IMG_0001': 1410000001500, // 500ms later
|
|
'IMG_0002': 1420000000500 // 500ms earlier
|
|
}
|
|
const res = delta.calculate(database, disk)
|
|
should(res).eql({
|
|
unchanged: ['IMG_0001', 'IMG_0002'],
|
|
added: [],
|
|
modified: [],
|
|
deleted: []
|
|
})
|
|
})
|
|
|
|
it('new files', () => {
|
|
const database = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000000000
|
|
}
|
|
const disk = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000000000,
|
|
'IMG_0003': 1430000000000
|
|
}
|
|
const res = delta.calculate(database, disk)
|
|
should(res).eql({
|
|
unchanged: ['IMG_0001', 'IMG_0002'],
|
|
added: ['IMG_0003'],
|
|
modified: [],
|
|
deleted: []
|
|
})
|
|
})
|
|
|
|
it('deleted files', () => {
|
|
const database = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000000000
|
|
}
|
|
const disk = {
|
|
'IMG_0001': 1410000000000
|
|
}
|
|
const res = delta.calculate(database, disk)
|
|
should(res).eql({
|
|
unchanged: ['IMG_0001'],
|
|
added: [],
|
|
modified: [],
|
|
deleted: ['IMG_0002']
|
|
})
|
|
})
|
|
|
|
it('modified files', () => {
|
|
const database = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000000000
|
|
}
|
|
const disk = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000002000
|
|
}
|
|
const res = delta.calculate(database, disk)
|
|
should(res).eql({
|
|
unchanged: ['IMG_0001'],
|
|
added: [],
|
|
modified: ['IMG_0002'],
|
|
deleted: []
|
|
})
|
|
})
|
|
|
|
it('all cases', () => {
|
|
const database = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000000000,
|
|
'IMG_0003': 1430000000000
|
|
}
|
|
const disk = {
|
|
'IMG_0001': 1410000000000,
|
|
'IMG_0002': 1420000002000,
|
|
'IMG_0004': 1445000000000
|
|
}
|
|
const res = delta.calculate(database, disk)
|
|
should(res).eql({
|
|
unchanged: ['IMG_0001'],
|
|
added: ['IMG_0004'],
|
|
modified: ['IMG_0002'],
|
|
deleted: ['IMG_0003']
|
|
})
|
|
})
|
|
})
|