From 0378297bd180fe0ca0316061677bca86bfc99a48 Mon Sep 17 00:00:00 2001 From: Toni Melisma Date: Sat, 10 Oct 2020 21:02:11 +0300 Subject: [PATCH] Refactored main program (ignore videos option, output HTML in root of gallery etc) --- .gitignore | 1 - cmd/gogallery/main.go | 684 +++++++++++++++++++++++++++++++++++++ cmd/gogallery/main_test.go | 9 + 3 files changed, 693 insertions(+), 1 deletion(-) create mode 100644 cmd/gogallery/main.go create mode 100644 cmd/gogallery/main_test.go diff --git a/.gitignore b/.gitignore index 28aa3c0..4bf8519 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ *.dll *.so *.dylib -gogallery # Test binary, built with `go test -c` *.test diff --git a/cmd/gogallery/main.go b/cmd/gogallery/main.go new file mode 100644 index 0000000..081a1c9 --- /dev/null +++ b/cmd/gogallery/main.go @@ -0,0 +1,684 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "text/template" + "time" + + "github.com/cheggaaa/pb/v3" + "github.com/h2non/bimg" +) + +// global defaults +const optSymlinkDir = "_original" +const optFullsizeDir = "_pictures" +const optThumbnailDir = "_thumbnails" + +const optDirectoryMode os.FileMode = 0755 +const optFileMode os.FileMode = 0644 + +const thumbnailExtension = ".jpg" +const fullsizePictureExtension = ".jpg" +const fullsizeVideoExtension = ".mp4" + +const videoWorkerPoolSize = 2 +const imageWorkerPoolSize = 5 + +var optIgnoreVideos = false +var optDryRun = false + +// templates +const rawTemplate = ` + + + +{{ .Title }} + + + + + {{range .Subdirectories}} + +
+ {{range .Thumbnails}} + + {{end}} +
+ + {{end}} + {{range .Files}} + +
+ {{ .Original }} + +
+ + {{end}} + + +` + +// this function parses command-line arguments +func parseArgs() (inputDirectory string, outputDirectory string) { + outputDirectoryPtr := flag.String("o", ".", "Output root directory for gallery") + optIgnoreVideosPtr := flag.Bool("v", false, "Ignore video files") + optDryRunPtr := flag.Bool("d", false, "Dry run - don't make changes, only explain what would be done") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "Usage: %s [OPTION]... DIRECTORY\n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Create a static photo and video gallery from all\nsubdirectories and files in directory.\n") + fmt.Fprintf(os.Stderr, "\n") + flag.PrintDefaults() + } + + flag.Parse() + + if flag.NArg() == 0 { + fmt.Fprintf(os.Stderr, "%s: missing directories to use as input\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Try '%s -h' for more information.\n", os.Args[0]) + os.Exit(1) + } + + if flag.NArg() != 1 { + fmt.Fprintf(os.Stderr, "%s: wrong number of arguments given for input (expected one)\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Supply all options before input directory (go standard library limitation).\n") + fmt.Fprintf(os.Stderr, "Try '%s -h' for more information.\n", os.Args[0]) + os.Exit(1) + } + + if _, err := os.Stat(flag.Args()[0]); os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "%s: Directory does not exist: %s\n", os.Args[0], flag.Args()[0]) + os.Exit(1) + } + + if _, err := os.Stat(*outputDirectoryPtr); os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "%s: Directory does not exist: %s\n", os.Args[0], *outputDirectoryPtr) + os.Exit(1) + } + + if isEmptyDir(flag.Args()[0]) { + fmt.Fprintf(os.Stderr, "%s: Input directory is empty: %s\n", os.Args[0], flag.Args()[0]) + os.Exit(1) + } + + if *optDryRunPtr { + optDryRun = true + } + + if *optIgnoreVideosPtr { + optIgnoreVideos = true + } else { + _, err := exec.LookPath("ffmpeg") + if err != nil { + fmt.Fprintf(os.Stderr, "%s: Can't find ffmpeg in path\n", os.Args[0]) + os.Exit(1) + } + } + + return *outputDirectoryPtr, flag.Args()[0] +} + +// each file has a corresponding struct with relative and absolute paths +// for source files, if a newer thumbnail exists in gallery we set the existing flag and don't copy it +// for gallery files, if no corresponding source file exists, the existing flag stays false +// all non-existing gallery files will be deleted in the end +type file struct { + name string + relPath string + absPath string + modTime time.Time + exists bool +} + +type directory struct { + name string + relPath string + absPath string + modTime time.Time + subdirectories []directory + files []file +} + +// struct used to fill in data for each html page +type htmlData struct { + Title string + Subdirectories []struct { + Name string + Thumbnails []string + } + Files []struct { + Filename string + Thumbnail string + Fullsize string + Original string + } +} + +// struct used to send jobs to workers via channels +type job struct { + source string + destination string +} + +func checkError(e error) { + if e != nil { + panic(e) + } +} + +func isEmptyDir(directory string) (isEmpty bool) { + // TODO figure out a faster way to check if directory is empty + list, err := ioutil.ReadDir(directory) + checkError(err) + + if len(list) == 0 { + return true + } + return false +} + +func isVideoFile(filename string) bool { + switch filepath.Ext(strings.ToLower(filename)) { + case ".mp4", ".mov", ".3gp", ".avi", ".mts", ".m4v", ".mpg": + return true + default: + return false + } +} + +func isImageFile(filename string) bool { + switch filepath.Ext(strings.ToLower(filename)) { + case ".jpg", ".jpeg", ".heic", ".png", ".gif", ".tif": + return true + case ".cr2", ".raw", ".arw": + return true + default: + return false + } +} + +func isMediaFile(filename string) bool { + if isImageFile(filename) { + return true + } + + if !optIgnoreVideos && isVideoFile(filename) { + return true + } + + return false +} + +func recurseDirectory(thisDirectory string, relativeDirectory string) (root directory) { + root.name = filepath.Base(thisDirectory) + asIsStat, _ := os.Stat(thisDirectory) + root.modTime = asIsStat.ModTime() + root.relPath = relativeDirectory + root.absPath, _ = filepath.Abs(thisDirectory) + + list, err := ioutil.ReadDir(thisDirectory) + checkError(err) + + for _, entry := range list { + if entry.IsDir() { + if !isEmptyDir(filepath.Join(thisDirectory, entry.Name())) { + root.subdirectories = append(root.subdirectories, recurseDirectory(filepath.Join(thisDirectory, entry.Name()), filepath.Join(relativeDirectory, root.name))) + } + } else { + if isMediaFile(entry.Name()) { + root.files = append(root.files, file{name: entry.Name(), modTime: entry.ModTime(), relPath: filepath.Join(relativeDirectory, root.name, entry.Name()), absPath: filepath.Join(thisDirectory, entry.Name()), exists: false}) + } + } + } + + return (root) +} + +func stripExtension(fullFilename string) (baseFilename string) { + extension := filepath.Ext(fullFilename) + return fullFilename[0 : len(fullFilename)-len(extension)] +} + +func compareDirectories(source *directory, gallery *directory) { + for i, inputFile := range source.files { + inputFileBasename := stripExtension(inputFile.name) + for j, outputFile := range gallery.files { + outputFileBasename := stripExtension(outputFile.name) + if inputFileBasename == outputFileBasename { + gallery.files[j].exists = true + if !outputFile.modTime.Before(inputFile.modTime) { + source.files[i].exists = true + } + } + } + } + + for k, inputDir := range source.subdirectories { + for l, outputDir := range gallery.subdirectories { + if inputDir.name == outputDir.name { + compareDirectories(&(source.subdirectories[l]), &(gallery.subdirectories[k])) + } + } + } +} + +func countChanges(source directory) (outputChanges int) { + outputChanges = 0 + for _, file := range source.files { + if !file.exists { + outputChanges++ + } + } + + for _, dir := range source.subdirectories { + outputChanges = outputChanges + countChanges(dir) + } + + return outputChanges +} + +func createGallery(source directory, sourceRootDir string, gallery directory, fullsizeImageJobs chan job, thumbnailImageJobs chan job, fullsizeVideoJobs chan job, thumbnailVideoJobs chan job) { + // Create directories if they don't exist + fullsizeDirectoryPath := strings.Replace(filepath.Join(gallery.absPath, source.relPath, source.name), sourceRootDir, optFullsizeDir, 1) + createDirectory(fullsizeDirectoryPath) + + thumbnailDirectoryPath := strings.Replace(filepath.Join(gallery.absPath, source.relPath, source.name), sourceRootDir, optThumbnailDir, 1) + createDirectory(thumbnailDirectoryPath) + + symlinkDirectoryPath := strings.Replace(filepath.Join(gallery.absPath, source.relPath, source.name), sourceRootDir, optSymlinkDir, 1) + createDirectory(symlinkDirectoryPath) + + htmlDirectoryPath := strings.Replace(filepath.Join(gallery.absPath, source.relPath, source.name), sourceRootDir, "", 1) + createDirectory(htmlDirectoryPath) + + for _, file := range source.files { + if !file.exists { + // Create full-size copy + fullsizeFilePath := strings.Replace(filepath.Join(gallery.absPath, file.relPath), sourceRootDir, optFullsizeDir, 1) + fullsizeCopyFile(file.absPath, fullsizeFilePath, fullsizeImageJobs, fullsizeVideoJobs) + + // Create thumbnail + thumbnailFilePath := strings.Replace(filepath.Join(gallery.absPath, file.relPath), sourceRootDir, optThumbnailDir, 1) + thumbnailCopyFile(file.absPath, thumbnailFilePath, thumbnailImageJobs, thumbnailVideoJobs) + + // Symlink each file + symlinkFilePath := strings.Replace(filepath.Join(gallery.absPath, file.relPath), sourceRootDir, optSymlinkDir, 1) + symlinkFile(file.absPath, symlinkFilePath) + } + } + + // Recurse into each subdirectory to continue creating symlinks + for _, dir := range source.subdirectories { + createGallery(dir, sourceRootDir, gallery, fullsizeImageJobs, thumbnailImageJobs, fullsizeVideoJobs, thumbnailVideoJobs) + } + + createHTML(source.subdirectories, source.files, sourceRootDir, htmlDirectoryPath) +} + +func getHTMLRelPath(originalRelPath string, newRootDir string, sourceRootDir string, folderThumbnail bool) (thumbnailRelPath string) { + // Calculate relative path to know how many /../ we need to put into URL to get to root of Gallery + directoryList := strings.Split(originalRelPath, "/") + // Substract filename from length + // HTML files have file thumbnails, pictures and links and folder thumbnails - the latter + // are one level deeper but linked on the same level, thus the hack below + var directoryDepth int + if folderThumbnail { + directoryDepth = len(directoryList) - 2 + } else { + directoryDepth = len(directoryList) - 1 + } + var escapeStringArray []string + for j := 0; j < directoryDepth; j++ { + escapeStringArray = append(escapeStringArray, "..") + } + + return filepath.Join(strings.Join(escapeStringArray, "/"), strings.Replace(originalRelPath, sourceRootDir, newRootDir, 1)) +} + +func createHTML(subdirectories []directory, files []file, sourceRootDir string, htmlDirectoryPath string) { + htmlFilePath := filepath.Join(htmlDirectoryPath, "index.html") + + var data htmlData + + data.Title = filepath.Base(htmlDirectoryPath) + for _, dir := range subdirectories { + var thumbnails []string + // Link four first thumbnails to folder image + for i := 0; i < len(dir.files) && i < 4; i++ { + thumbnailRelURL := getHTMLRelPath(dir.files[i].relPath, optThumbnailDir, sourceRootDir, true) + thumbnails = append(thumbnails, thumbnailRelURL) + } + + data.Subdirectories = append(data.Subdirectories, struct { + Name string + Thumbnails []string + }{Name: dir.name, Thumbnails: thumbnails}) + } + for _, file := range files { + + data.Files = append(data.Files, struct { + Filename string + Thumbnail string + Fullsize string + Original string + }{Filename: file.name, Thumbnail: getHTMLRelPath(file.relPath, optThumbnailDir, sourceRootDir, false), Fullsize: getHTMLRelPath(file.relPath, optFullsizeDir, sourceRootDir, false), Original: getHTMLRelPath(file.relPath, optSymlinkDir, sourceRootDir, false)}) + } + + if optDryRun { + fmt.Println("Would create HTML:", htmlFilePath) + } else { + cookedTemplate, err := template.New("index").Parse(rawTemplate) + checkError(err) + + htmlFileHandle, err := os.Create(htmlFilePath) + checkError(err) + defer htmlFileHandle.Close() + + err = cookedTemplate.Execute(htmlFileHandle, data) + checkError(err) + + htmlFileHandle.Sync() + htmlFileHandle.Close() + } +} + +func createDirectory(destination string) { + if _, err := os.Stat(destination); os.IsNotExist(err) { + if optDryRun { + fmt.Println("Would create dir", destination) + } else { + err := os.Mkdir(destination, optDirectoryMode) + checkError(err) + } + } +} + +func symlinkFile(source string, destination string) { + if optDryRun { + fmt.Println("Would link", source, "to", destination) + } else { + if _, err := os.Stat(destination); err == nil { + err := os.Remove(destination) + checkError(err) + } + err := os.Symlink(source, destination) + checkError(err) + } +} + +func resizeThumbnailVideo(source string, destination string) { + ffmpegCommand := exec.Command("ffmpeg", "-y", "-i", source, "-ss", "00:00:01", "-vframes", "1", "-vf", "scale=200:200:force_original_aspect_ratio=increase,crop=200:200", "-loglevel", "fatal", destination) + ffmpegCommand.Stdout = os.Stdout + ffmpegCommand.Stderr = os.Stderr + + err := ffmpegCommand.Run() + if err != nil { + fmt.Fprintf(os.Stderr, "Could create thumbnail of video %s", source) + } +} + +func resizeFullsizeVideo(source string, destination string) { + ffmpegCommand := exec.Command("ffmpeg", "-y", "-i", source, "-pix_fmt", "yuv420p", "-vcodec", "libx264", "-acodec", "aac", "-movflags", "faststart", "-r", "24", "-vf", "scale='min(640,iw)':'min(640,ih)':force_original_aspect_ratio=decrease", "-crf", "28", "-loglevel", "fatal", destination) + ffmpegCommand.Stdout = os.Stdout + ffmpegCommand.Stderr = os.Stderr + + err := ffmpegCommand.Run() + if err != nil { + fmt.Fprintf(os.Stderr, "Could create full-size video of %s", source) + } +} + +func resizeThumbnailImage(source string, destination string) { + buffer, err := bimg.Read(source) + checkError(err) + + newImage, err := bimg.NewImage(buffer).Thumbnail(200) + checkError(err) + + newImage2, err := bimg.NewImage(newImage).AutoRotate() + + bimg.Write(destination, newImage2) +} + +func resizeFullsizeImage(source string, destination string) { + buffer, err := bimg.Read(source) + checkError(err) + + bufferImageSize, err := bimg.Size(buffer) + ratio := bufferImageSize.Width / bufferImageSize.Height + + newImage, err := bimg.NewImage(buffer).Resize(ratio*1080, 1080) + checkError(err) + + newImage2, err := bimg.NewImage(newImage).AutoRotate() + + bimg.Write(destination, newImage2) +} + +func fullsizeImageWorker(wg *sync.WaitGroup, imageJobs chan job, progressBar *pb.ProgressBar) { + defer wg.Done() + for job := range imageJobs { + resizeFullsizeImage(job.source, job.destination) + if !optDryRun { + progressBar.Increment() + } + } +} + +func fullsizeVideoWorker(wg *sync.WaitGroup, videoJobs chan job, progressBar *pb.ProgressBar) { + defer wg.Done() + for job := range videoJobs { + resizeFullsizeVideo(job.source, job.destination) + if !optDryRun { + progressBar.Increment() + } + } +} + +func fullsizeCopyFile(source string, destination string, fullsizeImageJobs chan job, fullsizeVideoJobs chan job) { + if isImageFile(source) { + destination = stripExtension(destination) + fullsizePictureExtension + if optDryRun { + fmt.Println("Would full-size copy image", source, "to", destination) + } else { + var imageJob job + imageJob.source = source + imageJob.destination = destination + fullsizeImageJobs <- imageJob + } + } else if isVideoFile(source) { + destination = stripExtension(destination) + fullsizeVideoExtension + if optDryRun { + fmt.Println("Would full-size copy video", source, "to", destination) + } else { + var videoJob job + videoJob.source = source + videoJob.destination = destination + fullsizeVideoJobs <- videoJob + } + } else { + fmt.Println("can't recognize file type for full-size copy", source) + } +} + +func thumbnailImageWorker(wg *sync.WaitGroup, thumbnailImageJobs chan job) { + defer wg.Done() + for job := range thumbnailImageJobs { + resizeThumbnailImage(job.source, job.destination) + } +} + +func thumbnailVideoWorker(wg *sync.WaitGroup, thumbnailVideoJobs chan job) { + defer wg.Done() + for job := range thumbnailVideoJobs { + resizeThumbnailVideo(job.source, job.destination) + } +} + +func thumbnailCopyFile(source string, destination string, thumbnailImageJobs chan job, thumbnailVideoJobs chan job) { + if isImageFile(source) { + destination = stripExtension(destination) + thumbnailExtension + if optDryRun { + fmt.Println("Would thumbnail copy image", source, "to", destination) + } else { + var imageJob job + imageJob.source = source + imageJob.destination = destination + thumbnailImageJobs <- imageJob + } + } else if isVideoFile(source) { + destination = stripExtension(destination) + thumbnailExtension + if optDryRun { + fmt.Println("Would thumbnail copy video", source, "to", destination) + } else { + var videoJob job + videoJob.source = source + videoJob.destination = destination + thumbnailVideoJobs <- videoJob + } + } else { + fmt.Println("can't recognize file type for thumbnail copy", source) + } +} + +func cleanGallery(gallery directory) { + for _, file := range gallery.files { + if !file.exists { + if optDryRun { + fmt.Println("Would delete", file.absPath) + } else { + err := os.Remove(file.absPath) + checkError(err) + } + + } + } + + for _, dir := range gallery.subdirectories { + cleanGallery(dir) + } + + if isEmptyDir(gallery.absPath) { + if optDryRun { + fmt.Println("Would remove empty directory", gallery.absPath) + } else { + err := os.Remove(gallery.absPath) + checkError(err) + } + } +} + +// Check that source directory root doesn't contain a name reserved for our output directories +func checkReservedNames(inputDirectory string) { + list, err := ioutil.ReadDir(inputDirectory) + checkError(err) + + for _, entry := range list { + if entry.Name() == optSymlinkDir || entry.Name() == optFullsizeDir || entry.Name() == optThumbnailDir { + fmt.Fprintf(os.Stderr, "Source directory root cannot contain file or folder with\n") + fmt.Fprintf(os.Stderr, "reserved names '%s', '%s' or '%s'\n", optSymlinkDir, optFullsizeDir, optThumbnailDir) + os.Exit(1) + } + } +} + +func main() { + var inputDirectory string + var outputDirectory string + + var gallery directory + var source directory + + // parse command-line args and set HTML template ready + outputDirectory, inputDirectory = parseArgs() + + fmt.Println(os.Args[0], ": Creating photo gallery") + fmt.Println("") + fmt.Println("Gathering photos and videos from:", inputDirectory) + fmt.Println("Creating static gallery in:", outputDirectory) + if optDryRun { + fmt.Println("Only dry run, not actually changing anything") + } + fmt.Println("") + + // check that source directory doesn't have reserved directory or file names + checkReservedNames(inputDirectory) + + // create directory structs by recursing through source and gallery directories + gallery = recurseDirectory(outputDirectory, "") + source = recurseDirectory(inputDirectory, "") + + // check whether gallery already has up-to-date pictures of sources, + // mark existing pictures in structs + for _, dir := range gallery.subdirectories { + if dir.name == optSymlinkDir { + compareDirectories(&source, &dir) + } + } + for _, dir := range gallery.subdirectories { + if dir.name == optFullsizeDir { + compareDirectories(&source, &dir) + } + } + for _, dir := range gallery.subdirectories { + if dir.name == optThumbnailDir { + compareDirectories(&source, &dir) + } + } + + changes := countChanges(source) + if changes > 0 { + var progressBar *pb.ProgressBar + if !optDryRun { + progressBar = pb.StartNew(changes) + } + + fullsizeImageJobs := make(chan job, 100000) + thumbnailImageJobs := make(chan job, 100000) + fullsizeVideoJobs := make(chan job, 100000) + thumbnailVideoJobs := make(chan job, 100000) + + var wg sync.WaitGroup + + for i := 1; i <= imageWorkerPoolSize; i++ { + wg.Add(2) + go fullsizeImageWorker(&wg, fullsizeImageJobs, progressBar) + go thumbnailImageWorker(&wg, thumbnailImageJobs) + } + + if !optIgnoreVideos { + for i := 1; i <= videoWorkerPoolSize; i++ { + wg.Add(2) + go fullsizeVideoWorker(&wg, fullsizeVideoJobs, progressBar) + go thumbnailVideoWorker(&wg, thumbnailVideoJobs) + } + } + + // create the gallery + createGallery(source, source.name, gallery, fullsizeImageJobs, thumbnailImageJobs, fullsizeVideoJobs, thumbnailVideoJobs) + + close(fullsizeImageJobs) + close(fullsizeVideoJobs) + close(thumbnailImageJobs) + close(thumbnailVideoJobs) + wg.Wait() + + if !optDryRun { + progressBar.Finish() + } + fmt.Println("Gallery created!") + } else { + fmt.Println("No pictures to update!") + } + // delete stale pictures + fmt.Println("\nCleaning up...") + cleanGallery(gallery) + fmt.Println("\nDone!") +} diff --git a/cmd/gogallery/main_test.go b/cmd/gogallery/main_test.go new file mode 100644 index 0000000..40ff687 --- /dev/null +++ b/cmd/gogallery/main_test.go @@ -0,0 +1,9 @@ +package main + +import ( + "testing" +) + +func TestHello(t *testing.T) { + t.Logf("placeholder") +}