Loading cache/save almost

czkawka_core/src/broken_files.rs

@@ -21,7 +21,7 @@ use crate::common::{
     check_folder_children, create_crash_message, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS,
     IMAGE_RS_BROKEN_FILES_EXTENSIONS, PDF_FILES_EXTENSIONS, ZIP_FILES_EXTENSIONS,
 };
-use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized};
+use crate::common_cache::{get_broken_files_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
 use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
 use crate::common_tool::{CommonData, CommonToolData};
 use crate::common_traits::*;
@@ -357,7 +357,7 @@ impl BrokenFiles {
         let files_to_check = mem::take(&mut self.files_to_check);
 
         if self.common_data.use_cache {
-            let (messages, loaded_items) = load_cache_from_file_generalized::<FileEntry>(&get_broken_files_cache_file(), self.get_delete_outdated_cache(), &files_to_check);
+            let (messages, loaded_items) = load_cache_from_file_generalized_by_path::<FileEntry>(&get_broken_files_cache_file(), self.get_delete_outdated_cache(), &files_to_check);
             self.get_text_messages_mut().extend_with_another_messages(messages);
             loaded_hash_map = loaded_items.unwrap_or_default();
 

czkawka_core/src/common_cache.rs

@@ -83,7 +83,26 @@ where
     text_messages
 }
 
-pub fn load_cache_from_file_generalized<T>(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap<String, T>) -> (Messages, Option<BTreeMap<String, T>>)
+pub fn load_cache_from_file_generalized_by_path<T>(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap<String, T>) -> (Messages, Option<BTreeMap<String, T>>)
+where
+    for<'a> T: Deserialize<'a> + ResultEntry + Sized + Send + Sync,
+{
+    let (text_messages, vec_loaded_cache) = load_cache_from_file_generalized(cache_file_name, delete_outdated_cache, used_files);
+    let Some(vec_loaded_entries) = vec_loaded_cache else {
+        return (text_messages, None);
+    };
+
+    debug!("Converting cache vec into map");
+    let map_loaded_entries: BTreeMap<String, T> = vec_loaded_entries
+        .into_iter()
+        .map(|file_entry| (file_entry.get_path().to_string_lossy().into_owned(), file_entry))
+        .collect();
+    debug!("Converted cache vec into map");
+
+    (text_messages, Some(map_loaded_entries))
+}
+
+fn load_cache_from_file_generalized<T>(cache_file_name: &str, delete_outdated_cache: bool, used_files: &BTreeMap<String, T>) -> (Messages, Option<Vec<T>>)
 where
     for<'a> T: Deserialize<'a> + ResultEntry + Sized + Send + Sync,
 {
@@ -150,12 +169,8 @@ where
 
         text_messages.messages.push(format!("Properly loaded {} cache entries.", vec_loaded_entries.len()));
 
-        let map_loaded_entries: BTreeMap<_, _> = vec_loaded_entries
-            .into_iter()
-            .map(|file_entry| (file_entry.get_path().to_string_lossy().into_owned(), file_entry))
-            .collect();
-        debug!("Loaded cache from file {cache_file_name} (or json alternative) - {} results", map_loaded_entries.len());
-        return (text_messages, Some(map_loaded_entries));
+        debug!("Loaded cache from file {cache_file_name} (or json alternative) - {} results", vec_loaded_entries.len());
+        return (text_messages, Some(vec_loaded_entries));
     }
     debug!("Failed to load cache from file {cache_file_name} because not exists");
     (text_messages, None)

czkawka_core/src/duplicate.rs

@@ -19,6 +19,7 @@ use rayon::prelude::*;
 use xxhash_rust::xxh3::Xxh3;
 
 use crate::common::{open_cache_folder, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads};
+use crate::common_cache::{get_duplicate_cache_file, save_cache_to_file_generalized};
 use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
 use crate::common_messages::Messages;
 use crate::common_tool::{CommonData, CommonToolData};
@@ -596,13 +597,14 @@ impl DuplicateFinder {
                 }
             }
 
-            save_hashes_to_file(
+            let messages = save_cache_to_file_generalized(
+                &get_duplicate_cache_file(&self.hash_type, true),
                 &save_cache_to_hashmap,
-                &mut self.common_data.text_messages,
-                &self.hash_type,
-                true,
+                self.common_data.save_also_as_json,
                 self.minimal_prehash_cache_file_size,
             );
+            self.get_text_messages_mut().extend_with_another_messages(messages);
+
             debug!("prehash_save_cache_at_exit - saving prehash cache end");
         }
     }
@@ -771,7 +773,15 @@ impl DuplicateFinder {
                 }
             }
         }
-        save_hashes_to_file(&all_results, &mut self.common_data.text_messages, &self.hash_type, false, self.minimal_cache_file_size);
+
+        let messages = save_cache_to_file_generalized(
+            &get_duplicate_cache_file(&self.hash_type, false),
+            &all_results,
+            self.common_data.save_also_as_json,
+            self.minimal_cache_file_size,
+        );
+        self.get_text_messages_mut().extend_with_another_messages(messages);
+
         debug!("full_hashing_save_cache_at_exit - end");
     }
 
@@ -1318,32 +1328,6 @@ pub fn make_hard_link(src: &Path, dst: &Path) -> io::Result<()> {
     result
 }
 
-pub fn save_hashes_to_file(hashmap: &BTreeMap<String, FileEntry>, text_messages: &mut Messages, type_of_hash: &HashType, is_prehash: bool, minimal_cache_file_size: u64) {
-    if let Some(((file_handler, cache_file), (_json_file, _json_name))) = open_cache_folder(&get_file_hash_name(type_of_hash, is_prehash), true, false, &mut text_messages.warnings)
-    {
-        let mut writer = BufWriter::new(file_handler.unwrap()); // Unwrap cannot fail
-
-        let mut how_much = 0;
-        for file_entry in hashmap.values() {
-            if file_entry.size >= minimal_cache_file_size {
-                let string: String = format!("{}//{}//{}//{}", file_entry.path.display(), file_entry.size, file_entry.modified_date, file_entry.hash);
-
-                if let Err(e) = writeln!(writer, "{string}") {
-                    text_messages
-                        .warnings
-                        .push(format!("Failed to save some data to cache file {}, reason {}", cache_file.display(), e));
-                    return;
-                }
-                how_much += 1;
-            }
-        }
-
-        text_messages
-            .messages
-            .push(flc!("core_saving_to_cache", generate_translation_hashmap(vec![("number", how_much.to_string())])));
-    }
-}
-
 pub fn load_hashes_from_file(text_messages: &mut Messages, delete_outdated_cache: bool, type_of_hash: &HashType, is_prehash: bool) -> Option<BTreeMap<u64, Vec<FileEntry>>> {
     if let Some(((file_handler, cache_file), (_json_file, _json_name))) =
         open_cache_folder(&get_file_hash_name(type_of_hash, is_prehash), false, false, &mut text_messages.warnings)

czkawka_core/src/same_music.rs

@@ -24,7 +24,7 @@ use symphonia::core::meta::MetadataOptions;
 use symphonia::core::probe::Hint;
 
 use crate::common::{create_crash_message, filter_reference_folders_generic, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, AUDIO_FILES_EXTENSIONS};
-use crate::common_cache::{get_similar_music_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized};
+use crate::common_cache::{get_similar_music_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
 use crate::common_dir_traversal::{CheckingMethod, DirTraversalBuilder, DirTraversalResult, FileEntry, ProgressData, ToolType};
 use crate::common_tool::{CommonData, CommonToolData};
 use crate::common_traits::*;
@@ -234,7 +234,7 @@ impl SameMusic {
 
         if self.common_data.use_cache {
             let (messages, loaded_items) =
-                load_cache_from_file_generalized::<MusicEntry>(get_similar_music_cache_file(checking_tags), self.get_delete_outdated_cache(), &self.music_to_check);
+                load_cache_from_file_generalized_by_path::<MusicEntry>(get_similar_music_cache_file(checking_tags), self.get_delete_outdated_cache(), &self.music_to_check);
             self.get_text_messages_mut().extend_with_another_messages(messages);
             loaded_hash_map = loaded_items.unwrap_or_default();
 

czkawka_core/src/similar_images.rs

@@ -22,7 +22,7 @@ use crate::common::{
     check_folder_children, create_crash_message, get_dynamic_image_from_raw_image, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, HEIC_EXTENSIONS,
     IMAGE_RS_SIMILAR_IMAGES_EXTENSIONS, RAW_IMAGE_EXTENSIONS,
 };
-use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized};
+use crate::common_cache::{get_similar_images_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
 use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
 use crate::common_tool::{CommonData, CommonToolData};
 use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults};
@@ -287,7 +287,7 @@ impl SimilarImages {
         let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
 
         if self.common_data.use_cache {
-            let (messages, loaded_items) = load_cache_from_file_generalized::<FileEntry>(
+            let (messages, loaded_items) = load_cache_from_file_generalized_by_path::<FileEntry>(
                 &get_similar_images_cache_file(&self.hash_size, &self.hash_alg, &self.image_filter),
                 self.get_delete_outdated_cache(),
                 &self.images_to_check,

czkawka_core/src/similar_videos.rs

@@ -16,7 +16,7 @@ use vid_dup_finder_lib::HashCreationErrorKind::DetermineVideo;
 use vid_dup_finder_lib::{NormalizedTolerance, VideoHash};
 
 use crate::common::{check_folder_children, prepare_thread_handler_common, send_info_and_wait_for_ending_all_threads, VIDEO_FILES_EXTENSIONS};
-use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized};
+use crate::common_cache::{get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized};
 use crate::common_dir_traversal::{common_get_entry_data_metadata, common_read_dir, get_lowercase_name, get_modified_time, CheckingMethod, ProgressData, ToolType};
 use crate::common_tool::{CommonData, CommonToolData};
 use crate::common_traits::{DebugPrint, PrintResults, ResultEntry, SaveResults};
@@ -265,7 +265,8 @@ impl SimilarVideos {
         let mut non_cached_files_to_check: BTreeMap<String, FileEntry> = Default::default();
 
         if self.common_data.use_cache {
-            let (messages, loaded_items) = load_cache_from_file_generalized::<FileEntry>(&get_similar_videos_cache_file(), self.get_delete_outdated_cache(), &self.videos_to_check);
+            let (messages, loaded_items) =
+                load_cache_from_file_generalized_by_path::<FileEntry>(&get_similar_videos_cache_file(), self.get_delete_outdated_cache(), &self.videos_to_check);
             self.get_text_messages_mut().extend_with_another_messages(messages);
             loaded_hash_map = loaded_items.unwrap_or_default();
 

czkawka_gui/src/connect_things/connect_settings.rs

@@ -1,7 +1,9 @@
 use std::collections::BTreeMap;
 use std::default::Default;
 
-use czkawka_core::common_cache::{get_similar_images_cache_file, get_similar_videos_cache_file, load_cache_from_file_generalized, save_cache_to_file_generalized};
+use czkawka_core::common_cache::{
+    get_duplicate_cache_file, get_similar_images_cache_file, get_similar_videos_cache_file, load_cache_from_file_generalized_by_path, save_cache_to_file_generalized,
+};
 use directories_next::ProjectDirs;
 use gtk4::prelude::*;
 use gtk4::{Label, ResponseType, Window};
@@ -129,13 +131,10 @@ pub fn connect_settings(gui_data: &GuiData) {
                                             hashmap_to_save.insert(file_entry.path.to_string_lossy().to_string(), file_entry);
                                         }
                                     }
-                                    czkawka_core::duplicate::save_hashes_to_file(
-                                        &hashmap_to_save,
-                                        &mut messages,
-                                        type_of_hash,
-                                        use_prehash,
-                                        entry_settings_cache_file_minimal_size.text().as_str().parse::<u64>().unwrap_or(2 * 1024 * 1024),
-                                    );
+
+                                    let minimal_cache_size = entry_settings_cache_file_minimal_size.text().as_str().parse::<u64>().unwrap_or(2 * 1024 * 1024);
+
+                                    save_cache_to_file_generalized(&get_duplicate_cache_file(type_of_hash, use_prehash), &hashmap_to_save, false, minimal_cache_size);
                                 }
                             }
 
@@ -170,7 +169,7 @@ pub fn connect_settings(gui_data: &GuiData) {
                                 FilterType::Triangle,
                             ] {
                                 for hash_alg in &[HashAlg::Blockhash, HashAlg::Gradient, HashAlg::DoubleGradient, HashAlg::VertGradient, HashAlg::Mean] {
-                                    let (mut messages, loaded_items) = load_cache_from_file_generalized::<czkawka_core::similar_images::FileEntry>(
+                                    let (mut messages, loaded_items) = load_cache_from_file_generalized_by_path::<czkawka_core::similar_images::FileEntry>(
                                         &get_similar_images_cache_file(hash_size, hash_alg, image_filter),
                                         true,
                                         &Default::default(),
@@ -206,7 +205,7 @@ pub fn connect_settings(gui_data: &GuiData) {
                 dialog.connect_response(move |dialog, response_type| {
                     if response_type == ResponseType::Ok {
                         let (mut messages, loaded_items) =
-                            load_cache_from_file_generalized::<czkawka_core::similar_videos::FileEntry>(&get_similar_videos_cache_file(), true, &Default::default());
+                            load_cache_from_file_generalized_by_path::<czkawka_core::similar_videos::FileEntry>(&get_similar_videos_cache_file(), true, &Default::default());
 
                         if let Some(cache_entries) = loaded_items {
                             let save_messages = save_cache_to_file_generalized(&get_similar_videos_cache_file(), &cache_entries, false, 0);