add feature: advanced metadata filter mode (Obsidian user only)

web_ui/modules/utilities.py

@@ -1,45 +1,58 @@
 import json
 import os
 
-def extract_string(text, delimiter):
-    # Extract string between delimiters
-    start_index = text.index(delimiter) + len(delimiter)
-    end_index = text.index(delimiter, start_index)
-    return text[start_index:end_index]
 
+# def extract_string(text, delimiter):
+#     # Extract string between delimiters
+#     start_index = text.index(delimiter) + len(delimiter)
+#     end_index = text.index(delimiter, start_index)
+#     return text[start_index:end_index]
 
-def extract_string(text, delimiter, force=False):
+
+def extract_string(text, delimiter, force=False, join=True, split_mode=False):
     if not delimiter in text:
         if force:
             return ''
         else:
             return text
     else:
-        substring = text.split(delimiter)
-        result = []
-        for i in range(1, len(substring), 2):
-            result.append(substring[i])
-        return ''.join(result)
-        
+        if split_mode:
+            return text.split(delimiter)
+        else:
+            substring = text.split(delimiter)
+            result = []
+            for i in range(1, len(substring), 2):
+                result.append(substring[i])
+            if join:
+                return ''.join(result)
+            else:
+                return result
+
 
 def create_not_exist(path):
     dir = os.path.dirname(path)
     if not os.path.exists(dir):
         os.makedirs(dir)
 
+
 def create_file_not_exist(path):
     if not os.path.exists(path):
         write_file('', path)
 
+
 def read_file(filepath, delimiter='', force=False):
     with open(filepath, 'r', encoding='utf-8') as file:
         data = file.read()
-        if delimiter != '':
-            data = extract_string(data, delimiter, force)
-        return data
+        return parse_data(data, delimiter, force)
+
 
+def parse_data(data, delimiter='', force=False):
+    if delimiter != '':
+        data = extract_string(data, delimiter, force)
+    return data
 
-def read_files(file_dir, delimiter='', force=False):
+
+def read_files(file_dir, delimiter='', force=False, single_string=True):
     contents = []
 
     # Read all files in a directory
@@ -54,40 +67,62 @@ def read_files(file_dir, delimiter='', force=False):
             file_data = read_file(filepath, delimiter, force)
             if force and file_data == '':
                 continue
-            
+
             content = [f'[{filename}]', file_data]
             contents.append('\n\n'.join(content))
-        
-    result = '\n\n\n\n'.join(contents)
+    if single_string:
+        result = '\n\n\n\n'.join(contents)
+    else:
+        result = contents
     return result
 
+
 def write_file(content, filepath, mode='w'):
     create_not_exist(filepath)
     with open(filepath, mode, encoding='utf-8') as file:
         file.write(content)
 
+
 def create_json_not_exist(filepath, initial_value={}):
     if not os.path.exists(filepath):
         write_json_file(initial_value, filepath)
 
+
 def write_json_file(content, filepath, mode='w'):
     with open(filepath, mode) as file:
         json.dump(content, file, indent=2)
 
 
 def read_json_file(filepath):
-    with open(filepath, 'r') as file:
-        return json.load(file)
+    try:
+        with open(filepath, 'r') as file:
+            return json.load(file)
+    except FileNotFoundError:
+        return {}
 
-def read_json_at(filepath, key):
+
+def read_json_at(filepath, key, default_value=''):
     data = read_json_file(filepath)
-    if data[key] == 'True' or data[key] == 'true':
-        return True
-    elif data[key] == 'False' or data[key] == 'false':
-        return False
-    else:
+    try:
+        # if key is string, check if it is boolean or numeric
+        if isinstance(data[key], str):
+            if data[key] in ['true', 'false']:
+                return data[key] == 'true'
+            elif data[key].isnumeric():
+                return int(data[key])
+            elif data[key].replace('.', '', 1).isnumeric():
+                return float(data[key])
+            else:
+                return data[key]
+        else:
+            return data[key]
+    except KeyError:
+        # if key not found, create key with default value
+        data[key] = default_value
+        write_json_file(data, filepath)
         return data[key]
 
+
 def update_json(filepath, key, value):
     data = read_json_file(filepath)
     data[key] = value

web_ui/pages/1_Configs.py

@@ -15,16 +15,22 @@ prompt_dir = f'{user_dir}prompt/'
 brain_memo = f'{user_dir}brain-memo.json'
 
 
-def save(content, path, page=''):
+def save(content, path, page='', json_value: dict = None):
+    if json_value is None:
+        json_value = []
     save_but = st.button('💾Save')
     if save_but:
         util.write_file(content, path)
         st.success(f'✅File saved!')
         # write to json file
         if page == '💽Brain Memory':
-            util.update_json(brain_memo, 'delimiter', delimiter)
-            util.update_json(brain_memo, 'append_mode', append_mode)
-            util.update_json(brain_memo, 'force_mode', force_delimiter)
+            util.update_json(brain_memo, 'delimiter', json_value['delimiter'])
+            util.update_json(brain_memo, 'append_mode', json_value['append_mode'])
+            util.update_json(brain_memo, 'force_mode', json_value['force_mode'])
+            util.update_json(brain_memo, 'advanced_mode', json_value['advanced_mode'])
+            util.update_json(brain_memo, 'filter_keys', json_value['filter_keys'])
+            util.update_json(brain_memo, 'filter_logics', json_value['filter_logics'])
+            util.update_json(brain_memo, 'filter_values', json_value['filter_values'])
 
 
 def select_directory():
@@ -36,65 +42,143 @@ def select_directory():
     return directory
 
 
-with st.sidebar:
-    st.title('Settings')
-    menu = st.radio('Menu', [
-        '📝Prompts',
-        '💽Brain Memory',
-        '🔑API Keys'
-    ])
-
-with body:
-    match menu:
-        case '📝Prompts':
-            st.title('📝Prompts')
-            st.text('Configuration of prompts.')
-            selected_file = st.selectbox('Prompt File', os.listdir(prompt_dir))
-            selected_path = prompt_dir + selected_file
-            mod_text = st.text_area('Prompts', value=util.read_file(selected_path), height=500)
-            save(mod_text, selected_path)
-
-        case '💽Brain Memory':
-            st.title('💽Brain Memory')
-            st.text('Modify your brain knowledge base.')
-            memory_data = util.read_file(f'{user_dir}input.txt')
-
-            note_dir = ''
-
-            col1, col2 = st.columns(2)
-            with col1:
-                st.button('🔄Refresh')
-            with col2:
-                if st.button('📁Select Note Directory'):
-                    note_dir = select_directory()
-                    util.update_json(brain_memo, 'note_dir', note_dir)
-            note_dir = st.text_input('Note Directory', value=util.read_json_at(brain_memo, 'note_dir'),
-                                     placeholder='Select Note Directory', key='note_dir')
-
-            col1, col2 = st.columns(2)
-            with col1:
-                delimiter_memo = util.read_json_at(brain_memo, 'delimiter')
-                delimiter = st.text_input('Delimiter', delimiter_memo, placeholder='e.g. +++')
-
-            with col2:
-                append_mode = st.checkbox('Append Mode', value=util.read_json_at(brain_memo, 'append_mode'))
-
-                force_delimiter = st.checkbox('Force Delimiter', value=util.read_json_at(brain_memo, 'force_mode'))
-
-            # if note directory is selected
-            if note_dir != '':
-
-                note_data = util.read_files(note_dir, delimiter, force_delimiter)
-                if append_mode:
-                    memory_data += note_data
-                else:
-                    memory_data = note_data
-
-            mod_text = st.text_area('Raw Memory Inputs', value=memory_data, height=500)
-            save(mod_text, f'{user_dir}input.txt', '💽Brain Memory')
-
-        case '🔑API Keys':
-            st.title('🔑API Keys')
-            st.text('Configure your OpenAI API keys.')
-            mod_text = st.text_input('API Keys', value=util.read_file(f'{user_dir}API-KEYS.txt'))
-            save(mod_text, f'{user_dir}API-KEYS.txt')
+def match_logic(logic, filter_key, filter_val, key, value):
+    if logic == 'IS':
+        return filter_key == key and filter_val == value
+    elif logic == 'IS NOT':
+        return filter_key == key and filter_val != value
+
+
+def match_fields(contents: list, logic_select, filter_key, filter_val):
+    filtered_contents = []
+    for content in contents:
+        # extract metadata
+        try:
+            yaml = util.extract_string(content, '---', True, join=False, split_mode=True)[1]
+        except IndexError:
+            yaml = ''
+        fields = yaml.split('\n')
+        for field in fields:
+            if field == '':
+                continue
+            key, value = field.split(':')
+            key = key.strip()
+            value = value.strip()
+            if match_logic(logic_select, filter_key, filter_val, key, value):
+                filtered_contents.append(content)
+                break
+    return filtered_contents
+
+
+def filter_data(contents: list, append=True):
+
+    # filters
+    col1, col2, col3 = st.columns(3)
+    with col1:
+        filter_key = st.text_input('Key', placeholder='Key', value=util.read_json_at(brain_memo, 'filter_keys'))
+    with col2:
+        options = ['IS', 'IS NOT']
+        default_value = util.read_json_at(brain_memo, 'filter_logics')
+        logic_select = st.selectbox('Logic', options, index=options.index(default_value))
+    with col3:
+        value = util.read_json_at(brain_memo, 'filter_values')
+        if isinstance(value, int):
+            value = "{:02}".format(value)
+        filter_val = st.text_input('value', placeholder='Value', value=value)
+
+    # filter data
+    filtered_contents = match_fields(contents, logic_select, filter_key, filter_val)
+    result = filtered_contents
+    if append:
+        return '\n\n\n\n'.join(result), filter_key, logic_select, filter_val
+    else:
+        return result, filter_key, logic_select, filter_val
+
+
+
+def main():
+    with st.sidebar:
+        st.title('Settings')
+        menu = st.radio('Menu', [
+            '📝Prompts',
+            '💽Brain Memory',
+            '🔑API Keys'
+        ])
+
+    with body:
+        match menu:
+            case '📝Prompts':
+                st.title('📝Prompts')
+                st.text('Configuration of prompts.')
+                selected_file = st.selectbox('Prompt File', os.listdir(prompt_dir))
+                selected_path = prompt_dir + selected_file
+                mod_text = st.text_area('Prompts', value=util.read_file(selected_path), height=500)
+                save(mod_text, selected_path)
+
+            case '💽Brain Memory':
+                st.title('💽Brain Memory')
+                st.text('Modify your brain knowledge base.')
+                memory_data = util.read_file(f'{user_dir}input.txt')
+
+                col1, col2 = st.columns(2)
+                with col1:
+                    st.button('🔄Refresh')
+                with col2:
+                    if st.button('📁Select Note Directory'):
+                        note_dir = select_directory()
+                        util.update_json(brain_memo, 'note_dir', note_dir)
+                note_dir = st.text_input('Note Directory', value=util.read_json_at(brain_memo, 'note_dir'),
+                                         placeholder='Select Note Directory', key='note_dir')
+
+                col1, col2, col3 = st.columns(3)
+                with col1:
+                    delimiter_memo = util.read_json_at(brain_memo, 'delimiter')
+                    delimiter = st.text_input('Delimiter', delimiter_memo, placeholder='e.g. +++')
+
+                with col2:
+                    append_mode = st.checkbox('Append Mode', value=util.read_json_at(brain_memo, 'append_mode'))
+                    force_delimiter = st.checkbox('Force Delimiter', value=util.read_json_at(brain_memo, 'force_mode'))
+                with col3:
+                    advanced_mode = st.radio('Advanced Mode (Obsidian only)', ['Off', 'On'],
+                                             index=util.read_json_at(brain_memo, 'advanced_mode', 0))
+                    advanced_mode_index = 1 if advanced_mode == 'On' else 0
+
+                # if note directory is selected
+                if note_dir != '':
+
+                    filter_key = ''
+                    filter_logic = ''
+                    filter_val = ''
+                    # if advanced mode enabled
+                    if advanced_mode_index == 1:
+                        note_datas = util.read_files(note_dir, single_string=False)
+                        note_datas, filter_key, filter_logic, filter_val = filter_data(note_datas, True)
+                        modified_data = util.parse_data(note_datas, delimiter, force_delimiter)
+                    else:
+                        modified_data = util.read_files(note_dir, single_string=True, delimiter=delimiter,
+                                                        force=force_delimiter)
+                    if append_mode:
+                        memory_data += modified_data
+                    else:
+                        memory_data = modified_data
+
+                mod_text = st.text_area('Raw Memory Inputs', value=memory_data, height=500)
+                save(mod_text, f'{user_dir}input.txt', '💽Brain Memory', {
+                    'delimiter': delimiter,
+                    'append_mode': append_mode,
+                    'force_mode': force_delimiter,
+                    'advanced_mode': advanced_mode_index,
+                    'filter_keys': filter_key,
+                    'filter_logics': filter_logic,
+                    'filter_values': filter_val
+                })
+
+            case '🔑API Keys':
+                st.title('🔑API Keys')
+                st.text('Configure your OpenAI API keys.')
+                mod_text = st.text_input('API Keys', value=util.read_file(f'{user_dir}API-KEYS.txt'))
+                save(mod_text, f'{user_dir}API-KEYS.txt')
+
+
+if __name__ == '__main__':
+    main()