From 2f949ce1dda03d4c65016547489c79a9e415be01 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sun, 27 Mar 2022 14:07:58 +0200 Subject: [PATCH] Enabled search for text based custom column content in simple search (fix for #2279) --- cps/db.py | 75 +++++++++++++++++++++++++++------------------------ cps/helper.py | 18 ------------- cps/opds.py | 2 +- cps/web.py | 39 ++++++++------------------- 4 files changed, 52 insertions(+), 82 deletions(-) diff --git a/cps/db.py b/cps/db.py index 3b193e1a..3f64fbb5 100644 --- a/cps/db.py +++ b/cps/db.py @@ -819,38 +819,21 @@ class CalibreDB: def check_exists_book(self, authr, title): self.session.connection().connection.connection.create_function("lower", 1, lcase) q = list() - authorterms = re.split(r'\s*&\s*', authr) - for authorterm in authorterms: - q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) + author_terms = re.split(r'\s*&\s*', authr) + for author_term in author_terms: + q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + author_term + "%"))) return self.session.query(Books) \ .filter(and_(Books.authors.any(and_(*q)), func.lower(Books.title).ilike("%" + title + "%"))).first() - def search_query(self, term, config_read_column, *join): + def search_query(self, term, config, *join): term.strip().lower() self.session.connection().connection.connection.create_function("lower", 1, lcase) q = list() - authorterms = re.split("[, ]+", term) - for authorterm in authorterms: - q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + authorterm + "%"))) - query = self.generate_linked_query(config_read_column, Books) - '''if not config_read_column: - query = (self.session.query(Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(Books) - .outerjoin(ub.ReadBook, and_(Books.id == ub.ReadBook.book_id, - int(current_user.id) == ub.ReadBook.user_id))) - else: - try: - read_column = cc_classes[config_read_column] - query = (self.session.query(Books, ub.ArchivedBook.is_archived, read_column.value) - .select_from(Books) - .outerjoin(read_column, read_column.book == Books.id)) - except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(config_read_column)) - # Skip linking read column - query = self.session.query(Books, ub.ArchivedBook.is_archived, None) - query = query.outerjoin(ub.ArchivedBook, and_(Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id))''' - + author_terms = re.split("[, ]+", term) + for author_term in author_terms: + q.append(Books.authors.any(func.lower(Authors.name).ilike("%" + author_term + "%"))) + query = self.generate_linked_query(config.config_read_column, Books) if len(join) == 6: query = query.outerjoin(join[0], join[1]).outerjoin(join[2]).outerjoin(join[3], join[4]).outerjoin(join[5]) if len(join) == 3: @@ -859,20 +842,42 @@ class CalibreDB: query = query.outerjoin(join[0], join[1]) elif len(join) == 1: query = query.outerjoin(join[0]) - return query.filter(self.common_filters(True)).filter( - or_(Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")), - Books.series.any(func.lower(Series.name).ilike("%" + term + "%")), - Books.authors.any(and_(*q)), - Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")), - func.lower(Books.title).ilike("%" + term + "%") - )) + + cc = self.get_cc_columns(config, filter_config_custom_read=True) + filter_expression = [Books.tags.any(func.lower(Tags.name).ilike("%" + term + "%")), + Books.series.any(func.lower(Series.name).ilike("%" + term + "%")), + Books.authors.any(and_(*q)), + Books.publishers.any(func.lower(Publishers.name).ilike("%" + term + "%")), + func.lower(Books.title).ilike("%" + term + "%")] + for c in cc: + if c.datatype not in ["datetime", "rating", "bool", "int", "float"]: + filter_expression.append( + getattr(Books, + 'custom_column_' + str(c.id)).any( + func.lower(cc_classes[c.id].value).ilike("%" + term + "%"))) + return query.filter(self.common_filters(True)).filter(or_(*filter_expression)) + + def get_cc_columns(self, config, filter_config_custom_read=False): + tmp_cc = self.session.query(CustomColumns).filter(CustomColumns.datatype.notin_(cc_exceptions)).all() + cc = [] + r = None + if config.config_columns_to_ignore: + r = re.compile(config.config_columns_to_ignore) + + for col in tmp_cc: + if filter_config_custom_read and config.config_read_column and config.config_read_column == col.id: + continue + if r and r.match(col.name): + continue + cc.append(col) + + return cc # read search results from calibre-database and return it (function is used for feed and simple search - def get_search_results(self, term, offset=None, order=None, limit=None, - config_read_column=False, *join): + def get_search_results(self, term, config, offset=None, order=None, limit=None, *join): order = order[0] if order else [Books.sort] pagination = None - result = self.search_query(term, config_read_column, *join).order_by(*order).all() + result = self.search_query(term, config, *join).order_by(*order).all() result_count = len(result) if offset != None and limit != None: offset = int(offset) diff --git a/cps/helper.py b/cps/helper.py index 3a2276e9..765cf22b 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -953,24 +953,6 @@ def check_valid_domain(domain_text): return not len(result) -def get_cc_columns(filter_config_custom_read=False): - tmpcc = calibre_db.session.query(db.CustomColumns)\ - .filter(db.CustomColumns.datatype.notin_(db.cc_exceptions)).all() - cc = [] - r = None - if config.config_columns_to_ignore: - r = re.compile(config.config_columns_to_ignore) - - for col in tmpcc: - if filter_config_custom_read and config.config_read_column and config.config_read_column == col.id: - continue - if r and r.match(col.name): - continue - cc.append(col) - - return cc - - def get_download_link(book_id, book_format, client): book_format = book_format.split(".")[0] book = calibre_db.get_filtered_book(book_id, allow_show_archived=True) diff --git a/cps/opds.py b/cps/opds.py index 702ffe1e..8907f628 100644 --- a/cps/opds.py +++ b/cps/opds.py @@ -467,7 +467,7 @@ def feed_unread_books(): def feed_search(term): if term: - entries, __, ___ = calibre_db.get_search_results(term, config_read_column=config.config_read_column) + entries, __, ___ = calibre_db.get_search_results(term, config=config) entries_count = len(entries) if len(entries) > 0 else 1 pagination = Pagination(1, entries_count, entries_count) return render_xml_template('feed.xml', searchterm=term, entries=entries, pagination=pagination) diff --git a/cps/web.py b/cps/web.py index 525fccbe..e8bf48e3 100644 --- a/cps/web.py +++ b/cps/web.py @@ -50,7 +50,7 @@ from . import babel, db, ub, config, get_locale, app from . import calibre_db, kobo_sync_status from .gdriveutils import getFileFromEbooksFolder, do_gdrive_download from .helper import check_valid_domain, render_task_status, check_email, check_username, \ - get_cc_columns, get_book_cover, get_download_link, send_mail, generate_random_password, \ + get_book_cover, get_download_link, send_mail, generate_random_password, \ send_registration_mail, check_send_to_kindle, check_read_formats, tags_filters, reset_password, valid_email, \ edit_book_read_status from .pagination import Pagination @@ -724,10 +724,10 @@ def render_prepare_search_form(cc): def render_search_results(term, offset=None, order=None, limit=None): join = db.books_series_link, db.books_series_link.c.book == db.Books.id, db.Series entries, result_count, pagination = calibre_db.get_search_results(term, + config, offset, order, limit, - config.config_read_column, *join) return render_title_template('search.html', searchterm=term, @@ -765,7 +765,7 @@ def books_list(data, sort_param, book_id, page): @login_required def books_table(): visibility = current_user.view_settings.get('table', {}) - cc = get_cc_columns(filter_config_custom_read=True) + cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) return render_title_template('book_table.html', title=_(u"Books List"), cc=cc, page="book_table", visiblility=visibility) @@ -809,7 +809,7 @@ def list_books(): calibre_db.common_filters(allow_show_archived=True)).count() if state is not None: if search_param: - books = calibre_db.search_query(search_param, config.config_read_column).all() + books = calibre_db.search_query(search_param, config).all() filtered_count = len(books) else: query = calibre_db.generate_linked_query(config.config_read_column, db.Books) @@ -817,10 +817,10 @@ def list_books(): entries = calibre_db.get_checkbox_sorted(books, state, off, limit, order, True) elif search_param: entries, filtered_count, __ = calibre_db.get_search_results(search_param, + config, off, [order, ''], limit, - config.config_read_column, *join) else: entries, __, __ = calibre_db.fill_indexpage_with_archived_books((int(off) / (int(limit)) + 1), @@ -1232,26 +1232,9 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): sort_param = order[0] if order else [db.Books.sort] pagination = None - cc = get_cc_columns(filter_config_custom_read=True) + cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) calibre_db.session.connection().connection.connection.create_function("lower", 1, db.lcase) query = calibre_db.generate_linked_query(config.config_read_column, db.Books) - '''if not config.config_read_column: - query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, ub.ReadBook).select_from(db.Books) - .outerjoin(ub.ReadBook, and_(db.Books.id == ub.ReadBook.book_id, - int(current_user.id) == ub.ReadBook.user_id))) - else: - try: - read_column = db.cc_classes[config.config_read_column] - query = (calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, read_column.value) - .select_from(db.Books) - .outerjoin(read_column, read_column.book == db.Books.id)) - except (KeyError, AttributeError, IndexError): - log.error("Custom Column No.{} is not existing in calibre database".format(config.config_read_column)) - # Skip linking read column - query = calibre_db.session.query(db.Books, ub.ArchivedBook.is_archived, None) - query = query.outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, - int(current_user.id) == ub.ArchivedBook.user_id))''' - q = query.outerjoin(db.books_series_link, db.books_series_link.c.book == db.Books.id) \ .outerjoin(db.Series) \ .filter(calibre_db.common_filters(True)) @@ -1338,7 +1321,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): if description: q = q.filter(db.Books.comments.any(func.lower(db.Comments.text).ilike("%" + description + "%"))) - # search custom culumns + # search custom columns try: q = adv_search_custom_columns(cc, term, q) except AttributeError as ex: @@ -1370,7 +1353,7 @@ def render_adv_search_results(term, offset=None, order=None, limit=None): @login_required_if_no_ano def advanced_search_form(): # Build custom columns names - cc = get_cc_columns(filter_config_custom_read=True) + cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) return render_prepare_search_form(cc) @@ -1757,10 +1740,10 @@ def show_book(book_id): for lang_index in range(0, len(entry.languages)): entry.languages[lang_index].language_name = isoLanguages.get_language_name(get_locale(), entry.languages[ lang_index].lang_code) - cc = get_cc_columns(filter_config_custom_read=True) + cc = calibre_db.get_cc_columns(config, filter_config_custom_read=True) book_in_shelves = [] - shelfs = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all() - for sh in shelfs: + shelves = ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).all() + for sh in shelves: book_in_shelves.append(sh.shelf) entry.tags = sort(entry.tags, key=lambda tag: tag.name)