From 069dc2766fc82fd520a729113c709fb159cf81c3 Mon Sep 17 00:00:00 2001 From: Ozzie Isaacs Date: Sat, 23 Apr 2022 19:56:02 +0200 Subject: [PATCH] Update optional-requirements Bugfix with serializing tasks Bugfix order of tasks (id was used instead of task_id) Code cosmetics --- cps/editbooks.py | 36 +++++++++-------- cps/helper.py | 13 ++++--- cps/kobo.py | 82 ++++++++++++++++++++------------------- cps/schedule.py | 2 +- cps/services/worker.py | 2 +- cps/static/js/table.js | 4 +- cps/tasks/thumbnail.py | 6 +-- optional-requirements.txt | 4 +- 8 files changed, 79 insertions(+), 70 deletions(-) diff --git a/cps/editbooks.py b/cps/editbooks.py index 1d61c8dd..f2ad969c 100755 --- a/cps/editbooks.py +++ b/cps/editbooks.py @@ -241,7 +241,7 @@ def delete_book_ajax(book_id, book_format): def delete_whole_book(book_id, book): - # delete book from Shelfs, Downloads, Read list + # delete book from shelves, Downloads, Read list ub.session.query(ub.BookShelf).filter(ub.BookShelf.book_id == book_id).delete() ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id).delete() ub.delete_download(book_id) @@ -383,7 +383,7 @@ def render_edit_book(book_id): for authr in book.authors: author_names.append(authr.name.replace('|', ',')) - # Option for showing convertbook button + # Option for showing convert_book button valid_source_formats = list() allowed_conversion_formats = list() kepub_possible = None @@ -413,11 +413,11 @@ def render_edit_book(book_id): def edit_book_ratings(to_save, book): changed = False - if to_save.get("rating","").strip(): + if to_save.get("rating", "").strip(): old_rating = False if len(book.ratings) > 0: old_rating = book.ratings[0].rating - rating_x2 = int(float(to_save.get("rating","")) * 2) + rating_x2 = int(float(to_save.get("rating", "")) * 2) if rating_x2 != old_rating: changed = True is_rating = calibre_db.session.query(db.Ratings).filter(db.Ratings.rating == rating_x2).first() @@ -622,8 +622,9 @@ def edit_cc_data(book_id, book, to_save, cc): 'custom') return changed + # returns None if no file is uploaded -# returns False if an error occours, in all other cases the ebook metadata is returned +# returns False if an error occurs, in all other cases the ebook metadata is returned def upload_single_file(file_request, book, book_id): # Check and handle Uploaded file requested_file = file_request.files.get('btn-upload-format', None) @@ -676,7 +677,7 @@ def upload_single_file(file_request, book, book_id): calibre_db.session.rollback() log.error_or_exception("Database error: {}".format(e)) flash(_(u"Database error: %(error)s.", error=e.orig), category="error") - return False # return redirect(url_for('web.show_book', book_id=book.id)) + return False # return redirect(url_for('web.show_book', book_id=book.id)) # Queue uploader info link = '{}'.format(url_for('web.show_book', book_id=book.id), escape(book.title)) @@ -688,6 +689,7 @@ def upload_single_file(file_request, book, book_id): rarExecutable=config.config_rarfile_location) return None + def upload_cover(cover_request, book): requested_file = cover_request.files.get('btn-upload-cover', None) if requested_file: @@ -698,7 +700,7 @@ def upload_cover(cover_request, book): return False ret, message = helper.save_cover(requested_file, book.path) if ret is True: - helper.clear_cover_thumbnail_cache(book.id) + helper.replace_cover_thumbnail_cache(book.id) return True else: flash(message, category="error") @@ -739,6 +741,7 @@ def handle_author_on_edit(book, author_name, update_stored=True): change = True return input_authors, change, renamed + @EditBook.route("/admin/book/", methods=['GET']) @login_required_if_no_ano @edit_required @@ -815,6 +818,7 @@ def edit_book(book_id): if result is True: book.has_cover = 1 modify_date = True + helper.replace_cover_thumbnail_cache(book.id) else: flash(error, category="error") @@ -986,7 +990,7 @@ def create_book_on_upload(modify_date, meta): try: pubdate = datetime.strptime(meta.pubdate[:10], "%Y-%m-%d") - except: + except ValueError: pubdate = datetime(101, 1, 1) # Calibre adds books with utc as timezone @@ -1063,18 +1067,18 @@ def file_handling_on_upload(requested_file): def move_coverfile(meta, db_book): # move cover to final directory, including book id if meta.cover: - coverfile = meta.cover + cover_file = meta.cover else: - coverfile = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg') - new_coverpath = os.path.join(config.config_calibre_dir, db_book.path) + cover_file = os.path.join(constants.STATIC_DIR, 'generic_cover.jpg') + new_cover_path = os.path.join(config.config_calibre_dir, db_book.path) try: - os.makedirs(new_coverpath, exist_ok=True) - copyfile(coverfile, os.path.join(new_coverpath, "cover.jpg")) + os.makedirs(new_cover_path, exist_ok=True) + copyfile(cover_file, os.path.join(new_cover_path, "cover.jpg")) if meta.cover: os.unlink(meta.cover) except OSError as e: - log.error("Failed to move cover file %s: %s", new_coverpath, e) - flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_coverpath, + log.error("Failed to move cover file %s: %s", new_cover_path, e) + flash(_(u"Failed to Move Cover File %(file)s: %(error)s", file=new_cover_path, error=e), category="error") @@ -1193,7 +1197,7 @@ def edit_list_book(param): vals = request.form.to_dict() book = calibre_db.get_book(vals['pk']) sort_param = "" - # ret = "" + ret = "" try: if param == 'series_index': edit_book_series_index(vals['value'], book) diff --git a/cps/helper.py b/cps/helper.py index 43b7141d..ea5820dd 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -820,9 +820,6 @@ def save_cover_from_url(url, book_path): log.error("python modul advocate is not installed but is needed") return False, _("Python modul 'advocate' is not installed but is needed for cover downloads") img.raise_for_status() - # # cover_processing() - # move_coverfile(meta, db_book) - return save_cover(img, book_path) except (socket.gaierror, requests.exceptions.HTTPError, @@ -1020,7 +1017,7 @@ def render_task_status(tasklist): ret['user'] = escape(user) # prevent xss # Hidden fields - ret['id'] = task.id + ret['task_id'] = task.id ret['stat'] = task.stat ret['is_cancellable'] = task.is_cancellable @@ -1078,8 +1075,12 @@ def get_download_link(book_id, book_format, client): def clear_cover_thumbnail_cache(book_id): - WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id, _("Replace Thumbnail for book {}".format(book_id))), - hidden=True) + WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) + + +def replace_cover_thumbnail_cache(book_id): + WorkerThread.add(None, TaskClearCoverThumbnailCache(book_id), hidden=True) + WorkerThread.add(None, TaskGenerateCoverThumbnails(book_id), hidden=True) def delete_thumbnail_cache(): diff --git a/cps/kobo.py b/cps/kobo.py index f2a3b2f8..46e68acb 100644 --- a/cps/kobo.py +++ b/cps/kobo.py @@ -148,8 +148,8 @@ def HandleSyncRequest(): sync_token.books_last_created = datetime.datetime.min sync_token.reading_state_last_modified = datetime.datetime.min - new_books_last_modified = sync_token.books_last_modified # needed for sync selected shelfs only - new_books_last_created = sync_token.books_last_created # needed to distinguish between new and changed entitlement + new_books_last_modified = sync_token.books_last_modified # needed for sync selected shelfs only + new_books_last_created = sync_token.books_last_created # needed to distinguish between new and changed entitlement new_reading_state_last_modified = sync_token.reading_state_last_modified new_archived_last_modified = datetime.datetime.min @@ -176,18 +176,17 @@ def HandleSyncRequest(): .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, ub.ArchivedBook.user_id == current_user.id)) .filter(db.Books.id.notin_(calibre_db.session.query(ub.KoboSyncedBooks.book_id) - .filter(ub.KoboSyncedBooks.user_id == current_user.id))) - .filter(ub.BookShelf.date_added > sync_token.books_last_modified) - .filter(db.Data.format.in_(KOBO_FORMATS)) - .filter(calibre_db.common_filters(allow_show_archived=True)) - .order_by(db.Books.id) - .order_by(ub.ArchivedBook.last_modified) - .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id) - .join(ub.Shelf) - .filter(ub.Shelf.user_id == current_user.id) - .filter(ub.Shelf.kobo_sync) - .distinct() - ) + .filter(ub.KoboSyncedBooks.user_id == current_user.id))) + .filter(ub.BookShelf.date_added > sync_token.books_last_modified) + .filter(db.Data.format.in_(KOBO_FORMATS)) + .filter(calibre_db.common_filters(allow_show_archived=True)) + .order_by(db.Books.id) + .order_by(ub.ArchivedBook.last_modified) + .join(ub.BookShelf, db.Books.id == ub.BookShelf.book_id) + .join(ub.Shelf) + .filter(ub.Shelf.user_id == current_user.id) + .filter(ub.Shelf.kobo_sync) + .distinct()) else: if sqlalchemy_version2: changed_entries = select(db.Books, ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived) @@ -196,16 +195,14 @@ def HandleSyncRequest(): ub.ArchivedBook.last_modified, ub.ArchivedBook.is_archived) changed_entries = (changed_entries - .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, - ub.ArchivedBook.user_id == current_user.id)) - .filter(db.Books.id.notin_(calibre_db.session.query(ub.KoboSyncedBooks.book_id) - .filter(ub.KoboSyncedBooks.user_id == current_user.id))) - .filter(calibre_db.common_filters(allow_show_archived=True)) - .filter(db.Data.format.in_(KOBO_FORMATS)) - .order_by(db.Books.last_modified) - .order_by(db.Books.id) - ) - + .join(db.Data).outerjoin(ub.ArchivedBook, and_(db.Books.id == ub.ArchivedBook.book_id, + ub.ArchivedBook.user_id == current_user.id)) + .filter(db.Books.id.notin_(calibre_db.session.query(ub.KoboSyncedBooks.book_id) + .filter(ub.KoboSyncedBooks.user_id == current_user.id))) + .filter(calibre_db.common_filters(allow_show_archived=True)) + .filter(db.Data.format.in_(KOBO_FORMATS)) + .order_by(db.Books.last_modified) + .order_by(db.Books.id)) reading_states_in_new_entitlements = [] if sqlalchemy_version2: @@ -215,7 +212,7 @@ def HandleSyncRequest(): log.debug("Books to Sync: {}".format(len(books.all()))) for book in books: formats = [data.format for data in book.Books.data] - if not 'KEPUB' in formats and config.config_kepubifypath and 'EPUB' in formats: + if 'KEPUB' not in formats and config.config_kepubifypath and 'EPUB' in formats: helper.convert_book_format(book.Books.id, config.config_calibre_dir, 'EPUB', 'KEPUB', current_user.name) kobo_reading_state = get_or_create_reading_state(book.Books.id) @@ -262,7 +259,7 @@ def HandleSyncRequest(): .columns(db.Books).first() else: max_change = changed_entries.from_self().filter(ub.ArchivedBook.is_archived)\ - .filter(ub.ArchivedBook.user_id==current_user.id) \ + .filter(ub.ArchivedBook.user_id == current_user.id) \ .order_by(func.datetime(ub.ArchivedBook.last_modified).desc()).first() max_change = max_change.last_modified if max_change else new_archived_last_modified @@ -425,9 +422,9 @@ def get_author(book): author_list = [] autor_roles = [] for author in book.authors: - autor_roles.append({"Name":author.name}) #.encode('unicode-escape').decode('latin-1') + autor_roles.append({"Name": author.name}) author_list.append(author.name) - return {"ContributorRoles": autor_roles, "Contributors":author_list} + return {"ContributorRoles": autor_roles, "Contributors": author_list} def get_publisher(book): @@ -441,6 +438,7 @@ def get_series(book): return None return book.series[0].name + def get_seriesindex(book): return book.series_index or 1 @@ -485,7 +483,7 @@ def get_metadata(book): "Language": "en", "PhoneticPronunciations": {}, "PublicationDate": convert_to_kobo_timestamp_string(book.pubdate), - "Publisher": {"Imprint": "", "Name": get_publisher(book),}, + "Publisher": {"Imprint": "", "Name": get_publisher(book), }, "RevisionId": book_uuid, "Title": book.title, "WorkId": book_uuid, @@ -504,6 +502,7 @@ def get_metadata(book): return metadata + @csrf.exempt @kobo.route("/v1/library/tags", methods=["POST", "DELETE"]) @requires_kobo_auth @@ -718,7 +717,6 @@ def sync_shelves(sync_token, sync_results, only_kobo_shelves=False): *extra_filters ).distinct().order_by(func.datetime(ub.Shelf.last_modified).asc()) - for shelf in shelflist: if not shelf_lib.check_shelf_view_permissions(shelf): continue @@ -764,6 +762,7 @@ def create_kobo_tag(shelf): ) return {"Tag": tag} + @csrf.exempt @kobo.route("/v1/library//state", methods=["GET", "PUT"]) @requires_kobo_auth @@ -808,7 +807,7 @@ def HandleStateRequest(book_uuid): book_read = kobo_reading_state.book_read_link new_book_read_status = get_ub_read_status(request_status_info["Status"]) if new_book_read_status == ub.ReadBook.STATUS_IN_PROGRESS \ - and new_book_read_status != book_read.read_status: + and new_book_read_status != book_read.read_status: book_read.times_started_reading += 1 book_read.last_time_started_reading = datetime.datetime.utcnow() book_read.read_status = new_book_read_status @@ -848,7 +847,7 @@ def get_ub_read_status(kobo_read_status): def get_or_create_reading_state(book_id): book_read = ub.session.query(ub.ReadBook).filter(ub.ReadBook.book_id == book_id, - ub.ReadBook.user_id == int(current_user.id)).one_or_none() + ub.ReadBook.user_id == int(current_user.id)).one_or_none() if not book_read: book_read = ub.ReadBook(user_id=current_user.id, book_id=book_id) if not book_read.kobo_reading_state: @@ -912,6 +911,7 @@ def get_current_bookmark_response(current_bookmark): } return resp + @kobo.route("/////image.jpg", defaults={'Quality': ""}) @kobo.route("//////image.jpg") @requires_kobo_auth @@ -989,8 +989,8 @@ def handle_getests(): if config.config_kobo_proxy: return redirect_or_proxy_request() else: - testkey = request.headers.get("X-Kobo-userkey","") - return make_response(jsonify({"Result": "Success", "TestKey":testkey, "Tests": {}})) + testkey = request.headers.get("X-Kobo-userkey", "") + return make_response(jsonify({"Result": "Success", "TestKey": testkey, "Tests": {}})) @csrf.exempt @@ -1020,7 +1020,7 @@ def make_calibre_web_auth_response(): content = request.get_json() AccessToken = base64.b64encode(os.urandom(24)).decode('utf-8') RefreshToken = base64.b64encode(os.urandom(24)).decode('utf-8') - return make_response( + return make_response( jsonify( { "AccessToken": AccessToken, @@ -1158,14 +1158,16 @@ def NATIVE_KOBO_RESOURCES(): "eula_page": "https://www.kobo.com/termsofuse?style=onestore", "exchange_auth": "https://storeapi.kobo.com/v1/auth/exchange", "external_book": "https://storeapi.kobo.com/v1/products/books/external/{Ids}", - "facebook_sso_page": "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/", + "facebook_sso_page": + "https://authorize.kobo.com/signin/provider/Facebook/login?returnUrl=http://store.kobobooks.com/", "featured_list": "https://storeapi.kobo.com/v1/products/featured/{FeaturedListId}", "featured_lists": "https://storeapi.kobo.com/v1/products/featured", "free_books_page": { "EN": "https://www.kobo.com/{region}/{language}/p/free-ebooks", "FR": "https://www.kobo.com/{region}/{language}/p/livres-gratuits", "IT": "https://www.kobo.com/{region}/{language}/p/libri-gratuiti", - "NL": "https://www.kobo.com/{region}/{language}/List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg", + "NL": "https://www.kobo.com/{region}/{language}/" + "List/bekijk-het-overzicht-van-gratis-ebooks/QpkkVWnUw8sxmgjSlCbJRg", "PT": "https://www.kobo.com/{region}/{language}/p/livros-gratis", }, "fte_feedback": "https://storeapi.kobo.com/v1/products/ftefeedback", @@ -1190,7 +1192,8 @@ def NATIVE_KOBO_RESOURCES(): "library_stack": "https://storeapi.kobo.com/v1/user/library/stacks/{LibraryItemId}", "library_sync": "https://storeapi.kobo.com/v1/library/sync", "love_dashboard_page": "https://store.kobobooks.com/{culture}/kobosuperpoints", - "love_points_redemption_page": "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}", + "love_points_redemption_page": + "https://store.kobobooks.com/{culture}/KoboSuperPointsRedemption?productId={ProductId}", "magazine_landing_page": "https://store.kobobooks.com/emagazines", "notifications_registration_issue": "https://storeapi.kobo.com/v1/notifications/registration", "oauth_host": "https://oauth.kobo.com", @@ -1206,7 +1209,8 @@ def NATIVE_KOBO_RESOURCES(): "product_recommendations": "https://storeapi.kobo.com/v1/products/{ProductId}/recommendations", "product_reviews": "https://storeapi.kobo.com/v1/products/{ProductIds}/reviews", "products": "https://storeapi.kobo.com/v1/products", - "provider_external_sign_in_page": "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/", + "provider_external_sign_in_page": + "https://authorize.kobo.com/ExternalSignIn/{providerName}?returnUrl=http://store.kobobooks.com/", "purchase_buy": "https://www.kobo.com/checkout/createpurchase/", "purchase_buy_templated": "https://www.kobo.com/{culture}/checkout/createpurchase/{ProductId}", "quickbuy_checkout": "https://storeapi.kobo.com/v1/store/quickbuy/{PurchaseId}/checkout", diff --git a/cps/schedule.py b/cps/schedule.py index 79e0006c..5500f1a9 100644 --- a/cps/schedule.py +++ b/cps/schedule.py @@ -45,7 +45,7 @@ def get_scheduled_tasks(reconnect=True): def end_scheduled_tasks(): worker = WorkerThread.get_instance() - for __, __, __, task in worker.tasks: + for __, __, __, task, __ in worker.tasks: if task.scheduled and task.is_cancellable: worker.end_task(task.id) diff --git a/cps/services/worker.py b/cps/services/worker.py index c1a8c384..63d83bfb 100644 --- a/cps/services/worker.py +++ b/cps/services/worker.py @@ -156,7 +156,7 @@ class WorkerThread(threading.Thread): def end_task(self, task_id): ins = self.get_instance() - for __, __, __, task in ins.tasks: + for __, __, __, task, __ in ins.tasks: if str(task.id) == str(task_id) and task.is_cancellable: task.stat = STAT_CANCELLED if task.stat == STAT_WAITING else STAT_ENDED diff --git a/cps/static/js/table.js b/cps/static/js/table.js index bb38fc9f..47c3b70e 100644 --- a/cps/static/js/table.js +++ b/cps/static/js/table.js @@ -641,9 +641,9 @@ function UserActions (value, row) { /* Function for cancelling tasks */ function TaskActions (value, row) { var cancellableStats = [0, 1, 2]; - if (row.id && row.is_cancellable && cancellableStats.includes(row.stat)) { + if (row.task_id && row.is_cancellable && cancellableStats.includes(row.stat)) { return [ - "
", + "
", "", "
" ].join(""); diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index c66c30ec..d11ef50c 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -50,7 +50,7 @@ def get_best_fit(width, height, image_width, image_height): resize_height = int(height / 2.0) aspect_ratio = image_width / image_height - # If this image's aspect ratio is different than the first image, then resize this image + # If this image's aspect ratio is different from the first image, then resize this image # to fill the width and height of the first image if aspect_ratio < width / height: resize_width = int(width / 2.0) @@ -225,7 +225,7 @@ class TaskGenerateCoverThumbnails(CalibreTask): def __str__(self): if self.book_id > 0: - return "Add Thumbnail for book {}".format(self.book_id) + return "Add Cover Thumbnails for Book {}".format(self.book_id) else: return "Generate Cover Thumbnails" @@ -501,7 +501,7 @@ class TaskClearCoverThumbnailCache(CalibreTask): # needed for logging def __str__(self): if self.book_id > 0: - return "Replace Thumbnail cache for book " + str(self.book_id) + return "Replace/Delete Cover Thumbnails for book " + str(self.book_id) else: return "Delete Thumbnail cache directory" diff --git a/optional-requirements.txt b/optional-requirements.txt index fea410e5..07e1ad88 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,5 +1,5 @@ # GDrive Integration -google-api-python-client>=1.7.11,<2.44.0 +google-api-python-client>=1.7.11,<2.46.0 gevent>20.6.0,<22.0.0 greenlet>=0.4.17,<1.2.0 httplib2>=0.9.2,<0.21.0 @@ -13,7 +13,7 @@ rsa>=3.4.2,<4.9.0 # Gmail google-auth-oauthlib>=0.4.3,<0.6.0 -google-api-python-client>=1.7.11,<2.44.0 +google-api-python-client>=1.7.11,<2.46.0 # goodreads goodreads>=0.3.2,<0.4.0