diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 49ab5882..7bff089b 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -22,7 +22,9 @@ import shutil import chardet import ssl import sqlite3 +import mimetypes +from werkzeug.datastructures import Headers from flask import Response, stream_with_context from sqlalchemy import create_engine from sqlalchemy import Column, UniqueConstraint @@ -64,7 +66,7 @@ except ImportError as err: importError = err gdrive_support = False -from . import logger, cli_param, config +from . import logger, cli_param, config, db from .constants import CONFIG_DIR as _CONFIG_DIR @@ -265,7 +267,7 @@ def getFile(pathId, fileName, drive, nocase): if fileList.__len__() == 0: return None if nocase: - return fileList[0] + return fileList[0] if db.lcase(fileList[0]['title']) == db.lcase(fileName) else None for f in fileList: if f['title'] == fileName: return f @@ -273,8 +275,6 @@ def getFile(pathId, fileName, drive, nocase): def getFolderId(path, drive): - # drive = getDrive(drive) - log.info(f"GetFolder: {path}") currentFolderId = None try: currentFolderId = getEbooksFolderId(drive) @@ -348,16 +348,23 @@ def moveGdriveFolderRemote(origin_file, target_folder, single_book=False): previous_parents = ",".join([parent["id"] for parent in origin_file.get('parents')]) children = drive.auth.service.children().list(folderId=previous_parents).execute() if single_book: - # gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True) - gFileTargetDir = drive.CreateFile( - {'title': target_folder, 'parents': [{"kind": "drive#fileLink", 'id': getEbooksFolderId()}], - "mimeType": "application/vnd.google-apps.folder"}) - gFileTargetDir.Upload() - # Move the file to the new folder - drive.auth.service.files().update(fileId=origin_file['id'], - addParents=gFileTargetDir['id'], - removeParents=previous_parents, - fields='id, parents').execute() + gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True) + if gFileTargetDir: + # Move the file to the new folder + drive.auth.service.files().update(fileId=origin_file['id'], + addParents=gFileTargetDir['id'], + removeParents=previous_parents, + fields='id, parents').execute() + else: + gFileTargetDir = drive.CreateFile( + {'title': target_folder, 'parents': [{"kind": "drive#fileLink", 'id': getEbooksFolderId()}], + "mimeType": "application/vnd.google-apps.folder"}) + gFileTargetDir.Upload() + # Move the file to the new folder + drive.auth.service.files().update(fileId=origin_file['id'], + addParents=gFileTargetDir['id'], + removeParents=previous_parents, + fields='id, parents').execute() elif origin_file['title'] != target_folder: #gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True) #if gFileTargetDir: @@ -366,12 +373,7 @@ def moveGdriveFolderRemote(origin_file, target_folder, single_book=False): drive.auth.service.files().patch(fileId=origin_file['id'], body={'title': target_folder}, fields='title').execute() - '''else: - # Move the file to the new folder - drive.auth.service.files().update(fileId=origin_file['id'], - addParents=gFileTargetDir['id'], - removeParents=previous_parents, - fields='id, parents').execute()''' + # if previous_parents has no children anymore, delete original fileparent if len(children['items']) == 1: deleteDatabaseEntry(previous_parents) @@ -600,7 +602,10 @@ def get_cover_via_gdrive(cover_path): except (OperationalError, IntegrityError) as ex: log.error_or_exception('Database error: {}'.format(ex)) session.rollback() - return df.metadata.get('webContentLink') + headers = Headers() + headers["Content-Type"] = 'image/jpeg' + resp, content = df.auth.Get_Http_Object().request(df.metadata.get('downloadUrl'), headers=headers) + return content else: return None diff --git a/cps/helper.py b/cps/helper.py index 85f1689b..f4c226a6 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -30,7 +30,7 @@ import requests import unidecode from uuid import uuid4 -from flask import send_from_directory, make_response, redirect, abort, url_for +from flask import send_from_directory, make_response, abort, url_for, Response from flask_babel import gettext as _ from flask_babel import lazy_gettext as N_ from flask_babel import get_locale @@ -393,10 +393,8 @@ def rename_all_files_on_change(one_book, new_path, old_path, all_new_name, gdriv if not gdrive: if not os.path.exists(new_path): os.makedirs(new_path) - shutil.move(os.path.normcase( - os.path.join(old_path, file_format.name + '.' + file_format.format.lower())), - os.path.normcase( - os.path.join(new_path, all_new_name + '.' + file_format.format.lower()))) + shutil.move(os.path.join(old_path, file_format.name + '.' + file_format.format.lower()), + os.path.join(new_path, all_new_name + '.' + file_format.format.lower())) else: g_file = gd.getFileFromEbooksFolder(old_path, file_format.name + '.' + file_format.format.lower()) @@ -457,7 +455,7 @@ def rename_author_path(first_author, old_author_dir, renamed_author, calibre_pat old_author_path = os.path.join(calibre_path, old_author_dir) new_author_path = os.path.join(calibre_path, new_author_rename_dir) try: - shutil.move(os.path.normcase(old_author_path), os.path.normcase(new_author_path)) + shutil.move(old_author_path, new_author_path) except OSError as ex: log.error("Rename author from: %s to %s: %s", old_author_path, new_author_path, ex) log.debug(ex, exc_info=True) @@ -527,7 +525,7 @@ def update_dir_structure_gdrive(book_id, first_author): new_titledir = get_valid_filename(book.title, chars=96) + " (" + str(book_id) + ")" if titledir != new_titledir: - g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), titledir) + g_file = gd.getFileFromEbooksFolder(authordir, titledir) if g_file: gd.moveGdriveFileRemote(g_file, new_titledir) book.path = book.path.split('/')[0] + '/' + new_titledir @@ -559,21 +557,20 @@ def move_files_on_change(calibre_path, new_author_dir, new_titledir, localbook, if original_filepath: if not os.path.isdir(new_path): os.makedirs(new_path) - shutil.move(os.path.normcase(original_filepath), os.path.normcase(os.path.join(new_path, db_filename))) + shutil.move(original_filepath, os.path.join(new_path, db_filename)) log.debug("Moving title: %s to %s/%s", original_filepath, new_path) else: # Check new path is not valid path if not os.path.exists(new_path): # move original path to new path log.debug("Moving title: %s to %s", path, new_path) - shutil.move(os.path.normcase(path), os.path.normcase(new_path)) + shutil.move(path, new_path) else: # path is valid copy only files to new location (merge) log.info("Moving title: %s into existing: %s", path, new_path) # Take all files and subfolder from old path (strange command) for dir_name, __, file_list in os.walk(path): for file in file_list: - shutil.move(os.path.normcase(os.path.join(dir_name, file)), - os.path.normcase(os.path.join(new_path + dir_name[len(path):], file))) + shutil.move(os.path.join(dir_name, file), os.path.join(new_path + dir_name[len(path):], file)) if not os.listdir(os.path.split(path)[0]): try: shutil.rmtree(os.path.split(path)[0]) @@ -615,7 +612,7 @@ def delete_book_gdrive(book, book_format): for entry in book.data: if entry.format.upper() == book_format: name = entry.name + '.' + book_format - g_file = gd.getFileFromEbooksFolder(book.path, name) + g_file = gd.getFileFromEbooksFolder(book.path, name, nocase=True) else: g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), book.path.split('/')[1]) if g_file: @@ -790,9 +787,9 @@ def get_book_cover_internal(book, resolution=None): try: if not gd.is_gdrive_ready(): return get_cover_on_failure() - path = gd.get_cover_via_gdrive(book.path) - if path: - return redirect(path) + cover_file = gd.get_cover_via_gdrive(book.path) + if cover_file: + return Response(cover_file, mimetype='image/jpeg') else: log.error('{}/cover.jpg not found on Google Drive'.format(book.path)) return get_cover_on_failure() diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 54b68f40..2921ffc9 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -19,6 +19,7 @@ import os from shutil import copyfile, copyfileobj from urllib.request import urlopen +from io import BytesIO from .. import constants from cps import config, db, fs, gdriveutils, logger, ub @@ -182,13 +183,11 @@ class TaskGenerateCoverThumbnails(CalibreTask): if not gdriveutils.is_gdrive_ready(): raise Exception('Google Drive is configured but not ready') - web_content_link = gdriveutils.get_cover_via_gdrive(book.path) - if not web_content_link: + content = gdriveutils.get_cover_via_gdrive(book.path) + if not content: raise Exception('Google Drive cover url not found') - - stream = None try: - stream = urlopen(web_content_link) + stream = BytesIO(content) with Image(file=stream) as img: filename = self.cache.get_cache_file_path(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) diff --git a/optional-requirements.txt b/optional-requirements.txt index e097746f..77f41e1e 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,5 +1,5 @@ # GDrive Integration -google-api-python-client>=1.7.11,<2.120.0 +google-api-python-client>=1.7.11,<2.200.0 gevent>20.6.0,<24.3.0 greenlet>=0.4.17,<3.1.0 httplib2>=0.9.2,<0.23.0 @@ -13,7 +13,7 @@ rsa>=3.4.2,<4.10.0 # Gmail google-auth-oauthlib>=0.4.3,<1.3.0 -google-api-python-client>=1.7.11,<2.120.0 +google-api-python-client>=1.7.11,<2.200.0 # goodreads goodreads>=0.3.2,<0.4.0 diff --git a/setup.cfg b/setup.cfg index dcbbac10..5e0882e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,19 +46,19 @@ install_requires = Flask-Principal>=0.3.2,<0.5.1 Flask>=1.0.2,<3.1.0 iso-639>=0.4.5,<0.5.0 - PyPDF>=3.15.6,<4.1.0 + PyPDF>=3.15.6,<4.3.0 pytz>=2016.10 requests>=2.28.0,<2.32.0 SQLAlchemy>=1.3.0,<2.1.0 tornado>=6.3,<6.5 Wand>=0.4.4,<0.7.0 unidecode>=0.04.19,<1.4.0 - lxml>=4.9.1,<5.2.0 + lxml>=4.9.1,<5.3.0 flask-wtf>=0.14.2,<1.3.0 chardet>=3.0.0,<4.1.0 advocate>=1.0.0,<1.1.0 Flask-Limiter>=2.3.0,<3.6.0 - regex>=2022.3.2,<2024.2.25 + regex>=2022.3.2,<2024.6.25 bleach>=6.0.0,<6.2.0 python-magic>=0.4.27,<0.5.0 @@ -69,7 +69,7 @@ include = cps/services* [options.extras_require] gdrive = - google-api-python-client>=1.7.11,<2.120.0 + google-api-python-client>=1.7.11,<2.200.0 gevent>20.6.0,<24.3.0 greenlet>=0.4.17,<3.1.0 httplib2>=0.9.2,<0.23.0 @@ -82,7 +82,7 @@ gdrive = rsa>=3.4.2,<4.10.0 gmail = google-auth-oauthlib>=0.4.3,<1.3.0 - google-api-python-client>=1.7.11,<2.120.0 + google-api-python-client>=1.7.11,<2.200.0 goodreads = goodreads>=0.3.2,<0.4.0 python-Levenshtein>=0.12.0,<0.26.0 diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index b799e9aa..952b4f89 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
Start Time: 2024-06-29 20:48:57
+Start Time: 2024-07-02 20:54:09
Stop Time: 2024-06-30 03:48:15
+Stop Time: 2024-07-03 04:18:18
Duration: 5h 44 min
+Duration: 6h 10 min
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_backup_metadata.py", line 54, in test_backup_all + self.assertEqual(1, len(res)) +AssertionError: 1 != 0+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_ebook_convert_gdrive.py", line 206, in test_convert_email + self.assertTrue(self.check_element_on_page((By.ID, "flash_success"))) +AssertionError: False is not true+
Traceback (most recent call last): + File "/home/ozzie/Development/calibre-web-test/test/test_ebook_convert_gdrive.py", line 251, in test_convert_failed_and_email + select.select_by_visible_text('AZW3') + File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/selenium/webdriver/support/select.py", line 134, in select_by_visible_text + raise NoSuchElementException(f"Could not locate element with visible text: {text}") +selenium.common.exceptions.NoSuchElementException: Message: Could not locate element with visible text: AZW3+
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_author_gdrive.py", line 174, in test_change_capital_one_author_two_books - self.assertFalse(self.check_element_on_page((By.ID, "flash_danger"))) -AssertionError: <selenium.webdriver.remote.webelement.WebElement (session="75cab94e-bc8d-4b29-b4fa-bcb013c9fe05", element="4bd75a44-1bfc-4cf4-a39d-36e9675ec783")> is not false-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_author_gdrive.py", line 370, in test_change_capital_rename_co_author - self.assertTrue(fs.isfile(os.path.join('test', 'hector Gonçalves/book9 (11)', -AssertionError: False is not true-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_list.py", line 81, in test_bookslist_edit_title - bl = self.get_books_list(-1) - File "/home/ozzie/Development/calibre-web-test/test/helper_ui.py", line 1898, in get_books_list - element_text = "+" if "glyphicon-plus" in click_element.find_elements(By.XPATH, "./span")[0].get_attribute('class') else "" -IndexError: list index out of range-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_list.py", line 331, in test_list_visibility - self.assertEqual(22, len(bl['table'][0])) -AssertionError: 22 != 1-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_list.py", line 335, in test_restricted_rights - self.assertTrue('Delete' in bl['table'][0]) -AssertionError: False is not true-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_list.py", line 57, in test_search_books_list - self.assertEqual(10, len(bl['table'])) -AssertionError: 10 != 1-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_list.py", line 38, in tearDownClass - cls.driver.get("http://127.0.0.1:" + PORTS[0]) - File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 449, in get - self.execute(Command.GET, {"url": url}) - File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/selenium/webdriver/remote/webdriver.py", line 440, in execute - self.error_handler.check_response(response) - File "/home/ozzie/Development/calibre-web-test/venv/lib/python3.10/site-packages/selenium/webdriver/remote/errorhandler.py", line 245, in check_response - raise exception_class(message, screen, stacktrace) -selenium.common.exceptions.TimeoutException: Message: Navigation timed out after 300000 ms -Stacktrace: -RemoteError@chrome://remote/content/shared/RemoteError.sys.mjs:8:8 -WebDriverError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:193:5 -TimeoutError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:740:5 -bail@chrome://remote/content/marionette/sync.sys.mjs:211:19-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_pip_install.py", line 36, in setUpClass - args = make_release.parse_arguments(['-p']) -NameError: name 'make_release' is not defined-
Traceback (most recent call last): - File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_metadata.py", line 90, in test_load_metadata - elif 'https://amazon.com/' == results[20]['source']: -IndexError: list index out of range+ File "/home/ozzie/Development/calibre-web-test/test/test_edit_books_metadata.py", line 173, in test_load_metadata + self.assertGreaterEqual(diff(BytesIO(cover), BytesIO(original_cover), delete_diff_file=True), 0.05) +AssertionError: 0.0 not greater than or equal to 0.05