diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py index 49ab5882..7bff089b 100644 --- a/cps/gdriveutils.py +++ b/cps/gdriveutils.py @@ -22,7 +22,9 @@ import shutil import chardet import ssl import sqlite3 +import mimetypes +from werkzeug.datastructures import Headers from flask import Response, stream_with_context from sqlalchemy import create_engine from sqlalchemy import Column, UniqueConstraint @@ -64,7 +66,7 @@ except ImportError as err: importError = err gdrive_support = False -from . import logger, cli_param, config +from . import logger, cli_param, config, db from .constants import CONFIG_DIR as _CONFIG_DIR @@ -265,7 +267,7 @@ def getFile(pathId, fileName, drive, nocase): if fileList.__len__() == 0: return None if nocase: - return fileList[0] + return fileList[0] if db.lcase(fileList[0]['title']) == db.lcase(fileName) else None for f in fileList: if f['title'] == fileName: return f @@ -273,8 +275,6 @@ def getFile(pathId, fileName, drive, nocase): def getFolderId(path, drive): - # drive = getDrive(drive) - log.info(f"GetFolder: {path}") currentFolderId = None try: currentFolderId = getEbooksFolderId(drive) @@ -348,16 +348,23 @@ def moveGdriveFolderRemote(origin_file, target_folder, single_book=False): previous_parents = ",".join([parent["id"] for parent in origin_file.get('parents')]) children = drive.auth.service.children().list(folderId=previous_parents).execute() if single_book: - # gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True) - gFileTargetDir = drive.CreateFile( - {'title': target_folder, 'parents': [{"kind": "drive#fileLink", 'id': getEbooksFolderId()}], - "mimeType": "application/vnd.google-apps.folder"}) - gFileTargetDir.Upload() - # Move the file to the new folder - drive.auth.service.files().update(fileId=origin_file['id'], - addParents=gFileTargetDir['id'], - removeParents=previous_parents, - fields='id, parents').execute() + gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True) + if gFileTargetDir: + # Move the file to the new folder + drive.auth.service.files().update(fileId=origin_file['id'], + addParents=gFileTargetDir['id'], + removeParents=previous_parents, + fields='id, parents').execute() + else: + gFileTargetDir = drive.CreateFile( + {'title': target_folder, 'parents': [{"kind": "drive#fileLink", 'id': getEbooksFolderId()}], + "mimeType": "application/vnd.google-apps.folder"}) + gFileTargetDir.Upload() + # Move the file to the new folder + drive.auth.service.files().update(fileId=origin_file['id'], + addParents=gFileTargetDir['id'], + removeParents=previous_parents, + fields='id, parents').execute() elif origin_file['title'] != target_folder: #gFileTargetDir = getFileFromEbooksFolder(None, target_folder, nocase=True) #if gFileTargetDir: @@ -366,12 +373,7 @@ def moveGdriveFolderRemote(origin_file, target_folder, single_book=False): drive.auth.service.files().patch(fileId=origin_file['id'], body={'title': target_folder}, fields='title').execute() - '''else: - # Move the file to the new folder - drive.auth.service.files().update(fileId=origin_file['id'], - addParents=gFileTargetDir['id'], - removeParents=previous_parents, - fields='id, parents').execute()''' + # if previous_parents has no children anymore, delete original fileparent if len(children['items']) == 1: deleteDatabaseEntry(previous_parents) @@ -600,7 +602,10 @@ def get_cover_via_gdrive(cover_path): except (OperationalError, IntegrityError) as ex: log.error_or_exception('Database error: {}'.format(ex)) session.rollback() - return df.metadata.get('webContentLink') + headers = Headers() + headers["Content-Type"] = 'image/jpeg' + resp, content = df.auth.Get_Http_Object().request(df.metadata.get('downloadUrl'), headers=headers) + return content else: return None diff --git a/cps/helper.py b/cps/helper.py index 85f1689b..f4c226a6 100644 --- a/cps/helper.py +++ b/cps/helper.py @@ -30,7 +30,7 @@ import requests import unidecode from uuid import uuid4 -from flask import send_from_directory, make_response, redirect, abort, url_for +from flask import send_from_directory, make_response, abort, url_for, Response from flask_babel import gettext as _ from flask_babel import lazy_gettext as N_ from flask_babel import get_locale @@ -393,10 +393,8 @@ def rename_all_files_on_change(one_book, new_path, old_path, all_new_name, gdriv if not gdrive: if not os.path.exists(new_path): os.makedirs(new_path) - shutil.move(os.path.normcase( - os.path.join(old_path, file_format.name + '.' + file_format.format.lower())), - os.path.normcase( - os.path.join(new_path, all_new_name + '.' + file_format.format.lower()))) + shutil.move(os.path.join(old_path, file_format.name + '.' + file_format.format.lower()), + os.path.join(new_path, all_new_name + '.' + file_format.format.lower())) else: g_file = gd.getFileFromEbooksFolder(old_path, file_format.name + '.' + file_format.format.lower()) @@ -457,7 +455,7 @@ def rename_author_path(first_author, old_author_dir, renamed_author, calibre_pat old_author_path = os.path.join(calibre_path, old_author_dir) new_author_path = os.path.join(calibre_path, new_author_rename_dir) try: - shutil.move(os.path.normcase(old_author_path), os.path.normcase(new_author_path)) + shutil.move(old_author_path, new_author_path) except OSError as ex: log.error("Rename author from: %s to %s: %s", old_author_path, new_author_path, ex) log.debug(ex, exc_info=True) @@ -527,7 +525,7 @@ def update_dir_structure_gdrive(book_id, first_author): new_titledir = get_valid_filename(book.title, chars=96) + " (" + str(book_id) + ")" if titledir != new_titledir: - g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), titledir) + g_file = gd.getFileFromEbooksFolder(authordir, titledir) if g_file: gd.moveGdriveFileRemote(g_file, new_titledir) book.path = book.path.split('/')[0] + '/' + new_titledir @@ -559,21 +557,20 @@ def move_files_on_change(calibre_path, new_author_dir, new_titledir, localbook, if original_filepath: if not os.path.isdir(new_path): os.makedirs(new_path) - shutil.move(os.path.normcase(original_filepath), os.path.normcase(os.path.join(new_path, db_filename))) + shutil.move(original_filepath, os.path.join(new_path, db_filename)) log.debug("Moving title: %s to %s/%s", original_filepath, new_path) else: # Check new path is not valid path if not os.path.exists(new_path): # move original path to new path log.debug("Moving title: %s to %s", path, new_path) - shutil.move(os.path.normcase(path), os.path.normcase(new_path)) + shutil.move(path, new_path) else: # path is valid copy only files to new location (merge) log.info("Moving title: %s into existing: %s", path, new_path) # Take all files and subfolder from old path (strange command) for dir_name, __, file_list in os.walk(path): for file in file_list: - shutil.move(os.path.normcase(os.path.join(dir_name, file)), - os.path.normcase(os.path.join(new_path + dir_name[len(path):], file))) + shutil.move(os.path.join(dir_name, file), os.path.join(new_path + dir_name[len(path):], file)) if not os.listdir(os.path.split(path)[0]): try: shutil.rmtree(os.path.split(path)[0]) @@ -615,7 +612,7 @@ def delete_book_gdrive(book, book_format): for entry in book.data: if entry.format.upper() == book_format: name = entry.name + '.' + book_format - g_file = gd.getFileFromEbooksFolder(book.path, name) + g_file = gd.getFileFromEbooksFolder(book.path, name, nocase=True) else: g_file = gd.getFileFromEbooksFolder(os.path.dirname(book.path), book.path.split('/')[1]) if g_file: @@ -790,9 +787,9 @@ def get_book_cover_internal(book, resolution=None): try: if not gd.is_gdrive_ready(): return get_cover_on_failure() - path = gd.get_cover_via_gdrive(book.path) - if path: - return redirect(path) + cover_file = gd.get_cover_via_gdrive(book.path) + if cover_file: + return Response(cover_file, mimetype='image/jpeg') else: log.error('{}/cover.jpg not found on Google Drive'.format(book.path)) return get_cover_on_failure() diff --git a/cps/tasks/thumbnail.py b/cps/tasks/thumbnail.py index 54b68f40..2921ffc9 100644 --- a/cps/tasks/thumbnail.py +++ b/cps/tasks/thumbnail.py @@ -19,6 +19,7 @@ import os from shutil import copyfile, copyfileobj from urllib.request import urlopen +from io import BytesIO from .. import constants from cps import config, db, fs, gdriveutils, logger, ub @@ -182,13 +183,11 @@ class TaskGenerateCoverThumbnails(CalibreTask): if not gdriveutils.is_gdrive_ready(): raise Exception('Google Drive is configured but not ready') - web_content_link = gdriveutils.get_cover_via_gdrive(book.path) - if not web_content_link: + content = gdriveutils.get_cover_via_gdrive(book.path) + if not content: raise Exception('Google Drive cover url not found') - - stream = None try: - stream = urlopen(web_content_link) + stream = BytesIO(content) with Image(file=stream) as img: filename = self.cache.get_cache_file_path(thumbnail.filename, constants.CACHE_TYPE_THUMBNAILS) diff --git a/optional-requirements.txt b/optional-requirements.txt index e097746f..77f41e1e 100644 --- a/optional-requirements.txt +++ b/optional-requirements.txt @@ -1,5 +1,5 @@ # GDrive Integration -google-api-python-client>=1.7.11,<2.120.0 +google-api-python-client>=1.7.11,<2.200.0 gevent>20.6.0,<24.3.0 greenlet>=0.4.17,<3.1.0 httplib2>=0.9.2,<0.23.0 @@ -13,7 +13,7 @@ rsa>=3.4.2,<4.10.0 # Gmail google-auth-oauthlib>=0.4.3,<1.3.0 -google-api-python-client>=1.7.11,<2.120.0 +google-api-python-client>=1.7.11,<2.200.0 # goodreads goodreads>=0.3.2,<0.4.0 diff --git a/setup.cfg b/setup.cfg index dcbbac10..5e0882e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -46,19 +46,19 @@ install_requires = Flask-Principal>=0.3.2,<0.5.1 Flask>=1.0.2,<3.1.0 iso-639>=0.4.5,<0.5.0 - PyPDF>=3.15.6,<4.1.0 + PyPDF>=3.15.6,<4.3.0 pytz>=2016.10 requests>=2.28.0,<2.32.0 SQLAlchemy>=1.3.0,<2.1.0 tornado>=6.3,<6.5 Wand>=0.4.4,<0.7.0 unidecode>=0.04.19,<1.4.0 - lxml>=4.9.1,<5.2.0 + lxml>=4.9.1,<5.3.0 flask-wtf>=0.14.2,<1.3.0 chardet>=3.0.0,<4.1.0 advocate>=1.0.0,<1.1.0 Flask-Limiter>=2.3.0,<3.6.0 - regex>=2022.3.2,<2024.2.25 + regex>=2022.3.2,<2024.6.25 bleach>=6.0.0,<6.2.0 python-magic>=0.4.27,<0.5.0 @@ -69,7 +69,7 @@ include = cps/services* [options.extras_require] gdrive = - google-api-python-client>=1.7.11,<2.120.0 + google-api-python-client>=1.7.11,<2.200.0 gevent>20.6.0,<24.3.0 greenlet>=0.4.17,<3.1.0 httplib2>=0.9.2,<0.23.0 @@ -82,7 +82,7 @@ gdrive = rsa>=3.4.2,<4.10.0 gmail = google-auth-oauthlib>=0.4.3,<1.3.0 - google-api-python-client>=1.7.11,<2.120.0 + google-api-python-client>=1.7.11,<2.200.0 goodreads = goodreads>=0.3.2,<0.4.0 python-Levenshtein>=0.12.0,<0.26.0 diff --git a/test/Calibre-Web TestSummary_Linux.html b/test/Calibre-Web TestSummary_Linux.html index b799e9aa..952b4f89 100644 --- a/test/Calibre-Web TestSummary_Linux.html +++ b/test/Calibre-Web TestSummary_Linux.html @@ -37,20 +37,20 @@
-

Start Time: 2024-06-29 20:48:57

+

Start Time: 2024-07-02 20:54:09

-

Stop Time: 2024-06-30 03:48:15

+

Stop Time: 2024-07-03 04:18:18

-

Duration: 5h 44 min

+

Duration: 6h 10 min

@@ -234,11 +234,11 @@ - + TestBackupMetadata 21 - 21 - 0 + 20 + 1 0 0 @@ -248,11 +248,31 @@ - +
TestBackupMetadata - test_backup_all
- PASS + +
+ FAIL +
+ + + + @@ -852,34 +872,76 @@ - + TestEbookConvertCalibreGDrive - 6 - 6 - 0 - 0 + 7 + 5 + 1 + 1 0 - Detail + Detail - +
TestEbookConvertCalibreGDrive - test_convert_email
- PASS + +
+ FAIL +
+ + + + - +
TestEbookConvertCalibreGDrive - test_convert_failed_and_email
- PASS + +
+ ERROR +
+ + + + @@ -919,6 +981,15 @@ + + + +
TestEbookConvertCalibreGDrive - test_thumbnail_cache
+ + PASS + + + @@ -1695,11 +1766,11 @@ - + TestEditAuthorsGdrive 7 - 5 - 2 + 7 + 0 0 0 @@ -1727,31 +1798,11 @@ - +
TestEditAuthorsGdrive - test_change_capital_one_author_two_books
- -
- FAIL -
- - - - + PASS @@ -1765,31 +1816,11 @@ AssertionError: <selenium.webdriver.remote.webelement.WebElement (session= - +
TestEditAuthorsGdrive - test_change_capital_rename_co_author
- -
- FAIL -
- - - - + PASS @@ -1813,12 +1844,12 @@ AssertionError: False is not true - + TestEditBooksList 19 - 15 - 3 - 1 + 19 + 0 + 0 0 Detail @@ -1829,7 +1860,7 @@ AssertionError: False is not true -
TestEditBooksList - test_booklist_xss: r = requests.session()
+
TestEditBooksList - test_booklist_xss
PASS @@ -1962,243 +1993,77 @@ AssertionError: False is not true - +
TestEditBooksList - test_bookslist_edit_title
- -
- ERROR -
- - - - + PASS - +
TestEditBooksList - test_list_visibility
- -
- FAIL -
- - - - + PASS - +
TestEditBooksList - test_restricted_rights
- -
- FAIL -
- - - - + PASS - +
TestEditBooksList - test_search_books_list
- -
- FAIL -
- - - - - - - - - - - _ErrorHolder - 2 - 0 - 0 - 2 - 0 - - Detail - - - - - - - -
tearDownClass (test_edit_books_list)
- - -
- ERROR -
- - - - - - - - - - -
setUpClass (test_pip_install)
- - -
- ERROR -
- - - - + PASS - + TestLoadMetadata 1 0 - 0 1 0 + 0 - Detail + Detail - +
TestLoadMetadata - test_load_metadata
- ERROR + FAIL
-