diff --git a/cps/db.py b/cps/db.py index f6ee790e..56594de2 100755 --- a/cps/db.py +++ b/cps/db.py @@ -12,9 +12,9 @@ import ub session = None cc_exceptions = None -cc_classes = None -cc_ids = None -books_custom_column_links = None +cc_classes = {} +cc_ids = [] +books_custom_column_links = {} engine = None @@ -293,41 +293,40 @@ def setup_db(): cc = conn.execute("SELECT id, datatype FROM custom_columns") - cc_ids = [] cc_exceptions = ['datetime', 'int', 'comments', 'float', 'composite', 'series'] - books_custom_column_links = {} - cc_classes = {} for row in cc: if row.datatype not in cc_exceptions: - books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata, + if row.id not in books_custom_column_links: + books_custom_column_links[row.id] = Table('books_custom_column_' + str(row.id) + '_link', Base.metadata, Column('book', Integer, ForeignKey('books.id'), primary_key=True), Column('value', Integer, ForeignKey('custom_column_' + str(row.id) + '.id'), primary_key=True) ) - cc_ids.append([row.id, row.datatype]) - if row.datatype == 'bool': - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'book': Column(Integer, ForeignKey('books.id')), - 'value': Column(Boolean)} - else: - ccdict = {'__tablename__': 'custom_column_' + str(row.id), - 'id': Column(Integer, primary_key=True), - 'value': Column(String)} - cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict) + cc_ids.append([row.id, row.datatype]) + if row.datatype == 'bool': + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'book': Column(Integer, ForeignKey('books.id')), + 'value': Column(Boolean)} + else: + ccdict = {'__tablename__': 'custom_column_' + str(row.id), + 'id': Column(Integer, primary_key=True), + 'value': Column(String)} + cc_classes[row.id] = type('Custom_Column_' + str(row.id), (Base,), ccdict) for id in cc_ids: - if id[1] == 'bool': - setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]], - primaryjoin=( - Books.id == cc_classes[id[0]].book), - backref='books')) - else: - setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]], - secondary=books_custom_column_links[id[0]], - backref='books')) + if not hasattr(Books, 'custom_column_' + str(id[0])): + if id[1] == 'bool': + setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]], + primaryjoin=( + Books.id == cc_classes[id[0]].book), + backref='books')) + else: + setattr(Books, 'custom_column_' + str(id[0]), relationship(cc_classes[id[0]], + secondary=books_custom_column_links[id[0]], + backref='books')) # Base.metadata.create_all(engine) Session = sessionmaker() diff --git a/cps/gdriveutils.py b/cps/gdriveutils.py new file mode 100644 index 00000000..a0d1fa2f --- /dev/null +++ b/cps/gdriveutils.py @@ -0,0 +1,313 @@ +from pydrive.auth import GoogleAuth +from pydrive.drive import GoogleDrive +import os, time + +from ub import config + +from sqlalchemy import * +from sqlalchemy import exc +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import * + +from apiclient import errors + +import web + + +dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "gdrive.db") +engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False) +Base = declarative_base() + +# Open session for database connection +Session = sessionmaker() +Session.configure(bind=engine) +session = Session() + +class GdriveId(Base): + __tablename__='gdrive_ids' + + id = Column(Integer, primary_key=True) + gdrive_id = Column(Integer, unique=True) + path = Column(String) + + def __repr__(self): + return str(self.path) + +if not os.path.exists(dbpath): + try: + Base.metadata.create_all(engine) + except Exception: + raise + +def getDrive(gauth=None): + if not gauth: + gauth=GoogleAuth(settings_file='settings.yaml') + # Try to load saved client credentials + gauth.LoadCredentialsFile("gdrive_credentials") + if gauth.access_token_expired: + # Refresh them if expired + gauth.Refresh() + else: + # Initialize the saved creds + gauth.Authorize() + # Save the current credentials to a file + return GoogleDrive(gauth) + +def getEbooksFolder(drive=None): + if not drive: + drive = getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + ebooksFolder= "title = '%s' and 'root' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % config.config_google_drive_folder + + fileList = drive.ListFile({'q': ebooksFolder}).GetList() + return fileList[0] + +def getEbooksFolderId(drive=None): + storedPathName=session.query(GdriveId).filter(GdriveId.path == '/').first() + if storedPathName: + return storedPathName.gdrive_id + else: + gDriveId=GdriveId() + gDriveId.gdrive_id=getEbooksFolder(drive)['id'] + gDriveId.path='/' + session.merge(gDriveId) + session.commit() + return + +def getFolderInFolder(parentId, folderName, drive=None): + if not drive: + drive = getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + folder= "title = '%s' and '%s' in parents and mimeType = 'application/vnd.google-apps.folder' and trashed = false" % (folderName.replace("'", "\\'"), parentId) + fileList = drive.ListFile({'q': folder}).GetList() + return fileList[0] + +def getFile(pathId, fileName, drive=None): + if not drive: + drive = getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + metaDataFile="'%s' in parents and trashed = false and title = '%s'" % (pathId, fileName.replace("'", "\\'")) + + fileList = drive.ListFile({'q': metaDataFile}).GetList() + return fileList[0] + +def getFolderId(path, drive=None): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + currentFolderId=getEbooksFolderId(drive) + sqlCheckPath=path if path[-1] =='/' else path + '/' + storedPathName=session.query(GdriveId).filter(GdriveId.path == sqlCheckPath).first() + + if not storedPathName: + dbChange=False + s=path.split('/') + for i, x in enumerate(s): + if len(x) > 0: + currentPath="/".join(s[:i+1]) + if currentPath[-1] != '/': + currentPath = currentPath + '/' + storedPathName=session.query(GdriveId).filter(GdriveId.path == currentPath).first() + if storedPathName: + currentFolderId=storedPathName.gdrive_id + else: + currentFolderId=getFolderInFolder(currentFolderId, x, drive)['id'] + gDriveId=GdriveId() + gDriveId.gdrive_id=currentFolderId + gDriveId.path=currentPath + session.merge(gDriveId) + dbChange=True + if dbChange: + session.commit() + else: + currentFolderId=storedPathName.gdrive_id + return currentFolderId + + +def getFileFromEbooksFolder(drive, path, fileName): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + if path: + sqlCheckPath=path if path[-1] =='/' else path + '/' + folderId=getFolderId(path, drive) + else: + folderId=getEbooksFolderId(drive) + + return getFile(folderId, fileName, drive) + +def copyDriveFileRemote(drive, origin_file_id, copy_title): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + copied_file = {'title': copy_title} + try: + file_data = drive.auth.service.files().copy( + fileId=origin_file_id, body=copied_file).execute() + return drive.CreateFile({'id': file_data['id']}) + except errors.HttpError as error: + print ('An error occurred: %s' % error) + return None + +def downloadFile(drive, path, filename, output): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + f=getFileFromEbooksFolder(drive, path, filename) + f.GetContentFile(output) + +def backupCalibreDbAndOptionalDownload(drive, f=None): + pass + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + metaDataFile="'%s' in parents and title = 'metadata.db' and trashed = false" % getEbooksFolderId() + + fileList = drive.ListFile({'q': metaDataFile}).GetList() + + databaseFile=fileList[0] + + if f: + databaseFile.GetContentFile(f) + +def copyToDrive(drive, uploadFile, createRoot, replaceFiles, + ignoreFiles=[], + parent=None, prevDir=''): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + isInitial=not bool(parent) + if not parent: + parent=getEbooksFolder(drive) + if os.path.isdir(os.path.join(prevDir,uploadFile)): + existingFolder=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList() + if len(existingFolder) == 0 and (not isInitial or createRoot): + parent = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], + "mimeType": "application/vnd.google-apps.folder" }) + parent.Upload() + else: + if (not isInitial or createRoot) and len(existingFolder) > 0: + parent=existingFolder[0] + for f in os.listdir(os.path.join(prevDir,uploadFile)): + if f not in ignoreFiles: + copyToDrive(drive, f, True, replaceFiles, ignoreFiles, parent, os.path.join(prevDir,uploadFile)) + else: + if os.path.basename(uploadFile) not in ignoreFiles: + existingFiles=drive.ListFile({'q' : "title = '%s' and '%s' in parents and trashed = false" % (os.path.basename(uploadFile), parent['id'])}).GetList() + if len(existingFiles) > 0: + driveFile = drive.CreateFile({'title': os.path.basename(uploadFile), 'parents' : [{"kind": "drive#fileLink", 'id' : parent['id']}], }) + else: + driveFile=existingFiles[0] + driveFile.SetContentFile(os.path.join(prevDir,uploadFile)) + driveFile.Upload() + +def watchChange(drive, channel_id, channel_type, channel_address, + channel_token=None, expiration=None): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + """Watch for all changes to a user's Drive. + Args: + service: Drive API service instance. + channel_id: Unique string that identifies this channel. + channel_type: Type of delivery mechanism used for this channel. + channel_address: Address where notifications are delivered. + channel_token: An arbitrary string delivered to the target address with + each notification delivered over this channel. Optional. + channel_address: Address where notifications are delivered. Optional. + Returns: + The created channel if successful + Raises: + apiclient.errors.HttpError: if http request to create channel fails. + """ + body = { + 'id': channel_id, + 'type': channel_type, + 'address': channel_address + } + if channel_token: + body['token'] = channel_token + if expiration: + body['expiration'] = expiration + return drive.auth.service.changes().watch(body=body).execute() + +def watchFile(drive, file_id, channel_id, channel_type, channel_address, + channel_token=None, expiration=None): + """Watch for any changes to a specific file. + Args: + service: Drive API service instance. + file_id: ID of the file to watch. + channel_id: Unique string that identifies this channel. + channel_type: Type of delivery mechanism used for this channel. + channel_address: Address where notifications are delivered. + channel_token: An arbitrary string delivered to the target address with + each notification delivered over this channel. Optional. + channel_address: Address where notifications are delivered. Optional. + Returns: + The created channel if successful + Raises: + apiclient.errors.HttpError: if http request to create channel fails. + """ + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + + body = { + 'id': channel_id, + 'type': channel_type, + 'address': channel_address + } + if channel_token: + body['token'] = channel_token + if expiration: + body['expiration'] = expiration + return drive.auth.service.files().watch(fileId=file_id, body=body).execute() + +def stopChannel(drive, channel_id, resource_id): + """Stop watching to a specific channel. + Args: + service: Drive API service instance. + channel_id: ID of the channel to stop. + resource_id: Resource ID of the channel to stop. + Raises: + apiclient.errors.HttpError: if http request to create channel fails. + """ + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + service=drive.auth.service + body = { + 'id': channel_id, + 'resourceId': resource_id + } + return drive.auth.service.channels().stop(body=body).execute() + +def getChangeById (drive, change_id): + if not drive: + drive=getDrive() + if drive.auth.access_token_expired: + drive.auth.Refresh() + """Print a single Change resource information. + + Args: + service: Drive API service instance. + change_id: ID of the Change resource to retrieve. + """ + try: + change = drive.auth.service.changes().get(changeId=change_id).execute() + return change + except errors.HttpError, error: + web.app.logger.exception(error) + return None diff --git a/cps/templates/config_edit.html b/cps/templates/config_edit.html index 7824ddd0..9d107cb2 100644 --- a/cps/templates/config_edit.html +++ b/cps/templates/config_edit.html @@ -7,6 +7,45 @@ +
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ {% if show_authenticate_google_drive %} +
+ Authenticate Google Drive +
+ {% else %} + {% if content.config_google_drive_watch_changes_response %} + +
+ + + Revoke + + {% else %} + Enable watch of metadata.db + {% endif %} +
+ {% endif %} +
@@ -80,3 +119,22 @@
{% endblock %} +{% block js %} + +{% endblock %} diff --git a/cps/ub.py b/cps/ub.py index 19f85880..1addddcb 100644 --- a/cps/ub.py +++ b/cps/ub.py @@ -11,6 +11,7 @@ import traceback import logging from werkzeug.security import generate_password_hash from flask_babel import gettext as _ +import json dbpath = os.path.join(os.path.normpath(os.path.dirname(os.path.realpath(__file__)) + os.sep + ".." + os.sep), "app.db") engine = create_engine('sqlite:///{0}'.format(dbpath), echo=False) @@ -269,6 +270,12 @@ class Settings(Base): config_anonbrowse = Column(SmallInteger, default=0) config_public_reg = Column(SmallInteger, default=0) config_default_role = Column(SmallInteger, default=0) + config_use_google_drive = Column(Boolean) + config_google_drive_client_id = Column(String) + config_google_drive_client_secret = Column(String) + config_google_drive_folder = Column(String) + config_google_drive_calibre_url_base = Column(String) + config_google_drive_watch_changes_response = Column(String) def __repr__(self): pass @@ -295,6 +302,16 @@ class Config: self.config_anonbrowse = data.config_anonbrowse self.config_public_reg = data.config_public_reg self.config_default_role = data.config_default_role + self.config_use_google_drive = data.config_use_google_drive + self.config_google_drive_client_id = data.config_google_drive_client_id + self.config_google_drive_client_secret = data.config_google_drive_client_secret + self.config_google_drive_calibre_url_base = data.config_google_drive_calibre_url_base + self.config_google_drive_folder = data.config_google_drive_folder + if data.config_google_drive_watch_changes_response: + self.config_google_drive_watch_changes_response = json.loads(data.config_google_drive_watch_changes_response) + else: + self.config_google_drive_watch_changes_response=None + if self.config_calibre_dir is not None: self.db_configured = True else: @@ -379,6 +396,17 @@ def migrate_Database(): conn.execute("ALTER TABLE Settings ADD column `config_anonbrowse` SmallInteger DEFAULT 0") conn.execute("ALTER TABLE Settings ADD column `config_public_reg` SmallInteger DEFAULT 0") session.commit() + + try: + session.query(exists().where(Settings.config_use_google_drive)).scalar() + except exc.OperationalError: + conn = engine.connect() + conn.execute("ALTER TABLE Settings ADD column `config_use_google_drive` INTEGER DEFAULT 0") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_id` String DEFAULT ''") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_client_secret` String DEFAULT ''") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_calibre_url_base` INTEGER DEFAULT 0") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_folder` String DEFAULT ''") + conn.execute("ALTER TABLE Settings ADD column `config_google_drive_watch_changes_response` String DEFAULT ''") try: session.query(exists().where(Settings.config_default_role)).scalar() session.commit() diff --git a/cps/web.py b/cps/web.py index 4e644cc1..9a49ddf2 100755 --- a/cps/web.py +++ b/cps/web.py @@ -1,12 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from pydrive.auth import GoogleAuth + import mimetypes import logging from logging.handlers import RotatingFileHandler from tempfile import gettempdir import textwrap from flask import Flask, render_template, session, request, Response, redirect, url_for, send_from_directory, \ - make_response, g, flash, abort + make_response, g, flash, abort, send_file import ub from ub import config import helper @@ -42,6 +44,15 @@ import db from shutil import move, copyfile from tornado.ioloop import IOLoop import StringIO +from shutil import move +import gdriveutils +import io +import hashlib +import threading + +import time + +current_milli_time = lambda: int(round(time.time() * 1000)) try: from wand.image import Image @@ -52,13 +63,67 @@ except ImportError, e: from cgi import escape # Global variables +gdrive_watch_callback_token='target=calibreweb-watch_files' global_task = None +def md5(fname): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + +class Singleton: + """ + A non-thread-safe helper class to ease implementing singletons. + This should be used as a decorator -- not a metaclass -- to the + class that should be a singleton. + + The decorated class can define one `__init__` function that + takes only the `self` argument. Also, the decorated class cannot be + inherited from. Other than that, there are no restrictions that apply + to the decorated class. + + To get the singleton instance, use the `Instance` method. Trying + to use `__call__` will result in a `TypeError` being raised. + + """ + + def __init__(self, decorated): + self._decorated = decorated + + def Instance(self): + """ + Returns the singleton instance. Upon its first call, it creates a + new instance of the decorated class and calls its `__init__` method. + On all subsequent calls, the already created instance is returned. + + """ + try: + return self._instance + except AttributeError: + self._instance = self._decorated() + return self._instance + + def __call__(self): + raise TypeError('Singletons must be accessed through `Instance()`.') + + def __instancecheck__(self, inst): + return isinstance(inst, self._decorated) + +@Singleton +class Gauth: + def __init__(self): + self.auth=GoogleAuth(settings_file='settings.yaml') + +@Singleton +class Gdrive: + def __init__(self): + self.drive=gdriveutils.getDrive(Gauth.Instance().auth) -# Proxy Helper class class ReverseProxied(object): """Wrap the application in this middleware and configure the - front-end server to add these headers, to let you quietly bind + front-end server to add these headers, to let you quietly bind this to a URL other than / and to an HTTP scheme that is different than what is used locally. @@ -187,6 +252,12 @@ def authenticate(): 'You have to login with proper credentials', 401, {'WWW-Authenticate': 'Basic realm="Login Required"'}) +def updateGdriveCalibreFromLocal(): + gdriveutils.backupCalibreDbAndOptionalDownload(Gdrive.Instance().drive) + gdriveutils.copyToDrive(Gdrive.Instance().drive, config.config_calibre_dir, False, True) + for x in os.listdir(config.config_calibre_dir): + if os.path.isdir(os.path.join(config.config_calibre_dir,x)): + shutil.rmtree(os.path.join(config.config_calibre_dir,x)) def requires_basic_auth_if_no_ano(f): @wraps(f) @@ -286,6 +357,17 @@ def formatdate(val): formatdate = datetime.datetime.strptime(conformed_timestamp[:15], "%Y%m%d %H%M%S") return format_date(formatdate, format='medium',locale=get_locale()) +@app.template_filter('strftime') +def timestamptodate(date, fmt=None): + date=datetime.datetime.fromtimestamp( + int(date)/1000 + ) + native = date.replace(tzinfo=None) + if fmt: + format=fmt + else: + format='%d %m %Y - %H:%S' + return native.strftime(format) def admin_required(f): """ @@ -668,8 +750,15 @@ def get_opds_download_link(book_id, format): file_name = book.title if len(book.authors) > 0: file_name = book.authors[0].name + '-' + file_name - file_name = helper.get_valid_filename(file_name) - response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format)) + + if config.config_use_google_drive: + df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format)) + download_url = df.metadata.get('downloadUrl') + resp, content = df.auth.Get_Http_Object().request(download_url) + response=send_file(io.BytesIO(content)) + else: + file_name = helper.get_valid_filename(file_name) + response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format)) response.headers["Content-Disposition"] = "attachment; filename=\"%s.%s\"" % (data.name, format) return response @@ -802,7 +891,9 @@ def hot_books(page): hot_books = all_books.offset(off).limit(config.config_books_per_page) entries = list() for book in hot_books: - entries.append(db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first()) + entry=db.session.query(db.Books).filter(filter).filter(db.Books.id == book.Downloads.book_id).first() + if entry: + entries.append(entry) numBooks = entries.__len__() pagination = Pagination(page, config.config_books_per_page, numBooks) return render_title_template('index.html', random=random, entries=entries, pagination=pagination, @@ -1037,6 +1128,99 @@ def stats(): categorycounter=categorys, seriecounter=series, title=_(u"Statistics")) +#@app.route("/load_gdrive") +#@login_required +#@admin_required +#def load_all_gdrive_folder_ids(): +# books=db.session.query(db.Books).all() +# for book in books: +# gdriveutils.getFolderId(book.path, Gdrive.Instance().drive) +# return + +@app.route("/gdrive/authenticate") +@login_required +@admin_required +def authenticate_google_drive(): + authUrl=Gauth.Instance().auth.GetAuthUrl() + return redirect(authUrl) + +@app.route("/gdrive/callback") +def google_drive_callback(): + auth_code = request.args.get('code') + credentials = Gauth.Instance().auth.flow.step2_exchange(auth_code) + with open('gdrive_credentials' ,'w') as f: + f.write(credentials.to_json()) + return redirect(url_for('configuration')) + +@app.route("/gdrive/watch/subscribe") +@login_required +@admin_required +def watch_gdrive(): + if not config.config_google_drive_watch_changes_response: + address = '%scalibre-web/gdrive/watch/callback' % config.config_google_drive_calibre_url_base + notification_id=str(uuid4()) + result = gdriveutils.watchChange(Gdrive.Instance().drive, notification_id, + 'web_hook', address, gdrive_watch_callback_token, current_milli_time() + 604800*1000) + print (result) + settings = ub.session.query(ub.Settings).first() + settings.config_google_drive_watch_changes_response=json.dumps(result) + ub.session.merge(settings) + ub.session.commit() + settings = ub.session.query(ub.Settings).first() + config.loadSettings() + + print (settings.config_google_drive_watch_changes_response) + + return redirect(url_for('configuration')) + +@app.route("/gdrive/watch/revoke") +@login_required +@admin_required +def revoke_watch_gdrive(): + last_watch_response=config.config_google_drive_watch_changes_response + if last_watch_response: + response=gdriveutils.stopChannel(Gdrive.Instance().drive, last_watch_response['id'], last_watch_response['resourceId']) + settings = ub.session.query(ub.Settings).first() + settings.config_google_drive_watch_changes_response=None + ub.session.merge(settings) + ub.session.commit() + config.loadSettings() + return redirect(url_for('configuration')) + +@app.route("/gdrive/watch/callback", methods=['GET', 'POST']) +def on_received_watch_confirmation(): + app.logger.info (request.headers) + if request.headers.get('X-Goog-Channel-Token') == gdrive_watch_callback_token \ + and request.headers.get('X-Goog-Resource-State') == 'change' \ + and request.data: + + data=request.data + + def updateMetaData(): + app.logger.info ('Change received from gdrive') + app.logger.info (data) + try: + j=json.loads(data) + app.logger.info ('Getting change details') + response=gdriveutils.getChangeById(Gdrive.Instance().drive, j['id']) + app.logger.info (response) + if response: + dbpath = os.path.join(config.config_calibre_dir, "metadata.db") + if not response['deleted'] and response['file']['title'] == 'metadata.db' and response['file']['md5Checksum'] != md5(dbpath): + app.logger.info ('Database file updated') + copyfile (dbpath, config.config_calibre_dir + "/metadata.db_" + str(current_milli_time())) + app.logger.info ('Backing up existing and downloading updated metadata.db') + gdriveutils.downloadFile(Gdrive.Instance().drive, None, "metadata.db", config.config_calibre_dir + "/tmp_metadata.db") + app.logger.info ('Setting up new DB') + os.rename(config.config_calibre_dir + "/tmp_metadata.db", dbpath) + db.setup_db() + except Exception, e: + app.logger.exception(e) + + updateMetaData() + return '' + + @app.route("/shutdown") @login_required @admin_required @@ -1173,8 +1357,15 @@ def advanced_search(): @app.route("/cover/") @login_required_if_no_ano def get_cover(cover_path): - return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg") + if config.config_use_google_drive: + df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg') + download_url = df.metadata.get('webContentLink') + return redirect(download_url) + else: + return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg") + resp.headers['Content-Type']='image/jpeg' + return resp @app.route("/opds/thumb_240_240/") @app.route("/opds/cover_240_240/") @@ -1183,7 +1374,12 @@ def get_cover(cover_path): @requires_basic_auth_if_no_ano def feed_get_cover(book_id): book = db.session.query(db.Books).filter(db.Books.id == book_id).first() - return send_from_directory(os.path.join(config.config_calibre_dir, book.path), "cover.jpg") + if config.config_use_google_drive: + df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, cover_path, 'cover.jpg') + download_url = df.metadata.get('webContentLink') + return redirect(download_url) + else: + return send_from_directory(os.path.join(config.config_calibre_dir, cover_path), "cover.jpg") def render_read_books(page, are_read, as_xml=False): readBooks=ub.session.query(ub.ReadBook).filter(ub.ReadBook.user_id == int(current_user.id)).filter(ub.ReadBook.is_read == True).all() @@ -1308,8 +1504,13 @@ def get_download_link(book_id, format): if len(book.authors) > 0: file_name = book.authors[0].name + '-' + file_name file_name = helper.get_valid_filename(file_name) - response = make_response( - send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format)) + if config.config_use_google_drive: + df=gdriveutils.getFileFromEbooksFolder(Gdrive.Instance().drive, book.path, '%s.%s' % (data.name, format)) + download_url = df.metadata.get('downloadUrl') + resp, content = df.auth.Get_Http_Object().request(download_url) + response=send_file(io.BytesIO(content)) + else: + response = make_response(send_from_directory(os.path.join(config.config_calibre_dir, book.path), data.name + "." + format)) try: response.headers["Content-Type"] = mimetypes.types_map['.' + format] except: @@ -1682,6 +1883,38 @@ def configuration_helper(origin): if content.config_calibre_dir != to_save["config_calibre_dir"]: content.config_calibre_dir = to_save["config_calibre_dir"] db_change = True + ##Google drive setup + create_new_yaml=False + if "config_google_drive_client_id" in to_save: + if content.config_google_drive_client_id != to_save["config_google_drive_client_id"]: + content.config_google_drive_client_id = to_save["config_google_drive_client_id"] + create_new_yaml=True + db_change = True + if "config_google_drive_client_secret" in to_save: + if content.config_google_drive_client_secret != to_save["config_google_drive_client_secret"]: + content.config_google_drive_client_secret = to_save["config_google_drive_client_secret"] + create_new_yaml=True + db_change = True + if "config_google_drive_calibre_url_base" in to_save: + if content.config_google_drive_calibre_url_base != to_save["config_google_drive_calibre_url_base"]: + content.config_google_drive_calibre_url_base = to_save["config_google_drive_calibre_url_base"] + create_new_yaml=True + db_change = True + if ("config_use_google_drive" in to_save and not content.config_use_google_drive) or ("config_use_google_drive" not in to_save and content.config_use_google_drive): + content.config_use_google_drive = "config_use_google_drive" in to_save + db_change = True + if not content.config_use_google_drive: + create_new_yaml=False + if create_new_yaml: + with open('settings.yaml', 'w') as f: + with open('gdrive_template.yaml' ,'r') as t: + f.write(t.read() % {'client_id' : content.config_google_drive_client_id, 'client_secret' : content.config_google_drive_client_secret, + "redirect_uri" : content.config_google_drive_calibre_url_base + 'gdrive/callback'}) + if "config_google_drive_folder" in to_save: + if content.config_google_drive_folder != to_save["config_google_drive_folder"]: + content.config_google_drive_folder = to_save["config_google_drive_folder"] + db_change = True + ## if "config_port" in to_save: if content.config_port != int(to_save["config_port"]): content.config_port = int(to_save["config_port"]) @@ -1751,6 +1984,7 @@ def configuration_helper(origin): if origin: success = True return render_title_template("config_edit.html", origin=origin, success=success, content=config, + show_authenticate_google_drive=not os.path.exists('settings.yaml') or not os.path.exists('gdrive_credentials'), title=_(u"Basic Configuration")) @@ -1999,7 +2233,7 @@ def edit_book(book_id): modify_database_object(input_authors, book.authors, db.Authors, db.session, 'author') if author0_before_edit != book.authors[0].name: edited_books_id.add(book.id) - book.author_sort=helper.get_sorted_author(input_authors[0]) + book.author_sort=helper.get_sorted_author(input_authors[0]) if to_save["cover_url"] and os.path.splitext(to_save["cover_url"])[1].lower() == ".jpg": img = requests.get(to_save["cover_url"]) @@ -2163,6 +2397,8 @@ def edit_book(book_id): author_names.append(author.name) for b in edited_books_id: helper.update_dir_stucture(b, config.config_calibre_dir) + if config.config_use_google_drive: + updateGdriveCalibreFromLocal() if "detail_view" in to_save: return redirect(url_for('show_book', id=book.id)) else: @@ -2227,7 +2463,7 @@ def upload(): if is_author: db_author = is_author else: - db_author = db.Authors(author, helper.get_sorted_author(author), "") + db_author = db.Authors(author, helper.get_sorted_author(author), "") db.session.add(db_author) # combine path and normalize path from windows systems path = os.path.join(author_dir, title_dir).replace('\\','/') @@ -2242,6 +2478,9 @@ def upload(): author_names = [] for author in db_book.authors: author_names.append(author.name) + if config.config_use_google_drive: + if not current_user.role_edit() and not current_user.role_admin(): + updateGdriveCalibreFromLocal() cc = db.session.query(db.Custom_Columns).filter(db.Custom_Columns.datatype.notin_(db.cc_exceptions)).all() if current_user.role_edit() or current_user.role_admin(): return render_title_template('book_edit.html', book=db_book, authors=author_names, cc=cc, diff --git a/gdrive_template.yaml b/gdrive_template.yaml new file mode 100644 index 00000000..a6d77eee --- /dev/null +++ b/gdrive_template.yaml @@ -0,0 +1,14 @@ +client_config_backend: settings +client_config: + client_id: %(client_id)s + client_secret: %(client_secret)s + redirect_uri: %(redirect_uri)s + +save_credentials: True +save_credentials_backend: file +save_credentials_file: gdrive_credentials + +get_refresh_token: True + +oauth_scope: + - https://www.googleapis.com/auth/drive