From eb6e38e28ced19b164cda076c91db32eafcd369c Mon Sep 17 00:00:00 2001 From: sian1468 <58017832+sian1468@users.noreply.github.com> Date: Mon, 18 Dec 2023 18:07:56 +0700 Subject: [PATCH] [Backport] Add option to disable image file size check (close #170) (#228) This commit is backport from [saveweb/wikiteam3](https://github.com/saveweb/wikiteam3) all credit goes to the original author. Close #170 Fix size mismatch error when some wiki do server-side image resizing/compression without re-upload/update data in wiki. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- wikiteam3/dumpgenerator/cli/cli.py | 6 ++++++ wikiteam3/dumpgenerator/dump/image/image.py | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/wikiteam3/dumpgenerator/cli/cli.py b/wikiteam3/dumpgenerator/cli/cli.py index da4600e..79dadb5 100644 --- a/wikiteam3/dumpgenerator/cli/cli.py +++ b/wikiteam3/dumpgenerator/cli/cli.py @@ -129,6 +129,11 @@ def getArgumentParser(): action="store_true", help="Bypass CDN image compression. (CloudFlare Polish, etc.)", ) + groupDownload.add_argument( + "--disable-image-verify", + action="store_true", + help="Don't verify image size and hash while downloading. (useful for wikis with server-side image resizing)", + ) groupDownload.add_argument( "--namespaces", metavar="1,2,3", @@ -469,6 +474,7 @@ def getParameters(params=None) -> Tuple[Config, Dict]: "session": session, "stdout_log_path": args.stdout_log_path, "bypass_cdn_image_compression": args.bypass_cdn_image_compression, + "disable_image_verify": args.disable_image_verify, } # calculating path, if not defined by user with --path= diff --git a/wikiteam3/dumpgenerator/dump/image/image.py b/wikiteam3/dumpgenerator/dump/image/image.py index 256c568..3dde9a0 100644 --- a/wikiteam3/dumpgenerator/dump/image/image.py +++ b/wikiteam3/dumpgenerator/dump/image/image.py @@ -48,6 +48,7 @@ class Image: c_savedImageDescs = 0 bypass_cdn_image_compression: bool = other["bypass_cdn_image_compression"] + disable_image_verify: bool = other["disable_image_verify"] def modify_params(params: Optional[Dict] = None) -> Dict: """bypass Cloudflare Polish (image optimization)""" @@ -130,7 +131,11 @@ class Image: if r.status_code == 200: try: - if size == "False" or len(r.content) == int(size): + if ( + size == "False" + or len(r.content) == int(size) + or disable_image_verify + ): # size == 'False' means size is unknown with open(filename3, "wb") as imagefile: imagefile.write(r.content)