lint: added type hinting for dump & cli package

wikiteam3/dumpgenerator/cli/cli.py

@@ -15,8 +15,9 @@ from wikiteam3.dumpgenerator.version import getVersion
 from wikiteam3.dumpgenerator.api import getWikiEngine
 from wikiteam3.dumpgenerator.config import Config, DefaultConfig, newConfig
 
+from typing import *
 
-def getParameters(params=[]):
+def getParameters(params=None) -> Tuple[Config, Dict]:
     if not params:
         params = sys.argv
 

wikiteam3/dumpgenerator/config.py

@@ -63,7 +63,7 @@ class Config:
     templates: bool = False
 
 DefaultConfig = Config()
-def newConfig(configDict):
+def newConfig(configDict) -> Config:
     return _dataclass_from_dict(Config, configDict)
 
 def loadConfig(config: Config=None, configfilename=""):

wikiteam3/dumpgenerator/dump/generator.py

@@ -20,6 +20,8 @@ except ImportError:
     )
     sys.exit(1)
 
+from typing import *
+
 from wikiteam3.dumpgenerator.config import loadConfig, saveConfig
 from wikiteam3.dumpgenerator.config import Config, DefaultConfig
 from wikiteam3.dumpgenerator.cli import getParameters, bye, welcome
@@ -62,7 +64,7 @@ class Tee(object):
 
 class DumpGenerator:
     @staticmethod
-    def __init__(params=[]):
+    def __init__(params=None):
         """Main function"""
         configfilename = "config.json"
         config, other = getParameters(params=params)
@@ -81,7 +83,7 @@ class DumpGenerator:
                 print('\nWarning!: "%s" path exists' % (config.path))
                 reply = ""
                 if config.failfast:
-                    retry = "yes"
+                    reply = "yes"
                 while reply.lower() not in ["yes", "y", "no", "n"]:
                     reply = input(
                         'There is a dump in "%s", probably incomplete.\nIf you choose resume, to avoid conflicts, the parameters you have chosen in the current session will be ignored\nand the parameters available in "%s/%s" will be loaded.\nDo you want to resume ([yes, y], [no, n])? '
@@ -119,7 +121,7 @@ class DumpGenerator:
             bye()
 
     @staticmethod
-    def createNewDump(config: Config=None, other={}):
+    def createNewDump(config: Config=None, other: Dict=None):
         images = []
         print("Trying generating a new dump into a new directory...")
         if config.xml:
@@ -137,7 +139,7 @@ class DumpGenerator:
             saveLogs(config=config, session=other["session"])
 
     @staticmethod
-    def resumePreviousDump(config: Config=None, other={}):
+    def resumePreviousDump(config: Config=None, other: Dict=None):
         images = []
         print("Resuming previous dump process...")
         if config.xml:

wikiteam3/dumpgenerator/dump/misc/index_php.py

@@ -11,7 +11,7 @@ def saveIndexPHP(config: Config=None, session=None):
         print("index.html exists, do not overwrite")
     else:
         print("Downloading index.php (Main Page) as index.html")
-        r = session.post(url=config.index, params={}, timeout=10)
+        r = session.post(url=config.index, params: Dict=None, timeout=10)
         raw = str(r.text)
         Delay(config=config, session=session)
         raw = removeIP(raw=raw)

wikiteam3/dumpgenerator/dump/page/image.py

@@ -1,7 +1,8 @@
 import os
 import re
 import sys
-import urllib
+import urllib.parse
+from typing import *
 
 from wikiteam3.dumpgenerator.cli import Delay
 from wikiteam3.utils import domain2prefix
@@ -27,7 +28,7 @@ class Image:
             ]
         )
 
-    def generateImageDump(config: Config=None, other={}, images=[], start="", session=None):
+    def generateImageDump(config: Config=None, other: Dict=None, images: Iterable[str]=None, start="", session=None):
         """Save files and descriptions using a file list"""
 
         # fix use subdirectories md5
@@ -58,6 +59,7 @@ class Image:
                 print("Filename is too long, truncating. Now it is:", filename2)
             filename3 = f"{imagepath}/{filename2}"
 
+            original_url = url
             r = session.head(url=url, allow_redirects=True)
             original_url_redirected = len(r.history) > 0
 
@@ -95,8 +97,8 @@ class Image:
 
             Delay(config=config, session=session)
             # saving description if any
+            title = "Image:%s" % (filename)
             try:
-                title = "Image:%s" % (filename)
                 if (
                     config.xmlrevisions
                     and config.api
@@ -440,7 +442,7 @@ class Image:
 
         return images
 
-    def saveImageNames(config: Config=None, images=[], session=None):
+    def saveImageNames(config: Config=None, images: Iterable[str]=None, session=None):
         """Save image list in a file, including filename, url and uploader"""
 
         imagesfilename = "{}-{}-images.txt".format(

wikiteam3/dumpgenerator/dump/page/page_titles.py

@@ -48,7 +48,7 @@ def getPageTitlesScraper(config: Config=None, session=None):
     namespaces, namespacenames = getNamespacesScraper(config=config, session=session)
     for namespace in namespaces:
         print("    Retrieving titles in the namespace", namespace)
-        url = "{}?title=Special:Allpages&namespace={}".format(
+        url = "{}?title=Special:Allpages&namespace: Dict=None".format(
             config.index, namespace
         )
         r = session.get(url=url, timeout=30)
@@ -89,7 +89,7 @@ def getPageTitlesScraper(config: Config=None, session=None):
                 if r_suballpages == r_suballpages1:
                     to = i.group("to")
                     name = f"{fr}-{to}"
-                    url = "{}?title=Special:Allpages&namespace={}&from={}&to={}".format(
+                    url = "{}?title=Special:Allpages&namespace: Dict=None&from: Dict=None&to: Dict=None".format(
                         config.index,
                         namespace,
                         fr,
@@ -101,7 +101,7 @@ def getPageTitlesScraper(config: Config=None, session=None):
                     # clean &namespace=\d, sometimes happens
                     fr = fr.split("&namespace=")[0]
                     name = fr
-                    url = "{}?title=Special:Allpages/{}&namespace={}".format(
+                    url = "{}?title=Special:Allpages/{}&namespace: Dict=None".format(
                         config.index,
                         name,
                         namespace,
@@ -109,7 +109,7 @@ def getPageTitlesScraper(config: Config=None, session=None):
                 elif r_suballpages == r_suballpages3:
                     fr = fr.split("&namespace=")[0]
                     name = fr
-                    url = "{}?title=Special:Allpages&from={}&namespace={}".format(
+                    url = "{}?title=Special:Allpages&from: Dict=None&namespace: Dict=None".format(
                         config.index,
                         name,
                         namespace,

wikiteam3/dumpgenerator/dump/page/page_xml.py

@@ -13,7 +13,7 @@ from wikiteam3.utils import uprint
 from wikiteam3.dumpgenerator.config import Config, DefaultConfig
 
 
-def getXMLPageCore(headers={}, params={}, config: Config=None, session=None) -> str:
+def getXMLPageCore(headers: Dict=None, params: Dict=None, config: Config=None, session=None) -> str:
     """"""
     # returns a XML containing params['limit'] revisions (or current only), ending in </mediawiki>
     # if retrieving params['limit'] revisions fails, returns a current only version
@@ -196,7 +196,7 @@ def getXMLPage(config: Config=None, title="", verbose=True, session=None):
             uprint("    %s, %d edits" % (title.strip(), edit_count))
 
 
-def makeXmlPageFromRaw(xml):
+def makeXmlPageFromRaw(xml) -> str:
     """Discard the metadata around a <page> element in <mediawiki> string"""
     root = etree.XML(xml)
     find = etree.XPath("//*[local-name() = 'page']")

wikiteam3/dumpgenerator/dump/xmlrev/xml_dump.py

@@ -1,5 +1,6 @@
 import re
 import sys
+from typing import *
 
 from wikiteam3.dumpgenerator.cli import Delay
 from wikiteam3.utils import domain2prefix
@@ -13,7 +14,7 @@ from .xml_header import getXMLHeader
 from .xml_revisions import getXMLRevisions
 from .xml_truncate import truncateXMLDump
 
-def generateXMLDump(config: Config=None, titles=[], start=None, session=None):
+def generateXMLDump(config: Config=None, titles: Iterable[str]=None, start=None, session=None):
     """Generates a XML dump for a list of titles or from revision IDs"""
     # TODO: titles is now unused.
 

wikiteam3/uploader.py

@@ -348,7 +348,7 @@ def upload(wikis, logfile, config: Config=None, uploadeddumps=[]):
             c += 1
 
 
-def main(params=[]):
+def main(params=None):
     parser = argparse.ArgumentParser(
         """uploader.py