Merge pull request #452 from yzqzss/patch-4

Update dumpgenerator.py

dumpgenerator.py

@@ -170,7 +170,7 @@ def getNamespacesScraper(config={}, session=None):
 
         # [^>]*? to include selected="selected"
         m = re.compile(
-            r'<option [^>]*?value="(?P<namespaceid>\d+)"[^>]*?>(?P<namespacename>[^<]+)</option>').finditer(raw)
+            r'<option [^>]*?value=[\'"](?P<namespaceid>\d+)[\'"][^>]*?>(?P<namespacename>[^<]+)</option>').finditer(raw)
         if 'all' in namespaces:
             namespaces = []
             for i in m:
@@ -287,9 +287,9 @@ def getPageTitlesScraper(config={}, session=None):
 
         r_title = r'title="(?P<title>[^>]+)">'
         r_suballpages = ''
-        r_suballpages1 = r'&amp;from=(?P<from>[^>]+)&amp;to=(?P<to>[^>]+)">'
-        r_suballpages2 = r'Special:Allpages/(?P<from>[^>]+)">'
-        r_suballpages3 = r'&amp;from=(?P<from>[^>]+)" title="[^>]+">'
+        r_suballpages1 = r'&amp;from=(?P<from>[^>"]+)&amp;to=(?P<to>[^>"]+)">'
+        r_suballpages2 = r'Special:Allpages/(?P<from>[^>"]+)">'
+        r_suballpages3 = r'&amp;from=(?P<from>[^>"]+)" title="[^>]+">'
         if re.search(r_suballpages1, raw):
             r_suballpages = r_suballpages1
         elif re.search(r_suballpages2, raw):
@@ -299,7 +299,7 @@ def getPageTitlesScraper(config={}, session=None):
         else:
             pass  # perhaps no subpages
 
-        # Should be enought subpages on Special:Allpages
+        # Should be enough subpages on Special:Allpages
         deep = 50
         c = 0
         oldfr = ''
@@ -321,8 +321,8 @@ def getPageTitlesScraper(config={}, session=None):
                     name = '%s-%s' % (fr, to)
                     url = '%s?title=Special:Allpages&namespace=%s&from=%s&to=%s' % (
                         config['index'], namespace, fr, to)  # do not put urllib.quote in fr or to
-                # fix, esta regexp no carga bien todas? o falla el r_title en
-                # este tipo de subpag? (wikiindex)
+                # fix, this regexp doesn't properly save everything? or does r_title fail on this
+                # type of subpage? (wikiindex)
                 elif r_suballpages == r_suballpages2:
                     # clean &amp;namespace=\d, sometimes happens
                     fr = fr.split('&amp;namespace=')[0]
@@ -1519,7 +1519,7 @@ def generateImageDump(config={}, other={}, images=[], start='', session=None):
         title = u'Image:%s' % (filename)
         try:
             if config['xmlrevisions'] and config['api'] and config['api'].endswith("api.php"):
-                r = session.get(config['api'] + u"?action=query&export&exportnowrap&titles=%s" % title)
+                r = session.get(config['api'] + u"?action=query&export&exportnowrap&titles=%s" % urllib.parse.quote(title))
                 xmlfiledesc = r.text
             else:
                 xmlfiledesc = getXMLFileDesc(
@@ -2250,7 +2250,7 @@ def resumePreviousDump(config={}, other={}):
             listdir = os.listdir('%s/images' % (config['path']))
         except:
             pass  # probably directory does not exist
-        listdir.sort()
+        listdir = set(listdir)
         complete = True
         lastfilename = ''
         lastfilename2 = ''
@@ -2528,7 +2528,7 @@ def main(params=[]):
         print '\nWarning!: "%s" path exists' % (config['path'])
         reply = ''
         if config['failfast']:
-            retry = 'yes'
+            reply = 'yes'
         while reply.lower() not in ['yes', 'y', 'no', 'n']:
             reply = raw_input(
                 'There is a dump in "%s", probably incomplete.\nIf you choose resume, to avoid conflicts, the parameters you have chosen in the current session will be ignored\nand the parameters available in "%s/%s" will be loaded.\nDo you want to resume ([yes, y], [no, n])? ' %