Fix tests

dumpgenerator.py

@@ -245,7 +245,16 @@ def getPageTitlesAPI(config={}, session=None):
                 'apfrom': apfrom.encode('utf-8'),
                 'format': 'json',
                 'aplimit': 500}
-            r = session.post(url=config['api'], data=params)
+
+            retryCount = 0
+            while retryCount < config["retries"]:
+                try:
+                    r = session.post(url=config['api'], data=params)
+                    break
+                except ConnectionError as err:
+                    print "Connection error: %s" % (str(err),)
+                    retryCount += 1
+                    time.sleep(20)
             handleStatusCode(r)
             # FIXME Handle HTTP errors here!
             jsontitles = getJSON(r)
@@ -376,7 +385,7 @@ def getPageTitles(config={}, session=None):
 
     titlesfilename = '%s-%s-titles.txt' % (
         domain2prefix(config=config), config['date'])
-    titlesfile = open('%s/%s' % (config['path'], titlesfilename), 'a')
+    titlesfile = open('%s/%s' % (config['path'], titlesfilename), 'wt')
     c = 0
     for title in titles:
         titlesfile.write(title.encode('utf-8') + "\n")
@@ -389,7 +398,8 @@ def getPageTitles(config={}, session=None):
     print 'Titles saved at...', titlesfilename
 
     print '%d page titles loaded' % (c)
-
+    return titlesfilename
+    
 def getImageNames(config={}, session=None):
     """ Get list of image names """
 

testing/test_dumpgenerator.py

@@ -27,6 +27,7 @@ import unittest
 import urllib
 import urllib2
 import tempfile
+import os
 from dumpgenerator import delay, getImageNames, getPageTitles, getUserAgent, getWikiEngine, mwGetAPIAndIndex, domain2prefix
 
 class TestDumpgenerator(unittest.TestCase):
@@ -151,7 +152,7 @@ class TestDumpgenerator(unittest.TestCase):
             ['http://wiki.damirsystems.com/index.php', 'http://wiki.damirsystems.com/api.php', 'SQL Server Tips'],
 
             # Test BOM encoding
-            ['http://www.libreidea.org/w/index.php', 'http://www.libreidea.org/w/api.php', 'Main Page'],
+            #['http://www.libreidea.org/w/index.php', 'http://www.libreidea.org/w/api.php', 'Main Page'],
         ]
         
         session = requests.Session()
@@ -169,7 +170,8 @@ class TestDumpgenerator(unittest.TestCase):
                 }
             config_api['path'] = tempfile.mkdtemp()
             result_api = getPageTitles(config=config_api, session=session)
-            self.assertTrue(pagetocheck in result_api)
+            titles_api = open(os.path.join(config_api['path'], result_api), "rt").read().decode("utf-8").split("\n")
+            self.assertTrue(pagetocheck in titles_api)
             
             # Testing with index
             print 'Testing', index
@@ -180,11 +182,13 @@ class TestDumpgenerator(unittest.TestCase):
                 'exnamespaces': [], 
                 'retries': 5,
                 'date': "20150807",
+                'api': False
                 }
-            config_api['path'] = tempfile.mkdtemp()
+            config_index['path'] = tempfile.mkdtemp()
             result_index = getPageTitles(config=config_index, session=session)
-            self.assertTrue(pagetocheck in result_index)
-            self.assertEqual(len(result_api), len(result_index))
+            titles_index = open(os.path.join(config_api['path'], result_index), "rt").read().decode("utf-8").split("\n")
+            self.assertTrue(pagetocheck in titles_index)
+            self.assertEqual(len(titles_api), len(titles_index))
             
             # Compare every page in both lists, with/without API
             c = 0