|
|
|
@ -484,13 +484,12 @@ def getXMLPageCore(headers={}, params={}, config={}, session=None):
|
|
|
|
|
maxseconds = 100 # max seconds to wait in a single sleeping
|
|
|
|
|
maxretries = config['retries'] # x retries and skip
|
|
|
|
|
increment = 20 # increment every retry
|
|
|
|
|
abortonconnectionerror = False
|
|
|
|
|
|
|
|
|
|
while not re.search(r'</mediawiki>', xml):
|
|
|
|
|
if c > 0 and c < maxretries:
|
|
|
|
|
wait = increment * c < maxseconds and increment * \
|
|
|
|
|
c or maxseconds # incremental until maxseconds
|
|
|
|
|
print ' In attempt {0}, XML for "{1}" is wrong. Waiting {2} seconds and reloading...'.format(c, params['pages'], wait)
|
|
|
|
|
print ' In attempt %d, XML for "%s" is wrong. Waiting %d seconds and reloading...'%(c, params['pages'], wait)
|
|
|
|
|
time.sleep(wait)
|
|
|
|
|
# reducing server load requesting smallest chunks (if curonly then
|
|
|
|
|
# limit = 1 from mother function)
|
|
|
|
@ -532,9 +531,7 @@ def getXMLPageCore(headers={}, params={}, config={}, session=None):
|
|
|
|
|
handleStatusCode(r)
|
|
|
|
|
xml = fixBOM(r)
|
|
|
|
|
except requests.exceptions.ConnectionError as e:
|
|
|
|
|
print ' Connection error {0}'.format(e[0])
|
|
|
|
|
if abortonconnectionerror:
|
|
|
|
|
raise ExportAbortedError(config['index'])
|
|
|
|
|
print ' Connection error: %s'.%(str(e[0]))
|
|
|
|
|
xml = ''
|
|
|
|
|
c += 1
|
|
|
|
|
|
|
|
|
@ -1364,9 +1361,9 @@ def getParameters(params=[]):
|
|
|
|
|
check = checkAPI(api=api, session=session)
|
|
|
|
|
break
|
|
|
|
|
except requests.exceptions.ConnectionError as e:
|
|
|
|
|
print 'Connection error {0}'.format(e)
|
|
|
|
|
print 'Connection error: %s'%(str(e))
|
|
|
|
|
retry += 1
|
|
|
|
|
print "Start retry attempt {0} in {1} seconds.".format(retry+1, retrydelay)
|
|
|
|
|
print "Start retry attempt %d in %d seconds."%(retry+1, retrydelay)
|
|
|
|
|
time.sleep(retrydelay)
|
|
|
|
|
if api and check:
|
|
|
|
|
index2 = check[1]
|
|
|
|
@ -1456,7 +1453,7 @@ def getParameters(params=[]):
|
|
|
|
|
'path': args.path and os.path.normpath(args.path) or '',
|
|
|
|
|
'cookies': args.cookies or '',
|
|
|
|
|
'delay': args.delay,
|
|
|
|
|
'retries': int(args.retries),
|
|
|
|
|
'retries': int(args.retries),
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
other = {
|
|
|
|
|