|
|
@ -77,9 +77,9 @@ for wiki in wikis:
|
|
|
|
# typically they don't provide any crawl-delay value in their robots.txt).
|
|
|
|
# typically they don't provide any crawl-delay value in their robots.txt).
|
|
|
|
if started and wikidir: #then resume
|
|
|
|
if started and wikidir: #then resume
|
|
|
|
print 'Resuming download, using directory', wikidir
|
|
|
|
print 'Resuming download, using directory', wikidir
|
|
|
|
subprocess.call('python dumpgenerator.py --api=%s --xml --images --resume --path=%s' % (wiki, wikidir), shell=True)
|
|
|
|
subprocess.call('./dumpgenerator.py --api=%s --xml --images --resume --path=%s' % (wiki, wikidir), shell=True)
|
|
|
|
else: #download from scratch
|
|
|
|
else: #download from scratch
|
|
|
|
subprocess.call('python dumpgenerator.py --api=%s --xml --images' % wiki, shell=True)
|
|
|
|
subprocess.call('./dumpgenerator.py --api=%s --xml --images' % wiki, shell=True)
|
|
|
|
started = True
|
|
|
|
started = True
|
|
|
|
#save wikidir now
|
|
|
|
#save wikidir now
|
|
|
|
for dirname, dirnames, filenames in os.walk('.'):
|
|
|
|
for dirname, dirnames, filenames in os.walk('.'):
|
|
|
|