Remove duplicates from wiki-site.com

2 years ago · 4c970e358d
parent 74a8e9609f
commit 4c970e358d
2 changed files with 1 additions and 278 deletions
--- a/listsofwikis/mediawiki/wiki-site-spider.py
+++ b/listsofwikis/mediawiki/wiki-site-spider.py
@ -33,6 +33,7 @@ def main():
        req = requests.get(url, headers=headers)
        wikis.extend(re.findall(r'<td><a href="([^>]+?)"', req.text))

+    wikis = list(set(wikis))
    wikis.sort()
    with open('wiki-site.com', 'w') as f:
        for wiki in wikis:
--- a/listsofwikis/mediawiki/wiki-site.com
+++ b/listsofwikis/mediawiki/wiki-site.com