Update fandom.com spider and remove duplicates

listsofwikis/mediawiki/fandom-spider.py

@@ -40,6 +40,7 @@ def main():
             req = requests.get('https://community.fandom.com%s' % lvl3)
         wikis.extend([wiki.replace('http://', 'https://') for wiki in re.findall(r'<a class=\"title\" href=\"([^>]+?)\">', req.text)])
 
+    wikis = list(set(wikis))
     wikis.sort()
     with open('fandom.com', 'w') as f:
         for wiki in wikis:

listsofwikis/mediawiki/fandom.com

@@ -4533,6 +4533,7 @@ https://advivor.fandom.com/fr/api.php
 https://advn.fandom.com/api.php
 https://advogados-de-primeira.fandom.com/pt-br/api.php
 https://advopedia.fandom.com/api.php
+https://adwot.fandom.com/ru/api.php
 https://adwr.fandom.com/api.php
 https://adwshow.fandom.com/api.php
 https://adyhb.fandom.com/api.php
@@ -6031,7 +6032,6 @@ https://airline.fandom.com/api.php
 https://airlinecommander.fandom.com/api.php
 https://airlineempires.fandom.com/api.php
 https://airlinemanager2.fandom.com/api.php
-https://airlinermodels.fandom.com/ru/api.php
 https://airlines-fanfiction.fandom.com/api.php
 https://airlines-manager.fandom.com/api.php
 https://airlines-of-the-world.fandom.com/api.php
@@ -14082,6 +14082,7 @@ https://arena-of-fate.fandom.com/api.php
 https://arena-of-the-planeswalkers.fandom.com/api.php
 https://arena-of-valor-storylore.fandom.com/api.php
 https://arena-shooter.fandom.com/api.php
+https://arena-tower-defense-fanmade.fandom.com/api.php
 https://arena-tower-defense.fandom.com/api.php
 https://arena-x-2201549430.fandom.com/api.php
 https://arena-x-roblox-fakemon.fandom.com/api.php
@@ -27685,9 +27686,6 @@ https://bmoma.fandom.com/api.php
 https://bmovie-monsters.fandom.com/api.php
 https://bmovies.fandom.com/api.php
 https://bmovieselsite.fandom.com/el/api.php
-https://bmovieselsite.fandom.com/el/api.php
-https://bmovieselsite.fandom.com/el/api.php
-https://bmovieselsite.fandom.com/el/api.php
 https://bmp.fandom.com/api.php
 https://bmrct.fandom.com/api.php
 https://bmrpg.fandom.com/api.php
@@ -27912,8 +27910,6 @@ https://bobdylan.fandom.com/api.php
 https://bobelconstructor.fandom.com/es/api.php
 https://bober.fandom.com/pl/api.php
 https://bobesponja.fandom.com/api.php
-https://bobesponja.fandom.com/api.php
-https://bobesponja.fandom.com/api.php
 https://bobesponjabrasil.fandom.com/pt-br/api.php
 https://bobesponjaespana.fandom.com/es/api.php
 https://bobesponjafanon.fandom.com/es/api.php
@@ -41085,7 +41081,6 @@ https://codes-pokemon-rouge-feu-vert-feuille.fandom.com/fr/api.php
 https://codes-randomizer.fandom.com/api.php
 https://codes.fandom.com/api.php
 https://codesah.fandom.com/api.php
-https://codesah.fandom.com/api.php
 https://codeshare.fandom.com/api.php
 https://codesnippets.fandom.com/api.php
 https://codespark-academy.fandom.com/api.php
@@ -47042,6 +47037,7 @@ https://culdestuck.fandom.com/api.php
 https://cules.fandom.com/es/api.php
 https://culinary-community.fandom.com/api.php
 https://cullenism.fandom.com/api.php
+https://culler.fandom.com/api.php
 https://culling-streets.fandom.com/api.php
 https://culling.fandom.com/api.php
 https://culobatmangordo.fandom.com/es/api.php
@@ -49154,6 +49150,7 @@ https://danny-phantom-origins.fandom.com/api.php
 https://danny-phantom.fandom.com/de/api.php
 https://danny-pudi.fandom.com/api.php
 https://danny-von-koopa.fandom.com/api.php
+https://danny123d.fandom.com/api.php
 https://dannycity.fandom.com/api.php
 https://dannydocile.fandom.com/api.php
 https://dannyellison.fandom.com/api.php
@@ -49189,7 +49186,6 @@ https://danproud.fandom.com/api.php
 https://dans-big-brother-simulator-seasons.fandom.com/api.php
 https://dans-half-life.fandom.com/api.php
 https://dans-insane-fanfiction.fandom.com/api.php
-https://dans-long-terms.fandom.com/api.php
 https://dans-voyage.fandom.com/api.php
 https://dansari.fandom.com/api.php
 https://danse-macabre.fandom.com/api.php
@@ -53837,7 +53833,6 @@ https://detain.fandom.com/api.php
 https://detarame.fandom.com/api.php
 https://detatokoprincess.fandom.com/api.php
 https://detectiuconan.fandom.com/ca/api.php
-https://detectiuconan.fandom.com/ca/api.php
 https://detectiv.fandom.com/ru/api.php
 https://detective-alexis.fandom.com/api.php
 https://detective-austin-acnh.fandom.com/api.php
@@ -62981,7 +62976,6 @@ https://earth602.fandom.com/api.php
 https://earth6160.fandom.com/api.php
 https://earth62.fandom.com/api.php
 https://earth63.fandom.com/api.php
-https://earth63.fandom.com/api.php
 https://earth64.fandom.com/api.php
 https://earth710.fandom.com/api.php
 https://earth710database.fandom.com/api.php
@@ -76674,6 +76668,7 @@ https://fire-channel.fandom.com/api.php
 https://fire-clan.fandom.com/api.php
 https://fire-craft-battle-royale.fandom.com/es/api.php
 https://fire-dancer-by-ann-maxwell.fandom.com/api.php
+https://fire-emblem-a-new-dawn.fandom.com/api.php
 https://fire-emblem-awakening.fandom.com/api.php
 https://fire-emblem-civ-v.fandom.com/api.php
 https://fire-emblem-encyclopedie.fandom.com/fr/api.php
@@ -76706,7 +76701,6 @@ https://fire-force.fandom.com/ru/api.php
 https://fire-gaming-youtube.fandom.com/api.php
 https://fire-hellish-towers.fandom.com/api.php
 https://fire-ice-and-thunder-bgz.fandom.com/fr/api.php
-https://fire-ice-dungeons-dragons-campaign.fandom.com/api.php
 https://fire-lily.fandom.com/api.php
 https://fire-mage.fandom.com/api.php
 https://fire-of-bananas.fandom.com/api.php
@@ -77171,6 +77165,7 @@ https://five-hours-at-doritos-wiki.fandom.com/api.php
 https://five-intelligent-creatures-of-time-and-space.fandom.com/zh/api.php
 https://five-kingdoms.fandom.com/api.php
 https://five-lovestruck-nights.fandom.com/api.php
+https://five-luminos-at-night-5.fandom.com/api.php
 https://five-man-band.fandom.com/api.php
 https://five-more-nights-at-freddys.fandom.com/api.php
 https://five-nations.fandom.com/api.php
@@ -77213,7 +77208,6 @@ https://five-nights-at-cartoon-network.fandom.com/api.php
 https://five-nights-at-chavos.fandom.com/pt-br/api.php
 https://five-nights-at-chickens.fandom.com/api.php
 https://five-nights-at-chompys.fandom.com/api.php
-https://five-nights-at-chuck-e-cheese-s.fandom.com/api.php
 https://five-nights-at-chuck-e-cheeses-reboot.fandom.com/api.php
 https://five-nights-at-chuck-e-cheeses-rebooted.fandom.com/api.php
 https://five-nights-at-chuck-e-chesses-rebooted.fandom.com/api.php
@@ -78082,7 +78076,6 @@ https://fliplinestudios.fandom.com/api.php
 https://fliplinestudios.fandom.com/es/api.php
 https://fliplinestudios.fandom.com/fr/api.php
 https://fliplinestudios.fandom.com/no/api.php
-https://fliplinestudios.fandom.com/no/api.php
 https://fliplinestudios.fandom.com/pl/api.php
 https://fliplinestudios.fandom.com/zh/api.php
 https://fliplinestudiosfanon-rybakcmusic.fandom.com/api.php
@@ -78621,6 +78614,7 @@ https://fnaf-lost-media-and-unused-content.fandom.com/api.php
 https://fnaf-mashup-multiplayer.fandom.com/api.php
 https://fnaf-merch.fandom.com/api.php
 https://fnaf-multiplayer-community.fandom.com/api.php
+https://fnaf-multiplayer.fandom.com/api.php
 https://fnaf-mysterys.fandom.com/api.php
 https://fnaf-nextgen.fandom.com/api.php
 https://fnaf-origins.fandom.com/api.php
@@ -78640,7 +78634,6 @@ https://fnaf-roblox.fandom.com/api.php
 https://fnaf-roleplay.fandom.com/api.php
 https://fnaf-rp-ii.fandom.com/api.php
 https://fnaf-rp.fandom.com/api.php
-https://fnaf-rpg-adventures.fandom.com/api.php
 https://fnaf-rulez.fandom.com/api.php
 https://fnaf-sb-version-b.fandom.com/api.php
 https://fnaf-scott-cawthon.fandom.com/es/api.php
@@ -85233,7 +85226,6 @@ https://geek-girl.fandom.com/api.php
 https://geek-inc.fandom.com/pt/api.php
 https://geek.fandom.com/fr/api.php
 https://geekandsundry.fandom.com/api.php
-https://geekandsundry.fandom.com/api.php
 https://geekbynight.fandom.com/api.php
 https://geekcharming.fandom.com/api.php
 https://geekcraft.fandom.com/api.php
@@ -90247,6 +90239,7 @@ https://greatest-freak-out-ever.fandom.com/api.php
 https://greatest-gd-levels-list.fandom.com/api.php
 https://greatest-legends.fandom.com/api.php
 https://greatest-movies.fandom.com/api.php
+https://greatest-movietv-shows-of-all-time.fandom.com/api.php
 https://greatest-shows.fandom.com/api.php
 https://greatest.fandom.com/api.php
 https://greatestanimebattles.fandom.com/api.php
@@ -90294,7 +90287,6 @@ https://greatveemon-project-test.fandom.com/api.php
 https://greatverse.fandom.com/fr/api.php
 https://greatvision-song-contest.fandom.com/api.php
 https://greatwar.fandom.com/api.php
-https://greatwarforcybertron.fandom.com/api.php
 https://greatwolflodge.fandom.com/api.php
 https://greatwolju.fandom.com/ko/api.php
 https://greatworldhistory.fandom.com/api.php
@@ -101862,8 +101854,6 @@ https://ifbyluck.fandom.com/api.php
 https://ifcarchives.fandom.com/api.php
 https://ifdemonslayerhadadifferentanimecontext.fandom.com/api.php
 https://ifes.fandom.com/api.php
-https://ifes.fandom.com/api.php
-https://ifes.fandom.com/api.php
 https://iffermoon.fandom.com/api.php
 https://iffypedia.fandom.com/api.php
 https://ifheartswereunbreakable.fandom.com/api.php
@@ -106606,6 +106596,7 @@ https://jackinthebox.fandom.com/api.php
 https://jackipedia.fandom.com/api.php
 https://jackiverse.fandom.com/api.php
 https://jackkirbysfourthworld.fandom.com/api.php
+https://jackmasseywelsh.fandom.com/api.php
 https://jackninja5.fandom.com/api.php
 https://jackninja5test.fandom.com/api.php
 https://jacknjellify-and-carykh.fandom.com/api.php
@@ -106643,7 +106634,6 @@ https://jacksfilms.fandom.com/api.php
 https://jacksfilmsarg.fandom.com/api.php
 https://jacksgoodinformation.fandom.com/api.php
 https://jacksjourney.fandom.com/api.php
-https://jackskelethor-dubs.fandom.com/api.php
 https://jackskelethordubs.fandom.com/api.php
 https://jacksmith.fandom.com/api.php
 https://jacksmp.fandom.com/api.php
@@ -109258,7 +109248,6 @@ https://jorvikipedia.fandom.com/api.php
 https://jorvikipedia.fandom.com/cs/api.php
 https://jorvikipedia.fandom.com/pl/api.php
 https://jorvikipedia.fandom.com/sv/api.php
-https://jorvikipedia.fandom.com/sv/api.php
 https://jorvikpedia.fandom.com/de/api.php
 https://jorvinpedia.fandom.com/api.php
 https://jorvivor.fandom.com/api.php
@@ -115787,6 +115776,7 @@ https://korean-comic-affiliation.fandom.com/api.php
 https://korean-daebak.fandom.com/api.php
 https://korean-diepio.fandom.com/api.php
 https://korean-localization.fandom.com/ko/api.php
+https://korean-vtuber.fandom.com/api.php
 https://korean-worldpedia.fandom.com/ko/api.php
 https://korean.fandom.com/api.php
 https://korean.fandom.com/ja/api.php
@@ -115815,7 +115805,6 @@ https://korinitekraft.fandom.com/api.php
 https://koriolis-tretii-gorizont.fandom.com/ru/api.php
 https://koriri.fandom.com/api.php
 https://korits.fandom.com/api.php
-https://korkby.fandom.com/api.php
 https://korkology.fandom.com/api.php
 https://korkut.fandom.com/tr/api.php
 https://korliki.fandom.com/pl/api.php
@@ -133677,9 +133666,6 @@ https://memoriesofgacha.fandom.com/api.php
 https://memoriesofmars.fandom.com/api.php
 https://memoriesquestion.fandom.com/api.php
 https://memory-alpha.fandom.com/api.php
-https://memory-alpha.fandom.com/api.php
-https://memory-alpha.fandom.com/api.php
-https://memory-alpha.fandom.com/api.php
 https://memory-alpha.fandom.com/bg/api.php
 https://memory-alpha.fandom.com/ca/api.php
 https://memory-alpha.fandom.com/cs/api.php
@@ -152267,6 +152253,7 @@ https://npc-tower-defense.fandom.com/api.php
 https://npc-wcue.fandom.com/api.php
 https://npc.fandom.com/api.php
 https://npcarebecomingsmart.fandom.com/api.php
+https://npcc.fandom.com/api.php
 https://npcgenerator.fandom.com/api.php
 https://npcgm.fandom.com/api.php
 https://npcrepo.fandom.com/api.php
@@ -152351,7 +152338,6 @@ https://nseurope.fandom.com/api.php
 https://nsf.fandom.com/zh/api.php
 https://nsfleague.fandom.com/api.php
 https://nsfn.fandom.com/tr/api.php
-https://nsfrencoverse.fandom.com/api.php
 https://nsfs.fandom.com/pt-br/api.php
 https://nsg-senate.fandom.com/api.php
 https://nsg.fandom.com/ru/api.php
@@ -183864,6 +183850,7 @@ https://rtec.fandom.com/api.php
 https://rtechcc.fandom.com/api.php
 https://rteenagers.fandom.com/api.php
 https://rter.fandom.com/ru/api.php
+https://rtes-universe.fandom.com/pt-br/api.php
 https://rtest.fandom.com/api.php
 https://rtf-ctf.fandom.com/api.php
 https://rtf.fandom.com/api.php
@@ -185389,7 +185376,6 @@ https://sailor-eds.fandom.com/api.php
 https://sailor-luna.fandom.com/api.php
 https://sailor-mom.fandom.com/es/api.php
 https://sailor-momo.fandom.com/api.php
-https://sailor-moon-br.fandom.com/pt-br/api.php
 https://sailor-moon-crystal.fandom.com/fr/api.php
 https://sailor-moon-crystal.fandom.com/ru/api.php
 https://sailor-moon-dic-and-cloverway.fandom.com/api.php
@@ -193498,7 +193484,6 @@ https://simpleplanesjundroo.fandom.com/api.php
 https://simpleplaneswiki.fandom.com/api.php
 https://simplequest.fandom.com/api.php
 https://simplerockets.fandom.com/api.php
-https://simplerockets.fandom.com/api.php
 https://simplerockets2.fandom.com/api.php
 https://simplesandbox2.fandom.com/api.php
 https://simplescience.fandom.com/api.php
@@ -243422,6 +243407,7 @@ https://valdez-family.fandom.com/api.php
 https://valdisstory.fandom.com/api.php
 https://vale-city-egame.fandom.com/api.php
 https://vale-city-game.fandom.com/api.php
+https://vale-zephyr.fandom.com/api.php
 https://valeanchronicles.fandom.com/api.php
 https://valecnici.fandom.com/cs/api.php
 https://valedenrpg.fandom.com/api.php
@@ -243510,7 +243496,6 @@ https://valianthearts-archive.fandom.com/api.php
 https://valianthearts.fandom.com/api.php
 https://valiantminecraft.fandom.com/api.php
 https://valiantonline.fandom.com/api.php
-https://valiarch.fandom.com/api.php
 https://valid-wiki-of-the-taken.fandom.com/api.php
 https://validaine.fandom.com/api.php
 https://validate.fandom.com/api.php
@@ -254411,6 +254396,7 @@ https://womanizing-mage.fandom.com/api.php
 https://womans-world.fandom.com/api.php
 https://womantoons.fandom.com/api.php
 https://wombles.fandom.com/api.php
+https://wombo.fandom.com/api.php
 https://womby.fandom.com/api.php
 https://women-and-children-first-on-the-titanic.fandom.com/api.php
 https://women-and-girl.fandom.com/api.php
@@ -254473,7 +254459,6 @@ https://wonder5masters.fandom.com/api.php
 https://wonderbox.fandom.com/api.php
 https://wonderboy.fandom.com/api.php
 https://wondercastle.fandom.com/pt-br/api.php
-https://wondercinematicuniverse.fandom.com/api.php
 https://wondercombat-multiverse.fandom.com/api.php
 https://wonderdraft.fandom.com/api.php
 https://wonderend.fandom.com/api.php