Archives
/
wikiteam
mirror of https://github.com/WikiTeam/wikiteam
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Update neoseeker.com spider and list

Browse Source
pull/431/head
Liu 2 years ago
parent 6346fd6553
commit 49e41ee75d
3 changed files with 197 additions and 195 deletions
Show Stats Download Patch File Download Diff File

21
listsofwikis/mediawiki/neoseeker-spider.py
Unescape Escape View File

@ -1,17 +1,17 @@
#!/usr/bin/env python2
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright (C) 2014-2017 WikiTeam developers
# Copyright (C) 2014-2021 WikiTeam developers
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
@ -22,14 +22,15 @@ def main():
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:24.0) Gecko/20100101 Firefox/24.0',
}
url = 'http://neowiki.neoseeker.com/wiki/Special:WikiList'
url = 'https://neowiki.neoseeker.com/wiki/Special:WikiList'
r = requests.get(url, headers=headers)
raw = r.text
m = re.findall(ur'<li><a href=\'([^>]+?)/wiki/\'>', raw)
m = re.findall(r'<li><a href=\'([^>]+?)/wiki/\'>', raw)
m = [w.replace('http://', 'https://') + '/w/api.php' for w in m]
m.sort()
for i in m:
print i + '/w/api.php'
with open('neoseeker.com', 'w') as f:
f.write('\n'.join(m))
if __name__ == '__main__':
main()

366
listsofwikis/mediawiki/neoseeker.com vendored
Unescape Escape View File

@ -1,183 +1,183 @@
http://24.neoseeker.com/w/api.php
http://aceattorney.neoseeker.com/w/api.php
http://advancewars.neoseeker.com/w/api.php
http://adventuretime.neoseeker.com/w/api.php
http://animalcrossing.neoseeker.com/w/api.php
http://attackontitan.neoseeker.com/w/api.php
http://avatar.neoseeker.com/w/api.php
http://banished.neoseeker.com/w/api.php
http://banjokazooie.neoseeker.com/w/api.php
http://batman.neoseeker.com/w/api.php
http://battlefield.neoseeker.com/w/api.php
http://bioshock.neoseeker.com/w/api.php
http://bleach.neoseeker.com/w/api.php
http://boktai.neoseeker.com/w/api.php
http://bond.neoseeker.com/w/api.php
http://borderlands.neoseeker.com/w/api.php
http://boundbyflame.neoseeker.com/w/api.php
http://bravely.neoseeker.com/w/api.php
http://breathoffire.neoseeker.com/w/api.php
http://brink.neoseeker.com/w/api.php
http://callofduty.neoseeker.com/w/api.php
http://castlecrashers.neoseeker.com/w/api.php
http://castlevania.neoseeker.com/w/api.php
http://childoflight.neoseeker.com/w/api.php
http://chrono.neoseeker.com/w/api.php
http://cnc.neoseeker.com/w/api.php
http://cowboybebop.neoseeker.com/w/api.php
http://crash.neoseeker.com/w/api.php
http://crossedge.neoseeker.com/w/api.php
http://cubeworld.neoseeker.com/w/api.php
http://danganronpa.neoseeker.com/w/api.php
http://darksouls.neoseeker.com/w/api.php
http://deadoralive.neoseeker.com/w/api.php
http://deathnote.neoseeker.com/w/api.php
http://demonssouls.neoseeker.com/w/api.php
http://destiny.neoseeker.com/w/api.php
http://devilmaycry.neoseeker.com/w/api.php
http://digimon.neoseeker.com/w/api.php
http://disgaea.neoseeker.com/w/api.php
http://doctorwho.neoseeker.com/w/api.php
http://donkeykong.neoseeker.com/w/api.php
http://doom.neoseeker.com/w/api.php
http://dothack.neoseeker.com/w/api.php
http://doujin.neoseeker.com/w/api.php
http://dragonage.neoseeker.com/w/api.php
http://dragonball.neoseeker.com/w/api.php
http://dragonquest.neoseeker.com/w/api.php
http://dragonsdogma.neoseeker.com/w/api.php
http://dynastywarriors.neoseeker.com/w/api.php
http://elderscrolls.neoseeker.com/w/api.php
http://endlessocean.neoseeker.com/w/api.php
http://evangelion.neoseeker.com/w/api.php
http://fable.neoseeker.com/w/api.php
http://fairytail.neoseeker.com/w/api.php
http://fallout.neoseeker.com/w/api.php
http://familyguy.neoseeker.com/w/api.php
http://fatalfury.neoseeker.com/w/api.php
http://fifa.neoseeker.com/w/api.php
http://finalfantasy.neoseeker.com/w/api.php
http://fireemblem.neoseeker.com/w/api.php
http://footballmanager.neoseeker.com/w/api.php
http://fullmetalalchemist.neoseeker.com/w/api.php
http://futurama.neoseeker.com/w/api.php
http://fzero.neoseeker.com/w/api.php
http://gearsofwar.neoseeker.com/w/api.php
http://glee.neoseeker.com/w/api.php
http://godofwar.neoseeker.com/w/api.php
http://goldensun.neoseeker.com/w/api.php
http://granturismo.neoseeker.com/w/api.php
http://growlanser.neoseeker.com/w/api.php
http://gta.neoseeker.com/w/api.php
http://gta5.neoseeker.com/w/api.php
http://guitarhero.neoseeker.com/w/api.php
http://gundam.neoseeker.com/w/api.php
http://halflife.neoseeker.com/w/api.php
http://halo.neoseeker.com/w/api.php
http://harrypotter.neoseeker.com/w/api.php
http://haruhi.neoseeker.com/w/api.php
http://harvestmoon.neoseeker.com/w/api.php
http://hearthstone.neoseeker.com/w/api.php
http://heavyrain.neoseeker.com/w/api.php
http://heroesofruin.neoseeker.com/w/api.php
http://hitman.neoseeker.com/w/api.php
http://house.neoseeker.com/w/api.php
http://hungergames.neoseeker.com/w/api.php
http://infamous.neoseeker.com/w/api.php
http://inheritance.neoseeker.com/w/api.php
http://inuyasha.neoseeker.com/w/api.php
http://jakdaxter.neoseeker.com/w/api.php
http://kairosoft.neoseeker.com/w/api.php
http://kidicarus.neoseeker.com/w/api.php
http://kingdomhearts.neoseeker.com/w/api.php
http://kirby.neoseeker.com/w/api.php
http://koa.neoseeker.com/w/api.php
http://layton.neoseeker.com/w/api.php
http://leagueoflegends.neoseeker.com/w/api.php
http://legendofdragoon.neoseeker.com/w/api.php
http://littlebigplanet.neoseeker.com/w/api.php
http://lotr.neoseeker.com/w/api.php
http://magicalstarsign.neoseeker.com/w/api.php
http://maplestory.neoseeker.com/w/api.php
http://mario.neoseeker.com/w/api.php
http://masseffect.neoseeker.com/w/api.php
http://megaman.neoseeker.com/w/api.php
http://megamitensei.neoseeker.com/w/api.php
http://metalgear.neoseeker.com/w/api.php
http://metroid.neoseeker.com/w/api.php
http://minecraft.neoseeker.com/w/api.php
http://monsterhunter.neoseeker.com/w/api.php
http://mortalkombat.neoseeker.com/w/api.php
http://mother.neoseeker.com/w/api.php
http://mtg.neoseeker.com/w/api.php
http://mylittlepony.neoseeker.com/w/api.php
http://naruto.neoseeker.com/w/api.php
http://ncis.neoseeker.com/w/api.php
http://needforspeed.neoseeker.com/w/api.php
http://neopets.neoseeker.com/w/api.php
http://ninjagaiden.neoseeker.com/w/api.php
http://ninokuni.neoseeker.com/w/api.php
http://okami.neoseeker.com/w/api.php
http://onepiece.neoseeker.com/w/api.php
http://persona.neoseeker.com/w/api.php
http://pes.neoseeker.com/w/api.php
http://pikmin.neoseeker.com/w/api.php
http://pokemon.neoseeker.com/w/api.php
http://princeofpersia.neoseeker.com/w/api.php
http://ratchetclank.neoseeker.com/w/api.php
http://reborn.neoseeker.com/w/api.php
http://residentevil.neoseeker.com/w/api.php
http://resonance.neoseeker.com/w/api.php
http://rockband.neoseeker.com/w/api.php
http://rpgmaker.neoseeker.com/w/api.php
http://runefactory.neoseeker.com/w/api.php
http://runescape.neoseeker.com/w/api.php
http://sandbox.neoseeker.com/w/api.php
http://scottpilgrim.neoseeker.com/w/api.php
http://shadowofthecolossus.neoseeker.com/w/api.php
http://shadowrunreturns.neoseeker.com/w/api.php
http://shenmue.neoseeker.com/w/api.php
http://simpsons.neoseeker.com/w/api.php
http://skate.neoseeker.com/w/api.php
http://skylanders.neoseeker.com/w/api.php
http://skyrim.neoseeker.com/w/api.php
http://slycooper.neoseeker.com/w/api.php
http://smackdown.neoseeker.com/w/api.php
http://smashbros.neoseeker.com/w/api.php
http://sonic.neoseeker.com/w/api.php
http://soulcalibur.neoseeker.com/w/api.php
http://souleater.neoseeker.com/w/api.php
http://spiderman.neoseeker.com/w/api.php
http://spongebob.neoseeker.com/w/api.php
http://spyro.neoseeker.com/w/api.php
http://starcraft.neoseeker.com/w/api.php
http://starfox.neoseeker.com/w/api.php
http://stargate.neoseeker.com/w/api.php
http://starocean.neoseeker.com/w/api.php
http://starwars.neoseeker.com/w/api.php
http://streetfighter.neoseeker.com/w/api.php
http://tales.neoseeker.com/w/api.php
http://tekken.neoseeker.com/w/api.php
http://terraria.neoseeker.com/w/api.php
http://thedarkness.neoseeker.com/w/api.php
http://thesims.neoseeker.com/w/api.php
http://thewarriors.neoseeker.com/w/api.php
http://theworldendswithyou.neoseeker.com/w/api.php
http://thief.neoseeker.com/w/api.php
http://timesplitters.neoseeker.com/w/api.php
http://tonyhawk.neoseeker.com/w/api.php
http://twilight.neoseeker.com/w/api.php
http://twistedmetal.neoseeker.com/w/api.php
http://uncharted.neoseeker.com/w/api.php
http://valkyriachronicles.neoseeker.com/w/api.php
http://vivapinata.neoseeker.com/w/api.php
http://wakfu.neoseeker.com/w/api.php
http://warcraft.neoseeker.com/w/api.php
http://warhammer.neoseeker.com/w/api.php
http://watchdogs.neoseeker.com/w/api.php
http://whiteknightchronicles.neoseeker.com/w/api.php
http://wikiguides.neoseeker.com/w/api.php
http://wow.neoseeker.com/w/api.php
http://xenoblade.neoseeker.com/w/api.php
http://yugioh.neoseeker.com/w/api.php
http://zelda.neoseeker.com/w/api.php
https://24.neoseeker.com/w/api.php
https://aceattorney.neoseeker.com/w/api.php
https://advancewars.neoseeker.com/w/api.php
https://adventuretime.neoseeker.com/w/api.php
https://animalcrossing.neoseeker.com/w/api.php
https://attackontitan.neoseeker.com/w/api.php
https://avatar.neoseeker.com/w/api.php
https://banished.neoseeker.com/w/api.php
https://banjokazooie.neoseeker.com/w/api.php
https://batman.neoseeker.com/w/api.php
https://battlefield.neoseeker.com/w/api.php
https://bioshock.neoseeker.com/w/api.php
https://bleach.neoseeker.com/w/api.php
https://boktai.neoseeker.com/w/api.php
https://bond.neoseeker.com/w/api.php
https://borderlands.neoseeker.com/w/api.php
https://boundbyflame.neoseeker.com/w/api.php
https://bravely.neoseeker.com/w/api.php
https://breathoffire.neoseeker.com/w/api.php
https://brink.neoseeker.com/w/api.php
https://callofduty.neoseeker.com/w/api.php
https://castlecrashers.neoseeker.com/w/api.php
https://castlevania.neoseeker.com/w/api.php
https://childoflight.neoseeker.com/w/api.php
https://chrono.neoseeker.com/w/api.php
https://cnc.neoseeker.com/w/api.php
https://cowboybebop.neoseeker.com/w/api.php
https://crash.neoseeker.com/w/api.php
https://crossedge.neoseeker.com/w/api.php
https://cubeworld.neoseeker.com/w/api.php
https://danganronpa.neoseeker.com/w/api.php
https://darksouls.neoseeker.com/w/api.php
https://deadoralive.neoseeker.com/w/api.php
https://deathnote.neoseeker.com/w/api.php
https://demonssouls.neoseeker.com/w/api.php
https://destiny.neoseeker.com/w/api.php
https://devilmaycry.neoseeker.com/w/api.php
https://digimon.neoseeker.com/w/api.php
https://disgaea.neoseeker.com/w/api.php
https://doctorwho.neoseeker.com/w/api.php
https://donkeykong.neoseeker.com/w/api.php
https://doom.neoseeker.com/w/api.php
https://dothack.neoseeker.com/w/api.php
https://doujin.neoseeker.com/w/api.php
https://dragonage.neoseeker.com/w/api.php
https://dragonball.neoseeker.com/w/api.php
https://dragonquest.neoseeker.com/w/api.php
https://dragonsdogma.neoseeker.com/w/api.php
https://dynastywarriors.neoseeker.com/w/api.php
https://elderscrolls.neoseeker.com/w/api.php
https://endlessocean.neoseeker.com/w/api.php
https://evangelion.neoseeker.com/w/api.php
https://fable.neoseeker.com/w/api.php
https://fairytail.neoseeker.com/w/api.php
https://fallout.neoseeker.com/w/api.php
https://familyguy.neoseeker.com/w/api.php
https://fatalfury.neoseeker.com/w/api.php
https://fifa.neoseeker.com/w/api.php
https://finalfantasy.neoseeker.com/w/api.php
https://fireemblem.neoseeker.com/w/api.php
https://footballmanager.neoseeker.com/w/api.php
https://fullmetalalchemist.neoseeker.com/w/api.php
https://futurama.neoseeker.com/w/api.php
https://fzero.neoseeker.com/w/api.php
https://gearsofwar.neoseeker.com/w/api.php
https://glee.neoseeker.com/w/api.php
https://godofwar.neoseeker.com/w/api.php
https://goldensun.neoseeker.com/w/api.php
https://granturismo.neoseeker.com/w/api.php
https://growlanser.neoseeker.com/w/api.php
https://gta.neoseeker.com/w/api.php
https://gta5.neoseeker.com/w/api.php
https://guitarhero.neoseeker.com/w/api.php
https://gundam.neoseeker.com/w/api.php
https://halflife.neoseeker.com/w/api.php
https://halo.neoseeker.com/w/api.php
https://harrypotter.neoseeker.com/w/api.php
https://haruhi.neoseeker.com/w/api.php
https://harvestmoon.neoseeker.com/w/api.php
https://hearthstone.neoseeker.com/w/api.php
https://heavyrain.neoseeker.com/w/api.php
https://heroesofruin.neoseeker.com/w/api.php
https://hitman.neoseeker.com/w/api.php
https://house.neoseeker.com/w/api.php
https://hungergames.neoseeker.com/w/api.php
https://infamous.neoseeker.com/w/api.php
https://inheritance.neoseeker.com/w/api.php
https://inuyasha.neoseeker.com/w/api.php
https://jakdaxter.neoseeker.com/w/api.php
https://kairosoft.neoseeker.com/w/api.php
https://kidicarus.neoseeker.com/w/api.php
https://kingdomhearts.neoseeker.com/w/api.php
https://kirby.neoseeker.com/w/api.php
https://koa.neoseeker.com/w/api.php
https://layton.neoseeker.com/w/api.php
https://leagueoflegends.neoseeker.com/w/api.php
https://legendofdragoon.neoseeker.com/w/api.php
https://littlebigplanet.neoseeker.com/w/api.php
https://lotr.neoseeker.com/w/api.php
https://magicalstarsign.neoseeker.com/w/api.php
https://maplestory.neoseeker.com/w/api.php
https://mario.neoseeker.com/w/api.php
https://masseffect.neoseeker.com/w/api.php
https://megaman.neoseeker.com/w/api.php
https://megamitensei.neoseeker.com/w/api.php
https://metalgear.neoseeker.com/w/api.php
https://metroid.neoseeker.com/w/api.php
https://minecraft.neoseeker.com/w/api.php
https://monsterhunter.neoseeker.com/w/api.php
https://mortalkombat.neoseeker.com/w/api.php
https://mother.neoseeker.com/w/api.php
https://mtg.neoseeker.com/w/api.php
https://mylittlepony.neoseeker.com/w/api.php
https://naruto.neoseeker.com/w/api.php
https://ncis.neoseeker.com/w/api.php
https://needforspeed.neoseeker.com/w/api.php
https://neopets.neoseeker.com/w/api.php
https://ninjagaiden.neoseeker.com/w/api.php
https://ninokuni.neoseeker.com/w/api.php
https://okami.neoseeker.com/w/api.php
https://onepiece.neoseeker.com/w/api.php
https://persona.neoseeker.com/w/api.php
https://pes.neoseeker.com/w/api.php
https://pikmin.neoseeker.com/w/api.php
https://pokemon.neoseeker.com/w/api.php
https://princeofpersia.neoseeker.com/w/api.php
https://ratchetclank.neoseeker.com/w/api.php
https://reborn.neoseeker.com/w/api.php
https://residentevil.neoseeker.com/w/api.php
https://resonance.neoseeker.com/w/api.php
https://rockband.neoseeker.com/w/api.php
https://rpgmaker.neoseeker.com/w/api.php
https://runefactory.neoseeker.com/w/api.php
https://runescape.neoseeker.com/w/api.php
https://sandbox.neoseeker.com/w/api.php
https://scottpilgrim.neoseeker.com/w/api.php
https://shadowofthecolossus.neoseeker.com/w/api.php
https://shadowrunreturns.neoseeker.com/w/api.php
https://shenmue.neoseeker.com/w/api.php
https://simpsons.neoseeker.com/w/api.php
https://skate.neoseeker.com/w/api.php
https://skylanders.neoseeker.com/w/api.php
https://skyrim.neoseeker.com/w/api.php
https://slycooper.neoseeker.com/w/api.php
https://smackdown.neoseeker.com/w/api.php
https://smashbros.neoseeker.com/w/api.php
https://sonic.neoseeker.com/w/api.php
https://soulcalibur.neoseeker.com/w/api.php
https://souleater.neoseeker.com/w/api.php
https://spiderman.neoseeker.com/w/api.php
https://spongebob.neoseeker.com/w/api.php
https://spyro.neoseeker.com/w/api.php
https://starcraft.neoseeker.com/w/api.php
https://starfox.neoseeker.com/w/api.php
https://stargate.neoseeker.com/w/api.php
https://starocean.neoseeker.com/w/api.php
https://starwars.neoseeker.com/w/api.php
https://streetfighter.neoseeker.com/w/api.php
https://tales.neoseeker.com/w/api.php
https://tekken.neoseeker.com/w/api.php
https://terraria.neoseeker.com/w/api.php
https://thedarkness.neoseeker.com/w/api.php
https://thesims.neoseeker.com/w/api.php
https://thewarriors.neoseeker.com/w/api.php
https://theworldendswithyou.neoseeker.com/w/api.php
https://thief.neoseeker.com/w/api.php
https://timesplitters.neoseeker.com/w/api.php
https://tonyhawk.neoseeker.com/w/api.php
https://twilight.neoseeker.com/w/api.php
https://twistedmetal.neoseeker.com/w/api.php
https://uncharted.neoseeker.com/w/api.php
https://valkyriachronicles.neoseeker.com/w/api.php
https://vivapinata.neoseeker.com/w/api.php
https://wakfu.neoseeker.com/w/api.php
https://warcraft.neoseeker.com/w/api.php
https://warhammer.neoseeker.com/w/api.php
https://watchdogs.neoseeker.com/w/api.php
https://whiteknightchronicles.neoseeker.com/w/api.php
https://wikiguides.neoseeker.com/w/api.php
https://wow.neoseeker.com/w/api.php
https://xenoblade.neoseeker.com/w/api.php
https://yugioh.neoseeker.com/w/api.php
https://zelda.neoseeker.com/w/api.php

5
listsofwikis/mediawiki/shoutwiki-spider.py
Unescape Escape View File

@ -23,6 +23,7 @@ def main():
ids, wikis = [], []
gcont = 'tmp'
url = 'http://www.shoutwiki.com/w/api.php'
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:24.0) Gecko/20100101 Firefox/24.0'}
# grab wiki pages
params = {
@ -37,7 +38,7 @@ def main():
while gcont:
if gcont != 'tmp':
params['gcmcontinue'] = gcont
json = requests.get(url, params=params).json()
json = requests.get(url, params=params, headers=headers).json()
gcont = json['continue']['gcmcontinue'] if 'continue' in json else ''
query = json['query']['pages']
for wiki in query:
@ -54,7 +55,7 @@ def main():
}
for n in tqdm(range(0, len(ids), 50)):
params['pageids'] = '|'.join(ids[n:n+50])
json = requests.get(url, params=params).json()
json = requests.get(url, params=params, headers=headers).json()
for wiki in json['query']['pages']:
for val in wiki['revisions'][0]['slots']['main']['content'].split('\n|'):

Write Preview
Loading…
Cancel
Save