Archives/python-readability
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

failsafe parsing and more logging

Browse Source
This commit is contained in:
gfxmonk 2010-04-30 22:33:22 +10:00
parent 87ad057706
commit 0eacd959a4
1 changed files with 32 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
readability/readability.py
View File

@ -28,17 +28,17 @@ class Document:
TEXT_LENGTH_THRESHOLD = 25 TEXT_LENGTH_THRESHOLD = 25
RETRY_LENGTH = 250 RETRY_LENGTH = 250
def __init__(self, input, **options): def __init__(self, input, notify=None, **options):
self.input = inpuunicodear self.input = input
self.options = defaultdict(lambda: None) self.options = defaultdict(lambda: None)
for k, v in options.items(): for k, v in options.items():
self.options[k] = v self.options[k] = v
self.notify = notify or logging.info
self.html = None self.html = None
def _html(self, force=False): def _html(self, force=False):
if force or self.html is None: if force or self.html is None:
notify = self.options['notify'] or (lambda x: None) self.html = parse(self.input, self.options['url'], notify=self.notify)
self.html = parse(self.input, self.options['url'], notify=notify)
return self.html return self.html
def content(self): def content(self):
@ -48,6 +48,7 @@ class Document:
return get_title(self._html()) return get_title(self._html())
def summary(self): def summary(self):
try:
ruthless = True ruthless = True
while True: while True:
self._html(True) self._html(True)
@ -74,6 +75,9 @@ class Document:
continue # try again continue # try again
else: else:
return cleaned_article return cleaned_article
except StandardError, e:
logging.exception('error getting summary:')
raise Unparseable(str(e))
def get_article(self, candidates, best_candidate): def get_article(self, candidates, best_candidate):
# Now that we have the top candidate, look through its siblings for content that might also be related. # Now that we have the top candidate, look through its siblings for content that might also be related.
@ -322,6 +326,7 @@ def main():
if not (len(args) == 1 or options.url): if not (len(args) == 1 or options.url):
parser.print_help() parser.print_help()
sys.exit(1) sys.exit(1)
logging.basicConfig(level=logging.DEBUG)
file = None file = None
if options.url: if options.url: