This is a great amount of info
-And more content Home -
.""" - dom = document_fromstring( - "
text
"
- )
+ assert doc._original_document.title == 'Min Document Title'
- self.assertEqual(
- tounicode(
- leaf_div_elements_into_paragraphs(dom)),
- to_unicode(
- "child
"
- "aftertexttext
"
- )
- )
- def test_bad_links(self):
- """Some links should just not belong."""
- bad_links = [
- ' ',
- 'child
" - "aftertext
',
- ''
- ]
+def test_doc_no_scripts_styles():
+ """Step #1 remove all scripts from the document"""
+ doc = Article(load_snippet('document_scripts.html'))
+ readable = doc.readable_dom
+
+ assert readable.findall(".//script") == []
+ assert readable.findall(".//style") == []
+ assert readable.findall(".//link") == []
+
+
+def test_find_body_exists():
+ """If the document has a body, we store that as the readable html
+
+ No sense processing anything other than the body content.
+
+ """
+ doc = Article(load_snippet('document_min.html'))
+
+ assert doc.readable_dom.tag == 'div'
+ assert doc.readable_dom.get('id') == 'readabilityBody'
+
+
+def test_body_doesnt_exist():
+ """If we can't find a body, then we create one.
+
+ We build our doc around the rest of the html we parsed.
+
+ """
+ doc = Article(load_snippet('document_no_body.html'))
+
+ assert doc.readable_dom.tag == 'div'
+ assert doc.readable_dom.get('id') == 'readabilityBody'
+
+
+def test_bare_content():
+ """If the document is just pure content, no html tags we should be ok
+
+ We build our doc around the rest of the html we parsed.
+
+ """
+ doc = Article(load_snippet('document_only_content.html'))
+
+ assert doc.readable_dom.tag == 'div'
+ assert doc.readable_dom.get('id') == 'readabilityBody'
+
+
+def test_no_content():
+ """Without content we supply an empty unparsed doc."""
+ doc = Article('')
+
+ assert doc.readable_dom.tag == 'div'
+ assert doc.readable_dom.get('id') == 'readabilityBody'
+ assert doc.readable_dom.get('class') == 'parsing-error'
+
+
+# Test out our cleaning processing we do.
+
+
+def test_unlikely_hits():
+ """Verify we wipe out things from our unlikely list."""
+ doc = Article(load_snippet('test_readable_unlikely.html'))
+ readable = doc.readable_dom
+ must_not_appear = [
+ 'comment', 'community', 'disqus', 'extra', 'foot',
+ 'header', 'menu', 'remark', 'rss', 'shoutbox', 'sidebar',
+ 'sponsor', 'ad-break', 'agegate', 'pagination' '', 'pager',
+ 'popup', 'tweet', 'twitter', 'imgBlogpostPermalink']
+
+ want_to_appear = ['and', 'article', 'body', 'column', 'main', 'shadow']
+
+ for i in must_not_appear:
+ # we cannot find any class or id with this value
+ by_class = readable.find_class(i)
+
+ for test in by_class:
+ # if it's here it cannot have the must not class without the
+ # want to appear class
+ found = False
+ for cls in test.get('class').split():
+ if cls in want_to_appear:
+ found = True
+ assert found
+
+ by_ids = readable.get_element_by_id(i, False)
+ if by_ids is not False:
+ found = False
+ for ids in test.get('id').split():
+ if ids in want_to_appear:
+ found = True
+ assert found
+
+
+def test_misused_divs_transform():
+ """Verify we replace leaf node divs with p's
+
+ They should have the same content, just be a p vs a div
+
+ """
+ test_html = "simple
"
+ test_doc = document_fromstring(test_html)
+ assert tounicode(leaf_div_elements_into_paragraphs(test_doc)) == to_unicode(
+ "simple
" + ) + + test_html2 = ('simplelink'
+ '
')
+ test_doc2 = document_fromstring(test_html2)
+ assert tounicode(leaf_div_elements_into_paragraphs(test_doc2)) == to_unicode(
+ 'simplelink
' + ) + + +def test_dont_transform_div_with_div(): + """Verify that only child element is replaced by ',
+ ''
+ ]
+ for l in bad_links:
+ link = fragment_fromstring(l)
+ assert is_bad_link(link)
-class TestCandidateNodes(unittest.TestCase):
- """Candidate nodes are scoring containers we use."""
- def test_candidate_scores(self):
- """We should be getting back objects with some scores."""
- fives = ['']
- threes = ['', 'list item '
- node = fragment_fromstring(test_div)
- snode = ScoredNode(node)
- self.assertEqual(snode.content_score, -3)
-
-
-class TestScoreCandidates(unittest.TestCase):
- """The grand daddy of tests to make sure our scoring works
-
- Now scoring details will change over time, so the most important thing is
- to make sure candidates come out in the right order, not necessarily how
- they scored. Make sure to keep this in mind while getting tests going.
+
+def test_id_hits():
+ """If the id is in the list then it gets a weight"""
+ test_div = 'list item '
+ node = fragment_fromstring(test_div)
+ snode = ScoredNode(node)
+ assert snode.content_score == -3
+
+
+# The grand daddy of tests to make sure our scoring works
+# Now scoring details will change over time, so the most important thing is
+# to make sure candidates come out in the right order, not necessarily how
+# they scored. Make sure to keep this in mind while getting tests going.
+
+
+def test_simple_candidate_set():
+ """Tests a simple case of two candidate nodes"""
+ html = """
+
+
+
.""" + dom = document_fromstring( + "
text
"
+ )
+
+ assert tounicode(leaf_div_elements_into_paragraphs(dom)) == to_unicode(
+ "child
"
+ "aftertexttext
"
+ )
+
- for l in bad_links:
- link = fragment_fromstring(l)
- self.assertTrue(is_bad_link(link))
+def test_bad_links():
+ """Some links should just not belong."""
+ bad_links = [
+ ' ',
+ 'child
" + "aftertext',
+ ''
+ ]
+ for l in bad_links:
+ link = fragment_fromstring(l)
+ assert is_bad_link(link)
-class TestCandidateNodes(unittest.TestCase):
- """Candidate nodes are scoring containers we use."""
- def test_candidate_scores(self):
- """We should be getting back objects with some scores."""
- fives = ['']
- threes = ['', '') - self.assertEqual(get_class_weight(node), 25) +def test_article_enables_candidate_access(): + """Candidates are accessible after document processing.""" + doc = Article(load_article('ars.001.html')) - def test_positive_ids(self): - """Some ids get us bonus points.""" - node = fragment_fromstring('
') - self.assertEqual(get_class_weight(node), 25) + assert hasattr(doc, 'candidates') - def test_negative_class(self): - """Some classes get us negative points.""" - node = fragment_fromstring('
') - self.assertEqual(get_class_weight(node), -25) - def test_negative_ids(self): - """Some ids get us negative points.""" - node = fragment_fromstring('
') - self.assertEqual(get_class_weight(node), -25) +# Certain ids and classes get us bonus points. -class TestScoringNodes(unittest.TestCase): - """We take out list of potential nodes and score them up.""" +def test_positive_class(): + """Some classes get us bonus points.""" + node = fragment_fromstring('
') + assert get_class_weight(node) == 25 - def test_we_get_candidates(self): - """Processing candidates should get us a list of nodes to try out.""" - doc = document_fromstring(load_article("ars.001.html")) - test_nodes = tuple(doc.iter("p", "td", "pre")) - candidates = score_candidates(test_nodes) - # this might change as we tweak our algorithm, but if it does, - # it signifies we need to look at what we changed. - self.assertEqual(len(candidates.keys()), 37) +def test_positive_ids(): + """Some ids get us bonus points.""" + node = fragment_fromstring('
') + assert get_class_weight(node) == 25 - # one of these should have a decent score - scores = sorted(c.content_score for c in candidates.values()) - self.assertTrue(scores[-1] > 100) - - def test_bonus_score_per_100_chars_in_p(self): - """Nodes get 1 point per 100 characters up to max. 3 points.""" - def build_candidates(length): - html = "
%s
" % ("c" * length) - node = fragment_fromstring(html) - - return [node] - - test_nodes = build_candidates(50) - candidates = score_candidates(test_nodes) - pscore_50 = max(c.content_score for c in candidates.values()) - - test_nodes = build_candidates(100) - candidates = score_candidates(test_nodes) - pscore_100 = max(c.content_score for c in candidates.values()) - - test_nodes = build_candidates(300) - candidates = score_candidates(test_nodes) - pscore_300 = max(c.content_score for c in candidates.values()) - - test_nodes = build_candidates(400) - candidates = score_candidates(test_nodes) - pscore_400 = max(c.content_score for c in candidates.values()) - - self.assertAlmostEqual(pscore_50 + 0.5, pscore_100) - self.assertAlmostEqual(pscore_100 + 2.0, pscore_300) - self.assertAlmostEqual(pscore_300, pscore_400) - - -class TestLinkDensityScoring(unittest.TestCase): - """Link density will adjust out candidate scoresself.""" - - def test_link_density(self): - """Test that we get a link density""" - doc = document_fromstring(load_article('ars.001.html')) - for node in doc.iter('p', 'td', 'pre'): - density = get_link_density(node) - - # the density must be between 0, 1 - self.assertTrue(density >= 0.0 and density <= 1.0) - - -class TestSiblings(unittest.TestCase): - """Siblings will be included if their content is related.""" - - @unittest.skip("Not implemented yet.") - def test_bad_siblings_not_counted(self): - raise NotImplementedError() - - @unittest.skip("Not implemented yet.") - def test_good_siblings_counted(self): - raise NotImplementedError() - - -class TestMainText(unittest.TestCase): - def test_empty(self): - article = Article("") - annotated_text = article.main_text - - self.assertEqual(annotated_text, []) - - def test_no_annotations(self): - article = Article("This is text with no annotations
This is text\r\twith no annotations
') + assert get_class_weight(node) == -25 + + +def test_negative_ids(): + """Some ids get us negative points.""" + node = fragment_fromstring('
') + assert get_class_weight(node) == -25 + + +# We take out list of potential nodes and score them up. + + +def test_we_get_candidates(): + """Processing candidates should get us a list of nodes to try out.""" + doc = document_fromstring(load_article("ars.001.html")) + test_nodes = tuple(doc.iter("p", "td", "pre")) + candidates = score_candidates(test_nodes) + + # this might change as we tweak our algorithm, but if it does, + # it signifies we need to look at what we changed. + assert len(candidates.keys()) == 37 + + # one of these should have a decent score + scores = sorted(c.content_score for c in candidates.values()) + assert scores[-1] > 100 + + +def test_bonus_score_per_100_chars_in_p(): + """Nodes get 1 point per 100 characters up to max. 3 points.""" + def build_candidates(length): + html = "
%s
" % ("c" * length) + node = fragment_fromstring(html) + + return [node] + + test_nodes = build_candidates(50) + candidates = score_candidates(test_nodes) + pscore_50 = max(c.content_score for c in candidates.values()) + + test_nodes = build_candidates(100) + candidates = score_candidates(test_nodes) + pscore_100 = max(c.content_score for c in candidates.values()) + + test_nodes = build_candidates(300) + candidates = score_candidates(test_nodes) + pscore_300 = max(c.content_score for c in candidates.values()) + + test_nodes = build_candidates(400) + candidates = score_candidates(test_nodes) + pscore_400 = max(c.content_score for c in candidates.values()) + + assert pscore_50 + 0.5 == pscore_100 + assert pscore_100 + 2.0 == pscore_300 + assert pscore_300 == pscore_400 + + +# Link density will adjust out candidate scoresself. + + +def test_link_density(): + """Test that we get a link density""" + doc = document_fromstring(load_article('ars.001.html')) + for node in doc.iter('p', 'td', 'pre'): + density = get_link_density(node) + + # the density must be between 0, 1 + assert density >= 0.0 and density <= 1.0 + + +# Siblings will be included if their content is related. + + +@pytest.mark.skip("Not implemented yet.") +def test_bad_siblings_not_counted(): + raise NotImplementedError() + + +@pytest.mark.skip("Not implemented yet.") +def test_good_siblings_counted(): + raise NotImplementedError() + + +# TestMainText + +def test_empty(): + article = Article("") + annotated_text = article.main_text + + assert annotated_text == [] + + +def test_no_annotations(): + article = Article("This is text with no annotations
This is text\r\twith no annotations
ľščťžýáí
")
- generate_hash_id(dom)
+def test_generate_hash():
+ dom = fragment_fromstring("ľščťžýáí
")
+ generate_hash_id(dom)
- def test_hash_from_id_on_exception(self):
- generate_hash_id(None)
- def test_different_hashes(self):
- dom = fragment_fromstring("ľščťžýáí
")
- hash_dom = generate_hash_id(dom)
- hash_none = generate_hash_id(None)
+def test_hash_from_id_on_exception():
+ generate_hash_id(None)
- self.assertNotEqual(hash_dom, hash_none)
- def test_equal_hashes(self):
- dom1 = fragment_fromstring("ľščťžýáí
")
- dom2 = fragment_fromstring("ľščťžýáí
")
- hash_dom1 = generate_hash_id(dom1)
- hash_dom2 = generate_hash_id(dom2)
- self.assertEqual(hash_dom1, hash_dom2)
+def test_different_hashes():
+ dom = fragment_fromstring("ľščťžýáí
")
+ hash_dom = generate_hash_id(dom)
+ hash_none = generate_hash_id(None)
- hash_none1 = generate_hash_id(None)
- hash_none2 = generate_hash_id(None)
- self.assertEqual(hash_none1, hash_none2)
+ assert hash_dom != hash_none
-class TestCheckNodeAttr(unittest.TestCase):
- """Verify a node has a class/id in the given set.
+def test_equal_hashes():
+ dom1 = fragment_fromstring("ľščťžýáí
")
+ dom2 = fragment_fromstring("ľščťžýáí
")
+ hash_dom1 = generate_hash_id(dom1)
+ hash_dom2 = generate_hash_id(dom2)
+ assert hash_dom1 == hash_dom2
- The idea is that we have sets of known good/bad ids and classes and need
- to verify the given node does/doesn't have those classes/ids.
+ hash_none1 = generate_hash_id(None)
+ hash_none2 = generate_hash_id(None)
+ assert hash_none1 == hash_none2
+
+
+# Verify a node has a class/id in the given set.
+# The idea is that we have sets of known good/bad ids and classes and need
+# to verify the given node does/doesn't have those classes/ids.
+
+
+def test_has_class():
+ """Verify that a node has a class in our set."""
+ test_pattern = re.compile('test1|test2', re.I)
+ test_node = fragment_fromstring('')
+ test_node.set('class', 'test2 comment')
+
+ assert check_node_attributes(test_pattern, test_node, 'class')
+
+
+def test_has_id():
+ """Verify that a node has an id in our set."""
+ test_pattern = re.compile('test1|test2', re.I)
+ test_node = fragment_fromstring('')
+ test_node.set('id', 'test2')
+
+ assert check_node_attributes(test_pattern, test_node, 'id')
+
+
+def test_lacks_class():
+ """Verify that a node does not have a class in our set."""
+ test_pattern = re.compile('test1|test2', re.I)
+ test_node = fragment_fromstring('')
+ test_node.set('class', 'test4 comment')
+
+ assert not check_node_attributes(test_pattern, test_node, 'class')
+
+
+def test_lacks_id():
+ """Verify that a node does not have an id in our set."""
+ test_pattern = re.compile('test1|test2', re.I)
+ test_node = fragment_fromstring('')
+ test_node.set('id', 'test4')
+
+ assert not check_node_attributes(test_pattern, test_node, 'id')
+
+
+# Verify we calc our link density correctly.
+
+
+def test_empty_node():
+ """An empty node doesn't have much of a link density"""
+ doc = Article("")
+
+ assert get_link_density(doc.readable_dom) == 0.0
+
+
+def test_small_doc_no_links():
+ doc = Article(load_snippet('document_min.html'))
+
+ assert get_link_density(doc.readable_dom) == 0.0
+
+
+def test_several_links():
+ """This doc has a 3 links with the majority of content."""
+ doc = Article(load_snippet('document_absolute_url.html'))
+
+ assert get_link_density(doc.readable_dom) == 22/37
+
+
+# Verify we score nodes correctly based on their class/id attributes.
+
+
+def test_no_matches_zero():
+ """If you don't have the attribute then you get a weight of 0"""
+ node = fragment_fromstring("")
+
+ assert get_class_weight(node) == 0
- """
- def test_has_class(self):
- """Verify that a node has a class in our set."""
- test_pattern = re.compile('test1|test2', re.I)
- test_node = fragment_fromstring('')
- test_node.set('class', 'test2 comment')
-
- self.assertTrue(
- check_node_attributes(test_pattern, test_node, 'class'))
-
- def test_has_id(self):
- """Verify that a node has an id in our set."""
- test_pattern = re.compile('test1|test2', re.I)
- test_node = fragment_fromstring('')
- test_node.set('id', 'test2')
-
- self.assertTrue(check_node_attributes(test_pattern, test_node, 'id'))
-
- def test_lacks_class(self):
- """Verify that a node does not have a class in our set."""
- test_pattern = re.compile('test1|test2', re.I)
- test_node = fragment_fromstring('')
- test_node.set('class', 'test4 comment')
- self.assertFalse(
- check_node_attributes(test_pattern, test_node, 'class'))
-
- def test_lacks_id(self):
- """Verify that a node does not have an id in our set."""
- test_pattern = re.compile('test1|test2', re.I)
- test_node = fragment_fromstring('')
- test_node.set('id', 'test4')
- self.assertFalse(check_node_attributes(test_pattern, test_node, 'id'))
-
-
-class TestLinkDensity(unittest.TestCase):
- """Verify we calc our link density correctly."""
-
- def test_empty_node(self):
- """An empty node doesn't have much of a link density"""
- doc = Article("")
- self.assertEqual(get_link_density(doc.readable_dom), 0.0)
-
- def test_small_doc_no_links(self):
- doc = Article(load_snippet('document_min.html'))
- self.assertEqual(get_link_density(doc.readable_dom), 0.0)
-
- def test_several_links(self):
- """This doc has a 3 links with the majority of content."""
- doc = Article(load_snippet('document_absolute_url.html'))
- self.assertAlmostEqual(get_link_density(doc.readable_dom), 22/37)
-
-
-class TestClassWeight(unittest.TestCase):
- """Verify we score nodes correctly based on their class/id attributes."""
-
- def test_no_matches_zero(self):
- """If you don't have the attribute then you get a weight of 0"""
- node = fragment_fromstring("")
- self.assertEqual(get_class_weight(node), 0)
-
- def test_id_hits(self):
- """If the id is in the list then it gets a weight"""
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertEqual(get_class_weight(node), 25)
-
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertEqual(get_class_weight(node), -25)
-
- def test_class_hits(self):
- """If the class is in the list then it gets a weight"""
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertEqual(get_class_weight(node), 25)
-
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertEqual(get_class_weight(node), -25)
-
- def test_scores_collide(self):
- """We might hit both positive and negative scores.
-
- Positive and negative scoring is done independently so it's possible
- to hit both positive and negative scores and cancel each other out.
-
- """
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertEqual(get_class_weight(node), 0)
-
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertEqual(get_class_weight(node), 25)
-
- def test_scores_only_once(self):
- """Scoring is not cumulative within a class hit."""
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertEqual(get_class_weight(node), 25)
-
-
-class TestUnlikelyNode(unittest.TestCase):
- """is_unlikely_node should help verify our node is good/bad."""
-
- def test_body_is_always_likely(self):
- """The body tag is always a likely node."""
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertFalse(is_unlikely_node(node))
-
- def test_is_unlikely(self):
- "Keywords in the class/id will make us believe this is unlikely."
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertTrue(is_unlikely_node(node))
-
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertTrue(is_unlikely_node(node))
-
- def test_not_unlikely(self):
- """Suck it double negatives."""
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertFalse(is_unlikely_node(node))
-
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertFalse(is_unlikely_node(node))
-
- def test_maybe_hits(self):
- """We've got some maybes that will overrule an unlikely node."""
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- self.assertFalse(is_unlikely_node(node))
-
-
-class TestScoredNode(unittest.TestCase):
- """ScoredNodes constructed have initial content_scores, etc."""
-
- def test_hash_id(self):
- """ScoredNodes have a hash_id based on their content
-
- Since this is based on the html there are chances for collisions, but
- it helps us follow and identify nodes through the scoring process. Two
- identical nodes would score the same, so meh all good.
-
- """
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- snode = ScoredNode(node)
- self.assertEqual(snode.hash_id, 'ffa4c519')
-
- def test_div_content_score(self):
- """A div starts out with a score of 5 and modifies from there"""
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- snode = ScoredNode(node)
- self.assertEqual(snode.content_score, 5)
-
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- snode = ScoredNode(node)
- self.assertEqual(snode.content_score, 30)
-
- test_div = 'Content
'
- node = fragment_fromstring(test_div)
- snode = ScoredNode(node)
- self.assertEqual(snode.content_score, -20)
-
- def test_headings_score(self):
- """Heading tags aren't likely candidates, hurt their scores."""
- test_div = 'Heading
' - node = fragment_fromstring(test_div) - snode = ScoredNode(node) - self.assertEqual(snode.content_score, -5) - - def test_list_items(self): - """Heading tags aren't likely candidates, hurt their scores.""" - test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+
+ assert get_class_weight(node) == 25
+
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+
+ assert get_class_weight(node) == -25
+
+
+def test_class_hits():
+ """If the class is in the list then it gets a weight"""
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert get_class_weight(node) == 25
+
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert get_class_weight(node) == -25
+
+
+def test_scores_collide():
+ """We might hit both positive and negative scores.
+
+ Positive and negative scoring is done independently so it's possible
+ to hit both positive and negative scores and cancel each other out.
"""
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert get_class_weight(node) == 0
+
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert get_class_weight(node) == 25
+
+
+def test_scores_only_once():
+ """Scoring is not cumulative within a class hit."""
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+
+ assert get_class_weight(node) == 25
+
+
+# is_unlikely_node should help verify our node is good/bad.
+
- def test_simple_candidate_set(self):
- """Tests a simple case of two candidate nodes"""
- html = """
-
-
-
-
-
-
-
- """
- dom = document_fromstring(html)
- div_nodes = dom.findall(".//div")
-
- candidates = score_candidates(div_nodes)
- ordered = sorted(
- (c for c in candidates.values()), reverse=True,
- key=attrgetter("content_score"))
-
- self.assertEqual(ordered[0].node.tag, "div")
- self.assertEqual(ordered[0].node.attrib["class"], "content")
- self.assertEqual(ordered[1].node.tag, "body")
- self.assertEqual(ordered[2].node.tag, "html")
- self.assertEqual(ordered[3].node.tag, "div")
- self.assertEqual(ordered[3].node.attrib["class"], "footer")
+def test_body_is_always_likely():
+ """The body tag is always a likely node."""
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+
+ assert not is_unlikely_node(node)
+
+
+def test_is_unlikely():
+ """Keywords in the class/id will make us believe this is unlikely."""
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert is_unlikely_node(node)
+
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert is_unlikely_node(node)
+
+
+def test_not_unlikely():
+ """Suck it double negatives."""
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert not is_unlikely_node(node)
+
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert not is_unlikely_node(node)
+
+
+def test_maybe_hits():
+ """We've got some maybes that will overrule an unlikely node."""
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ assert not is_unlikely_node(node)
+
+
+# ScoredNodes constructed have initial content_scores, etc.
+
+
+def test_hash_id():
+ """ScoredNodes have a hash_id based on their content
+
+ Since this is based on the html there are chances for collisions, but
+ it helps us follow and identify nodes through the scoring process. Two
+ identical nodes would score the same, so meh all good.
+
+ """
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ snode = ScoredNode(node)
+
+ assert snode.hash_id == 'ffa4c519'
+
+
+def test_div_content_score():
+ """A div starts out with a score of 5 and modifies from there"""
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ snode = ScoredNode(node)
+ assert snode.content_score == 5
+
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ snode = ScoredNode(node)
+ assert snode.content_score == 30
+
+ test_div = 'Content
'
+ node = fragment_fromstring(test_div)
+ snode = ScoredNode(node)
+ assert snode.content_score == -20
+
+
+def test_headings_score():
+ """Heading tags aren't likely candidates, hurt their scores."""
+ test_div = 'Heading
' + node = fragment_fromstring(test_div) + snode = ScoredNode(node) + + assert snode.content_score == -5 + + +def test_list_items(): + """Heading tags aren't likely candidates, hurt their scores.""" + test_div = '
+
+
+
+
+ """
+ dom = document_fromstring(html)
+ div_nodes = dom.findall(".//div")
+
+ candidates = score_candidates(div_nodes)
+ ordered = sorted(
+ (c for c in candidates.values()), reverse=True,
+ key=attrgetter("content_score"))
+
+ assert ordered[0].node.tag == "div"
+ assert ordered[0].node.attrib["class"] == "content"
+ assert ordered[1].node.tag == "body"
+ assert ordered[2].node.tag == "html"
+ assert ordered[3].node.tag == "div"
+ assert ordered[3].node.attrib["class"] == "footer"
This is a great amount of info
+And more content Home +