feat: added linkDensity function

src/extractor/generic/utils/dom/link-density.js

@@ -0,0 +1,23 @@
+// Determines what percentage of the text
+// in a node is link text
+// Takes a node, returns a float
+export default function linkDensity(node) {
+  const totalTextLength = textLength(node.text())
+
+  const linkText = node.find('a').text()
+  const linkLength = textLength(linkText)
+
+  if (totalTextLength > 0) {
+    return linkLength / totalTextLength
+  } else if (totalTextLength === 0 && linkLength > 0) {
+    return 1
+  } else {
+    return 0
+  }
+}
+
+function textLength(text) {
+  return text.trim()
+             .replace(/\s+/g, ' ')
+             .length
+}

src/extractor/generic/utils/dom/link-density.test.js

@@ -0,0 +1,34 @@
+import assert from 'assert'
+import cheerio from 'cheerio'
+
+import HTML from '../fixtures/html'
+
+import { linkDensity } from './index'
+
+describe('linkDensity($)', () => {
+  it("returns 0.5 if half of the text is a link", () => {
+    const $ = cheerio.load(HTML.linkDensity5)
+
+    const density = linkDensity($('div').first(), $)
+
+    assert.equal(density, 0.5)
+  })
+
+  it("returns 1 if all of the text is a link", () => {
+    const $ = cheerio.load(HTML.linkDensity1)
+
+    const density = linkDensity($('div').first(), $)
+
+    assert.equal(density, 1)
+  })
+
+  it("returns 0 if there's no text", () => {
+    const $ = cheerio.load(HTML.linkDensity0)
+
+    const density = linkDensity($('div').first())
+
+    assert.equal(density, 0)
+  })
+
+})
+

src/extractor/generic/utils/fixtures/html.js

@@ -223,7 +223,18 @@ const HTML = {
   convertNodeTo: {
     before: '<div>Should become a p</div>',
     after: '<p>Should become a p</p>',
-  }
+  },
+
+  // linkDensity
+  linkDensity5: `
+    <div><p>Some text!</p><p><a href="">Some text!</a></p> </div>
+  `,
+  linkDensity1: `
+    <div><p><a href="">Some text!</a></p></div>
+  `,
+  linkDensity0: `
+    <div><p><a href=""></a></p></div>
+  `,
 }
 
 export default HTML