Move url extraction logic to extractor

internal/api/api.go

@@ -17,6 +17,10 @@ import (
 	"time"
 )
 
+var (
+	resourcesIndex = "resources"
+)
+
 // Represent a resource in elasticsearch
 type resourceIndex struct {
 	URL   string    `json:"url"`
@@ -100,7 +104,7 @@ func searchResources(es *elastic.Client) echo.HandlerFunc {
 		// Perform the search request.
 		query := elastic.NewMatchQuery("url", string(b))
 		res, err := es.Search().
-			Index("resource").
+			Index(resourcesIndex).
 			Query(query).
 			Do(context.Background())
 		if err != nil {
@@ -141,7 +145,7 @@ func addResource(es *elastic.Client) echo.HandlerFunc {
 		}
 
 		_, err := es.Index().
-			Index("resources").
+			Index(resourcesIndex).
 			BodyJson(doc).
 			Do(context.Background())
 		if err != nil {

internal/extractor/extractor.go

@@ -1,6 +1,8 @@
 package extractor
 
 import (
+	"fmt"
+	"github.com/PuerkitoBio/purell"
 	"github.com/creekorful/trandoshan/api"
 	"github.com/creekorful/trandoshan/internal/messaging"
 	"github.com/creekorful/trandoshan/internal/util/logging"
@@ -121,7 +123,21 @@ func extractResource(msg messaging.NewResourceMsg) (api.ResourceDto, []string, e
 
 	// Extract URLs
 	xu := xurls.Strict()
-	return resDto, xu.FindAllString(msg.Body, -1), nil
+
+	// Sanitize URLs
+	urls := xu.FindAllString(msg.Body, -1)
+	var normalizedURLS []string
+
+	for _, url := range urls {
+		normalizedURL, err := normalizeURL(url)
+		if err != nil {
+			continue
+		}
+
+		normalizedURLS = append(normalizedURLS, normalizedURL)
+	}
+
+	return resDto, normalizedURLS, nil
 }
 
 // extract title from html body
@@ -138,3 +154,13 @@ func extractTitle(body string) string {
 
 	return body[startPos:endPos]
 }
+
+func normalizeURL(u string) (string, error) {
+	normalizedURL, err := purell.NormalizeURLString(u, purell.FlagsUsuallySafeGreedy|
+		purell.FlagRemoveDirectoryIndex|purell.FlagRemoveFragment|purell.FlagRemoveDuplicateSlashes)
+	if err != nil {
+		return "", fmt.Errorf("error while normalizing URL %s: %s", u, err)
+	}
+
+	return normalizedURL, nil
+}

internal/extractor/extractor_test.go

@@ -8,10 +8,10 @@ import (
 func TestExtractResource(t *testing.T) {
 	msg := messaging.NewResourceMsg{
 		URL:  "https://example.org/300",
-		Body: "<title>Creekorful Inc</title>This is sparta",
+		Body: "<title>Creekorful Inc</title>This is sparta<a href\"https://google.com/test?test=test#12\"",
 	}
 
-	resDto, _, err := extractResource(msg)
+	resDto, urls, err := extractResource(msg)
 	if err != nil {
 		t.FailNow()
 	}
@@ -25,6 +25,13 @@ func TestExtractResource(t *testing.T) {
 	if resDto.Body != msg.Body {
 		t.Fail()
 	}
+
+	if len(urls) == 0 {
+		t.FailNow()
+	}
+	if urls[0] != "https://google.com/test?test=test" {
+		t.Fail()
+	}
 }
 
 func TestExtractTitle(t *testing.T) {
@@ -38,3 +45,14 @@ func TestExtractTitle(t *testing.T) {
 		t.Errorf("No matches should have been returned")
 	}
 }
+
+func TestNormalizeURL(t *testing.T) {
+	url, err := normalizeURL("https://this-is-sparta.de?url=url-query-param#fragment-23")
+	if err != nil {
+		t.FailNow()
+	}
+
+	if url != "https://this-is-sparta.de?url=url-query-param" {
+		t.Fail()
+	}
+}

internal/scheduler/scheduler.go

@@ -3,7 +3,6 @@ package scheduler
 import (
 	"encoding/base64"
 	"fmt"
-	"github.com/PuerkitoBio/purell"
 	"github.com/creekorful/trandoshan/api"
 	"github.com/creekorful/trandoshan/internal/messaging"
 	"github.com/creekorful/trandoshan/internal/util/logging"
@@ -74,19 +73,20 @@ func handleMessage(apiClient api.Client) natsutil.MsgHandler {
 		}
 
 		log.Debug().Str("url", urlMsg.URL).Msg("Processing URL: %s")
-		normalizedURL, err := normalizeURL(urlMsg.URL)
+
+		u, err := url.Parse(urlMsg.URL)
 		if err != nil {
-			log.Err(err).Msg("Error while normalizing URL")
+			log.Err(err).Msg("Error while parsing URL")
 			return err
 		}
 
 		// Make sure URL is valid .onion
-		if !strings.Contains(normalizedURL.Host, ".onion") {
+		if !strings.Contains(u.Host, ".onion") {
-			log.Debug().Stringer("url", normalizedURL).Msg("URL is not a valid hidden service")
+			log.Debug().Stringer("url", u).Msg("URL is not a valid hidden service")
 			return err
 		}
 
-		b64URI := base64.URLEncoding.EncodeToString([]byte(normalizedURL.String()))
+		b64URI := base64.URLEncoding.EncodeToString([]byte(u.String()))
 		urls, err := apiClient.SearchResources(b64URI)
 		if err != nil {
 			log.Err(err).Msg("Error while searching URL")
@@ -95,29 +95,14 @@ func handleMessage(apiClient api.Client) natsutil.MsgHandler {
 
 		// No matches: schedule!
 		if len(urls) == 0 {
-			log.Debug().Stringer("url", normalizedURL).Msg("URL should be scheduled")
+			log.Debug().Stringer("url", u).Msg("URL should be scheduled")
 			if err := natsutil.PublishMsg(nc, &messaging.URLTodoMsg{URL: urlMsg.URL}); err != nil {
 				return fmt.Errorf("error while publishing URL: %s", err)
 			}
 		} else {
-			log.Trace().Stringer("url", normalizedURL).Msg("URL should not be scheduled")
+			log.Trace().Stringer("url", u).Msg("URL should not be scheduled")
 		}
 
 		return nil
 	}
 }
-
-func normalizeURL(u string) (*url.URL, error) {
-	normalizedURL, err := purell.NormalizeURLString(u, purell.FlagsUsuallySafeGreedy|
-		purell.FlagRemoveDirectoryIndex|purell.FlagRemoveFragment|purell.FlagRemoveDuplicateSlashes)
-	if err != nil {
-		return nil, fmt.Errorf("error while normalizing URL %s: %s", u, err)
-	}
-
-	nu, err := url.Parse(normalizedURL)
-	if err != nil {
-		return nil, fmt.Errorf("error while parsing URL: %s", err)
-	}
-
-	return nu, nil
-}

internal/scheduler/scheduler_test.go

@@ -1,14 +1 @@
 package scheduler
-
-import "testing"
-
-func TestNormalizeURL(t *testing.T) {
-	url, err := normalizeURL("https://this-is-sparta.de?url=url-query-param#fragment-23")
-	if err != nil {
-		t.FailNow()
-	}
-
-	if url.String() != "https://this-is-sparta.de?url=url-query-param" {
-		t.Fail()
-	}
-}