fabric/jina/jina.go

package jina

// see https://jina.ai for more information

import (
	"fmt"
	"io"
	"net/http"

	"github.com/danielmiessler/fabric/common"
)

type Client struct {
	*common.Configurable
	ApiKey *common.SetupQuestion
}

func NewClient() (ret *Client) {

	label := "Jina AI"

	ret = &Client{
		Configurable: &common.Configurable{
			Label:         label,
			EnvNamePrefix: common.BuildEnvVariablePrefix(label),
		},
	}

	ret.ApiKey = ret.AddSetupQuestion("API Key", false)

	return
}

// ScrapeURL return the main content of a webpage in clean, LLM-friendly text.
func (jc *Client) ScrapeURL(url string) (ret string, err error) {
	return jc.request(fmt.Sprintf("https://r.jina.ai/%s", url))
}

func (jc *Client) ScrapeQuestion(question string) (ret string, err error) {
	return jc.request(fmt.Sprintf("https://s.jina.ai/%s", question))
}

func (jc *Client) request(requestURL string) (ret string, err error) {
	var req *http.Request
	if req, err = http.NewRequest("GET", requestURL, nil); err != nil {
		err = fmt.Errorf("error creating request: %w", err)
		return
	}

	// if api keys exist, set the header
	if jc.ApiKey.Value != "" {
		req.Header.Set("Authorization", "Bearer "+jc.ApiKey.Value)
	}

	client := &http.Client{}
	var resp *http.Response
	if resp, err = client.Do(req); err != nil {
		err = fmt.Errorf("error sending request: %w", err)
		return
	}
	defer resp.Body.Close()

	var body []byte
	if body, err = io.ReadAll(resp.Body); err != nil {
		err = fmt.Errorf("error reading response body: %w", err)
		return
	}
	ret = string(body)
	return
}