mirror of https://github.com/nomic-ai/gpt4all
nodejs bindings (#602)
* chore: boilerplate, refactor in future * chore: boilerplate * feat: can compile succesfully * document .gyp file * add src, test and fix gyp * progress on prompting and some helper methods * add destructor and basic prompting work, prepare download function * download function done * download function edits and adding documentation * fix bindings memory issue and add tests and specs * add more documentation and readme * add npmignore * Update README.md Signed-off-by: Jacob Nguyen <76754747+jacoobes@users.noreply.github.com> * Update package.json - redundant scripts Signed-off-by: Jacob Nguyen <76754747+jacoobes@users.noreply.github.com> --------- Signed-off-by: Jacob Nguyen <76754747+jacoobes@users.noreply.github.com>pull/681/head
Jacob Nguyen
1 year ago
committed by
GitHub
parent
5c1cf69b8b
commit
4462d2d755
@ -0,0 +1,2 @@
|
||||
node_modules/
|
||||
build/
|
||||
@ -0,0 +1,3 @@
|
||||
test/
|
||||
spec/
|
||||
|
||||
@ -1,4 +1,56 @@
|
||||
### Javascript Bindings
|
||||
The original [GPT4All typescript bindings](https://github.com/nomic-ai/gpt4all-ts) are now out of date.
|
||||
|
||||
The GPT4All community is looking for help in implemented a new set of bindings based on the C backend found [here](https://github.com/nomic-ai/gpt4all/tree/main/gpt4all-backend).
|
||||
- created by [jacoobes](https://github.com/jacoobes) and [nomic ai](https://home.nomic.ai) :D, for all to use.
|
||||
- will maintain this repository when possible, new feature requests will be handled through nomic
|
||||
|
||||
### Build Instructions
|
||||
|
||||
- As of 05/21/2023, Tested on windows (MSVC) only. (somehow got it to work on MSVC 🤯)
|
||||
- binding.gyp is compile config
|
||||
|
||||
### Requirements
|
||||
- git
|
||||
- [node.js >= 18.0.0](https://nodejs.org/en)
|
||||
- [yarn](https://yarnpkg.com/)
|
||||
- [node-gyp](https://github.com/nodejs/node-gyp)
|
||||
- all of its requirements.
|
||||
|
||||
### Build
|
||||
```sh
|
||||
git clone https://github.com/nomic-ai/gpt4all.git
|
||||
cd gpt4all-bindings/typescript
|
||||
```
|
||||
- The below shell commands assume the current working directory is `typescript`.
|
||||
|
||||
- To Build and Rebuild:
|
||||
```sh
|
||||
yarn
|
||||
```
|
||||
- llama.cpp git submodule for gpt4all can be possibly absent. If this is the case, make sure to run in llama.cpp parent directory
|
||||
```sh
|
||||
git submodule update --init --depth 1 --recursive`
|
||||
```
|
||||
### Test
|
||||
```sh
|
||||
yarn test
|
||||
```
|
||||
### Source Overview
|
||||
|
||||
#### src/
|
||||
- Extra functions to help aid devex
|
||||
- Typings for the native node addon
|
||||
- the javascript interface
|
||||
|
||||
#### test/
|
||||
- simple unit testings for some functions exported.
|
||||
- more advanced ai testing is not handled
|
||||
|
||||
#### spec/
|
||||
- Average look and feel of the api
|
||||
- Should work assuming a model is installed locally in working directory
|
||||
|
||||
#### index.cc
|
||||
- The bridge between nodejs and c. Where the bindings are.
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,46 @@
|
||||
{
|
||||
"targets": [
|
||||
{
|
||||
"target_name": "gpt4allts", # gpt4all-ts will cause compile error
|
||||
"cflags!": [ "-fno-exceptions" ],
|
||||
"cflags_cc!": [ "-fno-exceptions" ],
|
||||
"include_dirs": [
|
||||
"<!@(node -p \"require('node-addon-api').include\")",
|
||||
"../../gpt4all-backend/llama.cpp/", # need to include llama.cpp because the include paths for examples/common.h include llama.h relatively
|
||||
"../../gpt4all-backend",
|
||||
],
|
||||
"sources": [ # is there a better way to do this
|
||||
"../../gpt4all-backend/llama.cpp/examples/common.cpp",
|
||||
"../../gpt4all-backend/llama.cpp/ggml.c",
|
||||
"../../gpt4all-backend/llama.cpp/llama.cpp",
|
||||
"../../gpt4all-backend/utils.cpp",
|
||||
"../../gpt4all-backend/llmodel_c.cpp",
|
||||
"../../gpt4all-backend/gptj.cpp",
|
||||
"../../gpt4all-backend/llamamodel.cpp",
|
||||
"../../gpt4all-backend/mpt.cpp",
|
||||
"stdcapture.cc",
|
||||
"index.cc",
|
||||
],
|
||||
"conditions": [
|
||||
['OS=="mac"', {
|
||||
'defines': [
|
||||
'NAPI_CPP_EXCEPTIONS'
|
||||
],
|
||||
}],
|
||||
['OS=="win"', {
|
||||
'defines': [
|
||||
'NAPI_CPP_EXCEPTIONS',
|
||||
"__AVX2__" # allows SIMD: https://discord.com/channels/1076964370942267462/1092290790388150272/1107564673957630023
|
||||
],
|
||||
"msvs_settings": {
|
||||
"VCCLCompilerTool": {
|
||||
"AdditionalOptions": [
|
||||
"/std:c++20",
|
||||
"/EHsc"
|
||||
],
|
||||
},
|
||||
},
|
||||
}]
|
||||
]
|
||||
}]
|
||||
}
|
||||
@ -0,0 +1,227 @@
|
||||
#include <napi.h>
|
||||
#include <iostream>
|
||||
#include "llmodel_c.h"
|
||||
#include "llmodel.h"
|
||||
#include "gptj.h"
|
||||
#include "llamamodel.h"
|
||||
#include "mpt.h"
|
||||
#include "stdcapture.h"
|
||||
|
||||
class NodeModelWrapper : public Napi::ObjectWrap<NodeModelWrapper> {
|
||||
public:
|
||||
static Napi::Object Init(Napi::Env env, Napi::Object exports) {
|
||||
Napi::Function func = DefineClass(env, "LLModel", {
|
||||
InstanceMethod("type", &NodeModelWrapper::getType),
|
||||
InstanceMethod("name", &NodeModelWrapper::getName),
|
||||
InstanceMethod("stateSize", &NodeModelWrapper::StateSize),
|
||||
InstanceMethod("raw_prompt", &NodeModelWrapper::Prompt),
|
||||
InstanceMethod("setThreadCount", &NodeModelWrapper::SetThreadCount),
|
||||
InstanceMethod("threadCount", &NodeModelWrapper::ThreadCount),
|
||||
});
|
||||
|
||||
Napi::FunctionReference* constructor = new Napi::FunctionReference();
|
||||
*constructor = Napi::Persistent(func);
|
||||
env.SetInstanceData(constructor);
|
||||
|
||||
exports.Set("LLModel", func);
|
||||
return exports;
|
||||
}
|
||||
|
||||
Napi::Value getType(const Napi::CallbackInfo& info)
|
||||
{
|
||||
return Napi::String::New(info.Env(), type);
|
||||
}
|
||||
|
||||
NodeModelWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap<NodeModelWrapper>(info)
|
||||
{
|
||||
auto env = info.Env();
|
||||
std::string weights_path = info[0].As<Napi::String>().Utf8Value();
|
||||
|
||||
const char *c_weights_path = weights_path.c_str();
|
||||
|
||||
inference_ = create_model_set_type(c_weights_path);
|
||||
|
||||
auto success = llmodel_loadModel(inference_, c_weights_path);
|
||||
if(!success) {
|
||||
Napi::Error::New(env, "Failed to load model at given path").ThrowAsJavaScriptException();
|
||||
return;
|
||||
}
|
||||
name = weights_path.substr(weights_path.find_last_of("/\\") + 1);
|
||||
|
||||
};
|
||||
~NodeModelWrapper() {
|
||||
// destroying the model manually causes exit code 3221226505, why?
|
||||
// However, bindings seem to operate fine without destructing pointer
|
||||
//llmodel_model_destroy(inference_);
|
||||
}
|
||||
|
||||
Napi::Value IsModelLoaded(const Napi::CallbackInfo& info) {
|
||||
return Napi::Boolean::New(info.Env(), llmodel_isModelLoaded(inference_));
|
||||
}
|
||||
|
||||
Napi::Value StateSize(const Napi::CallbackInfo& info) {
|
||||
// Implement the binding for the stateSize method
|
||||
return Napi::Number::New(info.Env(), static_cast<int64_t>(llmodel_get_state_size(inference_)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a response using the model.
|
||||
* @param model A pointer to the llmodel_model instance.
|
||||
* @param prompt A string representing the input prompt.
|
||||
* @param prompt_callback A callback function for handling the processing of prompt.
|
||||
* @param response_callback A callback function for handling the generated response.
|
||||
* @param recalculate_callback A callback function for handling recalculation requests.
|
||||
* @param ctx A pointer to the llmodel_prompt_context structure.
|
||||
*/
|
||||
Napi::Value Prompt(const Napi::CallbackInfo& info) {
|
||||
|
||||
auto env = info.Env();
|
||||
|
||||
std::string question;
|
||||
if(info[0].IsString()) {
|
||||
question = info[0].As<Napi::String>().Utf8Value();
|
||||
} else {
|
||||
Napi::Error::New(env, "invalid string argument").ThrowAsJavaScriptException();
|
||||
return env.Undefined();
|
||||
}
|
||||
//defaults copied from python bindings
|
||||
llmodel_prompt_context promptContext = {
|
||||
.logits = nullptr,
|
||||
.tokens = nullptr,
|
||||
.n_past = 0,
|
||||
.n_ctx = 1024,
|
||||
.n_predict = 128,
|
||||
.top_k = 40,
|
||||
.top_p = 0.9f,
|
||||
.temp = 0.72f,
|
||||
.n_batch = 8,
|
||||
.repeat_penalty = 1.0f,
|
||||
.repeat_last_n = 10,
|
||||
.context_erase = 0.5
|
||||
};
|
||||
if(info[1].IsObject())
|
||||
{
|
||||
auto inputObject = info[1].As<Napi::Object>();
|
||||
|
||||
// Extract and assign the properties
|
||||
if (inputObject.Has("logits") || inputObject.Has("tokens")) {
|
||||
Napi::Error::New(env, "Invalid input: 'logits' or 'tokens' properties are not allowed").ThrowAsJavaScriptException();
|
||||
return env.Undefined();
|
||||
}
|
||||
// Assign the remaining properties
|
||||
if(inputObject.Has("n_past")) {
|
||||
promptContext.n_past = inputObject.Get("n_past").As<Napi::Number>().Int32Value();
|
||||
}
|
||||
if(inputObject.Has("n_ctx")) {
|
||||
promptContext.n_ctx = inputObject.Get("n_ctx").As<Napi::Number>().Int32Value();
|
||||
}
|
||||
if(inputObject.Has("n_predict")) {
|
||||
promptContext.n_predict = inputObject.Get("n_predict").As<Napi::Number>().Int32Value();
|
||||
}
|
||||
if(inputObject.Has("top_k")) {
|
||||
promptContext.top_k = inputObject.Get("top_k").As<Napi::Number>().Int32Value();
|
||||
}
|
||||
if(inputObject.Has("top_p")) {
|
||||
promptContext.top_p = inputObject.Get("top_p").As<Napi::Number>().FloatValue();
|
||||
}
|
||||
if(inputObject.Has("temp")) {
|
||||
promptContext.temp = inputObject.Get("temp").As<Napi::Number>().FloatValue();
|
||||
}
|
||||
if(inputObject.Has("n_batch")) {
|
||||
promptContext.n_batch = inputObject.Get("n_batch").As<Napi::Number>().Int32Value();
|
||||
}
|
||||
if(inputObject.Has("repeat_penalty")) {
|
||||
promptContext.repeat_penalty = inputObject.Get("repeat_penalty").As<Napi::Number>().FloatValue();
|
||||
}
|
||||
if(inputObject.Has("repeat_last_n")) {
|
||||
promptContext.repeat_last_n = inputObject.Get("repeat_last_n").As<Napi::Number>().Int32Value();
|
||||
}
|
||||
if(inputObject.Has("context_erase")) {
|
||||
promptContext.context_erase = inputObject.Get("context_erase").As<Napi::Number>().FloatValue();
|
||||
}
|
||||
}
|
||||
// custom callbacks are weird with the gpt4all c bindings: I need to turn Napi::Functions into raw c function pointers,
|
||||
// but it doesn't seem like its possible? (TODO, is it possible?)
|
||||
|
||||
// if(info[1].IsFunction()) {
|
||||
// Napi::Callback cb = *info[1].As<Napi::Function>();
|
||||
// }
|
||||
|
||||
|
||||
// For now, simple capture of stdout
|
||||
// possible TODO: put this on a libuv async thread. (AsyncWorker)
|
||||
CoutRedirect cr;
|
||||
llmodel_prompt(inference_, question.c_str(), &prompt_callback, &response_callback, &recalculate_callback, &promptContext);
|
||||
return Napi::String::New(env, cr.getString());
|
||||
}
|
||||
|
||||
void SetThreadCount(const Napi::CallbackInfo& info) {
|
||||
if(info[0].IsNumber()) {
|
||||
llmodel_setThreadCount(inference_, info[0].As<Napi::Number>().Int64Value());
|
||||
} else {
|
||||
Napi::Error::New(info.Env(), "Could not set thread count: argument 1 is NaN").ThrowAsJavaScriptException();
|
||||
return;
|
||||
}
|
||||
}
|
||||
Napi::Value getName(const Napi::CallbackInfo& info) {
|
||||
return Napi::String::New(info.Env(), name);
|
||||
}
|
||||
Napi::Value ThreadCount(const Napi::CallbackInfo& info) {
|
||||
return Napi::Number::New(info.Env(), llmodel_threadCount(inference_));
|
||||
}
|
||||
|
||||
private:
|
||||
llmodel_model inference_;
|
||||
std::string type;
|
||||
std::string name;
|
||||
|
||||
|
||||
//wrapper cb to capture output into stdout.then, CoutRedirect captures this
|
||||
// and writes it to a file
|
||||
static bool response_callback(int32_t tid, const char* resp)
|
||||
{
|
||||
if(tid != -1) {
|
||||
std::cout<<std::string(resp);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool prompt_callback(int32_t tid) { return true; }
|
||||
static bool recalculate_callback(bool isrecalculating) { return isrecalculating; }
|
||||
// Had to use this instead of the c library in order
|
||||
// set the type of the model loaded.
|
||||
// causes side effect: type is mutated;
|
||||
llmodel_model create_model_set_type(const char* c_weights_path)
|
||||
{
|
||||
|
||||
uint32_t magic;
|
||||
llmodel_model model;
|
||||
FILE *f = fopen(c_weights_path, "rb");
|
||||
fread(&magic, sizeof(magic), 1, f);
|
||||
|
||||
if (magic == 0x67676d6c) {
|
||||
model = llmodel_gptj_create();
|
||||
type = "gptj";
|
||||
}
|
||||
else if (magic == 0x67676a74) {
|
||||
model = llmodel_llama_create();
|
||||
type = "llama";
|
||||
}
|
||||
else if (magic == 0x67676d6d) {
|
||||
model = llmodel_mpt_create();
|
||||
type = "mpt";
|
||||
}
|
||||
else {fprintf(stderr, "Invalid model file\n");}
|
||||
fclose(f);
|
||||
|
||||
return model;
|
||||
}
|
||||
};
|
||||
|
||||
//Exports Bindings
|
||||
Napi::Object Init(Napi::Env env, Napi::Object exports) {
|
||||
return NodeModelWrapper::Init(env, exports);
|
||||
}
|
||||
|
||||
NODE_API_MODULE(NODE_GYP_MODULE_NAME, Init)
|
||||
@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "gpt4all-ts",
|
||||
"packageManager": "yarn@3.5.1",
|
||||
"gypfile": true,
|
||||
"scripts": {
|
||||
"test": "node ./test/index.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"bindings": "^1.5.0",
|
||||
"node-addon-api": "^6.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.1.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 18.x.x"
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
import { LLModel, prompt, createCompletion } from '../src/gpt4all.js'
|
||||
|
||||
|
||||
|
||||
const ll = new LLModel("./ggml-vicuna-7b-1.1-q4_2.bin");
|
||||
|
||||
|
||||
try {
|
||||
class Extended extends LLModel {
|
||||
|
||||
}
|
||||
} catch(e) {
|
||||
console.log("Extending from native class gone wrong " + e)
|
||||
}
|
||||
|
||||
console.log("state size " + ll.stateSize())
|
||||
|
||||
console.log("thread count " + ll.threadCount());
|
||||
ll.setThreadCount(5);
|
||||
console.log("thread count " + ll.threadCount());
|
||||
ll.setThreadCount(4);
|
||||
console.log("thread count " + ll.threadCount());
|
||||
|
||||
|
||||
console.log(createCompletion(
|
||||
ll,
|
||||
prompt`${"header"} ${"prompt"}`, {
|
||||
verbose: true,
|
||||
prompt: 'hello! Say something thought provoking.'
|
||||
}
|
||||
));
|
||||
|
||||
@ -0,0 +1,162 @@
|
||||
/// <reference types="node" />
|
||||
declare module 'gpt4all-ts';
|
||||
|
||||
|
||||
|
||||
|
||||
interface LLModelPromptContext {
|
||||
|
||||
// Size of the raw logits vector
|
||||
logits_size: number;
|
||||
|
||||
// Size of the raw tokens vector
|
||||
tokens_size: number;
|
||||
|
||||
// Number of tokens in past conversation
|
||||
n_past: number;
|
||||
|
||||
// Number of tokens possible in context window
|
||||
n_ctx: number;
|
||||
|
||||
// Number of tokens to predict
|
||||
n_predict: number;
|
||||
|
||||
// Top k logits to sample from
|
||||
top_k: number;
|
||||
|
||||
// Nucleus sampling probability threshold
|
||||
top_p: number;
|
||||
|
||||
// Temperature to adjust model's output distribution
|
||||
temp: number;
|
||||
|
||||
// Number of predictions to generate in parallel
|
||||
n_batch: number;
|
||||
|
||||
// Penalty factor for repeated tokens
|
||||
repeat_penalty: number;
|
||||
|
||||
// Last n tokens to penalize
|
||||
repeat_last_n: number;
|
||||
|
||||
// Percent of context to erase if we exceed the context window
|
||||
context_erase: number;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* LLModel class representing a language model.
|
||||
* This is a base class that provides common functionality for different types of language models.
|
||||
*/
|
||||
declare class LLModel {
|
||||
//either 'gpt', mpt', or 'llama'
|
||||
type() : ModelType;
|
||||
//The name of the model
|
||||
name(): ModelFile;
|
||||
constructor(path: string);
|
||||
/**
|
||||
* Get the size of the internal state of the model.
|
||||
* NOTE: This state data is specific to the type of model you have created.
|
||||
* @return the size in bytes of the internal state of the model
|
||||
*/
|
||||
stateSize(): number;
|
||||
/**
|
||||
* Get the number of threads used for model inference.
|
||||
* The default is the number of physical cores your computer has.
|
||||
* @returns The number of threads used for model inference.
|
||||
*/
|
||||
threadCount() : number;
|
||||
/**
|
||||
* Set the number of threads used for model inference.
|
||||
* @param newNumber The new number of threads.
|
||||
*/
|
||||
setThreadCount(newNumber: number): void;
|
||||
/**
|
||||
* Prompt the model with a given input and optional parameters.
|
||||
* This is the raw output from std out.
|
||||
* Use the prompt function exported for a value
|
||||
* @param q The prompt input.
|
||||
* @param params Optional parameters for the prompt context.
|
||||
* @returns The result of the model prompt.
|
||||
*/
|
||||
raw_prompt(q: string, params?: Partial<LLModelPromptContext>) : unknown; //todo work on return type
|
||||
|
||||
}
|
||||
|
||||
interface DownloadController {
|
||||
//Cancel the request to download from gpt4all website if this is called.
|
||||
cancel: () => void;
|
||||
//Convert the downloader into a promise, allowing people to await and manage its lifetime
|
||||
promise: () => Promise<void>
|
||||
}
|
||||
|
||||
|
||||
export interface DownloadConfig {
|
||||
/**
|
||||
* location to download the model.
|
||||
* Default is process.cwd(), or the current working directory
|
||||
*/
|
||||
location: string;
|
||||
/**
|
||||
* Debug mode -- check how long it took to download in seconds
|
||||
*/
|
||||
debug: boolean;
|
||||
/**
|
||||
* Default link = https://gpt4all.io/models`
|
||||
* This property overrides the default.
|
||||
*/
|
||||
link?: string
|
||||
}
|
||||
/**
|
||||
* Initiates the download of a model file of a specific model type.
|
||||
* By default this downloads without waiting. use the controller returned to alter this behavior.
|
||||
* @param {ModelFile[ModelType]} m - The model file to be downloaded.
|
||||
* @param {Record<string, unknown>} op - options to pass into the downloader. Default is { location: (cwd), debug: false }.
|
||||
* @returns {DownloadController} A DownloadController object that allows controlling the download process.
|
||||
*/
|
||||
declare function download(m: ModelFile[ModelType], op: { location: string, debug: boolean, link?:string }): DownloadController
|
||||
|
||||
|
||||
type ModelType = 'gptj' | 'llama' | 'mpt';
|
||||
|
||||
/*
|
||||
* A nice interface for intellisense of all possibly models.
|
||||
*/
|
||||
interface ModelFile {
|
||||
'gptj': | "ggml-gpt4all-j-v1.3-groovy.bin"
|
||||
| "ggml-gpt4all-j-v1.2-jazzy.bin"
|
||||
| "ggml-gpt4all-j-v1.1-breezy.bin"
|
||||
| "ggml-gpt4all-j.bin";
|
||||
'llama':| "ggml-gpt4all-l13b-snoozy.bin"
|
||||
| "ggml-vicuna-7b-1.1-q4_2.bin"
|
||||
| "ggml-vicuna-13b-1.1-q4_2.bin"
|
||||
| "ggml-wizardLM-7B.q4_2.bin"
|
||||
| "ggml-stable-vicuna-13B.q4_2.bin"
|
||||
| "ggml-nous-gpt4-vicuna-13b.bin"
|
||||
'mpt': | "ggml-mpt-7b-base.bin"
|
||||
| "ggml-mpt-7b-chat.bin"
|
||||
| "ggml-mpt-7b-instruct.bin"
|
||||
}
|
||||
|
||||
interface ExtendedOptions {
|
||||
verbose?: boolean;
|
||||
system?: string;
|
||||
header?: string;
|
||||
prompt: string;
|
||||
promptEntries?: Record<string, unknown>
|
||||
}
|
||||
|
||||
type PromptTemplate = (...args: string[]) => string;
|
||||
|
||||
declare function createCompletion(
|
||||
model: LLModel,
|
||||
pt: PromptTemplate,
|
||||
options: LLModelPromptContext&ExtendedOptions
|
||||
) : string
|
||||
|
||||
function prompt(
|
||||
strings: TemplateStringsArray
|
||||
): PromptTemplate
|
||||
|
||||
|
||||
export { LLModel, LLModelPromptContext, ModelType, download, DownloadController, prompt, ExtendedOptions, createCompletion }
|
||||
@ -0,0 +1,112 @@
|
||||
/// This file implements the gpt4all.d.ts file endings.
|
||||
/// Written in commonjs to support both ESM and CJS projects.
|
||||
|
||||
const { LLModel } = require('bindings')('../build/Release/gpt4allts');
|
||||
const { createWriteStream, existsSync } = require('fs');
|
||||
const { join } = require('path');
|
||||
const { performance } = require('node:perf_hooks');
|
||||
|
||||
|
||||
|
||||
// readChunks() reads from the provided reader and yields the results into an async iterable
|
||||
// https://css-tricks.com/web-streams-everywhere-and-fetch-for-node-js/
|
||||
function readChunks(reader) {
|
||||
return {
|
||||
async* [Symbol.asyncIterator]() {
|
||||
let readResult = await reader.read();
|
||||
while (!readResult.done) {
|
||||
yield readResult.value;
|
||||
readResult = await reader.read();
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
exports.LLModel = LLModel;
|
||||
|
||||
|
||||
exports.download = function (
|
||||
name,
|
||||
options = { debug: false, location: process.cwd(), link: undefined }
|
||||
) {
|
||||
const abortController = new AbortController();
|
||||
const signal = abortController.signal;
|
||||
|
||||
const pathToModel = join(options.location, name);
|
||||
if(existsSync(pathToModel)) {
|
||||
throw Error("Path to model already exists");
|
||||
}
|
||||
|
||||
//wrapper function to get the readable stream from request
|
||||
const fetcher = (name) => fetch(options.link ?? `https://gpt4all.io/models/${name}`, {
|
||||
signal,
|
||||
})
|
||||
.then(res => {
|
||||
if(!res.ok) {
|
||||
throw Error("Could not find "+ name + " from " + `https://gpt4all.io/models/` )
|
||||
}
|
||||
return res.body.getReader()
|
||||
})
|
||||
|
||||
//a promise that executes and writes to a stream. Resolves when done writing.
|
||||
const res = new Promise((resolve, reject) => {
|
||||
fetcher(name)
|
||||
//Resolves an array of a reader and writestream.
|
||||
.then(reader => [reader, createWriteStream(pathToModel)])
|
||||
.then(
|
||||
async ([readable, wstream]) => {
|
||||
console.log('(CLI might hang) Downloading @ ', pathToModel);
|
||||
let perf;
|
||||
if(options.debug) {
|
||||
perf = performance.now();
|
||||
}
|
||||
for await (const chunk of readChunks(readable)) {
|
||||
wstream.write(chunk);
|
||||
}
|
||||
if(options.debug) {
|
||||
console.log("Time taken: ", (performance.now()-perf).toFixed(2), " ms");
|
||||
}
|
||||
resolve();
|
||||
}
|
||||
).catch(reject);
|
||||
});
|
||||
|
||||
return {
|
||||
cancel : () => abortController.abort(),
|
||||
promise: () => res
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals#tagged_templates
|
||||
exports.prompt = function prompt(strings, ...keys) {
|
||||
return (...values) => {
|
||||
const dict = values[values.length - 1] || {};
|
||||
const result = [strings[0]];
|
||||
keys.forEach((key, i) => {
|
||||
const value = Number.isInteger(key) ? values[key] : dict[key];
|
||||
result.push(value, strings[i + 1]);
|
||||
});
|
||||
return result.join("");
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
|
||||
exports.createCompletion = function (llmodel, promptMaker, options) {
|
||||
//creating the keys to insert into promptMaker.
|
||||
const entries = {
|
||||
system: options.system ?? '',
|
||||
header: options.header ?? "### Instruction: The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.\n### Prompt: ",
|
||||
prompt: options.prompt,
|
||||
...(options.promptEntries ?? {})
|
||||
};
|
||||
|
||||
const fullPrompt = promptMaker(entries)+'\n### Response:';
|
||||
|
||||
if(options.verbose) {
|
||||
console.log("sending prompt: " + `"${fullPrompt}"`)
|
||||
}
|
||||
|
||||
return llmodel.raw_prompt(fullPrompt, options);
|
||||
}
|
||||
@ -0,0 +1,14 @@
|
||||
|
||||
#include "stdcapture.h"
|
||||
|
||||
CoutRedirect::CoutRedirect() {
|
||||
old = std::cout.rdbuf(buffer.rdbuf()); // redirect cout to buffer stream
|
||||
}
|
||||
|
||||
std::string CoutRedirect::getString() {
|
||||
return buffer.str(); // get string
|
||||
}
|
||||
|
||||
CoutRedirect::~CoutRedirect() {
|
||||
std::cout.rdbuf(old); // reverse redirect
|
||||
}
|
||||
@ -0,0 +1,21 @@
|
||||
//https://stackoverflow.com/questions/5419356/redirect-stdout-stderr-to-a-string
|
||||
#ifndef COUTREDIRECT_H
|
||||
#define COUTREDIRECT_H
|
||||
|
||||
#include <iostream>
|
||||
#include <streambuf>
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
|
||||
class CoutRedirect {
|
||||
public:
|
||||
CoutRedirect();
|
||||
std::string getString();
|
||||
~CoutRedirect();
|
||||
|
||||
private:
|
||||
std::stringstream buffer;
|
||||
std::streambuf* old;
|
||||
};
|
||||
|
||||
#endif // COUTREDIRECT_H
|
||||
@ -0,0 +1,41 @@
|
||||
import * as assert from 'node:assert'
|
||||
import { prompt, download } from '../src/gpt4all.js'
|
||||
|
||||
{
|
||||
|
||||
const somePrompt = prompt`${"header"} Hello joe, my name is Ron. ${"prompt"}`;
|
||||
assert.equal(
|
||||
somePrompt({ header: 'oompa', prompt: 'holy moly' }),
|
||||
'oompa Hello joe, my name is Ron. holy moly'
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
{
|
||||
|
||||
const indexedPrompt = prompt`${0}, ${1} ${0}`;
|
||||
assert.equal(
|
||||
indexedPrompt('hello', 'world'),
|
||||
'hello, world hello'
|
||||
);
|
||||
|
||||
assert.notEqual(
|
||||
indexedPrompt(['hello', 'world']),
|
||||
'hello, world hello'
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
{
|
||||
assert.equal(
|
||||
(prompt`${"header"} ${"prompt"}`)({ header: 'hello', prompt: 'poo' }), 'hello poo',
|
||||
"Template prompt not equal"
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
|
||||
assert.rejects(async () => download('poo.bin').promise());
|
||||
|
||||
|
||||
console.log('OK')
|
||||
@ -0,0 +1,25 @@
|
||||
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||
# yarn lockfile v1
|
||||
|
||||
|
||||
"@types/node@^20.1.5":
|
||||
version "20.1.5"
|
||||
resolved "https://registry.yarnpkg.com/@types/node/-/node-20.1.5.tgz#e94b604c67fc408f215fcbf3bd84d4743bf7f710"
|
||||
integrity sha512-IvGD1CD/nego63ySR7vrAKEX3AJTcmrAN2kn+/sDNLi1Ff5kBzDeEdqWDplK+0HAEoLYej137Sk0cUU8OLOlMg==
|
||||
|
||||
bindings@^1.5.0:
|
||||
version "1.5.0"
|
||||
resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df"
|
||||
integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==
|
||||
dependencies:
|
||||
file-uri-to-path "1.0.0"
|
||||
|
||||
file-uri-to-path@1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
|
||||
integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==
|
||||
|
||||
node-addon-api@^6.1.0:
|
||||
version "6.1.0"
|
||||
resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76"
|
||||
integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==
|
||||
Loading…
Reference in New Issue