Archives/gpt4all
2
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all synced 2024-11-02 09:40:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a collection list to support a UI.

Browse Source
This commit is contained in:
Adam Treat 2023-05-21 15:13:28 -04:00 committed by AT
parent 68ba9c564b
commit d9eddbec45
2 changed files with 118 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
gpt4all-chat/localdocs.cpp
View File

@ -179,7 +179,10 @@ const auto SELECT_COLLECTIONS_FROM_FOLDER_SQL = QLatin1String(R"(
)");
const auto SELECT_COLLECTIONS_SQL = QLatin1String(R"(
select collection_name, folder_id from collections;
select c.collection_name, f.folder_path, f.id
from collections c
join folders f on c.folder_id = f.id
order by c.collection_name asc, f.folder_path asc;
)");
bool addCollection(QSqlQuery &q, const QString &collection_name, int folder_id)
@ -222,13 +225,24 @@ bool selectCollectionsFromFolder(QSqlQuery &q, int folder_id, QList<QString> *co
return true;
}
bool selectAllFromCollections(QSqlQuery &q, QList<QPair<QString, int>> *collections) {
struct CollectionEntry {
QString collection;
QString folder_path;
int folder_id = -1;
};
bool selectAllFromCollections(QSqlQuery &q, QList<CollectionEntry> *collections) {
if (!q.prepare(SELECT_COLLECTIONS_SQL))
return false;
if (!q.exec())
return false;
while (q.next())
collections->append(qMakePair(q.value(0).toString(), q.value(1).toInt()));
while (q.next()) {
CollectionEntry e;
e.collection = q.value(0).toString();
e.folder_path = q.value(1).toString();
e.folder_id = q.value(0).toInt();
collections->append(e);
}
return true;
}
@ -248,6 +262,10 @@ const auto SELECT_FOLDERS_FROM_ID_SQL = QLatin1String(R"(
select folder_path from folders where id = ?;
)");
const auto SELECT_ALL_FOLDERPATHS_SQL = QLatin1String(R"(
select folder_path from folders;
)");
const auto FOLDERS_SQL = QLatin1String(R"(
create table folders(id integer primary key, folder_path varchar unique);
)");
@ -294,6 +312,16 @@ bool selectFolder(QSqlQuery &q, int id, QString *folder_path) {
return true;
}
bool selectAllFolderPaths(QSqlQuery &q, QList<QString> *folder_paths) {
if (!q.prepare(SELECT_ALL_FOLDERPATHS_SQL))
return false;
if (!q.exec())
return false;
while (q.next())
folder_paths->append(q.value(0).toString());
return true;
}
const auto INSERT_DOCUMENTS_SQL = QLatin1String(R"(
insert into documents(folder_id, document_time, document_path) values(?, ?, ?);
)");
@ -658,6 +686,57 @@ void Database::start()
if (err.type() != QSqlError::NoError)
qWarning() << "ERROR: initializing db" << err.text();
}
addCurrentFolders();
}
void Database::addCurrentFolders()
{
#if defined(DEBUG)
qDebug() << "addCurrentFolders";
#endif
QSqlQuery q;
QList<CollectionEntry> collections;
if (!selectAllFromCollections(q, &collections)) {
qWarning() << "ERROR: Cannot select collections" << q.lastError();
return;
}
for (auto e : collections)
addFolder(e.collection, e.folder_path);
}
void Database::updateCollectionList()
{
#if defined(DEBUG)
qDebug() << "updateCollectionList";
#endif
QSqlQuery q;
QList<CollectionEntry> collections;
if (!selectAllFromCollections(q, &collections)) {
qWarning() << "ERROR: Cannot select collections" << q.lastError();
return;
}
QList<CollectionInfo> collectionList;
QString currentCollectionName;
CollectionInfo currentCollectionInfo;
for (auto e : collections) {
if (e.collection != currentCollectionName) {
if (!currentCollectionInfo.name.isEmpty())
collectionList.append(currentCollectionInfo);
currentCollectionName = e.collection;
currentCollectionInfo.name = e.collection;
currentCollectionInfo.folders.clear();
}
currentCollectionInfo.folders.append(e.folder_path);
}
if (!currentCollectionInfo.name.isEmpty())
collectionList.append(currentCollectionInfo);
emit collectionListUpdated(collectionList);
}
void Database::addFolder(const QString &collection, const QString &path)
@ -701,6 +780,7 @@ void Database::addFolder(const QString &collection, const QString &path)
return;
scanDocuments(folder_id, path);
updateCollectionList();
}
void Database::removeFolder(const QString &collection, const QString &path)
@ -785,6 +865,7 @@ void Database::removeFolderInternal(const QString &collection, int folder_id, co
}
removeFolderFromWatch(path);
updateCollectionList();
}
bool Database::addFolderToWatch(const QString &path)
@ -848,28 +929,20 @@ void Database::cleanDB()
// Scan all folders in db to make sure they still exist
QSqlQuery q;
QList<QPair<QString, int>> collections;
QList<CollectionEntry> collections;
if (!selectAllFromCollections(q, &collections)) {
qWarning() << "ERROR: Cannot select collections" << q.lastError();
return;
}
for (auto pair : collections) {
for (auto e : collections) {
// Find the path for the folder
QString collection = pair.first;
int folder_id = pair.second;
QString folder_path;
if (!selectFolder(q, folder_id, &folder_path)) {
qWarning() << "ERROR: Cannot select folder from id" << folder_id << q.lastError();
return;
}
QFileInfo info(folder_path);
QFileInfo info(e.folder_path);
if (!info.exists() || !info.isReadable()) {
#if defined(DEBUG)
qDebug() << "clean db removing folder" << folder_id << folder_path;
qDebug() << "clean db removing folder" << e.folder_id << e.folder_path;
#endif
removeFolderInternal(collection, folder_id, folder_path);
removeFolderInternal(e.collection, e.folder_id, e.folder_path);
}
}
@ -905,6 +978,7 @@ void Database::cleanDB()
qWarning() << "ERROR: Cannot remove document_id" << document_id << query.lastError();
}
}
updateCollectionList();
}
void Database::directoryChanged(const QString &path)
@ -955,7 +1029,7 @@ LocalDocs::LocalDocs()
connect(this, &LocalDocs::requestRetrieveFromDB, m_database,
&Database::retrieveFromDB, Qt::QueuedConnection);
connect(m_database, &Database::retrieveResult, this,
&LocalDocs::retrieveResult, Qt::QueuedConnection);
&LocalDocs::handleRetrieveResult, Qt::QueuedConnection);
addFolder("localdocs", "/home/atreat/dev/large_language_models/localdocs");
}
@ -976,8 +1050,14 @@ void LocalDocs::requestRetrieve(const QList<QString> &collections, const QString
emit requestRetrieveFromDB(collections, text);
}
void LocalDocs::retrieveResult(const QList<QString> &result)
void LocalDocs::handleRetrieveResult(const QList<QString> &result)
{
m_retrieveResult = result;
emit receivedResult();
}
void LocalDocs::handleCollectionListUpdated(const QList<CollectionInfo> &collectionList)
{
m_collectionList = collectionList;
emit collectionListChanged();
}

20
gpt4all-chat/localdocs.h
View File

@ -14,6 +14,15 @@ struct DocumentInfo
QFileInfo doc;
};
struct CollectionInfo {
Q_GADGET
Q_PROPERTY(QString name MEMBER name)
public:
QString name;
QList<QString> folders;
};
Q_DECLARE_METATYPE(CollectionInfo)
class Database : public QObject
{
Q_OBJECT
@ -31,12 +40,15 @@ public Q_SLOTS:
Q_SIGNALS:
void docsToScanChanged();
void retrieveResult(const QList<QString> &result);
void collectionListUpdated(const QList<CollectionInfo> &collectionList);
private Q_SLOTS:
void start();
void directoryChanged(const QString &path);
bool addFolderToWatch(const QString &path);
bool removeFolderFromWatch(const QString &path);
void addCurrentFolders();
void updateCollectionList();
private:
void removeFolderInternal(const QString &collection, int folder_id, const QString &path);
@ -54,10 +66,13 @@ private:
class LocalDocs : public QObject
{
Q_OBJECT
Q_PROPERTY(QList<CollectionInfo> collectionList READ collectionList NOTIFY collectionListChanged)
public:
static LocalDocs *globalInstance();
QList<CollectionInfo> collectionList() const { return m_collectionList; }
void addFolder(const QString &collection, const QString &path);
void removeFolder(const QString &collection, const QString &path);
@ -69,13 +84,16 @@ Q_SIGNALS:
void requestRemoveFolder(const QString &collection, const QString &path);
void requestRetrieveFromDB(const QList<QString> &collections, const QString &text);
void receivedResult();
void collectionListChanged();
private Q_SLOTS:
void retrieveResult(const QList<QString> &result);
void handleRetrieveResult(const QList<QString> &result);
void handleCollectionListUpdated(const QList<CollectionInfo> &collectionList);
private:
Database *m_database;
QList<QString> m_retrieveResult;
QList<CollectionInfo> m_collectionList;
private:
explicit LocalDocs();