diff --git a/gpt4all-chat/localdocs.cpp b/gpt4all-chat/localdocs.cpp index 6ad63046..13648dc2 100644 --- a/gpt4all-chat/localdocs.cpp +++ b/gpt4all-chat/localdocs.cpp @@ -179,7 +179,10 @@ const auto SELECT_COLLECTIONS_FROM_FOLDER_SQL = QLatin1String(R"( )"); const auto SELECT_COLLECTIONS_SQL = QLatin1String(R"( - select collection_name, folder_id from collections; + select c.collection_name, f.folder_path, f.id + from collections c + join folders f on c.folder_id = f.id + order by c.collection_name asc, f.folder_path asc; )"); bool addCollection(QSqlQuery &q, const QString &collection_name, int folder_id) @@ -222,13 +225,24 @@ bool selectCollectionsFromFolder(QSqlQuery &q, int folder_id, QList *co return true; } -bool selectAllFromCollections(QSqlQuery &q, QList> *collections) { +struct CollectionEntry { + QString collection; + QString folder_path; + int folder_id = -1; +}; + +bool selectAllFromCollections(QSqlQuery &q, QList *collections) { if (!q.prepare(SELECT_COLLECTIONS_SQL)) return false; if (!q.exec()) return false; - while (q.next()) - collections->append(qMakePair(q.value(0).toString(), q.value(1).toInt())); + while (q.next()) { + CollectionEntry e; + e.collection = q.value(0).toString(); + e.folder_path = q.value(1).toString(); + e.folder_id = q.value(0).toInt(); + collections->append(e); + } return true; } @@ -248,6 +262,10 @@ const auto SELECT_FOLDERS_FROM_ID_SQL = QLatin1String(R"( select folder_path from folders where id = ?; )"); +const auto SELECT_ALL_FOLDERPATHS_SQL = QLatin1String(R"( + select folder_path from folders; + )"); + const auto FOLDERS_SQL = QLatin1String(R"( create table folders(id integer primary key, folder_path varchar unique); )"); @@ -294,6 +312,16 @@ bool selectFolder(QSqlQuery &q, int id, QString *folder_path) { return true; } +bool selectAllFolderPaths(QSqlQuery &q, QList *folder_paths) { + if (!q.prepare(SELECT_ALL_FOLDERPATHS_SQL)) + return false; + if (!q.exec()) + return false; + while (q.next()) + folder_paths->append(q.value(0).toString()); + return true; +} + const auto INSERT_DOCUMENTS_SQL = QLatin1String(R"( insert into documents(folder_id, document_time, document_path) values(?, ?, ?); )"); @@ -658,6 +686,57 @@ void Database::start() if (err.type() != QSqlError::NoError) qWarning() << "ERROR: initializing db" << err.text(); } + addCurrentFolders(); +} + +void Database::addCurrentFolders() +{ +#if defined(DEBUG) + qDebug() << "addCurrentFolders"; +#endif + + QSqlQuery q; + QList collections; + if (!selectAllFromCollections(q, &collections)) { + qWarning() << "ERROR: Cannot select collections" << q.lastError(); + return; + } + + for (auto e : collections) + addFolder(e.collection, e.folder_path); +} + +void Database::updateCollectionList() +{ +#if defined(DEBUG) + qDebug() << "updateCollectionList"; +#endif + + QSqlQuery q; + QList collections; + if (!selectAllFromCollections(q, &collections)) { + qWarning() << "ERROR: Cannot select collections" << q.lastError(); + return; + } + + QList collectionList; + QString currentCollectionName; + CollectionInfo currentCollectionInfo; + + for (auto e : collections) { + if (e.collection != currentCollectionName) { + if (!currentCollectionInfo.name.isEmpty()) + collectionList.append(currentCollectionInfo); + currentCollectionName = e.collection; + currentCollectionInfo.name = e.collection; + currentCollectionInfo.folders.clear(); + } + currentCollectionInfo.folders.append(e.folder_path); + } + if (!currentCollectionInfo.name.isEmpty()) + collectionList.append(currentCollectionInfo); + + emit collectionListUpdated(collectionList); } void Database::addFolder(const QString &collection, const QString &path) @@ -701,6 +780,7 @@ void Database::addFolder(const QString &collection, const QString &path) return; scanDocuments(folder_id, path); + updateCollectionList(); } void Database::removeFolder(const QString &collection, const QString &path) @@ -785,6 +865,7 @@ void Database::removeFolderInternal(const QString &collection, int folder_id, co } removeFolderFromWatch(path); + updateCollectionList(); } bool Database::addFolderToWatch(const QString &path) @@ -848,28 +929,20 @@ void Database::cleanDB() // Scan all folders in db to make sure they still exist QSqlQuery q; - QList> collections; + QList collections; if (!selectAllFromCollections(q, &collections)) { qWarning() << "ERROR: Cannot select collections" << q.lastError(); return; } - for (auto pair : collections) { + for (auto e : collections) { // Find the path for the folder - QString collection = pair.first; - int folder_id = pair.second; - QString folder_path; - if (!selectFolder(q, folder_id, &folder_path)) { - qWarning() << "ERROR: Cannot select folder from id" << folder_id << q.lastError(); - return; - } - - QFileInfo info(folder_path); + QFileInfo info(e.folder_path); if (!info.exists() || !info.isReadable()) { #if defined(DEBUG) - qDebug() << "clean db removing folder" << folder_id << folder_path; + qDebug() << "clean db removing folder" << e.folder_id << e.folder_path; #endif - removeFolderInternal(collection, folder_id, folder_path); + removeFolderInternal(e.collection, e.folder_id, e.folder_path); } } @@ -905,6 +978,7 @@ void Database::cleanDB() qWarning() << "ERROR: Cannot remove document_id" << document_id << query.lastError(); } } + updateCollectionList(); } void Database::directoryChanged(const QString &path) @@ -955,7 +1029,7 @@ LocalDocs::LocalDocs() connect(this, &LocalDocs::requestRetrieveFromDB, m_database, &Database::retrieveFromDB, Qt::QueuedConnection); connect(m_database, &Database::retrieveResult, this, - &LocalDocs::retrieveResult, Qt::QueuedConnection); + &LocalDocs::handleRetrieveResult, Qt::QueuedConnection); addFolder("localdocs", "/home/atreat/dev/large_language_models/localdocs"); } @@ -976,8 +1050,14 @@ void LocalDocs::requestRetrieve(const QList &collections, const QString emit requestRetrieveFromDB(collections, text); } -void LocalDocs::retrieveResult(const QList &result) +void LocalDocs::handleRetrieveResult(const QList &result) { m_retrieveResult = result; emit receivedResult(); } + +void LocalDocs::handleCollectionListUpdated(const QList &collectionList) +{ + m_collectionList = collectionList; + emit collectionListChanged(); +} diff --git a/gpt4all-chat/localdocs.h b/gpt4all-chat/localdocs.h index 11d618c4..af81775c 100644 --- a/gpt4all-chat/localdocs.h +++ b/gpt4all-chat/localdocs.h @@ -14,6 +14,15 @@ struct DocumentInfo QFileInfo doc; }; +struct CollectionInfo { + Q_GADGET + Q_PROPERTY(QString name MEMBER name) +public: + QString name; + QList folders; +}; +Q_DECLARE_METATYPE(CollectionInfo) + class Database : public QObject { Q_OBJECT @@ -31,12 +40,15 @@ public Q_SLOTS: Q_SIGNALS: void docsToScanChanged(); void retrieveResult(const QList &result); + void collectionListUpdated(const QList &collectionList); private Q_SLOTS: void start(); void directoryChanged(const QString &path); bool addFolderToWatch(const QString &path); bool removeFolderFromWatch(const QString &path); + void addCurrentFolders(); + void updateCollectionList(); private: void removeFolderInternal(const QString &collection, int folder_id, const QString &path); @@ -54,10 +66,13 @@ private: class LocalDocs : public QObject { Q_OBJECT + Q_PROPERTY(QList collectionList READ collectionList NOTIFY collectionListChanged) public: static LocalDocs *globalInstance(); + QList collectionList() const { return m_collectionList; } + void addFolder(const QString &collection, const QString &path); void removeFolder(const QString &collection, const QString &path); @@ -69,13 +84,16 @@ Q_SIGNALS: void requestRemoveFolder(const QString &collection, const QString &path); void requestRetrieveFromDB(const QList &collections, const QString &text); void receivedResult(); + void collectionListChanged(); private Q_SLOTS: - void retrieveResult(const QList &result); + void handleRetrieveResult(const QList &result); + void handleCollectionListUpdated(const QList &collectionList); private: Database *m_database; QList m_retrieveResult; + QList m_collectionList; private: explicit LocalDocs();