gpt4all/gpt4all-chat/localdocs.cpp

#include "localdocs.h"
#include "download.h"

#include <QTimer>
#include <QPdfDocument>

//#define DEBUG
//#define DEBUG_EXAMPLE

#define LOCALDOCS_VERSION 0

const auto INSERT_CHUNK_SQL = QLatin1String(R"(
    insert into chunks(document_id, chunk_id, chunk_text,
        embedding_id, embedding_path) values(?, ?, ?, ?, ?);
    )");

const auto INSERT_CHUNK_FTS_SQL = QLatin1String(R"(
    insert into chunks_fts(document_id, chunk_id, chunk_text,
        embedding_id, embedding_path) values(?, ?, ?, ?, ?);
    )");

const auto DELETE_CHUNKS_SQL = QLatin1String(R"(
    delete from chunks WHERE document_id = ?;
    )");

const auto DELETE_CHUNKS_FTS_SQL = QLatin1String(R"(
    delete from chunks_fts WHERE document_id = ?;
    )");

const auto CHUNKS_SQL = QLatin1String(R"(
    create table chunks(document_id integer, chunk_id integer, chunk_text varchar,
        embedding_id integer, embedding_path varchar);
    )");

const auto FTS_CHUNKS_SQL = QLatin1String(R"(
    create virtual table chunks_fts using fts5(document_id unindexed, chunk_id unindexed, chunk_text,
        embedding_id unindexed, embedding_path unindexed, tokenize="trigram");
    )");

const auto SELECT_SQL = QLatin1String(R"(
    select chunks_fts.rowid, chunks_fts.document_id, chunks_fts.chunk_text
    from chunks_fts
    join documents ON chunks_fts.document_id = documents.id
    join folders ON documents.folder_id = folders.id
    join collections ON folders.id = collections.folder_id
    where chunks_fts match ? and collections.collection_name in (%1)
    order by bm25(chunks_fts) desc
    limit 3;
    )");

bool addChunk(QSqlQuery &q, int document_id, int chunk_id, const QString &chunk_text, int embedding_id,
    const QString &embedding_path)
{
    {
        if (!q.prepare(INSERT_CHUNK_SQL))
            return false;
        q.addBindValue(document_id);
        q.addBindValue(chunk_id);
        q.addBindValue(chunk_text);
        q.addBindValue(embedding_id);
        q.addBindValue(embedding_path);
        if (!q.exec())
            return false;
    }
    {
        if (!q.prepare(INSERT_CHUNK_FTS_SQL))
            return false;
        q.addBindValue(document_id);
        q.addBindValue(chunk_id);
        q.addBindValue(chunk_text);
        q.addBindValue(embedding_id);
        q.addBindValue(embedding_path);
        if (!q.exec())
            return false;
    }
    return true;
}

bool removeChunksByDocumentId(QSqlQuery &q, int document_id)
{
    {
        if (!q.prepare(DELETE_CHUNKS_SQL))
            return false;
        q.addBindValue(document_id);
        if (!q.exec())
            return false;
    }

    {
        if (!q.prepare(DELETE_CHUNKS_FTS_SQL))
            return false;
        q.addBindValue(document_id);
        if (!q.exec())
            return false;
    }

    return true;
}

QStringList generateGrams(const QString &input, int N)
{
    // Remove common English punctuation using QRegularExpression
    QRegularExpression punctuation(R"([.,;:!?'"()\-])");
    QString cleanedInput = input;
    cleanedInput = cleanedInput.remove(punctuation);

    // Split the cleaned input into words using whitespace
    QStringList words = cleanedInput.split(QRegularExpression("\\s+"), Qt::SkipEmptyParts);
    N = qMin(words.size(), N);

    // Generate all possible N-grams
    QStringList ngrams;
    for (int i = 0; i < words.size() - (N - 1); ++i) {
        QStringList currentNgram;
        for (int j = 0; j < N; ++j) {
            currentNgram.append("\"" + words[i + j] + "\"");
        }
        ngrams.append("NEAR(" + currentNgram.join(" ") + ", " + QString::number(N) + ")");
    }
    return ngrams;
}

bool selectChunk(QSqlQuery &q, const QList<QString> &collection_names, const QString &chunk_text)
{
    const int N_WORDS = chunk_text.split(QRegularExpression("\\s+")).size();
    for (int N = N_WORDS; N > 2; N--) {
        // first try trigrams
        QList<QString> text = generateGrams(chunk_text, N);
        QString orText = text.join(" OR ");
        const QString collection_names_str = collection_names.join("', '");
        const QString formatted_query = SELECT_SQL.arg("'" + collection_names_str + "'");
        if (!q.prepare(formatted_query))
            return false;
        q.addBindValue(orText);
        bool success = q.exec();
        if (!success) return false;
        if (q.next()) {
#if defined(DEBUG)
            qDebug() << "hit on" << N << "before" << chunk_text << "after" << orText;
#endif
            q.previous();
            return true;
        }
    }
    return true;
}

void printResults(QSqlQuery &q)
{
    while (q.next()) {
        int rowid = q.value(0).toInt();
        QString collection_name = q.value(1).toString();
        QString chunk_text = q.value(2).toString();

        qDebug() << "rowid:" << rowid
                 << "collection_name:" << collection_name
                 << "chunk_text:" << chunk_text;
    }
}

const auto INSERT_COLLECTION_SQL = QLatin1String(R"(
    insert into collections(collection_name, folder_id) values(?, ?);
    )");

const auto DELETE_COLLECTION_SQL = QLatin1String(R"(
    delete from collections where collection_name = ? and folder_id = ?;
    )");

const auto COLLECTIONS_SQL = QLatin1String(R"(
    create table collections(collection_name varchar, folder_id integer, unique(collection_name, folder_id));
    )");

const auto SELECT_FOLDERS_FROM_COLLECTIONS_SQL = QLatin1String(R"(
    select folder_id from collections where collection_name = ?;
    )");

const auto SELECT_COLLECTIONS_FROM_FOLDER_SQL = QLatin1String(R"(
    select collection_name from collections where folder_id = ?;
    )");

const auto SELECT_COLLECTIONS_SQL = QLatin1String(R"(
    select c.collection_name, f.folder_path, f.id
    from collections c
    join folders f on c.folder_id = f.id
    order by c.collection_name asc, f.folder_path asc;
    )");

bool addCollection(QSqlQuery &q, const QString &collection_name, int folder_id)
{
    if (!q.prepare(INSERT_COLLECTION_SQL))
        return false;
    q.addBindValue(collection_name);
    q.addBindValue(folder_id);
    return q.exec();
}

bool removeCollection(QSqlQuery &q, const QString &collection_name, int folder_id)
{
    if (!q.prepare(DELETE_COLLECTION_SQL))
        return false;
    q.addBindValue(collection_name);
    q.addBindValue(folder_id);
    return q.exec();
}

bool selectFoldersFromCollection(QSqlQuery &q, const QString &collection_name, QList<int> *folderIds) {
    if (!q.prepare(SELECT_FOLDERS_FROM_COLLECTIONS_SQL))
        return false;
    q.addBindValue(collection_name);
    if (!q.exec())
        return false;
    while (q.next())
        folderIds->append(q.value(0).toInt());
    return true;
}

bool selectCollectionsFromFolder(QSqlQuery &q, int folder_id, QList<QString> *collections) {
    if (!q.prepare(SELECT_COLLECTIONS_FROM_FOLDER_SQL))
        return false;
    q.addBindValue(folder_id);
    if (!q.exec())
        return false;
    while (q.next())
        collections->append(q.value(0).toString());
    return true;
}

struct CollectionEntry {
    QString collection;
    QString folder_path;
    int folder_id = -1;
};

bool selectAllFromCollections(QSqlQuery &q, QList<CollectionEntry> *collections) {
    if (!q.prepare(SELECT_COLLECTIONS_SQL))
        return false;
    if (!q.exec())
        return false;
    while (q.next()) {
        CollectionEntry e;
        e.collection = q.value(0).toString();
        e.folder_path = q.value(1).toString();
        e.folder_id = q.value(0).toInt();
        collections->append(e);
    }
    return true;
}

const auto INSERT_FOLDERS_SQL = QLatin1String(R"(
    insert into folders(folder_path) values(?);
    )");

const auto DELETE_FOLDERS_SQL = QLatin1String(R"(
    delete from folders where id = ?;
    )");

const auto SELECT_FOLDERS_FROM_PATH_SQL = QLatin1String(R"(
    select id from folders where folder_path = ?;
    )");

const auto SELECT_FOLDERS_FROM_ID_SQL = QLatin1String(R"(
    select folder_path from folders where id = ?;
    )");

const auto SELECT_ALL_FOLDERPATHS_SQL = QLatin1String(R"(
    select folder_path from folders;
    )");

const auto FOLDERS_SQL = QLatin1String(R"(
    create table folders(id integer primary key, folder_path varchar unique);
    )");

bool addFolderToDB(QSqlQuery &q, const QString &folder_path, int *folder_id)
{
    if (!q.prepare(INSERT_FOLDERS_SQL))
        return false;
    q.addBindValue(folder_path);
    if (!q.exec())
        return false;
    *folder_id = q.lastInsertId().toInt();
    return true;
}

bool removeFolderFromDB(QSqlQuery &q, int folder_id) {
    if (!q.prepare(DELETE_FOLDERS_SQL))
        return false;
    q.addBindValue(folder_id);
    return q.exec();
}

bool selectFolder(QSqlQuery &q, const QString &folder_path, int *id) {
    if (!q.prepare(SELECT_FOLDERS_FROM_PATH_SQL))
        return false;
    q.addBindValue(folder_path);
    if (!q.exec())
        return false;
    Q_ASSERT(q.size() < 2);
    if (q.next())
        *id = q.value(0).toInt();
    return true;
}

bool selectFolder(QSqlQuery &q, int id, QString *folder_path) {
    if (!q.prepare(SELECT_FOLDERS_FROM_ID_SQL))
        return false;
    q.addBindValue(id);
    if (!q.exec())
        return false;
    Q_ASSERT(q.size() < 2);
    if (q.next())
        *folder_path = q.value(0).toString();
    return true;
}

bool selectAllFolderPaths(QSqlQuery &q, QList<QString> *folder_paths) {
    if (!q.prepare(SELECT_ALL_FOLDERPATHS_SQL))
        return false;
    if (!q.exec())
        return false;
    while (q.next())
        folder_paths->append(q.value(0).toString());
    return true;
}

const auto INSERT_DOCUMENTS_SQL = QLatin1String(R"(
    insert into documents(folder_id, document_time, document_path) values(?, ?, ?);
    )");

const auto UPDATE_DOCUMENT_TIME_SQL = QLatin1String(R"(
    update documents set document_time = ? where id = ?;
    )");

const auto DELETE_DOCUMENTS_SQL = QLatin1String(R"(
    delete from documents where id = ?;
    )");

const auto DOCUMENTS_SQL = QLatin1String(R"(
    create table documents(id integer primary key, folder_id integer, document_time integer, document_path varchar unique);
    )");

const auto SELECT_DOCUMENT_SQL = QLatin1String(R"(
    select id, document_time from documents where document_path = ?;
    )");

const auto SELECT_DOCUMENTS_SQL = QLatin1String(R"(
    select id from documents where folder_id = ?;
    )");

const auto SELECT_ALL_DOCUMENTS_SQL = QLatin1String(R"(
    select id, document_path from documents;
    )");

bool addDocument(QSqlQuery &q, int folder_id, qint64 document_time, const QString &document_path, int *document_id)
{
    if (!q.prepare(INSERT_DOCUMENTS_SQL))
        return false;
    q.addBindValue(folder_id);
    q.addBindValue(document_time);
    q.addBindValue(document_path);
    if (!q.exec())
        return false;
    *document_id = q.lastInsertId().toInt();
    return true;
}

bool removeDocument(QSqlQuery &q, int document_id) {
    if (!q.prepare(DELETE_DOCUMENTS_SQL))
        return false;
    q.addBindValue(document_id);
    return q.exec();
}

bool updateDocument(QSqlQuery &q, int id, qint64 document_time)
{
    if (!q.prepare(UPDATE_DOCUMENT_TIME_SQL))
        return false;
    q.addBindValue(id);
    q.addBindValue(document_time);
    return q.exec();
}

bool selectDocument(QSqlQuery &q, const QString &document_path, int *id, qint64 *document_time) {
    if (!q.prepare(SELECT_DOCUMENT_SQL))
        return false;
    q.addBindValue(document_path);
    if (!q.exec())
        return false;
    Q_ASSERT(q.size() < 2);
    if (q.next()) {
        *id = q.value(0).toInt();
        *document_time = q.value(1).toLongLong();
    }
    return true;
}

bool selectDocuments(QSqlQuery &q, int folder_id, QList<int> *documentIds) {
    if (!q.prepare(SELECT_DOCUMENTS_SQL))
        return false;
    q.addBindValue(folder_id);
    if (!q.exec())
        return false;
    while (q.next())
        documentIds->append(q.value(0).toInt());
    return true;
}

QSqlError initDb()
{
    QString dbPath = Download::globalInstance()->downloadLocalModelsPath()
        + QString("localdocs_v%1.db").arg(LOCALDOCS_VERSION);
    QSqlDatabase db = QSqlDatabase::addDatabase("QSQLITE");
    db.setDatabaseName(dbPath);

    if (!db.open())
        return db.lastError();

    QStringList tables = db.tables();
    if (tables.contains("chunks", Qt::CaseInsensitive))
        return QSqlError();

    QSqlQuery q;
    if (!q.exec(CHUNKS_SQL))
        return q.lastError();

    if (!q.exec(FTS_CHUNKS_SQL))
        return q.lastError();

    if (!q.exec(COLLECTIONS_SQL))
        return q.lastError();

    if (!q.exec(FOLDERS_SQL))
        return q.lastError();

    if (!q.exec(DOCUMENTS_SQL))
        return q.lastError();

#if defined(DEBUG_EXAMPLE)
    // Add a folder
    QString folder_path = "/example/folder";
    int folder_id;
    if (!addFolderToDB(q, folder_path, &folder_id)) {
        qDebug() << "Error adding folder:" << q.lastError().text();
        return q.lastError();
    }

    // Add a collection
    QString collection_name = "Example Collection";
    if (!addCollection(q, collection_name, folder_id)) {
        qDebug() << "Error adding collection:" << q.lastError().text();
        return q.lastError();
    }

    // Add a document
    int document_time = 123456789;
    int document_id;
    QString document_path = "/example/folder/document1.txt";
    if (!addDocument(q, folder_id, document_time, document_path, &document_id)) {
        qDebug() << "Error adding document:" << q.lastError().text();
        return q.lastError();
    }

    // Add chunks to the document
    QString chunk_text1 = "This is an example chunk.";
    QString chunk_text2 = "Another example chunk.";
    QString embedding_path = "/example/embeddings/embedding1.bin";
    int embedding_id = 1;

    if (!addChunk(q, document_id, 1, chunk_text1, embedding_id, embedding_path) ||
        !addChunk(q, document_id, 2, chunk_text2, embedding_id, embedding_path)) {
        qDebug() << "Error adding chunks:" << q.lastError().text();
        return q.lastError();
    }

    // Perform a search
    QList<QString> collection_names = {collection_name};
    QString search_text = "example";
    if (!selectChunk(q, collection_names, search_text)) {
        qDebug() << "Error selecting chunks:" << q.lastError().text();
        return q.lastError();
    }

    // Print the results
    printResults(q);
#endif

    return QSqlError();
}

Database::Database()
    : QObject(nullptr)
    , m_watcher(new QFileSystemWatcher(this))
{
    moveToThread(&m_dbThread);
    connect(&m_dbThread, &QThread::started, this, &Database::start);
    m_dbThread.setObjectName("database");
    m_dbThread.start();
}

void Database::handleDocumentErrorAndScheduleNext(const QString &errorMessage,
    int document_id, const QString &document_path, const QSqlError &error)
{
    qWarning() << errorMessage << document_id << document_path << error.text();
    if (!m_docsToScan.isEmpty())
        QTimer::singleShot(0, this, &Database::scanQueue);
}

void Database::chunkStream(QTextStream &stream, int document_id)
{
    const int chunkSize = 256;
    int chunk_id = 0;
    int charCount = 0;
    QList<QString> words;

    while (!stream.atEnd()) {
        QString word;
        stream >> word;
        charCount += word.length();
        words.append(word);
        if (charCount + words.size() - 1 >= chunkSize || stream.atEnd()) {
            const QString chunk = words.join(" ");
            QSqlQuery q;
            if (!addChunk(q,
                document_id,
                ++chunk_id,
                chunk,
                0 /*embedding_id*/,
                QString() /*embedding_path*/
            )) {
                qWarning() << "ERROR: Could not insert chunk into db" << q.lastError();
            }
            words.clear();
            charCount = 0;
        }
    }
}

void Database::scanQueue()
{
    if (m_docsToScan.isEmpty())
        return;

    DocumentInfo info = m_docsToScan.dequeue();

    // Update info
    info.doc.stat();

    // If the doc has since been deleted or no longer readable, then we schedule more work and return
    // leaving the cleanup for the cleanup handler
    if (!info.doc.exists() || !info.doc.isReadable()) {
        if (!m_docsToScan.isEmpty()) QTimer::singleShot(0, this, &Database::scanQueue);
        return;
    }

    const int folder_id = info.folder;
    const qint64 document_time = info.doc.fileTime(QFile::FileModificationTime).toMSecsSinceEpoch();
    const QString document_path = info.doc.canonicalFilePath();

#if defined(DEBUG)
    qDebug() << "scanning document" << document_path;
#endif

    // Check and see if we already have this document
    QSqlQuery q;
    int existing_id = -1;
    qint64 existing_time = -1;
    if (!selectDocument(q, document_path, &existing_id, &existing_time)) {
        return handleDocumentErrorAndScheduleNext("ERROR: Cannot select document",
            existing_id, document_path, q.lastError());
    }

    // If we have the document, we need to compare the last modification time and if it is newer
    // we must rescan the document, otherwise return
    if (existing_id != -1) {
        Q_ASSERT(existing_time != -1);
        if (document_time == existing_time) {
            // No need to rescan, but we do have to schedule next
            if (!m_docsToScan.isEmpty()) QTimer::singleShot(0, this, &Database::scanQueue);
            return;
        } else {
            if (!removeChunksByDocumentId(q, existing_id)) {
                return handleDocumentErrorAndScheduleNext("ERROR: Cannot remove chunks of document",
                    existing_id, document_path, q.lastError());
            }
        }
    }

    // Update the document_time for an existing document, or add it for the first time now
    int document_id = existing_id;
    if (document_id != -1) {
        if (!updateDocument(q, document_id, document_time)) {
            return handleDocumentErrorAndScheduleNext("ERROR: Could not update document_time",
                document_id, document_path, q.lastError());
        }
    } else {
        if (!addDocument(q, folder_id, document_time, document_path, &document_id)) {
            return handleDocumentErrorAndScheduleNext("ERROR: Could not add document",
                document_id, document_path, q.lastError());
        }
    }

    QElapsedTimer timer;
    timer.start();

    QSqlDatabase::database().transaction();
    Q_ASSERT(document_id != -1);
    if (info.doc.suffix() == QLatin1String("pdf")) {
        QPdfDocument doc;
        if (QPdfDocument::Error::None != doc.load(info.doc.canonicalFilePath())) {
            return handleDocumentErrorAndScheduleNext("ERROR: Could not load pdf",
                document_id, document_path, q.lastError());
            return;
        }
        QString text;
        for (int i = 0; i < doc.pageCount(); ++i) {
            const QPdfSelection selection = doc.getAllText(i);
            text.append(selection.text());
        }
        QTextStream stream(&text);
        chunkStream(stream, document_id);
    } else {
        QFile file(document_path);
        if (!file.open( QIODevice::ReadOnly)) {
            return handleDocumentErrorAndScheduleNext("ERROR: Cannot open file for scanning",
                                                      existing_id, document_path, q.lastError());
        }
        QTextStream stream(&file);
        chunkStream(stream, document_id);
        file.close();
    }
    QSqlDatabase::database().commit();

#if defined(DEBUG)
    qDebug() << "chunking" << document_path << "took" << timer.elapsed() << "ms";
#endif

    if (!m_docsToScan.isEmpty()) QTimer::singleShot(0, this, &Database::scanQueue);
}

void Database::scanDocuments(int folder_id, const QString &folder_path)
{
#if defined(DEBUG)
    qDebug() << "scanning folder for documents" << folder_path;
#endif

    static const QList<QString> extensions { "txt", "doc", "docx", "pdf", "rtf", "odt", "html", "htm",
    "xls", "xlsx", "csv", "ods", "ppt", "pptx", "odp", "xml", "json", "log", "md", "tex", "asc", "wks",
    "wpd", "wps", "wri", "xhtml", "xht", "xslt", "yaml", "yml", "dtd", "sgml", "tsv", "strings", "resx",
    "plist", "properties", "ini", "config", "bat", "sh", "ps1", "cmd", "awk", "sed", "vbs", "ics", "mht",
    "mhtml", "epub", "djvu", "azw", "azw3", "mobi", "fb2", "prc", "lit", "lrf", "tcr", "pdb", "oxps",
    "xps", "pages", "numbers", "key", "keynote", "abw", "zabw", "123", "wk1", "wk3", "wk4", "wk5", "wq1",
    "wq2", "xlw", "xlr", "dif", "slk", "sylk", "wb1", "wb2", "wb3", "qpw", "wdb", "wks", "wku", "wr1",
    "wrk", "xlk", "xlt", "xltm", "xltx", "xlsm", "xla", "xlam", "xll", "xld", "xlv", "xlw", "xlc", "xlm",
    "xlt", "xln" };

    QDir dir(folder_path);
    Q_ASSERT(dir.exists());
    Q_ASSERT(dir.isReadable());
    QDirIterator it(folder_path, QDir::Readable | QDir::Files, QDirIterator::Subdirectories);
    while (it.hasNext()) {
        it.next();
        QFileInfo fileInfo = it.fileInfo();
        if (fileInfo.isDir()) {
            addFolderToWatch(fileInfo.canonicalFilePath());
            continue;
        }

        if (!extensions.contains(fileInfo.suffix()))
            continue;

        DocumentInfo info;
        info.folder = folder_id;
        info.doc = fileInfo;
        m_docsToScan.enqueue(info);
    }
    emit docsToScanChanged();
}

void Database::start()
{
    connect(m_watcher, &QFileSystemWatcher::directoryChanged, this, &Database::directoryChanged);
    connect(this, &Database::docsToScanChanged, this, &Database::scanQueue);
    if (!QSqlDatabase::drivers().contains("QSQLITE")) {
        qWarning() << "ERROR: missing sqllite driver";
    } else {
        QSqlError err = initDb();
        if (err.type() != QSqlError::NoError)
            qWarning() << "ERROR: initializing db" << err.text();
    }
    addCurrentFolders();
}

void Database::addCurrentFolders()
{
#if defined(DEBUG)
    qDebug() << "addCurrentFolders";
#endif

    QSqlQuery q;
    QList<CollectionEntry> collections;
    if (!selectAllFromCollections(q, &collections)) {
        qWarning() << "ERROR: Cannot select collections" << q.lastError();
        return;
    }

    for (auto e : collections)
        addFolder(e.collection, e.folder_path);
}

void Database::updateCollectionList()
{
#if defined(DEBUG)
    qDebug() << "updateCollectionList";
#endif

    QSqlQuery q;
    QList<CollectionEntry> collections;
    if (!selectAllFromCollections(q, &collections)) {
        qWarning() << "ERROR: Cannot select collections" << q.lastError();
        return;
    }

    QList<CollectionInfo> collectionList;
    QString currentCollectionName;
    CollectionInfo currentCollectionInfo;

    for (auto e : collections) {
        if (e.collection != currentCollectionName) {
            if (!currentCollectionInfo.name.isEmpty())
                collectionList.append(currentCollectionInfo);
            currentCollectionName = e.collection;
            currentCollectionInfo.name = e.collection;
            currentCollectionInfo.folders.clear();
        }
        currentCollectionInfo.folders.append(e.folder_path);
    }
    if (!currentCollectionInfo.name.isEmpty())
        collectionList.append(currentCollectionInfo);

    emit collectionListUpdated(collectionList);
}

void Database::addFolder(const QString &collection, const QString &path)
{
    QFileInfo info(path);
    if (!info.exists() || !info.isReadable()) {
        qWarning() << "ERROR: Cannot add folder that doesn't exist or not readable" << path;
        return;
    }

    QSqlQuery q;
    int folder_id = -1;

    // See if the folder exists in the db
    if (!selectFolder(q, path, &folder_id)) {
        qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError();
        return;
    }

    // Add the folder
    if (folder_id == -1 && !addFolderToDB(q, path, &folder_id)) {
        qWarning() << "ERROR: Cannot add folder to db with path" << path << q.lastError();
        return;
    }

    Q_ASSERT(folder_id != -1);

    // See if the folder has already been added to the collection
    QList<int> folders;
    if (!selectFoldersFromCollection(q, collection, &folders)) {
        qWarning() << "ERROR: Cannot select folders from collections" << collection << q.lastError();
        return;
    }

    if (!folders.contains(folder_id) && !addCollection(q, collection, folder_id)) {
        qWarning() << "ERROR: Cannot add folder to collection" << collection << path << q.lastError();
        return;
    }

    if (!addFolderToWatch(path))
        return;

    scanDocuments(folder_id, path);
    updateCollectionList();
}

void Database::removeFolder(const QString &collection, const QString &path)
{
#if defined(DEBUG)
    qDebug() << "removeFolder" << path;
#endif

    QSqlQuery q;
    int folder_id = -1;

    // See if the folder exists in the db
    if (!selectFolder(q, path, &folder_id)) {
        qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError();
        return;
    }

    // If we don't have a folder_id in the db, then something bad has happened
    Q_ASSERT(folder_id != -1);
    if (folder_id == -1) {
        qWarning() << "ERROR: Collected folder does not exist in db" << path;
        m_watcher->removePath(path);
        return;
    }

    removeFolderInternal(collection, folder_id, path);
}

void Database::removeFolderInternal(const QString &collection, int folder_id, const QString &path)
{
    // Determine if the folder is used by more than one collection
    QSqlQuery q;
    QList<QString> collections;
    if (!selectCollectionsFromFolder(q, folder_id, &collections)) {
        qWarning() << "ERROR: Cannot select collections from folder" << folder_id << q.lastError();
        return;
    }

    // Remove it from the collections
    if (!removeCollection(q, collection, folder_id)) {
        qWarning() << "ERROR: Cannot remove collection" << collection << folder_id << q.lastError();
        return;
    }

    // If the folder is associated with more than one collection, then return
    if (collections.count() > 1)
        return;

    // First remove all upcoming jobs associated with this folder by performing an opt-in filter
    QQueue<DocumentInfo> docsToScan;
    for (DocumentInfo info : m_docsToScan) {
        if (info.folder == folder_id)
            continue;
        docsToScan.append(info);
    }
    m_docsToScan = docsToScan;
    emit docsToScanChanged();

    // Get a list of all documents associated with folder
    QList<int> documentIds;
    if (!selectDocuments(q, folder_id, &documentIds)) {
        qWarning() << "ERROR: Cannot select documents" << folder_id << q.lastError();
        return;
    }

    // Remove all chunks and documents associated with this folder
    for (int document_id : documentIds) {
        if (!removeChunksByDocumentId(q, document_id)) {
            qWarning() << "ERROR: Cannot remove chunks of document_id" << document_id << q.lastError();
            return;
        }

        if (!removeDocument(q, document_id)) {
            qWarning() << "ERROR: Cannot remove document_id" << document_id << q.lastError();
            return;
        }
    }

    if (!removeFolderFromDB(q, folder_id)) {
        qWarning() << "ERROR: Cannot remove folder_id" << folder_id << q.lastError();
        return;
    }

    removeFolderFromWatch(path);
    updateCollectionList();
}

bool Database::addFolderToWatch(const QString &path)
{
#if defined(DEBUG)
    qDebug() << "addFolderToWatch" << path;
#endif

    if (!m_watcher->addPath(path)) {
        qWarning() << "ERROR: Cannot add path to file watcher:" << path;
        return false;
    }
    return true;
}

bool Database::removeFolderFromWatch(const QString &path)
{
#if defined(DEBUG)
    qDebug() << "removeFolderFromWatch" << path;
#endif
    if (!m_watcher->removePath(path)) {
        qWarning() << "ERROR: Cannot remove path from file watcher:" << path;
        return false;
    }
    return true;
}

void Database::retrieveFromDB(const QList<QString> &collections, const QString &text)
{
#if defined(DEBUG)
    qDebug() << "retrieveFromDB" << collections << text;
#endif

    QSqlQuery q;
    if (!selectChunk(q, collections, text)) {
        qDebug() << "ERROR: selecting chunks:" << q.lastError().text();
        return;
    }

    QList<QString> results;
    while (q.next()) {
        int rowid = q.value(0).toInt();
        QString collection_name = q.value(1).toString();
        QString chunk_text = q.value(2).toString();
        results.append(chunk_text);
#if defined(DEBUG)
        qDebug() << "retrieve rowid:" << rowid
                 << "collection_name:" << collection_name
                 << "chunk_text:" << chunk_text;
#endif
    }

    emit retrieveResult(results);
}

void Database::cleanDB()
{
#if defined(DEBUG)
    qDebug() << "cleanDB";
#endif

    // Scan all folders in db to make sure they still exist
    QSqlQuery q;
    QList<CollectionEntry> collections;
    if (!selectAllFromCollections(q, &collections)) {
        qWarning() << "ERROR: Cannot select collections" << q.lastError();
        return;
    }

    for (auto e : collections) {
        // Find the path for the folder
        QFileInfo info(e.folder_path);
        if (!info.exists() || !info.isReadable()) {
#if defined(DEBUG)
            qDebug() << "clean db removing folder" << e.folder_id << e.folder_path;
#endif
            removeFolderInternal(e.collection, e.folder_id, e.folder_path);
        }
    }

    // Scan all documents in db to make sure they still exist
    if (!q.prepare(SELECT_ALL_DOCUMENTS_SQL)) {
        qWarning() << "ERROR: Cannot prepare sql for select all documents" << q.lastError();
        return;
    }

    if (!q.exec()) {
        qWarning() << "ERROR: Cannot exec sql for select all documents" << q.lastError();
        return;
    }

    while (q.next()) {
        int document_id = q.value(0).toInt();
        QString document_path = q.value(1).toString();
        QFileInfo info(document_path);
        if (info.exists() && info.isReadable())
            continue;

#if defined(DEBUG)
        qDebug() << "clean db removing document" << document_id << document_path;
#endif

        // Remove all chunks and documents that either don't exist or have become unreadable
        QSqlQuery query;
        if (!removeChunksByDocumentId(query, document_id)) {
            qWarning() << "ERROR: Cannot remove chunks of document_id" << document_id << query.lastError();
        }

        if (!removeDocument(query, document_id)) {
            qWarning() << "ERROR: Cannot remove document_id" << document_id << query.lastError();
        }
    }
    updateCollectionList();
}

void Database::directoryChanged(const QString &path)
{
#if defined(DEBUG)
    qDebug() << "directoryChanged" << path;
#endif

    QSqlQuery q;
    int folder_id = -1;

    // Lookup the folder_id in the db
    if (!selectFolder(q, path, &folder_id)) {
        qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError();
        return;
    }

    // If we don't have a folder_id in the db, then something bad has happened
    Q_ASSERT(folder_id != -1);
    if (folder_id == -1) {
        qWarning() << "ERROR: Watched folder does not exist in db" << path;
        m_watcher->removePath(path);
        return;
    }

    // Clean the database
    cleanDB();

    // Rescan the documents associated with the folder
    scanDocuments(folder_id, path);
}

class MyLocalDocs: public LocalDocs { };
Q_GLOBAL_STATIC(MyLocalDocs, localDocsInstance)
LocalDocs *LocalDocs::globalInstance()
{
    return localDocsInstance();
}

LocalDocs::LocalDocs()
    : QObject(nullptr)
    , m_database(new Database)
{
    connect(this, &LocalDocs::requestAddFolder, m_database,
        &Database::addFolder, Qt::QueuedConnection);
    connect(this, &LocalDocs::requestRemoveFolder, m_database,
        &Database::removeFolder, Qt::QueuedConnection);
    connect(this, &LocalDocs::requestRetrieveFromDB, m_database,
        &Database::retrieveFromDB, Qt::QueuedConnection);
    connect(m_database, &Database::retrieveResult, this,
        &LocalDocs::handleRetrieveResult, Qt::QueuedConnection);

    addFolder("localdocs", "/home/atreat/dev/large_language_models/localdocs");
}

void LocalDocs::addFolder(const QString &collection, const QString &path)
{
    emit requestAddFolder(collection, path);
}

void LocalDocs::removeFolder(const QString &collection, const QString &path)
{
    emit requestRemoveFolder(collection, path);
}

void LocalDocs::requestRetrieve(const QList<QString> &collections, const QString &text)
{
    m_retrieveResult = QList<QString>();
    emit requestRetrieveFromDB(collections, text);
}

void LocalDocs::handleRetrieveResult(const QList<QString> &result)
{
    m_retrieveResult = result;
    emit receivedResult();
}

void LocalDocs::handleCollectionListUpdated(const QList<CollectionInfo> &collectionList)
{
    m_collectionList = collectionList;
    emit collectionListChanged();
}