Version 0.3.3

- Add streaming response to chat
- Add stopping chat request and button

CMakeLists.txt

@@ -42,6 +42,7 @@ add_qtc_plugin(QodeAssist
     templates/DeepSeekCoderChat.hpp
     templates/CodeLlamaChat.hpp
     templates/QwenChat.hpp
+    templates/StarCoderChat.hpp
     providers/OllamaProvider.hpp providers/OllamaProvider.cpp
     providers/LMStudioProvider.hpp providers/LMStudioProvider.cpp
     providers/OpenAICompatProvider.hpp providers/OpenAICompatProvider.cpp

QodeAssist.json.in

@@ -1,6 +1,6 @@
 {
     "Name" : "QodeAssist",
-    "Version" : "0.3.1",
+    "Version" : "0.3.3",
     "CompatVersion" : "${IDE_VERSION_COMPAT}",
     "Vendor" : "Petr Mironychev",
     "Copyright" : "(C) ${IDE_COPYRIGHT_YEAR} Petr Mironychev, (C) ${IDE_COPYRIGHT_YEAR} The Qt Company Ltd",

README.md

@@ -69,6 +69,7 @@ QodeAssist currently supports the following LLM (Large Language Model) providers
 
 QodeAssist has been thoroughly tested and optimized for use with the following language models:
 
+- Llama
 - CodeLlama
 - StarCoder2
 - DeepSeek-Coder-V2

chatview/ChatModel.cpp

@@ -68,6 +68,28 @@ QHash<int, QByteArray> ChatModel::roleNames() const
     return roles;
 }
 
+void ChatModel::addMessage(const QString &content, ChatRole role, const QString &id)
+{
+    int tokenCount = estimateTokenCount(content);
+
+    if (!m_messages.isEmpty() && !id.isEmpty() && m_messages.last().id == id) {
+        Message &lastMessage = m_messages.last();
+        int oldTokenCount = lastMessage.tokenCount;
+        lastMessage.content = content;
+        lastMessage.tokenCount = tokenCount;
+        m_totalTokens += (tokenCount - oldTokenCount);
+        emit dataChanged(index(m_messages.size() - 1), index(m_messages.size() - 1));
+    } else {
+        beginInsertRows(QModelIndex(), m_messages.size(), m_messages.size());
+        m_messages.append({role, content, tokenCount, id});
+        m_totalTokens += tokenCount;
+        endInsertRows();
+    }
+
+    trim();
+    emit totalTokensChanged();
+}
+
 QVector<ChatModel::Message> ChatModel::getChatHistory() const
 {
     return m_messages;
@@ -92,17 +114,6 @@ int ChatModel::estimateTokenCount(const QString &text) const
     return text.length() / 4;
 }
 
-void ChatModel::addMessage(const QString &content, ChatRole role)
-{
-    int tokenCount = estimateTokenCount(content);
-    beginInsertRows(QModelIndex(), m_messages.size(), m_messages.size());
-    m_messages.append({role, content, tokenCount});
-    m_totalTokens += tokenCount;
-    endInsertRows();
-    trim();
-    emit totalTokensChanged();
-}
-
 void ChatModel::clear()
 {
     beginResetModel();
@@ -176,4 +187,9 @@ int ChatModel::tokensThreshold() const
     return settings.chatTokensThreshold();
 }
 
+QString ChatModel::lastMessageId() const
+{
+    return !m_messages.isEmpty() ? m_messages.last().id : "";
+}
+
 } // namespace QodeAssist::Chat

chatview/ChatModel.hpp

@@ -46,6 +46,7 @@ public:
         ChatRole role;
         QString content;
         int tokenCount;
+        QString id;
     };
 
     explicit ChatModel(QObject *parent = nullptr);
@@ -54,7 +55,7 @@ public:
     QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override;
     QHash<int, QByteArray> roleNames() const override;
 
-    Q_INVOKABLE void addMessage(const QString &content, ChatRole role);
+    Q_INVOKABLE void addMessage(const QString &content, ChatRole role, const QString &id);
     Q_INVOKABLE void clear();
     Q_INVOKABLE QList<MessagePart> processMessageContent(const QString &content) const;
 
@@ -65,6 +66,7 @@ public:
     int tokensThreshold() const;
 
     QString currentModel() const;
+    QString lastMessageId() const;
 
 signals:
     void totalTokensChanged();

chatview/ChatRootView.cpp

@@ -60,6 +60,11 @@ void ChatRootView::copyToClipboard(const QString &text)
     QGuiApplication::clipboard()->setText(text);
 }
 
+void ChatRootView::cancelRequest()
+{
+    m_clientInterface->cancelRequest();
+}
+
 void ChatRootView::generateColors()
 {
     QColor baseColor = backgroundColor();

chatview/ChatRootView.hpp

@@ -52,6 +52,7 @@ public:
 public slots:
     void sendMessage(const QString &message) const;
     void copyToClipboard(const QString &text);
+    void cancelRequest();
 
 signals:
     void chatModelChanged();

chatview/ClientInterface.cpp

@@ -38,8 +38,8 @@ ClientInterface::ClientInterface(ChatModel *chatModel, QObject *parent)
     connect(m_requestHandler,
             &LLMCore::RequestHandler::completionReceived,
             this,
-            [this](const QString &completion, const QJsonObject &, bool isComplete) {
-                handleLLMResponse(completion, isComplete);
+            [this](const QString &completion, const QJsonObject &request, bool isComplete) {
+                handleLLMResponse(completion, request, isComplete);
             });
 
     connect(m_requestHandler,
@@ -56,6 +56,8 @@ ClientInterface::~ClientInterface() = default;
 
 void ClientInterface::sendMessage(const QString &message)
 {
+    cancelRequest();
+
     LOG_MESSAGE("Sending message: " + message);
     LOG_MESSAGE("chatProvider " + Settings::generalSettings().chatLlmProviders.stringValue());
     LOG_MESSAGE("chatTemplate " + Settings::generalSettings().chatPrompts.stringValue());
@@ -74,6 +76,9 @@ void ClientInterface::sendMessage(const QString &message)
     providerRequest["stream"] = true;
     providerRequest["messages"] = m_chatModel->prepareMessagesForRequest(context);
 
+    if (!chatTemplate || !chatProvider) {
+        LOG_MESSAGE("Check settings, provider or template are not set");
+    }
     chatTemplate->prepareRequest(providerRequest, context);
     chatProvider->prepareRequest(providerRequest, LLMCore::RequestType::Chat);
 
@@ -89,28 +94,31 @@ void ClientInterface::sendMessage(const QString &message)
     QJsonObject request;
     request["id"] = QUuid::createUuid().toString();
 
-    m_accumulatedResponse.clear();
-    m_chatModel->addMessage(message, ChatModel::ChatRole::User);
+    m_chatModel->addMessage(message, ChatModel::ChatRole::User, "");
     m_requestHandler->sendLLMRequest(config, request);
 }
 
 void ClientInterface::clearMessages()
 {
     m_chatModel->clear();
-    m_accumulatedResponse.clear();
     LOG_MESSAGE("Chat history cleared");
 }
 
-void ClientInterface::handleLLMResponse(const QString &response, bool isComplete)
+void ClientInterface::cancelRequest()
 {
-    m_accumulatedResponse += response;
+    auto id = m_chatModel->lastMessageId();
+    m_requestHandler->cancelRequest(id);
+}
+
+void ClientInterface::handleLLMResponse(const QString &response,
+                                        const QJsonObject &request,
+                                        bool isComplete)
+{
+    QString messageId = request["id"].toString();
+    m_chatModel->addMessage(response.trimmed(), ChatModel::ChatRole::Assistant, messageId);
 
     if (isComplete) {
-        LOG_MESSAGE("Message completed. Final response: " + m_accumulatedResponse);
-        emit messageReceived(m_accumulatedResponse.trimmed());
-
-        m_chatModel->addMessage(m_accumulatedResponse.trimmed(), ChatModel::ChatRole::Assistant);
-        m_accumulatedResponse.clear();
+        LOG_MESSAGE("Message completed. Final response for message " + messageId + ": " + response);
     }
 }
 

chatview/ClientInterface.hpp

@@ -38,16 +38,15 @@ public:
 
     void sendMessage(const QString &message);
     void clearMessages();
+    void cancelRequest();
 
 signals:
-    void messageReceived(const QString &message);
     void errorOccurred(const QString &error);
 
 private:
-    void handleLLMResponse(const QString &response, bool isComplete);
+    void handleLLMResponse(const QString &response, const QJsonObject &request, bool isComplete);
 
     LLMCore::RequestHandler *m_requestHandler;
-    QString m_accumulatedResponse;
     ChatModel *m_chatModel;
 };
 

chatview/qml/RootItem.qml

@@ -121,6 +121,15 @@ ChatRootView {
                 text: qsTr("Send")
                 onClicked: sendChatMessage()
             }
+
+            Button {
+                id: stopButton
+
+                Layout.alignment: Qt.AlignBottom
+                text: qsTr("Stop")
+                onClicked: root.cancelRequest()
+            }
+
             Button {
                 id: clearButton
 

llmcore/PromptTemplateManager.cpp

@@ -43,8 +43,8 @@ void PromptTemplateManager::setCurrentFimTemplate(const QString &name)
 PromptTemplate *PromptTemplateManager::getCurrentFimTemplate()
 {
     if (m_currentFimTemplate == nullptr) {
-        LOG_MESSAGE("Current fim provider is null");
-        return nullptr;
+        LOG_MESSAGE("Current fim provider is null, return first");
+        return m_fimTemplates.first();
     }
 
     return m_currentFimTemplate;
@@ -63,8 +63,10 @@ void PromptTemplateManager::setCurrentChatTemplate(const QString &name)
 
 PromptTemplate *PromptTemplateManager::getCurrentChatTemplate()
 {
-    if (m_currentChatTemplate == nullptr)
-        LOG_MESSAGE("Current chat provider is null");
+    if (m_currentChatTemplate == nullptr) {
+        LOG_MESSAGE("Current chat provider is null, return first");
+        return m_chatTemplates.first();
+    }
 
     return m_currentChatTemplate;
 }

llmcore/ProvidersManager.cpp

@@ -56,8 +56,8 @@ Provider *ProvidersManager::setCurrentChatProvider(const QString &name)
 Provider *ProvidersManager::getCurrentFimProvider()
 {
     if (m_currentFimProvider == nullptr) {
-        LOG_MESSAGE("Current fim provider is null");
-        return nullptr;
+        LOG_MESSAGE("Current fim provider is null, return first");
+        return m_providers.first();
     }
 
     return m_currentFimProvider;
@@ -66,8 +66,8 @@ Provider *ProvidersManager::getCurrentFimProvider()
 Provider *ProvidersManager::getCurrentChatProvider()
 {
     if (m_currentChatProvider == nullptr) {
-        LOG_MESSAGE("Current chat provider is null");
-        return nullptr;
+        LOG_MESSAGE("Current chat provider is null, return first");
+        return m_providers.first();
     }
 
     return m_currentChatProvider;

llmcore/RequestHandler.cpp

@@ -80,22 +80,18 @@ void RequestHandler::handleLLMResponse(QNetworkReply *reply,
             && processSingleLineCompletion(reply, request, accumulatedResponse, config)) {
             return;
         }
+
+        if (isComplete) {
+            auto cleanedCompletion = removeStopWords(accumulatedResponse,
+                                                     config.promptTemplate->stopWords());
+            emit completionReceived(cleanedCompletion, request, true);
+        }
+    } else if (config.requestType == RequestType::Chat) {
+        emit completionReceived(accumulatedResponse, request, isComplete);
     }
 
-    if (isComplete || reply->isFinished()) {
-        if (isComplete) {
-            if (config.requestType == RequestType::Fim) {
-                auto cleanedCompletion = removeStopWords(accumulatedResponse,
-                                                         config.promptTemplate->stopWords());
-                emit completionReceived(cleanedCompletion, request, true);
-            } else {
-                emit completionReceived(accumulatedResponse, request, true);
-            }
-        } else {
-            emit completionReceived(accumulatedResponse, request, false);
-        }
+    if (isComplete)
         m_accumulatedResponses.remove(reply);
-    }
 }
 
 bool RequestHandler::cancelRequest(const QString &id)

qodeassist.cpp

@@ -55,6 +55,7 @@
 #include "templates/DeepSeekCoderFim.hpp"
 #include "templates/QwenChat.hpp"
 #include "templates/StarCoder2Fim.hpp"
+#include "templates/StarCoderChat.hpp"
 
 using namespace Utils;
 using namespace Core;
@@ -95,6 +96,7 @@ public:
         templateManager.registerTemplate<Templates::CodeLlamaChat>();
         templateManager.registerTemplate<Templates::QwenChat>();
         templateManager.registerTemplate<Templates::LlamaChat>();
+        templateManager.registerTemplate<Templates::StarCoderChat>();
 
         Utils::Icon QCODEASSIST_ICON(
             {{":/resources/images/qoderassist-icon.png", Utils::Theme::IconsBaseColor}});

settings/GeneralSettings.cpp

@@ -134,7 +134,7 @@ GeneralSettings::GeneralSettings()
     chatTokensThreshold.setToolTip(Tr::tr("Maximum number of tokens in chat history. When "
                                           "exceeded, oldest messages will be removed."));
     chatTokensThreshold.setRange(1000, 16000);
-    chatTokensThreshold.setDefaultValue(4000);
+    chatTokensThreshold.setDefaultValue(8000);
 
     loadProviders();
     loadPrompts();

templates/StarCoderChat.hpp

@@ -0,0 +1,51 @@
+/* 
+ * Copyright (C) 2024 Petr Mironychev
+ *
+ * This file is part of QodeAssist.
+ *
+ * QodeAssist is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * QodeAssist is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with QodeAssist. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <QJsonArray>
+#include "llmcore/PromptTemplate.hpp"
+
+namespace QodeAssist::Templates {
+
+class StarCoderChat : public LLMCore::PromptTemplate
+{
+public:
+    QString name() const override { return "StarCoder Chat"; }
+    LLMCore::TemplateType type() const override { return LLMCore::TemplateType::Chat; }
+    QString promptTemplate() const override { return "### Instruction:\n%1\n### Response:\n"; }
+    QStringList stopWords() const override
+    {
+        return QStringList() << "###"
+                             << "<|endoftext|>" << "<file_sep>";
+    }
+    void prepareRequest(QJsonObject &request, const LLMCore::ContextData &context) const override
+    {
+        QString formattedPrompt = promptTemplate().arg(context.prefix);
+        QJsonArray messages = request["messages"].toArray();
+
+        QJsonObject newMessage;
+        newMessage["role"] = "user";
+        newMessage["content"] = formattedPrompt;
+        messages.append(newMessage);
+
+        request["messages"] = messages;
+    }
+};
+} // namespace QodeAssist::Templates