From fbe363689f93bf07ff93bb4f3fcdd2c3c205b7a3 Mon Sep 17 00:00:00 2001 From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com> Date: Tue, 1 Oct 2024 00:20:00 +0200 Subject: [PATCH 1/5] Change message history --- chat/ChatClientInterface.cpp | 110 ++++++++++++++++++++++------------- chat/ChatClientInterface.hpp | 37 ++++++++++-- 2 files changed, 104 insertions(+), 43 deletions(-) diff --git a/chat/ChatClientInterface.cpp b/chat/ChatClientInterface.cpp index 265f330..de6808f 100644 --- a/chat/ChatClientInterface.cpp +++ b/chat/ChatClientInterface.cpp @@ -31,6 +31,51 @@ namespace QodeAssist::Chat { +int ChatHistory::estimateTokenCount(const QString &text) const +{ + return text.length() / 4; +} + +void ChatHistory::addMessage(ChatMessage::Role role, const QString &content) +{ + int tokenCount = estimateTokenCount(content); + m_messages.append({role, content, tokenCount}); + m_totalTokens += tokenCount; + trim(); +} +void ChatHistory::clear() +{ + m_messages.clear(); + m_totalTokens = 0; +} + +QVector ChatHistory::getMessages() const +{ + return m_messages; +} + +QString ChatHistory::getSystemPrompt() const +{ + return m_systemPrompt; +} + +void ChatHistory::setSystemPrompt(const QString &prompt) +{ + m_systemPrompt = prompt; +} + +void ChatHistory::trim() +{ + while (m_messages.size() > MAX_HISTORY_SIZE || m_totalTokens > MAX_TOKENS) { + if (!m_messages.isEmpty()) { + m_totalTokens -= m_messages.first().tokenCount; + m_messages.removeFirst(); + } else { + break; + } + } +} + ChatClientInterface::ChatClientInterface(QObject *parent) : QObject(parent) , m_requestHandler(new LLMRequestHandler(this)) @@ -51,15 +96,10 @@ ChatClientInterface::ChatClientInterface(QObject *parent) } }); - // QJsonObject systemMessage; - // systemMessage["role"] = "system"; - // systemMessage["content"] = "You are a helpful C++ and QML programming assistant."; - // m_chatHistory.append(systemMessage); + m_chatHistory.setSystemPrompt("You are a helpful C++ and QML programming assistant."); } -ChatClientInterface::~ChatClientInterface() -{ -} +ChatClientInterface::~ChatClientInterface() = default; void ChatClientInterface::sendMessage(const QString &message) { @@ -79,14 +119,11 @@ void ChatClientInterface::sendMessage(const QString &message) QJsonObject providerRequest; providerRequest["model"] = Settings::generalSettings().chatModelName(); providerRequest["stream"] = true; - - providerRequest["messages"] = m_chatHistory; + providerRequest["messages"] = prepareMessagesForRequest(); chatTemplate->prepareRequest(providerRequest, context); chatProvider->prepareRequest(providerRequest); - m_chatHistory = providerRequest["messages"].toArray(); - LLMConfig config; config.requestType = RequestType::Chat; config.provider = chatProvider; @@ -99,18 +136,22 @@ void ChatClientInterface::sendMessage(const QString &message) request["id"] = QUuid::createUuid().toString(); m_accumulatedResponse.clear(); - m_pendingMessage = message; + m_chatHistory.addMessage(ChatMessage::Role::User, message); m_requestHandler->sendLLMRequest(config, request); } void ChatClientInterface::clearMessages() { - m_chatHistory = {}; + m_chatHistory.clear(); m_accumulatedResponse.clear(); - m_pendingMessage.clear(); logMessage("Chat history cleared"); } +QVector ChatClientInterface::getChatHistory() const +{ + return m_chatHistory.getMessages(); +} + void ChatClientInterface::handleLLMResponse(const QString &response, bool isComplete) { m_accumulatedResponse += response; @@ -119,42 +160,33 @@ void ChatClientInterface::handleLLMResponse(const QString &response, bool isComp logMessage("Message completed. Final response: " + m_accumulatedResponse); emit messageReceived(m_accumulatedResponse.trimmed()); - QJsonObject assistantMessage; - assistantMessage["role"] = "assistant"; - assistantMessage["content"] = m_accumulatedResponse.trimmed(); - m_chatHistory.append(assistantMessage); - - m_pendingMessage.clear(); + m_chatHistory.addMessage(ChatMessage::Role::Assistant, m_accumulatedResponse.trimmed()); m_accumulatedResponse.clear(); - - trimChatHistory(); } } -void ChatClientInterface::trimChatHistory() +QJsonArray ChatClientInterface::prepareMessagesForRequest() const { - int maxTokens = 4000; - int totalTokens = 0; - QJsonArray newHistory; + QJsonArray messages; - if (!m_chatHistory.isEmpty() - && m_chatHistory.first().toObject()["role"].toString() == "system") { - newHistory.append(m_chatHistory.first()); - } + messages.append(QJsonObject{{"role", "system"}, {"content", m_chatHistory.getSystemPrompt()}}); - for (int i = m_chatHistory.size() - 1; i >= 0; --i) { - QJsonObject message = m_chatHistory[i].toObject(); - int messageTokens = message["content"].toString().length() / 4; - - if (totalTokens + messageTokens > maxTokens) { + for (const auto &message : m_chatHistory.getMessages()) { + QString role; + switch (message.role) { + case ChatMessage::Role::User: + role = "user"; break; + case ChatMessage::Role::Assistant: + role = "assistant"; + break; + default: + continue; } - - newHistory.prepend(message); - totalTokens += messageTokens; + messages.append(QJsonObject{{"role", role}, {"content", message.content}}); } - m_chatHistory = newHistory; + return messages; } } // namespace QodeAssist::Chat diff --git a/chat/ChatClientInterface.hpp b/chat/ChatClientInterface.hpp index 16267dc..5bb3d71 100644 --- a/chat/ChatClientInterface.hpp +++ b/chat/ChatClientInterface.hpp @@ -20,12 +20,41 @@ #pragma once #include -#include +#include +#include #include "QodeAssistData.hpp" #include "core/LLMRequestHandler.hpp" namespace QodeAssist::Chat { +struct ChatMessage +{ + enum class Role { System, User, Assistant }; + Role role; + QString content; + int tokenCount; +}; + +class ChatHistory +{ +public: + void addMessage(ChatMessage::Role role, const QString &content); + void clear(); + QVector getMessages() const; + QString getSystemPrompt() const; + void setSystemPrompt(const QString &prompt); + void trim(); + +private: + QVector m_messages; + QString m_systemPrompt; + int m_totalTokens = 0; + static const int MAX_HISTORY_SIZE = 50; + static const int MAX_TOKENS = 4000; + + int estimateTokenCount(const QString &text) const; +}; + class ChatClientInterface : public QObject { Q_OBJECT @@ -36,6 +65,7 @@ public: void sendMessage(const QString &message); void clearMessages(); + QVector getChatHistory() const; signals: void messageReceived(const QString &message); @@ -43,12 +73,11 @@ signals: private: void handleLLMResponse(const QString &response, bool isComplete); - void trimChatHistory(); + QJsonArray prepareMessagesForRequest() const; LLMRequestHandler *m_requestHandler; QString m_accumulatedResponse; - QString m_pendingMessage; - QJsonArray m_chatHistory; + ChatHistory m_chatHistory; }; } // namespace QodeAssist::Chat From 9903ac8f7b420ba200ff3c68d687e91fc6c363b7 Mon Sep 17 00:00:00 2001 From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com> Date: Wed, 2 Oct 2024 21:44:15 +0200 Subject: [PATCH 2/5] Fix system prompt for FIM --- DocumentContextReader.cpp | 3 --- LLMClientInterface.cpp | 4 ++++ QodeAssistConstants.hpp | 4 ++-- providers/LMStudioProvider.cpp | 15 +++++++++++++-- providers/OpenAICompatProvider.cpp | 16 +++++++++++++--- settings/ContextSettings.cpp | 8 +++----- 6 files changed, 35 insertions(+), 15 deletions(-) diff --git a/DocumentContextReader.cpp b/DocumentContextReader.cpp index b3c2707..5bbde27 100644 --- a/DocumentContextReader.cpp +++ b/DocumentContextReader.cpp @@ -250,9 +250,6 @@ QString DocumentContextReader::getInstructions() const { QString instructions; - if (Settings::contextSettings().useSpecificInstructions()) - instructions += getSpecificInstructions(); - if (Settings::contextSettings().useFilePathInContext()) instructions += getLanguageAndFileInfo(); diff --git a/LLMClientInterface.cpp b/LLMClientInterface.cpp index 6dd25ae..b9b0230 100644 --- a/LLMClientInterface.cpp +++ b/LLMClientInterface.cpp @@ -30,6 +30,7 @@ #include "PromptTemplateManager.hpp" #include "QodeAssistUtils.hpp" #include "core/LLMRequestConfig.hpp" +#include "settings/ContextSettings.hpp" #include "settings/GeneralSettings.hpp" namespace QodeAssist { @@ -159,6 +160,9 @@ void LLMClientInterface::handleCompletion(const QJsonObject &request) {"stop", QJsonArray::fromStringList(config.promptTemplate->stopWords())}}; + if (Settings::contextSettings().useSpecificInstructions()) + config.providerRequest["system"] = Settings::contextSettings().specificInstractions(); + config.promptTemplate->prepareRequest(config.providerRequest, updatedContext); config.provider->prepareRequest(config.providerRequest); diff --git a/QodeAssistConstants.hpp b/QodeAssistConstants.hpp index fc5b6d8..dd4c88d 100644 --- a/QodeAssistConstants.hpp +++ b/QodeAssistConstants.hpp @@ -53,10 +53,10 @@ const char AUTO_COMPLETION_CHAR_THRESHOLD[] = "QodeAssist.autoCompletionCharThre const char AUTO_COMPLETION_TYPING_INTERVAL[] = "QodeAssist.autoCompletionTypingInterval"; const char MAX_FILE_THRESHOLD[] = "QodeAssist.maxFileThreshold"; const char OLLAMA_LIVETIME[] = "QodeAssist.ollamaLivetime"; -const char SPECIFIC_INSTRUCTIONS[] = "QodeAssist.specificInstractions"; +const char SYSTEM_PROMPT[] = "QodeAssist.systemPrompt"; const char MULTILINE_COMPLETION[] = "QodeAssist.multilineCompletion"; const char API_KEY[] = "QodeAssist.apiKey"; -const char USE_SPECIFIC_INSTRUCTIONS[] = "QodeAssist.useSpecificInstructions"; +const char USE_SYSTEM_PROMPT[] = "QodeAssist.useSystemPrompt"; const char USE_FILE_PATH_IN_CONTEXT[] = "QodeAssist.useFilePathInContext"; const char CUSTOM_JSON_TEMPLATE[] = "QodeAssist.customJsonTemplate"; const char USE_PROJECT_CHANGES_CACHE[] = "QodeAssist.useProjectChangesCache"; diff --git a/providers/LMStudioProvider.cpp b/providers/LMStudioProvider.cpp index 75afce1..67651b2 100644 --- a/providers/LMStudioProvider.cpp +++ b/providers/LMStudioProvider.cpp @@ -55,9 +55,20 @@ QString LMStudioProvider::chatEndpoint() const void LMStudioProvider::prepareRequest(QJsonObject &request) { auto &settings = Settings::presetPromptsSettings(); + QJsonArray messages; + + if (request.contains("system")) { + QJsonObject systemMessage{{"role", "system"}, + {"content", request.take("system").toString()}}; + messages.append(systemMessage); + } + if (request.contains("prompt")) { - QJsonArray messages{ - {QJsonObject{{"role", "user"}, {"content", request.take("prompt").toString()}}}}; + QJsonObject userMessage{{"role", "user"}, {"content", request.take("prompt").toString()}}; + messages.append(userMessage); + } + + if (!messages.isEmpty()) { request["messages"] = std::move(messages); } diff --git a/providers/OpenAICompatProvider.cpp b/providers/OpenAICompatProvider.cpp index 36960f6..8d322f6 100644 --- a/providers/OpenAICompatProvider.cpp +++ b/providers/OpenAICompatProvider.cpp @@ -24,7 +24,6 @@ #include #include -#include "PromptTemplateManager.hpp" #include "settings/PresetPromptsSettings.hpp" namespace QodeAssist::Providers { @@ -54,9 +53,20 @@ QString OpenAICompatProvider::chatEndpoint() const void OpenAICompatProvider::prepareRequest(QJsonObject &request) { auto &settings = Settings::presetPromptsSettings(); + QJsonArray messages; + + if (request.contains("system")) { + QJsonObject systemMessage{{"role", "system"}, + {"content", request.take("system").toString()}}; + messages.append(systemMessage); + } + if (request.contains("prompt")) { - QJsonArray messages{ - {QJsonObject{{"role", "user"}, {"content", request.take("prompt").toString()}}}}; + QJsonObject userMessage{{"role", "user"}, {"content", request.take("prompt").toString()}}; + messages.append(userMessage); + } + + if (!messages.isEmpty()) { request["messages"] = std::move(messages); } diff --git a/settings/ContextSettings.cpp b/settings/ContextSettings.cpp index c76e902..d14aed9 100644 --- a/settings/ContextSettings.cpp +++ b/settings/ContextSettings.cpp @@ -17,8 +17,6 @@ * along with QodeAssist. If not, see . */ -#pragma once - #include "ContextSettings.hpp" #include @@ -60,11 +58,11 @@ ContextSettings::ContextSettings() useFilePathInContext.setDefaultValue(false); useFilePathInContext.setLabelText(Tr::tr("Use File Path in Context")); - useSpecificInstructions.setSettingsKey(Constants::USE_SPECIFIC_INSTRUCTIONS); + useSpecificInstructions.setSettingsKey(Constants::USE_SYSTEM_PROMPT); useSpecificInstructions.setDefaultValue(true); - useSpecificInstructions.setLabelText(Tr::tr("Use Specific Instructions")); + useSpecificInstructions.setLabelText(Tr::tr("Use System Prompt")); - specificInstractions.setSettingsKey(Constants::SPECIFIC_INSTRUCTIONS); + specificInstractions.setSettingsKey(Constants::SYSTEM_PROMPT); specificInstractions.setDisplayStyle(Utils::StringAspect::TextEditDisplay); specificInstractions.setLabelText( Tr::tr("Instructions: Please keep %1 for languge name, warning, it shouldn't too big")); From 1cbde3d55b369109fc95f61d1ec955d9c400d5d9 Mon Sep 17 00:00:00 2001 From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com> Date: Wed, 2 Oct 2024 21:44:32 +0200 Subject: [PATCH 3/5] Fix ollama model livetime --- providers/OllamaProvider.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/OllamaProvider.cpp b/providers/OllamaProvider.cpp index 661f944..5daf0d2 100644 --- a/providers/OllamaProvider.cpp +++ b/providers/OllamaProvider.cpp @@ -59,7 +59,6 @@ void OllamaProvider::prepareRequest(QJsonObject &request) QJsonObject options; options["num_predict"] = settings.maxTokens(); - options["keep_alive"] = settings.ollamaLivetime(); options["temperature"] = settings.temperature(); if (settings.useTopP()) options["top_p"] = settings.topP(); @@ -70,6 +69,7 @@ void OllamaProvider::prepareRequest(QJsonObject &request) if (settings.usePresencePenalty()) options["presence_penalty"] = settings.presencePenalty(); request["options"] = options; + request["keep_alive"] = settings.ollamaLivetime(); } bool OllamaProvider::handleResponse(QNetworkReply *reply, QString &accumulatedResponse) From d235d0fcdfa852136930cfe1e1b6d6788f31c4fb Mon Sep 17 00:00:00 2001 From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com> Date: Wed, 2 Oct 2024 22:25:53 +0200 Subject: [PATCH 4/5] Fix providers and prompt default value --- README.md | 10 ++++++++-- settings/GeneralSettings.cpp | 13 +++++++++---- templates/CodeLlamaFimTemplate.hpp | 2 +- templates/CodeLlamaInstruct.hpp | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 58bfac2..52182a3 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,15 @@ If you've successfully used a model that's not listed here, please let us know b 1. Install QtCreator 14.0 2. Install [Ollama](https://ollama.com). Make sure to review the system requirements before installation. -3. Install a language model in Ollama. For example, you can run: +3. Install a language models in Ollama. For example, you can run: + +For suggestions: ``` -ollama run starcoder2:7b +ollama run codellama:7b-code +``` +For chat: +``` +ollama run codellama:7b-instruct ``` 4. Download the QodeAssist plugin. 5. Launch Qt Creator and install the plugin: diff --git a/settings/GeneralSettings.cpp b/settings/GeneralSettings.cpp index 631080e..b06e78b 100644 --- a/settings/GeneralSettings.cpp +++ b/settings/GeneralSettings.cpp @@ -131,6 +131,11 @@ GeneralSettings::GeneralSettings() loadProviders(); loadPrompts(); + llmProviders.setDefaultValue(llmProviders.indexForDisplay("Ollama")); + chatLlmProviders.setDefaultValue(chatLlmProviders.indexForDisplay("Ollama")); + fimPrompts.setDefaultValue(fimPrompts.indexForDisplay("CodeLLama FIM")); + chatPrompts.setDefaultValue(chatPrompts.indexForDisplay("CodeLLama Chat")); + readSettings(); auto fimProviderName = llmProviders.displayForIndex(llmProviders.value()); @@ -273,12 +278,12 @@ void GeneralSettings::resetPageToDefaults() resetAspect(startSuggestionTimer); resetAspect(autoCompletionTypingInterval); resetAspect(autoCompletionCharThreshold); + resetAspect(llmProviders); + resetAspect(chatLlmProviders); + resetAspect(fimPrompts); + resetAspect(chatPrompts); } - int fimIndex = llmProviders.indexForDisplay("Ollama"); - llmProviders.setVolatileValue(fimIndex); - int chatIndex = chatLlmProviders.indexForDisplay("Ollama"); - chatLlmProviders.setVolatileValue(chatIndex); modelName.setVolatileValue(""); chatModelName.setVolatileValue(""); diff --git a/templates/CodeLlamaFimTemplate.hpp b/templates/CodeLlamaFimTemplate.hpp index 48fedfc..729f79e 100644 --- a/templates/CodeLlamaFimTemplate.hpp +++ b/templates/CodeLlamaFimTemplate.hpp @@ -27,7 +27,7 @@ class CodeLlamaFimTemplate : public PromptTemplate { public: TemplateType type() const override { return TemplateType::Fim; } - QString name() const override { return "CodeLlama FIM"; } + QString name() const override { return "CodeLLama FIM"; } QString promptTemplate() const override { return "%1
 %2 %3 "; }
     QStringList stopWords() const override
     {
diff --git a/templates/CodeLlamaInstruct.hpp b/templates/CodeLlamaInstruct.hpp
index 96d1f22..4c29f72 100644
--- a/templates/CodeLlamaInstruct.hpp
+++ b/templates/CodeLlamaInstruct.hpp
@@ -28,7 +28,7 @@ class CodeLlamaInstructTemplate : public PromptTemplate
 {
 public:
     TemplateType type() const override { return TemplateType::Chat; }
-    QString name() const override { return "CodeLlama Chat"; }
+    QString name() const override { return "CodeLLama Chat"; }
     QString promptTemplate() const override { return "[INST] %1 [/INST]"; }
     QStringList stopWords() const override { return QStringList() << "[INST]" << "[/INST]"; }
 

From 0ab4b51520bb38af69ac6cecda9e99433c7ee9bd Mon Sep 17 00:00:00 2001
From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com>
Date: Wed, 2 Oct 2024 22:28:52 +0200
Subject: [PATCH 5/5] Upgrade plugin version

---
 QodeAssist.json.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/QodeAssist.json.in b/QodeAssist.json.in
index 6a1f752..1e877d7 100644
--- a/QodeAssist.json.in
+++ b/QodeAssist.json.in
@@ -1,6 +1,6 @@
 {
     "Name" : "QodeAssist",
-    "Version" : "0.2.1",
+    "Version" : "0.2.2",
     "CompatVersion" : "${IDE_VERSION_COMPAT}",
     "Vendor" : "Petr Mironychev",
     "Copyright" : "(C) ${IDE_COPYRIGHT_YEAR} Petr Mironychev, (C) ${IDE_COPYRIGHT_YEAR} The Qt Company Ltd",