diff --git a/ChatView/ChatModel.cpp b/ChatView/ChatModel.cpp index ff2111d..dbe316c 100644 --- a/ChatView/ChatModel.cpp +++ b/ChatView/ChatModel.cpp @@ -336,7 +336,10 @@ void ChatModel::resetModelTo(int index) } void ChatModel::addToolExecutionStatus( - const QString &requestId, const QString &toolId, const QString &toolName) + const QString &requestId, + const QString &toolId, + const QString &toolName, + const QJsonObject &toolArguments) { QString content = toolName; @@ -347,11 +350,15 @@ void ChatModel::addToolExecutionStatus( && m_messages.last().role == ChatRole::Tool) { Message &lastMessage = m_messages.last(); lastMessage.content = content; + lastMessage.toolName = toolName; + lastMessage.toolArguments = toolArguments; LOG_MESSAGE(QString("Updated existing tool message at index %1").arg(m_messages.size() - 1)); emit dataChanged(index(m_messages.size() - 1), index(m_messages.size() - 1)); } else { beginInsertRows(QModelIndex(), m_messages.size(), m_messages.size()); Message newMessage{ChatRole::Tool, content, toolId}; + newMessage.toolName = toolName; + newMessage.toolArguments = toolArguments; m_messages.append(newMessage); endInsertRows(); LOG_MESSAGE(QString("Created new tool message at index %1 with toolId=%2") @@ -360,6 +367,38 @@ void ChatModel::addToolExecutionStatus( } } +void ChatModel::dropTrailingAssistantMessage(const QString &requestId) +{ + if (m_messages.isEmpty()) + return; + + const Message &last = m_messages.last(); + if (last.role != ChatRole::Assistant || last.id != requestId) + return; + + const int idx = m_messages.size() - 1; + beginRemoveRows(QModelIndex(), idx, idx); + m_messages.removeLast(); + endRemoveRows(); + LOG_MESSAGE(QString("Dropped leaked pre-tool assistant message at index %1").arg(idx)); +} + +void ChatModel::setToolMessageData( + const QString &toolId, + const QString &toolName, + const QJsonObject &toolArguments, + const QString &toolResult) +{ + for (int i = 0; i < m_messages.size(); ++i) { + if (m_messages[i].role == ChatRole::Tool && m_messages[i].id == toolId) { + m_messages[i].toolName = toolName; + m_messages[i].toolArguments = toolArguments; + m_messages[i].toolResult = toolResult; + return; + } + } +} + void ChatModel::updateToolResult( const QString &requestId, const QString &toolId, const QString &toolName, const QString &result) { @@ -379,6 +418,8 @@ void ChatModel::updateToolResult( for (int i = m_messages.size() - 1; i >= 0; --i) { if (m_messages[i].id == toolId && m_messages[i].role == ChatRole::Tool) { m_messages[i].content = toolName + "\n" + result; + m_messages[i].toolName = toolName; + m_messages[i].toolResult = result; emit dataChanged(index(i), index(i)); toolMessageFound = true; LOG_MESSAGE(QString("Updated tool result at index %1").arg(i)); diff --git a/ChatView/ChatModel.hpp b/ChatView/ChatModel.hpp index b3f7c73..ad4f02e 100644 --- a/ChatView/ChatModel.hpp +++ b/ChatView/ChatModel.hpp @@ -8,6 +8,7 @@ #include #include +#include #include #include "context/ContentFile.hpp" @@ -59,6 +60,10 @@ public: QList attachments; QList images; + QString toolName; + QJsonObject toolArguments; + QString toolResult; + int promptTokens = 0; int completionTokens = 0; int cachedPromptTokens = 0; @@ -91,7 +96,16 @@ public: Q_INVOKABLE void resetModelTo(int index); void addToolExecutionStatus( - const QString &requestId, const QString &toolId, const QString &toolName); + const QString &requestId, + const QString &toolId, + const QString &toolName, + const QJsonObject &toolArguments); + void dropTrailingAssistantMessage(const QString &requestId); + void setToolMessageData( + const QString &toolId, + const QString &toolName, + const QJsonObject &toolArguments, + const QString &toolResult); void updateToolResult( const QString &requestId, const QString &toolId, diff --git a/ChatView/ChatSerializer.cpp b/ChatView/ChatSerializer.cpp index 26ce8ae..f9f6548 100644 --- a/ChatView/ChatSerializer.cpp +++ b/ChatView/ChatSerializer.cpp @@ -80,6 +80,15 @@ QJsonObject ChatSerializer::serializeMessage( messageObj["signature"] = message.signature; } + if (message.role == ChatModel::ChatRole::Tool) { + if (!message.toolName.isEmpty()) + messageObj["toolName"] = message.toolName; + if (!message.toolArguments.isEmpty()) + messageObj["toolArguments"] = message.toolArguments; + if (!message.toolResult.isEmpty()) + messageObj["toolResult"] = message.toolResult; + } + if (!message.attachments.isEmpty()) { QJsonArray attachmentsArray; for (const auto &attachment : message.attachments) { @@ -126,6 +135,9 @@ ChatModel::Message ChatSerializer::deserializeMessage( message.id = json["id"].toString(); message.isRedacted = json["isRedacted"].toBool(false); message.signature = json["signature"].toString(); + message.toolName = json["toolName"].toString(); + message.toolArguments = json["toolArguments"].toObject(); + message.toolResult = json["toolResult"].toString(); if (json.contains("attachments")) { QJsonArray attachmentsArray = json["attachments"].toArray(); @@ -199,6 +211,10 @@ bool ChatSerializer::deserializeChat( message.images, message.isRedacted, message.signature); + if (message.role == ChatModel::ChatRole::Tool) { + model->setToolMessageData( + message.id, message.toolName, message.toolArguments, message.toolResult); + } LOG_MESSAGE(QString("Loaded message with %1 image(s), isRedacted=%2, signature length=%3") .arg(message.images.size()) .arg(message.isRedacted) diff --git a/ChatView/ClientInterface.cpp b/ChatView/ClientInterface.cpp index 8030776..6ce1f64 100644 --- a/ChatView/ClientInterface.cpp +++ b/ChatView/ClientInterface.cpp @@ -62,6 +62,11 @@ void ClientInterface::sendMessage( bool useTools, bool useThinking) { + if (message.trimmed().isEmpty() && attachments.isEmpty()) { + LOG_MESSAGE("Ignoring empty chat message"); + return; + } + cancelRequest(); m_accumulatedResponses.clear(); @@ -187,9 +192,41 @@ void ClientInterface::sendMessage( context.systemPrompt = systemPrompt; } + const bool toolHistory = promptTemplate->supportsToolHistory(); + QVector messages; + int toolCallMsgIdx = -1; for (const auto &msg : m_chatModel->getChatHistory()) { - if (msg.role == ChatModel::ChatRole::Tool || msg.role == ChatModel::ChatRole::FileEdit) { + if (msg.role == ChatModel::ChatRole::Tool) { + if (!toolHistory || msg.toolName.isEmpty()) { + continue; + } + + if (toolCallMsgIdx < 0) { + PluginLLMCore::Message assistantCall; + assistantCall.role = "assistant"; + messages.append(assistantCall); + toolCallMsgIdx = messages.size() - 1; + } + + PluginLLMCore::ToolCall call; + call.id = msg.id; + call.name = msg.toolName; + call.arguments = msg.toolArguments; + messages[toolCallMsgIdx].toolCalls.append(call); + + PluginLLMCore::Message toolResult; + toolResult.role = "tool"; + toolResult.toolCallId = msg.id; + toolResult.toolName = msg.toolName; + toolResult.content = msg.toolResult; + messages.append(toolResult); + continue; + } + + toolCallMsgIdx = -1; + + if (msg.role == ChatModel::ChatRole::FileEdit) { continue; } @@ -296,7 +333,7 @@ void ClientInterface::sendMessage( = provider->sendRequest(QUrl(Settings::generalSettings().caUrl()), payload, endpoint); QJsonObject request{{"id", requestId}}; - m_activeRequests[requestId] = {request, provider}; + m_activeRequests[requestId] = {request, provider, !toolHistory}; emit requestStarted(requestId); @@ -519,14 +556,21 @@ void ClientInterface::handleThinkingBlockReceived( } void ClientInterface::handleToolExecutionStarted( - const QString &requestId, const QString &toolId, const QString &toolName) + const QString &requestId, + const QString &toolId, + const QString &toolName, + const QJsonObject &arguments) { - if (!m_activeRequests.contains(requestId)) { + const auto requestIt = m_activeRequests.constFind(requestId); + if (requestIt == m_activeRequests.constEnd()) { LOG_MESSAGE(QString("Ignoring tool execution start for non-chat request: %1").arg(requestId)); return; } - m_chatModel->addToolExecutionStatus(requestId, toolId, toolName); + if (requestIt->dropPreToolText) { + m_chatModel->dropTrailingAssistantMessage(requestId); + } + m_chatModel->addToolExecutionStatus(requestId, toolId, toolName, arguments); m_awaitingContinuation.insert(requestId); } diff --git a/ChatView/ClientInterface.hpp b/ChatView/ClientInterface.hpp index 433961e..53f498e 100644 --- a/ChatView/ClientInterface.hpp +++ b/ChatView/ClientInterface.hpp @@ -54,7 +54,10 @@ private slots: void handleThinkingBlockReceived( const QString &requestId, const QString &thinking, const QString &signature); void handleToolExecutionStarted( - const QString &requestId, const QString &toolId, const QString &toolName); + const QString &requestId, + const QString &toolId, + const QString &toolName, + const QJsonObject &arguments); void handleToolExecutionCompleted( const QString &requestId, const QString &toolId, @@ -75,6 +78,7 @@ private: { QJsonObject originalRequest; PluginLLMCore::Provider *provider; + bool dropPreToolText = false; }; PluginLLMCore::IPromptProvider *m_promptProvider = nullptr; diff --git a/pluginllmcore/ContextData.hpp b/pluginllmcore/ContextData.hpp index 9116993..4fce5c8 100644 --- a/pluginllmcore/ContextData.hpp +++ b/pluginllmcore/ContextData.hpp @@ -3,6 +3,7 @@ #pragma once +#include #include #include @@ -17,6 +18,15 @@ struct ImageAttachment bool operator==(const ImageAttachment &) const = default; }; +struct ToolCall +{ + QString id; + QString name; + QJsonObject arguments; + + bool operator==(const ToolCall &) const = default; +}; + struct Message { QString role; @@ -26,6 +36,10 @@ struct Message bool isRedacted = false; std::optional> images; + QVector toolCalls; + QString toolCallId; + QString toolName; + // clang-format off bool operator==(const Message&) const = default; // clang-format on diff --git a/pluginllmcore/PromptTemplate.hpp b/pluginllmcore/PromptTemplate.hpp index d873239..7098ac4 100644 --- a/pluginllmcore/PromptTemplate.hpp +++ b/pluginllmcore/PromptTemplate.hpp @@ -25,12 +25,8 @@ public: virtual QString description() const = 0; virtual bool isSupportProvider(ProviderID id) const = 0; - // Endpoint path this template expects to be sent to. Empty string - // (default) means "let the provider's client use its standard chat - // path" (/chat/completions, /api/chat, /v1/messages, ...). Templates - // producing non-chat payload shapes (e.g. {prompt, suffix} for - // Mistral FIM, {input_prefix, input_suffix} for llama.cpp infill) - // must override this to the path their payload is valid for. virtual QString endpoint() const { return {}; } + + virtual bool supportsToolHistory() const { return false; } }; } // namespace QodeAssist::PluginLLMCore diff --git a/sources/external/llmqore b/sources/external/llmqore index 6b62a2d..ddbc38f 160000 --- a/sources/external/llmqore +++ b/sources/external/llmqore @@ -1 +1 @@ -Subproject commit 6b62a2d57c0e22952d15e783f341a5d895258dd6 +Subproject commit ddbc38ffbd47d553774ec1895fce53911cf3da73 diff --git a/templates/Claude.hpp b/templates/Claude.hpp index eea55ae..4496083 100644 --- a/templates/Claude.hpp +++ b/templates/Claude.hpp @@ -15,6 +15,7 @@ public: PluginLLMCore::TemplateType type() const override { return PluginLLMCore::TemplateType::Chat; } QString name() const override { return "Claude"; } QStringList stopWords() const override { return QStringList(); } + bool supportsToolHistory() const override { return true; } void prepareRequest(QJsonObject &request, const PluginLLMCore::ContextData &context) const override { QJsonArray messages; @@ -24,9 +25,48 @@ public: } if (context.history) { + int toolResultUserIdx = -1; for (const auto &msg : context.history.value()) { if (msg.role == "system") continue; - + + if (!msg.toolCalls.isEmpty()) { + toolResultUserIdx = -1; + QJsonArray content; + if (!msg.content.isEmpty()) { + content.append(QJsonObject{{"type", "text"}, {"text", msg.content}}); + } + for (const auto &call : msg.toolCalls) { + content.append(QJsonObject{ + {"type", "tool_use"}, + {"id", call.id}, + {"name", call.name}, + {"input", call.arguments}}); + } + messages.append(QJsonObject{{"role", "assistant"}, {"content", content}}); + continue; + } + + if (msg.role == "tool") { + QJsonObject resultBlock{ + {"type", "tool_result"}, + {"tool_use_id", msg.toolCallId}, + {"content", msg.content}}; + if (toolResultUserIdx >= 0) { + QJsonObject userMsg = messages[toolResultUserIdx].toObject(); + QJsonArray content = userMsg["content"].toArray(); + content.append(resultBlock); + userMsg["content"] = content; + messages[toolResultUserIdx] = userMsg; + } else { + messages.append(QJsonObject{ + {"role", "user"}, {"content", QJsonArray{resultBlock}}}); + toolResultUserIdx = messages.size() - 1; + } + continue; + } + + toolResultUserIdx = -1; + if (msg.isThinking) { // Claude API requires signature for thinking blocks if (msg.signature.isEmpty()) { diff --git a/templates/GoogleAI.hpp b/templates/GoogleAI.hpp index cdf6c25..bd57106 100644 --- a/templates/GoogleAI.hpp +++ b/templates/GoogleAI.hpp @@ -16,6 +16,7 @@ public: PluginLLMCore::TemplateType type() const override { return PluginLLMCore::TemplateType::Chat; } QString name() const override { return "Google AI"; } QStringList stopWords() const override { return QStringList(); } + bool supportsToolHistory() const override { return true; } void prepareRequest(QJsonObject &request, const PluginLLMCore::ContextData &context) const override { @@ -26,7 +27,45 @@ public: {"parts", QJsonObject{{"text", context.systemPrompt.value()}}}}; } + int toolResultIdx = -1; for (const auto &msg : context.history.value()) { + if (!msg.toolCalls.isEmpty()) { + toolResultIdx = -1; + QJsonArray callParts; + if (!msg.content.isEmpty()) { + callParts.append(QJsonObject{{"text", msg.content}}); + } + for (const auto &call : msg.toolCalls) { + callParts.append(QJsonObject{ + {"functionCall", + QJsonObject{{"name", call.name}, {"args", call.arguments}}}}); + } + contents.append(QJsonObject{{"role", "model"}, {"parts", callParts}}); + continue; + } + + if (msg.role == "tool") { + QJsonObject responsePart{ + {"functionResponse", + QJsonObject{ + {"name", msg.toolName}, + {"response", QJsonObject{{"result", msg.content}}}}}}; + if (toolResultIdx >= 0) { + QJsonObject fnMsg = contents[toolResultIdx].toObject(); + QJsonArray fnParts = fnMsg["parts"].toArray(); + fnParts.append(responsePart); + fnMsg["parts"] = fnParts; + contents[toolResultIdx] = fnMsg; + } else { + contents.append( + QJsonObject{{"role", "function"}, {"parts", QJsonArray{responsePart}}}); + toolResultIdx = contents.size() - 1; + } + continue; + } + + toolResultIdx = -1; + QJsonObject content; QJsonArray parts; diff --git a/templates/MistralAI.hpp b/templates/MistralAI.hpp index 3bc49fb..d5b5d75 100644 --- a/templates/MistralAI.hpp +++ b/templates/MistralAI.hpp @@ -6,6 +6,7 @@ #include #include "pluginllmcore/PromptTemplate.hpp" +#include "templates/ToolMessages.hpp" namespace QodeAssist::Templates { @@ -47,6 +48,7 @@ public: PluginLLMCore::TemplateType type() const override { return PluginLLMCore::TemplateType::Chat; } QString name() const override { return "Mistral AI Chat"; } QStringList stopWords() const override { return QStringList(); } + bool supportsToolHistory() const override { return true; } void prepareRequest(QJsonObject &request, const PluginLLMCore::ContextData &context) const override { @@ -59,6 +61,9 @@ public: if (context.history) { for (const auto &msg : context.history.value()) { + if (appendOpenAIToolMessage(messages, msg)) { + continue; + } if (msg.images && !msg.images->isEmpty()) { QJsonArray content; diff --git a/templates/Ollama.hpp b/templates/Ollama.hpp index 4386117..74e08d8 100644 --- a/templates/Ollama.hpp +++ b/templates/Ollama.hpp @@ -49,6 +49,7 @@ public: PluginLLMCore::TemplateType type() const override { return PluginLLMCore::TemplateType::Chat; } QString name() const override { return "Ollama Chat"; } QStringList stopWords() const override { return QStringList(); } + bool supportsToolHistory() const override { return true; } void prepareRequest(QJsonObject &request, const PluginLLMCore::ContextData &context) const override { @@ -63,8 +64,28 @@ public: for (const auto &msg : context.history.value()) { QJsonObject messageObj; messageObj["role"] = msg.role; - messageObj["content"] = msg.content; - + + if (!msg.toolCalls.isEmpty()) { + QJsonArray toolCalls; + for (const auto &call : msg.toolCalls) { + toolCalls.append(QJsonObject{ + {"type", "function"}, + {"function", + QJsonObject{{"name", call.name}, {"arguments", call.arguments}}}}); + } + messageObj["tool_calls"] = toolCalls; + if (!msg.content.isEmpty()) { + messageObj["content"] = msg.content; + } + } else { + messageObj["content"] = msg.content; + // Ollama correlates a tool result to its originating + // call by tool_name; omitting it breaks multi-tool turns. + if (msg.role == QLatin1String("tool") && !msg.toolName.isEmpty()) { + messageObj["tool_name"] = msg.toolName; + } + } + if (msg.images && !msg.images->isEmpty()) { QJsonArray images; for (const auto &image : msg.images.value()) { diff --git a/templates/OpenAI.hpp b/templates/OpenAI.hpp index 16bc53b..1957e1e 100644 --- a/templates/OpenAI.hpp +++ b/templates/OpenAI.hpp @@ -6,6 +6,7 @@ #include #include "pluginllmcore/PromptTemplate.hpp" +#include "templates/ToolMessages.hpp" namespace QodeAssist::Templates { @@ -15,6 +16,7 @@ public: PluginLLMCore::TemplateType type() const override { return PluginLLMCore::TemplateType::Chat; } QString name() const override { return "OpenAI"; } QStringList stopWords() const override { return QStringList(); } + bool supportsToolHistory() const override { return true; } void prepareRequest(QJsonObject &request, const PluginLLMCore::ContextData &context) const override { QJsonArray messages; @@ -26,6 +28,9 @@ public: if (context.history) { for (const auto &msg : context.history.value()) { + if (appendOpenAIToolMessage(messages, msg)) { + continue; + } if (msg.images && !msg.images->isEmpty()) { QJsonArray content; diff --git a/templates/OpenAICompatible.hpp b/templates/OpenAICompatible.hpp index 6fa9443..ee33563 100644 --- a/templates/OpenAICompatible.hpp +++ b/templates/OpenAICompatible.hpp @@ -6,6 +6,7 @@ #include #include "pluginllmcore/PromptTemplate.hpp" +#include "templates/ToolMessages.hpp" namespace QodeAssist::Templates { @@ -15,6 +16,7 @@ public: PluginLLMCore::TemplateType type() const override { return PluginLLMCore::TemplateType::Chat; } QString name() const override { return "OpenAI Compatible"; } QStringList stopWords() const override { return QStringList(); } + bool supportsToolHistory() const override { return true; } void prepareRequest(QJsonObject &request, const PluginLLMCore::ContextData &context) const override { QJsonArray messages; @@ -26,6 +28,9 @@ public: if (context.history) { for (const auto &msg : context.history.value()) { + if (appendOpenAIToolMessage(messages, msg)) { + continue; + } if (msg.images && !msg.images->isEmpty()) { QJsonArray content; diff --git a/templates/OpenAIResponses.hpp b/templates/OpenAIResponses.hpp index e218f02..cd6b9dd 100644 --- a/templates/OpenAIResponses.hpp +++ b/templates/OpenAIResponses.hpp @@ -6,6 +6,7 @@ #include "pluginllmcore/PromptTemplate.hpp" #include +#include #include namespace QodeAssist::Templates { @@ -22,6 +23,8 @@ public: QStringList stopWords() const override { return {}; } + bool supportsToolHistory() const override { return true; } + void prepareRequest( QJsonObject &request, const PluginLLMCore::ContextData &context) const override { @@ -39,6 +42,30 @@ public: continue; } + if (!msg.toolCalls.isEmpty()) { + if (!msg.content.isEmpty()) { + input.append(QJsonObject{{"role", "assistant"}, {"content", msg.content}}); + } + for (const auto &call : msg.toolCalls) { + input.append(QJsonObject{ + {"type", "function_call"}, + {"call_id", call.id}, + {"name", call.name}, + {"arguments", + QString::fromUtf8( + QJsonDocument(call.arguments).toJson(QJsonDocument::Compact))}}); + } + continue; + } + + if (msg.role == "tool") { + input.append(QJsonObject{ + {"type", "function_call_output"}, + {"call_id", msg.toolCallId}, + {"output", msg.content}}); + continue; + } + QJsonObject message; message["role"] = msg.role; diff --git a/templates/ToolMessages.hpp b/templates/ToolMessages.hpp new file mode 100644 index 0000000..d8c5b54 --- /dev/null +++ b/templates/ToolMessages.hpp @@ -0,0 +1,45 @@ +// Copyright (C) 2024-2026 Petr Mironychev +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include + +#include "pluginllmcore/ContextData.hpp" + +namespace QodeAssist::Templates { + +inline bool appendOpenAIToolMessage(QJsonArray &messages, const PluginLLMCore::Message &msg) +{ + if (!msg.toolCalls.isEmpty()) { + QJsonArray toolCalls; + for (const auto &call : msg.toolCalls) { + toolCalls.append(QJsonObject{ + {"id", call.id}, + {"type", "function"}, + {"function", + QJsonObject{ + {"name", call.name}, + {"arguments", + QString::fromUtf8( + QJsonDocument(call.arguments).toJson(QJsonDocument::Compact))}}}}); + } + QJsonObject toolMessage{{"role", "assistant"}, {"tool_calls", toolCalls}}; + toolMessage["content"] = msg.content.isEmpty() ? QJsonValue() : QJsonValue(msg.content); + messages.append(toolMessage); + return true; + } + + if (msg.role == QLatin1String("tool")) { + messages.append(QJsonObject{ + {"role", "tool"}, {"tool_call_id", msg.toolCallId}, {"content", msg.content}}); + return true; + } + + return false; +} + +} // namespace QodeAssist::Templates