feat: Add possibility to link files to the current system prompt

- Add linking files to chat
- Rework tokens counting
This commit is contained in:
Petr Mironychev 2025-01-23 10:17:38 +01:00 committed by GitHub
parent add86d2e67
commit 9add61c805
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 354 additions and 48 deletions

View File

@ -18,9 +18,12 @@ qt_add_qml_module(QodeAssistChatView
qml/parts/BottomBar.qml
qml/parts/AttachedFilesPlace.qml
RESOURCES
icons/attach-file.svg
icons/attach-file-light.svg
icons/attach-file-dark.svg
icons/close-dark.svg
icons/close-light.svg
icons/link-file-light.svg
icons/link-file-dark.svg
SOURCES
ChatWidget.hpp ChatWidget.cpp
ChatModel.hpp ChatModel.cpp

View File

@ -28,7 +28,6 @@ namespace QodeAssist::Chat {
ChatModel::ChatModel(QObject *parent)
: QAbstractListModel(parent)
, m_totalTokens(0)
{
auto &settings = Settings::chatAssistantSettings();
@ -90,26 +89,19 @@ void ChatModel::addMessage(
.arg(attachment.filename, attachment.content);
}
}
int tokenCount = estimateTokenCount(fullContent);
if (!m_messages.isEmpty() && !id.isEmpty() && m_messages.last().id == id) {
Message &lastMessage = m_messages.last();
int oldTokenCount = lastMessage.tokenCount;
lastMessage.content = content;
lastMessage.attachments = attachments;
lastMessage.tokenCount = tokenCount;
m_totalTokens += (tokenCount - oldTokenCount);
emit dataChanged(index(m_messages.size() - 1), index(m_messages.size() - 1));
} else {
beginInsertRows(QModelIndex(), m_messages.size(), m_messages.size());
Message newMessage{role, content, tokenCount, id};
Message newMessage{role, content, id};
newMessage.attachments = attachments;
m_messages.append(newMessage);
m_totalTokens += tokenCount;
endInsertRows();
}
emit totalTokensChanged();
}
QVector<ChatModel::Message> ChatModel::getChatHistory() const
@ -117,18 +109,11 @@ QVector<ChatModel::Message> ChatModel::getChatHistory() const
return m_messages;
}
int ChatModel::estimateTokenCount(const QString &text) const
{
return text.length() / 4;
}
void ChatModel::clear()
{
beginResetModel();
m_messages.clear();
m_totalTokens = 0;
endResetModel();
emit totalTokensChanged();
emit modelReseted();
}
@ -199,11 +184,6 @@ QJsonArray ChatModel::prepareMessagesForRequest(const QString &systemPrompt) con
return messages;
}
int ChatModel::totalTokens() const
{
return m_totalTokens;
}
int ChatModel::tokensThreshold() const
{
auto &settings = Settings::chatAssistantSettings();

View File

@ -33,7 +33,6 @@ namespace QodeAssist::Chat {
class ChatModel : public QAbstractListModel
{
Q_OBJECT
Q_PROPERTY(int totalTokens READ totalTokens NOTIFY totalTokensChanged FINAL)
Q_PROPERTY(int tokensThreshold READ tokensThreshold NOTIFY tokensThresholdChanged FINAL)
QML_ELEMENT
@ -47,7 +46,6 @@ public:
{
ChatRole role;
QString content;
int tokenCount;
QString id;
QList<Context::ContentFile> attachments;
@ -70,22 +68,17 @@ public:
QVector<Message> getChatHistory() const;
QJsonArray prepareMessagesForRequest(const QString &systemPrompt) const;
int totalTokens() const;
int tokensThreshold() const;
QString currentModel() const;
QString lastMessageId() const;
signals:
void totalTokensChanged();
void tokensThresholdChanged();
void modelReseted();
private:
int estimateTokenCount(const QString &text) const;
QVector<Message> m_messages;
int m_totalTokens = 0;
};
} // namespace QodeAssist::Chat

View File

@ -35,6 +35,8 @@
#include "GeneralSettings.hpp"
#include "Logger.hpp"
#include "ProjectSettings.hpp"
#include "context/TokenUtils.hpp"
#include "context/ContextManager.hpp"
namespace QodeAssist::Chat {
@ -61,7 +63,20 @@ ChatRootView::ChatRootView(QQuickItem *parent)
this,
&ChatRootView::autosave);
connect(
m_clientInterface,
&ClientInterface::messageReceivedCompletely,
this,
&ChatRootView::updateInputTokensCount);
connect(m_chatModel, &ChatModel::modelReseted, [this]() { m_recentFilePath = QString(); });
connect(this, &ChatRootView::attachmentFilesChanged, &ChatRootView::updateInputTokensCount);
connect(this, &ChatRootView::linkedFilesChanged, &ChatRootView::updateInputTokensCount);
connect(&Settings::chatAssistantSettings().useSystemPrompt, &Utils::BaseAspect::changed,
this, &ChatRootView::updateInputTokensCount);
connect(&Settings::chatAssistantSettings().systemPrompt, &Utils::BaseAspect::changed,
this, &ChatRootView::updateInputTokensCount);
updateInputTokensCount();
}
ChatModel *ChatRootView::chatModel() const
@ -71,7 +86,7 @@ ChatModel *ChatRootView::chatModel() const
void ChatRootView::sendMessage(const QString &message, bool sharingCurrentFile)
{
if (m_chatModel->totalTokens() > m_chatModel->tokensThreshold()) {
if (m_inputTokensCount > m_chatModel->tokensThreshold()) {
QMessageBox::StandardButton reply = QMessageBox::question(
Core::ICore::dialogParent(),
tr("Token Limit Exceeded"),
@ -87,7 +102,7 @@ void ChatRootView::sendMessage(const QString &message, bool sharingCurrentFile)
}
}
m_clientInterface->sendMessage(message, m_attachmentFiles, sharingCurrentFile);
m_clientInterface->sendMessage(message, m_attachmentFiles, m_linkedFiles, sharingCurrentFile);
clearAttachmentFiles();
}
@ -109,6 +124,14 @@ void ChatRootView::clearAttachmentFiles()
}
}
void ChatRootView::clearLinkedFiles()
{
if (!m_linkedFiles.isEmpty()) {
m_linkedFiles.clear();
emit linkedFilesChanged();
}
}
QString ChatRootView::getChatsHistoryDir() const
{
QString path;
@ -156,6 +179,7 @@ void ChatRootView::loadHistory(const QString &filePath)
} else {
m_recentFilePath = filePath;
}
updateInputTokensCount();
}
void ChatRootView::showSaveDialog()
@ -254,6 +278,16 @@ QString ChatRootView::getAutosaveFilePath() const
return QDir(dir).filePath(getSuggestedFileName() + ".json");
}
QStringList ChatRootView::attachmentFiles() const
{
return m_attachmentFiles;
}
QStringList ChatRootView::linkedFiles() const
{
return m_linkedFiles;
}
void ChatRootView::showAttachFilesDialog()
{
QFileDialog dialog(nullptr, tr("Select Files to Attach"));
@ -280,4 +314,86 @@ void ChatRootView::showAttachFilesDialog()
}
}
void ChatRootView::removeFileFromAttachList(int index)
{
if (index >= 0 && index < m_attachmentFiles.size()) {
m_attachmentFiles.removeAt(index);
emit attachmentFilesChanged();
}
}
void ChatRootView::showLinkFilesDialog()
{
QFileDialog dialog(nullptr, tr("Select Files to Attach"));
dialog.setFileMode(QFileDialog::ExistingFiles);
if (auto project = ProjectExplorer::ProjectManager::startupProject()) {
dialog.setDirectory(project->projectDirectory().toString());
}
if (dialog.exec() == QDialog::Accepted) {
QStringList newFilePaths = dialog.selectedFiles();
if (!newFilePaths.isEmpty()) {
bool filesAdded = false;
for (const QString &filePath : newFilePaths) {
if (!m_linkedFiles.contains(filePath)) {
m_linkedFiles.append(filePath);
filesAdded = true;
}
}
if (filesAdded) {
emit linkedFilesChanged();
}
}
}
}
void ChatRootView::removeFileFromLinkList(int index)
{
if (index >= 0 && index < m_linkedFiles.size()) {
m_linkedFiles.removeAt(index);
emit linkedFilesChanged();
}
}
void ChatRootView::calculateMessageTokensCount(const QString &message)
{
m_messageTokensCount = Context::TokenUtils::estimateTokens(message);
updateInputTokensCount();
}
void ChatRootView::updateInputTokensCount()
{
int inputTokens = m_messageTokensCount;
auto& settings = Settings::chatAssistantSettings();
if (settings.useSystemPrompt()) {
inputTokens += Context::TokenUtils::estimateTokens(settings.systemPrompt());
}
if (!m_attachmentFiles.isEmpty()) {
auto attachFiles = Context::ContextManager::instance().getContentFiles(m_attachmentFiles);
inputTokens += Context::TokenUtils::estimateFilesTokens(attachFiles);
}
if (!m_linkedFiles.isEmpty()) {
auto linkFiles = Context::ContextManager::instance().getContentFiles(m_linkedFiles);
inputTokens += Context::TokenUtils::estimateFilesTokens(linkFiles);
}
const auto& history = m_chatModel->getChatHistory();
for (const auto& message : history) {
inputTokens += Context::TokenUtils::estimateTokens(message.content);
inputTokens += 4; // + role
}
m_inputTokensCount = inputTokens;
emit inputTokensCountChanged();
}
int ChatRootView::inputTokensCount() const
{
return m_inputTokensCount;
}
} // namespace QodeAssist::Chat

View File

@ -33,7 +33,9 @@ class ChatRootView : public QQuickItem
Q_PROPERTY(QString currentTemplate READ currentTemplate NOTIFY currentTemplateChanged FINAL)
Q_PROPERTY(bool isSharingCurrentFile READ isSharingCurrentFile NOTIFY
isSharingCurrentFileChanged FINAL)
Q_PROPERTY(QStringList attachmentFiles MEMBER m_attachmentFiles NOTIFY attachmentFilesChanged)
Q_PROPERTY(QStringList attachmentFiles READ attachmentFiles NOTIFY attachmentFilesChanged FINAL)
Q_PROPERTY(QStringList linkedFiles READ linkedFiles NOTIFY linkedFilesChanged FINAL)
Q_PROPERTY(int inputTokensCount READ inputTokensCount NOTIFY inputTokensCountChanged FINAL)
QML_ELEMENT
@ -54,19 +56,32 @@ public:
void autosave();
QString getAutosaveFilePath() const;
QStringList attachmentFiles() const;
QStringList linkedFiles() const;
Q_INVOKABLE void showAttachFilesDialog();
Q_INVOKABLE void removeFileFromAttachList(int index);
Q_INVOKABLE void showLinkFilesDialog();
Q_INVOKABLE void removeFileFromLinkList(int index);
Q_INVOKABLE void calculateMessageTokensCount(const QString &message);
void updateInputTokensCount();
int inputTokensCount() const;
public slots:
void sendMessage(const QString &message, bool sharingCurrentFile = false);
void copyToClipboard(const QString &text);
void cancelRequest();
void clearAttachmentFiles();
void clearLinkedFiles();
signals:
void chatModelChanged();
void currentTemplateChanged();
void isSharingCurrentFileChanged();
void attachmentFilesChanged();
void linkedFilesChanged();
void inputTokensCountChanged();
private:
QString getChatsHistoryDir() const;
@ -77,6 +92,9 @@ private:
QString m_currentTemplate;
QString m_recentFilePath;
QStringList m_attachmentFiles;
QStringList m_linkedFiles;
int m_messageTokensCount{0};
int m_inputTokensCount{0};
};
} // namespace QodeAssist::Chat

View File

@ -82,7 +82,6 @@ QJsonObject ChatSerializer::serializeMessage(const ChatModel::Message &message)
QJsonObject messageObj;
messageObj["role"] = static_cast<int>(message.role);
messageObj["content"] = message.content;
messageObj["tokenCount"] = message.tokenCount;
messageObj["id"] = message.id;
return messageObj;
}
@ -92,7 +91,6 @@ ChatModel::Message ChatSerializer::deserializeMessage(const QJsonObject &json)
ChatModel::Message message;
message.role = static_cast<ChatModel::ChatRole>(json["role"].toInt());
message.content = json["content"].toString();
message.tokenCount = json["tokenCount"].toInt();
message.id = json["id"].toString();
return message;
}
@ -107,7 +105,6 @@ QJsonObject ChatSerializer::serializeChat(const ChatModel *model)
QJsonObject root;
root["version"] = VERSION;
root["messages"] = messagesArray;
root["totalTokens"] = model->totalTokens();
return root;
}

View File

@ -66,7 +66,7 @@ ClientInterface::ClientInterface(ChatModel *chatModel, QObject *parent)
ClientInterface::~ClientInterface() = default;
void ClientInterface::sendMessage(
const QString &message, const QList<QString> &attachments, bool includeCurrentFile)
const QString &message, const QList<QString> &attachments, const QList<QString> &linkedFiles, bool includeCurrentFile)
{
cancelRequest();
@ -107,6 +107,10 @@ void ClientInterface::sendMessage(
}
}
if (!linkedFiles.isEmpty()) {
systemPrompt = getSystemPromptWithLinkedFiles(systemPrompt, linkedFiles);
}
QJsonObject providerRequest;
providerRequest["model"] = Settings::generalSettings().caModel();
providerRequest["stream"] = chatAssistantSettings.stream();
@ -198,4 +202,21 @@ QString ClientInterface::getCurrentFileContext() const
return QString("Current file context:\n%1\nFile content:\n%2").arg(fileInfo, content);
}
QString ClientInterface::getSystemPromptWithLinkedFiles(const QString &basePrompt, const QList<QString> &linkedFiles) const
{
QString updatedPrompt = basePrompt;
if (!linkedFiles.isEmpty()) {
updatedPrompt += "\n\nLinked files for reference:\n";
auto contentFiles = Context::ContextManager::instance().getContentFiles(linkedFiles);
for (const auto &file : contentFiles) {
updatedPrompt += QString("\nFile: %1\nContent:\n%2\n")
.arg(file.filename, file.content);
}
}
return updatedPrompt;
}
} // namespace QodeAssist::Chat

View File

@ -39,6 +39,7 @@ public:
void sendMessage(
const QString &message,
const QList<QString> &attachments = {},
const QList<QString> &linkedFiles = {},
bool includeCurrentFile = false);
void clearMessages();
void cancelRequest();
@ -50,6 +51,9 @@ signals:
private:
void handleLLMResponse(const QString &response, const QJsonObject &request, bool isComplete);
QString getCurrentFileContext() const;
QString getSystemPromptWithLinkedFiles(
const QString &basePrompt,
const QList<QString> &linkedFiles) const;
LLMCore::RequestHandler *m_requestHandler;
ChatModel *m_chatModel;

View File

@ -1,6 +1,7 @@
<svg width="24" height="48" viewBox="0 0 24 48" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_37_14)">
<path d="M22 10.1053V36.7368C22 41.8547 17.525 46 12 46C6.475 46 2 41.8547 2 36.7368V7.78947C2 4.59368 4.8 2 8.25 2C11.7 2 15.75 4.59368 15.75 7.78947V35.5789C15.75 36.8526 13.375 39.0526 12 39.0526C10.625 39.0526 8.25 36.8526 8.25 35.5789V21.6842V8.94737" stroke="black" stroke-width="3" stroke-linecap="round"/>
<path d="M22 10.1053V36.7368C22 41.8547 17.525 46 12 46C6.475 46 2 41.8547 2 36.7368V7.78947C2 4.59368 4.8 2 8.25 2C11.7 2 15.75 4.59368 15.75 7.78947V35.5789C15.75 36.8526 13.375 39.0526 12 39.0526C10.625 39.0526 8.25 36.8526 8.25 35.5789V21.6842V8.94737" stroke="black" stroke-width="3" stroke-linecap="round"/>
</g>
<defs>
<clipPath id="clip0_37_14">

Before

Width:  |  Height:  |  Size: 555 B

After

Width:  |  Height:  |  Size: 869 B

View File

@ -0,0 +1,11 @@
<svg width="24" height="48" viewBox="0 0 24 48" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_51_20)">
<path d="M22 10.1053V36.7368C22 41.8547 17.525 46 12 46C6.475 46 2 41.8547 2 36.7368V7.78947C2 4.59368 4.8 2 8.25 2C11.7 2 15.75 4.59368 15.75 7.78947V35.5789C15.75 36.8526 13.375 39.0526 12 39.0526C10.625 39.0526 8.25 36.8526 8.25 35.5789V21.6842V8.94737" stroke="white" stroke-width="3" stroke-linecap="round"/>
<path d="M22 10.1053V36.7368C22 41.8547 17.525 46 12 46C6.475 46 2 41.8547 2 36.7368V7.78947C2 4.59368 4.8 2 8.25 2C11.7 2 15.75 4.59368 15.75 7.78947V35.5789C15.75 36.8526 13.375 39.0526 12 39.0526C10.625 39.0526 8.25 36.8526 8.25 35.5789V21.6842V8.94737" stroke="white" stroke-width="3" stroke-linecap="round"/>
</g>
<defs>
<clipPath id="clip0_51_20">
<rect width="24" height="48" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 869 B

View File

@ -0,0 +1,12 @@
<svg width="20" height="44" viewBox="0 0 20 44" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_49_24)">
<path d="M10 12L10 32L10 12Z" fill="black"/>
<path d="M10 12L10 32" stroke="black" stroke-width="3"/>
<path d="M1.50001 12.484C1.50001 -1.99999 18.5 -1.99999 18.5 12.484M1.5 31.5334C1.50001 46 18.5 46 18.5 31.5334" stroke="black" stroke-width="3" stroke-linecap="round"/>
</g>
<defs>
<clipPath id="clip0_49_24">
<rect width="20" height="44" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 513 B

View File

@ -0,0 +1,12 @@
<svg width="20" height="44" viewBox="0 0 20 44" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_51_24)">
<path d="M10 12L10 32Z" fill="white"/>
<path d="M10 12L10 32" stroke="white" stroke-width="3"/>
<path d="M1.50001 12.484C1.50001 -1.99999 18.5 -1.99999 18.5 12.484M1.5 31.5334C1.50001 46 18.5 46 18.5 31.5334" stroke="white" stroke-width="3" stroke-linecap="round"/>
</g>
<defs>
<clipPath id="clip0_51_24">
<rect width="20" height="44" fill="white"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 507 B

View File

@ -70,7 +70,7 @@ ChatRootView {
loadButton.onClicked: root.showLoadDialog()
clearButton.onClicked: root.clearChat()
tokensBadge {
text: qsTr("tokens:%1/%2").arg(root.chatModel.totalTokens).arg(root.chatModel.tokensThreshold)
text: qsTr("tokens:%1/%2").arg(root.inputTokensCount).arg(root.chatModel.tokensThreshold)
}
}
@ -147,6 +147,8 @@ ChatRootView {
}
}
onTextChanged: root.calculateMessageTokensCount(messageInput.text)
Keys.onPressed: function(event) {
if ((event.key === Qt.Key_Return || event.key === Qt.Key_Enter) && !(event.modifiers & Qt.ShiftModifier)) {
root.sendChatMessage()
@ -161,6 +163,21 @@ ChatRootView {
Layout.fillWidth: true
attachedFilesModel: root.attachmentFiles
iconPath: palette.window.hslLightness > 0.5 ? "qrc:/qt/qml/ChatView/icons/attach-file-dark.svg"
: "qrc:/qt/qml/ChatView/icons/attach-file-light.svg"
accentColor: Qt.tint(palette.mid, Qt.rgba(0, 0.8, 0.3, 0.4))
onRemoveFileFromListByIndex: (index) => root.removeFileFromAttachList(index)
}
AttachedFilesPlace {
id: linkedFilesPlace
Layout.fillWidth: true
attachedFilesModel: root.linkedFiles
iconPath: palette.window.hslLightness > 0.5 ? "qrc:/qt/qml/ChatView/icons/link-file-dark.svg"
: "qrc:/qt/qml/ChatView/icons/link-file-light.svg"
accentColor: Qt.tint(palette.mid, Qt.rgba(0, 0.3, 0.8, 0.4))
onRemoveFileFromListByIndex: (index) => root.removeFileFromLinkList(index)
}
BottomBar {
@ -173,11 +190,14 @@ ChatRootView {
stopButton.onClicked: root.cancelRequest()
sharingCurrentFile.checked: root.isSharingCurrentFile
attachFiles.onClicked: root.showAttachFilesDialog()
linkFiles.onClicked: root.showLinkFilesDialog()
}
}
function clearChat() {
root.chatModel.clear()
root.clearAttachmentFiles()
root.clearLinkedFiles()
}
function scrollToBottom() {

View File

@ -23,9 +23,13 @@ import QtQuick.Layouts
import ChatView
Flow {
id: attachFilesPlace
id: root
property alias attachedFilesModel: attachRepeater.model
property color accentColor: palette.mid
property string iconPath
signal removeFileFromListByIndex(index: int)
spacing: 5
leftPadding: 5
@ -41,18 +45,33 @@ Flow {
required property string modelData
height: 30
width: fileNameText.width + closeButton.width + 20
width: contentRow.width + 10
radius: 4
color: palette.button
border.width: 1
border.color: palette.mid
border.color: mouse.hovered ? palette.highlight : root.accentColor
HoverHandler {
id: mouse
}
Row {
id: contentRow
spacing: 5
anchors.verticalCenter: parent.verticalCenter
anchors.left: parent.left
anchors.leftMargin: 5
Image {
id: icon
anchors.verticalCenter: parent.verticalCenter
source: root.iconPath
sourceSize.width: 8
sourceSize.height: 15
}
Text {
id: fileNameText
@ -69,14 +88,10 @@ Flow {
id: closeButton
anchors.verticalCenter: parent.verticalCenter
width: closeIcon.width
height: closeButton.width
width: closeIcon.width + 5
height: closeButton.width + 5
onClicked: {
const newList = [...root.attachmentFiles];
newList.splice(index, 1);
root.attachmentFiles = newList;
}
onClicked: root.removeFileFromListByIndex(index)
Image {
id: closeIcon

View File

@ -29,6 +29,7 @@ Rectangle {
property alias stopButton: stopButtonId
property alias sharingCurrentFile: sharingCurrentFileId
property alias attachFiles: attachFilesId
property alias linkFiles: linkFilesId
color: palette.window.hslLightness > 0.5 ?
Qt.darker(palette.window, 1.1) :
@ -69,13 +70,24 @@ Rectangle {
id: attachFilesId
icon {
source: "qrc:/qt/qml/ChatView/icons/attach-file.svg"
source: "qrc:/qt/qml/ChatView/icons/attach-file-dark.svg"
height: 15
width: 8
}
text: qsTr("Attach files")
}
QoAButton {
id: linkFilesId
icon {
source: "qrc:/qt/qml/ChatView/icons/link-file-dark.svg"
height: 15
width: 8
}
text: qsTr("Link files")
}
Item {
Layout.fillWidth: true
}

View File

@ -3,6 +3,7 @@ add_library(Context STATIC
ChangesManager.h ChangesManager.cpp
ContextManager.hpp ContextManager.cpp
ContentFile.hpp
TokenUtils.hpp TokenUtils.cpp
)
target_link_libraries(Context

54
context/TokenUtils.cpp Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (C) 2024 Petr Mironychev
*
* This file is part of QodeAssist.
*
* QodeAssist is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* QodeAssist is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with QodeAssist. If not, see <https://www.gnu.org/licenses/>.
*/
#include "TokenUtils.hpp"
namespace QodeAssist::Context {
int TokenUtils::estimateTokens(const QString& text)
{
if (text.isEmpty()) {
return 0;
}
// TODO: need to improve
return text.length() / 4;
}
int TokenUtils::estimateFileTokens(const Context::ContentFile& file)
{
int total = 0;
total += estimateTokens(file.filename);
total += estimateTokens(file.content);
total += 5;
return total;
}
int TokenUtils::estimateFilesTokens(const QList<Context::ContentFile>& files)
{
int total = 0;
for (const auto& file : files) {
total += estimateFileTokens(file);
}
return total;
}
}

36
context/TokenUtils.hpp Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (C) 2024 Petr Mironychev
*
* This file is part of QodeAssist.
*
* QodeAssist is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* QodeAssist is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with QodeAssist. If not, see <https://www.gnu.org/licenses/>.
*/
#pragma once
#include <QString>
#include "ContentFile.hpp"
#include <QList>
namespace QodeAssist::Context {
class TokenUtils
{
public:
static int estimateTokens(const QString& text);
static int estimateFileTokens(const Context::ContentFile& file);
static int estimateFilesTokens(const QList<Context::ContentFile>& files);
};
}