From 61aae8e3bae16cbd8eb26e71024f132395cad8dd Mon Sep 17 00:00:00 2001 From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com> Date: Thu, 29 Aug 2024 00:28:54 +0200 Subject: [PATCH 1/4] Remove copyright from request --- DocumentContextReader.cpp | 148 +++++++++++++++++++++++--------------- DocumentContextReader.hpp | 17 +++-- LLMClientInterface.cpp | 5 ++ QodeAssistSettings.cpp | 2 +- 4 files changed, 110 insertions(+), 62 deletions(-) diff --git a/DocumentContextReader.cpp b/DocumentContextReader.cpp index dd0e394..056857c 100644 --- a/DocumentContextReader.cpp +++ b/DocumentContextReader.cpp @@ -27,6 +27,13 @@ namespace QodeAssist { +DocumentContextReader::DocumentContextReader(TextEditor::TextDocument *textDocument) + : m_textDocument(textDocument) + , m_document(textDocument->document()) +{ + m_copyrightInfo = findCopyright(); +} + QString DocumentContextReader::getLineText(int lineNumber, int cursorPosition) const { if (!m_document || lineNumber < 0) @@ -55,76 +62,41 @@ QString DocumentContextReader::getContextBefore(int lineNumber, int cursorPosition, int linesCount) const { - QString context; - for (int i = qMax(0, lineNumber - linesCount); i <= lineNumber; ++i) { - QString line = getLineText(i, i == lineNumber ? cursorPosition : -1); - context += line; - if (i < lineNumber) - context += "\n"; + int effectiveStartLine; + if (m_copyrightInfo.found) { + effectiveStartLine = qMax(m_copyrightInfo.endLine + 1, lineNumber - linesCount); + } else { + effectiveStartLine = qMax(0, lineNumber - linesCount); } - return context; + + return getContextBetween(effectiveStartLine, lineNumber, cursorPosition); } QString DocumentContextReader::getContextAfter(int lineNumber, int cursorPosition, int linesCount) const { - QString context; - int maxLine = lineNumber + linesCount; - for (int i = lineNumber; i <= maxLine; ++i) { - QString line = getLineText(i); - if (i == lineNumber && cursorPosition >= 0) { - line = line.mid(cursorPosition); - } - context += line; - if (i < maxLine && !line.isEmpty()) - context += "\n"; - } - return context; + int endLine = qMin(m_document->blockCount() - 1, lineNumber + linesCount); + return getContextBetween(lineNumber + 1, endLine, cursorPosition); } QString DocumentContextReader::readWholeFileBefore(int lineNumber, int cursorPosition) const { - QString content; - QTextBlock block = m_document->begin(); - int currentLine = 0; - - while (block.isValid() && currentLine <= lineNumber) { - if (currentLine == lineNumber) { - content += block.text().left(cursorPosition); - break; - } else { - content += block.text() + "\n"; - } - block = block.next(); - currentLine++; + int startLine = 0; + if (m_copyrightInfo.found) { + startLine = m_copyrightInfo.endLine + 1; } - return content; + startLine = qMin(startLine, lineNumber); + + QString result = getContextBetween(startLine, lineNumber, cursorPosition); + + return result; } QString DocumentContextReader::readWholeFileAfter(int lineNumber, int cursorPosition) const { - QString content; - QTextBlock block = m_document->begin(); - int currentLine = 0; - - while (block.isValid() && currentLine < lineNumber) { - block = block.next(); - currentLine++; - } - - while (block.isValid()) { - if (currentLine == lineNumber) { - content += block.text().mid(cursorPosition) + "\n"; - } else { - content += block.text() + "\n"; - } - block = block.next(); - currentLine++; - } - - return content.trimmed(); + return getContextBetween(lineNumber, m_document->blockCount() - 1, cursorPosition); } QString DocumentContextReader::getLanguageAndFileInfo() const @@ -139,10 +111,7 @@ QString DocumentContextReader::getLanguageAndFileInfo() const QString fileExtension = QFileInfo(filePath).suffix(); return QString("//Language: %1 (MIME: %2) filepath: %3(%4)\n\n") - .arg(language) - .arg(mimeType) - .arg(filePath) - .arg(fileExtension); + .arg(language, mimeType, filePath, fileExtension); } QString DocumentContextReader::getSpecificInstructions() const @@ -152,4 +121,69 @@ QString DocumentContextReader::getSpecificInstructions() const return QString("//Instructions: %1").arg(specificInstruction); } +CopyrightInfo DocumentContextReader::findCopyright() +{ + CopyrightInfo result = {-1, -1, false}; + + QString text = m_document->toPlainText(); + QRegularExpressionMatchIterator matchIterator = getCopyrightRegex().globalMatch(text); + + QList copyrightBlocks; + + while (matchIterator.hasNext()) { + QRegularExpressionMatch match = matchIterator.next(); + int startPos = match.capturedStart(); + int endPos = match.capturedEnd(); + + CopyrightInfo info; + info.startLine = m_document->findBlock(startPos).blockNumber(); + info.endLine = m_document->findBlock(endPos).blockNumber(); + info.found = true; + + copyrightBlocks.append(info); + } + + for (int i = 0; i < copyrightBlocks.size() - 1; ++i) { + if (copyrightBlocks[i].endLine + 1 >= copyrightBlocks[i + 1].startLine) { + copyrightBlocks[i].endLine = copyrightBlocks[i + 1].endLine; + copyrightBlocks.removeAt(i + 1); + --i; + } + } + + if (!copyrightBlocks.isEmpty()) { // temproary solution, need cache + return copyrightBlocks.first(); + } + + return result; +} + +QString DocumentContextReader::getContextBetween(int startLine, + int endLine, + int cursorPosition) const +{ + QString context; + for (int i = startLine; i <= endLine; ++i) { + QTextBlock block = m_document->findBlockByNumber(i); + if (!block.isValid()) { + break; + } + if (i == endLine) { + context += block.text().left(cursorPosition); + } else { + context += block.text() + "\n"; + } + } + + return context; +} + +const QRegularExpression &DocumentContextReader::getCopyrightRegex() +{ + static const QRegularExpression copyrightRegex( + R"((?:/\*[\s\S]*?Copyright[\s\S]*?\*/| // Copyright[\s\S]*?(?:\n\s*//.*)*|///.*Copyright[\s\S]*?(?:\n\s*///.*)*)|(?://))", + QRegularExpression::MultilineOption | QRegularExpression::CaseInsensitiveOption); + return copyrightRegex; +} + } // namespace QodeAssist diff --git a/DocumentContextReader.hpp b/DocumentContextReader.hpp index 7a2c229..b43a9f4 100644 --- a/DocumentContextReader.hpp +++ b/DocumentContextReader.hpp @@ -24,13 +24,17 @@ namespace QodeAssist { +struct CopyrightInfo +{ + int startLine; + int endLine; + bool found; +}; + class DocumentContextReader { public: - DocumentContextReader(TextEditor::TextDocument *textDocument) - : m_textDocument(textDocument) - , m_document(textDocument->document()) - {} + DocumentContextReader(TextEditor::TextDocument *textDocument); QString getLineText(int lineNumber, int cursorPosition = -1) const; QString getContextBefore(int lineNumber, int cursorPosition, int linesCount) const; @@ -39,10 +43,15 @@ public: QString readWholeFileAfter(int lineNumber, int cursorPosition) const; QString getLanguageAndFileInfo() const; QString getSpecificInstructions() const; + CopyrightInfo findCopyright(); + QString getContextBetween(int startLine, int endLine, int cursorPosition) const; private: TextEditor::TextDocument *m_textDocument; QTextDocument *m_document; + CopyrightInfo m_copyrightInfo; + + static const QRegularExpression &getCopyrightRegex(); }; } // namespace QodeAssist diff --git a/LLMClientInterface.cpp b/LLMClientInterface.cpp index 064c9ef..dd2d3a7 100644 --- a/LLMClientInterface.cpp +++ b/LLMClientInterface.cpp @@ -101,6 +101,9 @@ QString LLMClientInterface::сontextBefore(TextEditor::TextEditorWidget *widget, DocumentContextReader reader(widget->textDocument()); QString languageAndFileInfo = reader.getLanguageAndFileInfo(); + if (lineNumber < reader.findCopyright().endLine) + return QString(); + QString contextBefore; if (settings().readFullFile()) { contextBefore = reader.readWholeFileBefore(lineNumber, cursorPosition); @@ -124,6 +127,8 @@ QString LLMClientInterface::сontextAfter(TextEditor::TextEditorWidget *widget, return QString(); DocumentContextReader reader(widget->textDocument()); + if (lineNumber < reader.findCopyright().endLine) + return QString(); QString contextAfter; if (settings().readFullFile()) { diff --git a/QodeAssistSettings.cpp b/QodeAssistSettings.cpp index becb576..9fd5437 100644 --- a/QodeAssistSettings.cpp +++ b/QodeAssistSettings.cpp @@ -211,12 +211,12 @@ QodeAssistSettings::QodeAssistSettings() Form{Column{Row{selectModels, modelName}}}}, Group{title(Tr::tr("FIM Prompt Settings")), Form{Column{fimPrompts, - readFullFile, maxFileThreshold, ollamaLivetime, specificInstractions, temperature, maxTokens, + readFullFile, readStringsBeforeCursor, readStringsAfterCursor, startSuggestionTimer, From 6bd6edf54dc6a17dd5129c8a2da89efce6062088 Mon Sep 17 00:00:00 2001 From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com> Date: Thu, 29 Aug 2024 00:45:41 +0200 Subject: [PATCH 2/4] Add default key sequence --- qodeassist.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qodeassist.cpp b/qodeassist.cpp index e774f0d..9308730 100644 --- a/qodeassist.cpp +++ b/qodeassist.cpp @@ -82,9 +82,11 @@ public: ActionBuilder requestAction(this, Constants::QODE_ASSIST_REQUEST_SUGGESTION); requestAction.setToolTip( - Tr::tr("Request Ollama suggestion at the current editor's cursor position.")); + Tr::tr("Generate Qode Assist suggestion at the current cursor position.")); requestAction.setText(Tr::tr("Request Ollama Suggestion")); requestAction.setIcon(QCODEASSIST_ICON.icon()); + const QKeySequence defaultShortcut = QKeySequence(Qt::CTRL | Qt::ALT | Qt::Key_Q); + requestAction.setDefaultKeySequence(defaultShortcut); requestAction.addOnTriggered(this, [this] { if (auto editor = TextEditor::TextEditorWidget::currentTextEditorWidget()) { if (m_qodeAssistClient && m_qodeAssistClient->reachable()) { From 1201da6af399a7f05df6b333929f283461cd085b Mon Sep 17 00:00:00 2001 From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:27:50 +0200 Subject: [PATCH 3/4] Fix typo in logs and settings --- QodeAssist.json.in | 4 ++-- QodeAssistUtils.hpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/QodeAssist.json.in b/QodeAssist.json.in index ae7fa42..0f31cc6 100644 --- a/QodeAssist.json.in +++ b/QodeAssist.json.in @@ -1,6 +1,6 @@ { "Name" : "QodeAssist", - "Version" : "0.0.3", + "Version" : "0.0.4", "CompatVersion" : "${IDE_VERSION_COMPAT}", "Vendor" : "Petr Mironychev", "Copyright" : "(C) ${IDE_COPYRIGHT_YEAR} Petr Mironychev, (C) ${IDE_COPYRIGHT_YEAR} The Qt Company Ltd", @@ -11,6 +11,6 @@ Alternatively, this file may be used under the terms of the GNU General Public L "Prerequisites:", "- One of the supported LLM providers installed (e.g., Ollama or LM Studio)", "- A compatible large language model downloaded for your chosen provider (e.g., CodeLlama, StarCoder2)"], - "Url" : "https://github.com/Palm1r", + "Url" : "https://github.com/Palm1r/QodeAssist", ${IDE_PLUGIN_DEPENDENCIES} } diff --git a/QodeAssistUtils.hpp b/QodeAssistUtils.hpp index 2ce2570..4ad1267 100644 --- a/QodeAssistUtils.hpp +++ b/QodeAssistUtils.hpp @@ -41,7 +41,7 @@ inline void logMessage(const QString &message, bool silent = true) if (!loggingEnabled()) return; - const QString prefixedMessage = QLatin1String("[QLLamaAssist] ") + message; + const QString prefixedMessage = QLatin1String("[Qode Assist] ") + message; if (silent) { Core::MessageManager::writeSilently(prefixedMessage); } else { From a613ea19f48f226fcc84e7c64e6bbdeaacf616ec Mon Sep 17 00:00:00 2001 From: Petr Mironychev <9195189+Palm1r@users.noreply.github.com> Date: Thu, 29 Aug 2024 09:46:41 +0200 Subject: [PATCH 4/4] Add singleline completion --- LLMClientInterface.cpp | 34 ++++++++++++++++++++++++++++------ LLMClientInterface.hpp | 3 +++ QodeAssistConstants.hpp | 1 + QodeAssistSettings.cpp | 4 ++++ QodeAssistSettings.hpp | 1 + 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/LLMClientInterface.cpp b/LLMClientInterface.cpp index dd2d3a7..686f4cc 100644 --- a/LLMClientInterface.cpp +++ b/LLMClientInterface.cpp @@ -91,6 +91,27 @@ void LLMClientInterface::handleCancelRequest(const QJsonObject &request) } } +bool LLMClientInterface::processSingleLineCompletion(QNetworkReply *reply, + const QJsonObject &request, + const QString &accumulatedCompletion) +{ + int newlinePos = accumulatedCompletion.indexOf('\n'); + + if (newlinePos != -1) { + QString singleLineCompletion = accumulatedCompletion.left(newlinePos).trimmed(); + singleLineCompletion = removeStopWords(singleLineCompletion); + + QJsonObject position = request["params"].toObject()["doc"].toObject()["position"].toObject(); + + sendCompletionToClient(singleLineCompletion, request, position, true); + m_accumulatedResponses.remove(reply); + reply->abort(); + + return true; + } + return false; +} + QString LLMClientInterface::сontextBefore(TextEditor::TextEditorWidget *widget, int lineNumber, int cursorPosition) @@ -175,9 +196,7 @@ void LLMClientInterface::handleShutdown(const QJsonObject &request) emit messageReceived(LanguageServerProtocol::JsonRpcMessage(response)); } -void LLMClientInterface::handleTextDocumentDidOpen(const QJsonObject &request) -{ -} +void LLMClientInterface::handleTextDocumentDidOpen(const QJsonObject &request) {} void LLMClientInterface::handleInitialized(const QJsonObject &request) { @@ -207,6 +226,11 @@ void LLMClientInterface::handleLLMResponse(QNetworkReply *reply, const QJsonObje QJsonObject position = request["params"].toObject()["doc"].toObject()["position"].toObject(); + if (!settings().multiLineCompletion() + && processSingleLineCompletion(reply, request, accumulatedResponse)) { + return; + } + if (isComplete || reply->isFinished()) { if (isComplete) { auto cleanedCompletion = removeStopWords(accumulatedResponse); @@ -353,8 +377,6 @@ QString LLMClientInterface::removeStopWords(const QString &completion) return filteredCompletion; } -void LLMClientInterface::parseCurrentMessage() -{ -} +void LLMClientInterface::parseCurrentMessage() {} } // namespace QodeAssist diff --git a/LLMClientInterface.hpp b/LLMClientInterface.hpp index b0a11a0..6fef777 100644 --- a/LLMClientInterface.hpp +++ b/LLMClientInterface.hpp @@ -69,6 +69,9 @@ private: void handleInitialized(const QJsonObject &request); void handleExit(const QJsonObject &request); void handleCancelRequest(const QJsonObject &request); + bool processSingleLineCompletion(QNetworkReply *reply, + const QJsonObject &request, + const QString &accumulatedCompletion); QString сontextBefore(TextEditor::TextEditorWidget *widget, int lineNumber, int cursorPosition); QString сontextAfter(TextEditor::TextEditorWidget *widget, int lineNumber, int cursorPosition); diff --git a/QodeAssistConstants.hpp b/QodeAssistConstants.hpp index 3254c4f..3478190 100644 --- a/QodeAssistConstants.hpp +++ b/QodeAssistConstants.hpp @@ -53,6 +53,7 @@ const char START_SUGGESTION_TIMER[] = "QodeAssist.startSuggestionTimer"; const char MAX_FILE_THRESHOLD[] = "QodeAssist.maxFileThreshold"; const char OLLAMA_LIVETIME[] = "QodeAssist.ollamaLivetime"; const char SPECIFIC_INSTRUCTIONS[] = "QodeAssist.specificInstractions"; +const char MULTILINE_COMPLETION[] = "QodeAssist.multilineCompletion"; const char QODE_ASSIST_GENERAL_OPTIONS_ID[] = "QodeAssist.GeneralOptions"; const char QODE_ASSIST_GENERAL_OPTIONS_CATEGORY[] = "QodeAssist.Category"; diff --git a/QodeAssistSettings.cpp b/QodeAssistSettings.cpp index 9fd5437..77b3b59 100644 --- a/QodeAssistSettings.cpp +++ b/QodeAssistSettings.cpp @@ -166,6 +166,9 @@ QodeAssistSettings::QodeAssistSettings() "CRITICAL: Please provide minimal the best possible code completion suggestions.\n"); resetToDefaults.m_buttonText = Tr::tr("Reset to Defaults"); + multiLineCompletion.setSettingsKey(Constants::MULTILINE_COMPLETION); + multiLineCompletion.setDefaultValue(true); + multiLineCompletion.setLabelText(Tr::tr("Enable Multiline Completion")); const auto &manager = LLMProvidersManager::instance(); if (!manager.getProviderNames().isEmpty()) { @@ -203,6 +206,7 @@ QodeAssistSettings::QodeAssistSettings() return Column{Group{title(Tr::tr("General Settings")), Form{Column{enableQodeAssist, enableAutoComplete, + multiLineCompletion, enableLogging, Row{Stretch{1}, resetToDefaults}}}}, Group{title(Tr::tr("LLM Providers")), diff --git a/QodeAssistSettings.hpp b/QodeAssistSettings.hpp index b512234..32cba11 100644 --- a/QodeAssistSettings.hpp +++ b/QodeAssistSettings.hpp @@ -96,6 +96,7 @@ public: Utils::StringAspect ollamaLivetime{this}; Utils::StringAspect specificInstractions{this}; + Utils::BoolAspect multiLineCompletion{this}; ButtonAspect resetToDefaults{this};