feat: Add llama.cpp provider and fim template (#118)

CMakeLists.txt

@@ -69,6 +69,7 @@ add_qtc_plugin(QodeAssist
     templates/Llama2.hpp
     templates/CodeLlamaQMLFim.hpp
     templates/GoogleAI.hpp
+    templates/LlamaCppFim.hpp
     providers/Providers.hpp
     providers/OllamaProvider.hpp providers/OllamaProvider.cpp
     providers/ClaudeProvider.hpp providers/ClaudeProvider.cpp
@@ -78,6 +79,7 @@ add_qtc_plugin(QodeAssist
     providers/OpenAICompatProvider.hpp providers/OpenAICompatProvider.cpp
     providers/OpenRouterAIProvider.hpp providers/OpenRouterAIProvider.cpp
     providers/GoogleAIProvider.hpp providers/GoogleAIProvider.cpp
+    providers/LlamaCppProvider.hpp providers/LlamaCppProvider.cpp
     QodeAssist.qrc
     LSPCompletion.hpp
     LLMSuggestion.hpp LLMSuggestion.cpp

llmcore/ContextData.hpp

@@ -30,7 +30,7 @@ struct Message
     QString content;
 
     // clang-format off
-    auto operator<=>(const Message&) const = default;
+    bool operator==(const Message&) const = default;
     // clang-format on
 };
 

llmcore/ProviderID.hpp

@@ -28,6 +28,7 @@ enum class ProviderID {
     OpenAICompatible,
     MistralAI,
     OpenRouter,
-    GoogleAI
+    GoogleAI,
+    LlamaCpp
 };
 }

providers/LlamaCppProvider.cpp

@@ -0,0 +1,213 @@
+/* 
+ * Copyright (C) 2024 Petr Mironychev
+ *
+ * This file is part of QodeAssist.
+ *
+ * QodeAssist is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * QodeAssist is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with QodeAssist. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "LlamaCppProvider.hpp"
+
+#include <QEventLoop>
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QNetworkReply>
+
+#include "llmcore/OpenAIMessage.hpp"
+#include "llmcore/ValidationUtils.hpp"
+#include "logger/Logger.hpp"
+#include "settings/ChatAssistantSettings.hpp"
+#include "settings/CodeCompletionSettings.hpp"
+
+namespace QodeAssist::Providers {
+
+QString LlamaCppProvider::name() const
+{
+    return "llama.cpp";
+}
+
+QString LlamaCppProvider::url() const
+{
+    return "http://localhost:8080";
+}
+
+QString LlamaCppProvider::completionEndpoint() const
+{
+    return "/infill";
+}
+
+QString LlamaCppProvider::chatEndpoint() const
+{
+    return "/v1/chat/completions";
+}
+
+bool LlamaCppProvider::supportsModelListing() const
+{
+    return false;
+}
+
+void LlamaCppProvider::prepareRequest(
+    QJsonObject &request,
+    LLMCore::PromptTemplate *prompt,
+    LLMCore::ContextData context,
+    LLMCore::RequestType type)
+{
+    if (!prompt->isSupportProvider(providerID())) {
+        LOG_MESSAGE(QString("Template %1 doesn't support %2 provider").arg(name(), prompt->name()));
+    }
+
+    prompt->prepareRequest(request, context);
+
+    auto applyModelParams = [&request](const auto &settings) {
+        request["max_tokens"] = settings.maxTokens();
+        request["temperature"] = settings.temperature();
+
+        if (settings.useTopP())
+            request["top_p"] = settings.topP();
+        if (settings.useTopK())
+            request["top_k"] = settings.topK();
+        if (settings.useFrequencyPenalty())
+            request["frequency_penalty"] = settings.frequencyPenalty();
+        if (settings.usePresencePenalty())
+            request["presence_penalty"] = settings.presencePenalty();
+    };
+
+    if (type == LLMCore::RequestType::CodeCompletion) {
+        applyModelParams(Settings::codeCompletionSettings());
+    } else {
+        applyModelParams(Settings::chatAssistantSettings());
+    }
+}
+
+bool LlamaCppProvider::handleResponse(QNetworkReply *reply, QString &accumulatedResponse)
+{
+    QByteArray data = reply->readAll();
+    if (data.isEmpty()) {
+        return false;
+    }
+
+    bool isDone = data.contains("\"stop\":true") || data.contains("data: [DONE]");
+
+    QByteArrayList lines = data.split('\n');
+    for (const QByteArray &line : lines) {
+        if (line.trimmed().isEmpty()) {
+            continue;
+        }
+
+        if (line == "data: [DONE]") {
+            isDone = true;
+            continue;
+        }
+
+        QByteArray jsonData = line;
+        if (line.startsWith("data: ")) {
+            jsonData = line.mid(6);
+        }
+
+        QJsonParseError error;
+        QJsonDocument doc = QJsonDocument::fromJson(jsonData, &error);
+        if (doc.isNull()) {
+            continue;
+        }
+
+        QJsonObject obj = doc.object();
+
+        if (obj.contains("content")) {
+            QString content = obj["content"].toString();
+            if (!content.isEmpty()) {
+                accumulatedResponse += content;
+            }
+        } else if (obj.contains("choices")) {
+            auto message = LLMCore::OpenAIMessage::fromJson(obj);
+            if (message.hasError()) {
+                LOG_MESSAGE("Error in llama.cpp response: " + message.error);
+                continue;
+            }
+
+            QString content = message.getContent();
+            if (!content.isEmpty()) {
+                accumulatedResponse += content;
+            }
+
+            if (message.isDone()) {
+                isDone = true;
+            }
+        }
+
+        if (obj["stop"].toBool()) {
+            isDone = true;
+        }
+    }
+
+    return isDone;
+}
+
+QList<QString> LlamaCppProvider::getInstalledModels(const QString &url)
+{
+    return {};
+}
+
+QList<QString> LlamaCppProvider::validateRequest(
+    const QJsonObject &request, LLMCore::TemplateType type)
+{
+    if (type == LLMCore::TemplateType::FIM) {
+        const auto infillReq = QJsonObject{
+            {"model", {}},
+            {"input_prefix", {}},
+            {"input_suffix", {}},
+            {"prompt", {}},
+            {"temperature", {}},
+            {"top_p", {}},
+            {"top_k", {}},
+            {"max_tokens", {}},
+            {"frequency_penalty", {}},
+            {"presence_penalty", {}},
+            {"stop", QJsonArray{}},
+            {"stream", {}}};
+
+        return LLMCore::ValidationUtils::validateRequestFields(request, infillReq);
+    } else {
+        const auto chatReq = QJsonObject{
+            {"model", {}},
+            {"messages", QJsonArray{{QJsonObject{{"role", {}}, {"content", {}}}}}},
+            {"temperature", {}},
+            {"max_tokens", {}},
+            {"top_p", {}},
+            {"top_k", {}},
+            {"frequency_penalty", {}},
+            {"presence_penalty", {}},
+            {"stop", QJsonArray{}},
+            {"stream", {}}};
+
+        return LLMCore::ValidationUtils::validateRequestFields(request, chatReq);
+    }
+}
+
+QString LlamaCppProvider::apiKey() const
+{
+    return {};
+}
+
+void LlamaCppProvider::prepareNetworkRequest(QNetworkRequest &networkRequest) const
+{
+    networkRequest.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
+}
+
+LLMCore::ProviderID LlamaCppProvider::providerID() const
+{
+    return LLMCore::ProviderID::LlamaCpp;
+}
+
+} // namespace QodeAssist::Providers

providers/LlamaCppProvider.hpp

@@ -0,0 +1,47 @@
+/* 
+ * Copyright (C) 2024 Petr Mironychev
+ *
+ * This file is part of QodeAssist.
+ *
+ * QodeAssist is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * QodeAssist is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with QodeAssist. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "llmcore/Provider.hpp"
+
+namespace QodeAssist::Providers {
+
+class LlamaCppProvider : public LLMCore::Provider
+{
+public:
+    QString name() const override;
+    QString url() const override;
+    QString completionEndpoint() const override;
+    QString chatEndpoint() const override;
+    bool supportsModelListing() const override;
+    void prepareRequest(
+        QJsonObject &request,
+        LLMCore::PromptTemplate *prompt,
+        LLMCore::ContextData context,
+        LLMCore::RequestType type) override;
+    bool handleResponse(QNetworkReply *reply, QString &accumulatedResponse) override;
+    QList<QString> getInstalledModels(const QString &url) override;
+    QList<QString> validateRequest(const QJsonObject &request, LLMCore::TemplateType type) override;
+    QString apiKey() const override;
+    void prepareNetworkRequest(QNetworkRequest &networkRequest) const override;
+    LLMCore::ProviderID providerID() const override;
+};
+
+} // namespace QodeAssist::Providers

providers/Providers.hpp

@@ -23,6 +23,7 @@
 #include "providers/ClaudeProvider.hpp"
 #include "providers/GoogleAIProvider.hpp"
 #include "providers/LMStudioProvider.hpp"
+#include "providers/LlamaCppProvider.hpp"
 #include "providers/MistralAIProvider.hpp"
 #include "providers/OllamaProvider.hpp"
 #include "providers/OpenAICompatProvider.hpp"
@@ -42,6 +43,7 @@ inline void registerProviders()
     providerManager.registerProvider<OpenRouterProvider>();
     providerManager.registerProvider<MistralAIProvider>();
     providerManager.registerProvider<GoogleAIProvider>();
+    providerManager.registerProvider<LlamaCppProvider>();
 }
 
 } // namespace QodeAssist::Providers

templates/Alpaca.hpp

@@ -75,10 +75,11 @@ public:
     bool isSupportProvider(LLMCore::ProviderID id) const override
     {
         switch (id) {
-        case QodeAssist::LLMCore::ProviderID::Ollama:
+        case LLMCore::ProviderID::Ollama:
-        case QodeAssist::LLMCore::ProviderID::LMStudio:
+        case LLMCore::ProviderID::LMStudio:
-        case QodeAssist::LLMCore::ProviderID::OpenRouter:
+        case LLMCore::ProviderID::OpenRouter:
-        case QodeAssist::LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::LlamaCpp:
             return true;
         default:
             return false;

templates/ChatML.hpp

@@ -76,10 +76,11 @@ public:
     bool isSupportProvider(LLMCore::ProviderID id) const override
     {
         switch (id) {
-        case QodeAssist::LLMCore::ProviderID::Ollama:
+        case LLMCore::ProviderID::Ollama:
-        case QodeAssist::LLMCore::ProviderID::LMStudio:
+        case LLMCore::ProviderID::LMStudio:
-        case QodeAssist::LLMCore::ProviderID::OpenRouter:
+        case LLMCore::ProviderID::OpenRouter:
-        case QodeAssist::LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::LlamaCpp:
             return true;
         default:
             return false;

templates/Llama2.hpp

@@ -73,10 +73,11 @@ public:
     bool isSupportProvider(LLMCore::ProviderID id) const override
     {
         switch (id) {
-        case QodeAssist::LLMCore::ProviderID::Ollama:
+        case LLMCore::ProviderID::Ollama:
-        case QodeAssist::LLMCore::ProviderID::LMStudio:
+        case LLMCore::ProviderID::LMStudio:
-        case QodeAssist::LLMCore::ProviderID::OpenRouter:
+        case LLMCore::ProviderID::OpenRouter:
-        case QodeAssist::LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::LlamaCpp:
             return true;
         default:
             return false;

templates/Llama3.hpp

@@ -80,10 +80,11 @@ public:
     bool isSupportProvider(LLMCore::ProviderID id) const override
     {
         switch (id) {
-        case QodeAssist::LLMCore::ProviderID::Ollama:
+        case LLMCore::ProviderID::Ollama:
-        case QodeAssist::LLMCore::ProviderID::LMStudio:
+        case LLMCore::ProviderID::LMStudio:
-        case QodeAssist::LLMCore::ProviderID::OpenRouter:
+        case LLMCore::ProviderID::OpenRouter:
-        case QodeAssist::LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::LlamaCpp:
             return true;
         default:
             return false;

templates/LlamaCppFim.hpp

@@ -0,0 +1,56 @@
+/* 
+ * Copyright (C) 2024 Petr Mironychev
+ *
+ * This file is part of QodeAssist.
+ *
+ * QodeAssist is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * QodeAssist is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with QodeAssist. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "llmcore/PromptTemplate.hpp"
+
+namespace QodeAssist::Templates {
+
+class LlamaCppFim : public LLMCore::PromptTemplate
+{
+public:
+    LLMCore::TemplateType type() const override { return LLMCore::TemplateType::FIM; }
+    QString name() const override { return "llama.cpp FIM"; }
+    QStringList stopWords() const override { return {}; }
+
+    void prepareRequest(QJsonObject &request, const LLMCore::ContextData &context) const override
+    {
+        request["input_prefix"] = context.prefix.value_or("");
+        request["input_suffix"] = context.suffix.value_or("");
+    }
+
+    QString description() const override
+    {
+        return "Default llama.cpp FIM (Fill-in-Middle) /infill template with native format:\n\n"
+               "{\n"
+               "  \"input_prefix\": \"<code prefix>\",\n"
+               "  \"input_suffix\": \"<code suffix>\",\n"
+               "  \"input_extra\": \"<system prompt>\"\n"
+               "}\n\n"
+               "Recommended for models with FIM capability.";
+    }
+
+    bool isSupportProvider(LLMCore::ProviderID id) const override
+    {
+        return id == QodeAssist::LLMCore::ProviderID::LlamaCpp;
+    }
+};
+
+} // namespace QodeAssist::Templates

templates/OpenAICompatible.hpp

@@ -63,9 +63,10 @@ public:
     bool isSupportProvider(LLMCore::ProviderID id) const override
     {
         switch (id) {
-        case QodeAssist::LLMCore::ProviderID::OpenAICompatible:
+        case LLMCore::ProviderID::OpenAICompatible:
-        case QodeAssist::LLMCore::ProviderID::OpenRouter:
+        case LLMCore::ProviderID::OpenRouter:
-        case QodeAssist::LLMCore::ProviderID::LMStudio:
+        case LLMCore::ProviderID::LMStudio:
+        case LLMCore::ProviderID::LlamaCpp:
             return true;
         default:
             return false;

templates/Templates.hpp

@@ -34,6 +34,7 @@
 #include "templates/GoogleAI.hpp"
 #include "templates/Llama2.hpp"
 #include "templates/Llama3.hpp"
+#include "templates/LlamaCppFim.hpp"
 #include "templates/Qwen.hpp"
 #include "templates/StarCoder2Fim.hpp"
 
@@ -60,6 +61,7 @@ inline void registerTemplates()
     templateManager.registerTemplate<OpenAICompatible>();
     templateManager.registerTemplate<Alpaca>();
     templateManager.registerTemplate<GoogleAI>();
+    templateManager.registerTemplate<LlamaCppFim>();
 }
 
 } // namespace QodeAssist::Templates