feat: Add possibility to link files to the current system prompt

- Add linking files to chat
- Rework tokens counting

context/CMakeLists.txt

@@ -3,6 +3,7 @@ add_library(Context STATIC
     ChangesManager.h ChangesManager.cpp
     ContextManager.hpp ContextManager.cpp
     ContentFile.hpp
+    TokenUtils.hpp TokenUtils.cpp
 )
 
 target_link_libraries(Context

context/TokenUtils.cpp

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2024 Petr Mironychev
+ *
+ * This file is part of QodeAssist.
+ *
+ * QodeAssist is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * QodeAssist is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with QodeAssist. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "TokenUtils.hpp"
+
+namespace QodeAssist::Context {
+
+int TokenUtils::estimateTokens(const QString& text)
+{
+    if (text.isEmpty()) {
+        return 0;
+    }
+
+    // TODO: need to improve
+    return text.length() / 4;
+}
+
+int TokenUtils::estimateFileTokens(const Context::ContentFile& file)
+{
+    int total = 0;
+
+    total += estimateTokens(file.filename);
+    total += estimateTokens(file.content);
+    total += 5;
+
+    return total;
+}
+
+int TokenUtils::estimateFilesTokens(const QList<Context::ContentFile>& files)
+{
+    int total = 0;
+    for (const auto& file : files) {
+        total += estimateFileTokens(file);
+    }
+    return total;
+}
+
+}

context/TokenUtils.hpp

@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2024 Petr Mironychev
+ *
+ * This file is part of QodeAssist.
+ *
+ * QodeAssist is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * QodeAssist is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with QodeAssist. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <QString>
+#include "ContentFile.hpp"
+#include <QList>
+
+namespace QodeAssist::Context {
+
+class TokenUtils
+{
+public:
+    static int estimateTokens(const QString& text);
+    static int estimateFileTokens(const Context::ContentFile& file);
+    static int estimateFilesTokens(const QList<Context::ContentFile>& files);
+};
+
+}