refactor: Update token usage api (#347)

* refactor: Improve token usage api
* refactor: Image recognition to tokens
This commit is contained in:
Petr Mironychev
2026-05-14 21:19:12 +02:00
committed by GitHub
parent 86f4635080
commit 3b421f60af
20 changed files with 518 additions and 60 deletions

View File

@@ -3,6 +3,14 @@
#include "TokenUtils.hpp"
#include <QFileInfo>
#include <QImageReader>
#include <QSet>
#include <QSize>
#include <algorithm>
#include <cmath>
namespace QodeAssist::Context {
int TokenUtils::estimateTokens(const QString &text)
@@ -15,8 +23,48 @@ int TokenUtils::estimateTokens(const QString &text)
return text.length() / 4;
}
bool TokenUtils::isImageFilePath(const QString &filePath)
{
static const QSet<QString> imageExtensions = {"png", "jpg", "jpeg", "gif", "webp", "bmp"};
return imageExtensions.contains(QFileInfo(filePath).suffix().toLower());
}
int TokenUtils::estimateImageAttachmentTokens(const QString &filePath)
{
QImageReader reader(filePath);
QSize size = reader.size();
if (!size.isValid() || size.isEmpty())
return 1500;
double w = size.width();
double h = size.height();
const double longSide = std::max(w, h);
if (longSide > 2048.0) {
const double s = 2048.0 / longSide;
w *= s;
h *= s;
}
const double shortSide = std::min(w, h);
if (shortSide > 768.0) {
const double s = 768.0 / shortSide;
w *= s;
h *= s;
}
const int tilesW = static_cast<int>(std::ceil(w / 512.0));
const int tilesH = static_cast<int>(std::ceil(h / 512.0));
const int tiles = std::max(1, tilesW * tilesH);
return 85 + tiles * 170;
}
int TokenUtils::estimateFileTokens(const Context::ContentFile &file)
{
if (isImageFilePath(file.filename))
return estimateImageAttachmentTokens(QString());
int total = 0;
total += estimateTokens(file.filename);