mirror of
https://github.com/Palm1r/QodeAssist.git
synced 2026-05-30 02:49:12 -04:00
refactor: Update token usage api (#347)
* refactor: Improve token usage api * refactor: Image recognition to tokens
This commit is contained in:
@@ -3,6 +3,14 @@
|
||||
|
||||
#include "TokenUtils.hpp"
|
||||
|
||||
#include <QFileInfo>
|
||||
#include <QImageReader>
|
||||
#include <QSet>
|
||||
#include <QSize>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
|
||||
namespace QodeAssist::Context {
|
||||
|
||||
int TokenUtils::estimateTokens(const QString &text)
|
||||
@@ -15,8 +23,48 @@ int TokenUtils::estimateTokens(const QString &text)
|
||||
return text.length() / 4;
|
||||
}
|
||||
|
||||
bool TokenUtils::isImageFilePath(const QString &filePath)
|
||||
{
|
||||
static const QSet<QString> imageExtensions = {"png", "jpg", "jpeg", "gif", "webp", "bmp"};
|
||||
return imageExtensions.contains(QFileInfo(filePath).suffix().toLower());
|
||||
}
|
||||
|
||||
int TokenUtils::estimateImageAttachmentTokens(const QString &filePath)
|
||||
{
|
||||
QImageReader reader(filePath);
|
||||
QSize size = reader.size();
|
||||
if (!size.isValid() || size.isEmpty())
|
||||
return 1500;
|
||||
|
||||
double w = size.width();
|
||||
double h = size.height();
|
||||
|
||||
const double longSide = std::max(w, h);
|
||||
if (longSide > 2048.0) {
|
||||
const double s = 2048.0 / longSide;
|
||||
w *= s;
|
||||
h *= s;
|
||||
}
|
||||
|
||||
const double shortSide = std::min(w, h);
|
||||
if (shortSide > 768.0) {
|
||||
const double s = 768.0 / shortSide;
|
||||
w *= s;
|
||||
h *= s;
|
||||
}
|
||||
|
||||
const int tilesW = static_cast<int>(std::ceil(w / 512.0));
|
||||
const int tilesH = static_cast<int>(std::ceil(h / 512.0));
|
||||
const int tiles = std::max(1, tilesW * tilesH);
|
||||
|
||||
return 85 + tiles * 170;
|
||||
}
|
||||
|
||||
int TokenUtils::estimateFileTokens(const Context::ContentFile &file)
|
||||
{
|
||||
if (isImageFilePath(file.filename))
|
||||
return estimateImageAttachmentTokens(QString());
|
||||
|
||||
int total = 0;
|
||||
|
||||
total += estimateTokens(file.filename);
|
||||
|
||||
Reference in New Issue
Block a user