mirror/QodeAssist
1
0
Fork 0
mirror of https://github.com/Palm1r/QodeAssist.git synced 2025-11-22 02:22:44 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add image support for Claude, OpenAI and Google (#268)

Browse Source 
* feat: Add image support for Claude
* feat: Add images support for OpenAI
* feat: Add support images for google ai
* refactor: Separate ImageComponent
* feat: Add attach image button
* feat: Add support image for Mistral provider
* feat: Add support images for OpenAI compatible providers
* feat: Add support images for Ollama
This commit is contained in:
Petr Mironychev
2025-11-20 15:49:39 +01:00
committed by GitHub
parent ce9e2717d6
commit 55b6080273
41 changed files with 860 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

154
ChatView/ClientInterface.cpp
View File

@ -20,9 +20,12 @@
#include "ClientInterface.hpp"
#include <texteditor/textdocument.h>
#include <QFile>
#include <QFileInfo>
#include <QImageReader>
#include <QJsonArray>
#include <QJsonDocument>
#include <QMimeDatabase>
#include <QUuid>
#include <coreplugin/editormanager/editormanager.h>
@ -36,6 +39,7 @@
#include <texteditor/texteditor.h>
#include "ChatAssistantSettings.hpp"
#include "ChatSerializer.hpp"
#include "GeneralSettings.hpp"
#include "ToolsSettings.hpp"
#include "Logger.hpp"
@ -70,8 +74,44 @@ void ClientInterface::sendMessage(
Context::ChangesManager::instance().archiveAllNonArchivedEdits();
auto attachFiles = m_contextManager->getContentFiles(attachments);
m_chatModel->addMessage(message, ChatModel::ChatRole::User, "", attachFiles);
QList<QString> imageFiles;
QList<QString> textFiles;
for (const QString &filePath : attachments) {
if (isImageFile(filePath)) {
imageFiles.append(filePath);
} else {
textFiles.append(filePath);
}
}
auto attachFiles = m_contextManager->getContentFiles(textFiles);
QList<ChatModel::ImageAttachment> imageAttachments;
if (!imageFiles.isEmpty() && !m_chatFilePath.isEmpty()) {
for (const QString &imagePath : imageFiles) {
QString base64Data = encodeImageToBase64(imagePath);
if (base64Data.isEmpty()) {
continue;
}
QString storedPath;
QFileInfo fileInfo(imagePath);
if (ChatSerializer::saveImageToStorage(m_chatFilePath, fileInfo.fileName(), base64Data, storedPath)) {
ChatModel::ImageAttachment imageAttachment;
imageAttachment.fileName = fileInfo.fileName();
imageAttachment.storedPath = storedPath;
imageAttachment.mediaType = getMediaTypeForImage(imagePath);
imageAttachments.append(imageAttachment);
LOG_MESSAGE(QString("Stored image %1 as %2").arg(fileInfo.fileName(), storedPath));
}
}
} else if (!imageFiles.isEmpty()) {
LOG_MESSAGE(QString("Warning: Chat file path not set, cannot save %1 image(s)").arg(imageFiles.size()));
}
m_chatModel->addMessage(message, ChatModel::ChatRole::User, "", attachFiles, imageAttachments);
auto &chatAssistantSettings = Settings::chatAssistantSettings();
@ -133,8 +173,36 @@ void ClientInterface::sendMessage(
apiMessage.isRedacted = msg.isRedacted;
apiMessage.signature = msg.signature;
if (provider->supportImage() && !m_chatFilePath.isEmpty() && !msg.images.isEmpty()) {
auto apiImages = loadImagesFromStorage(msg.images);
if (!apiImages.isEmpty()) {
apiMessage.images = apiImages;
}
}
messages.append(apiMessage);
}
if (!imageAttachments.isEmpty() && provider->supportImage() && !messages.isEmpty()) {
for (int i = messages.size() - 1; i >= 0; --i) {
if (messages[i].role == "user") {
auto newImages = loadImagesFromStorage(imageAttachments);
if (!newImages.isEmpty()) {
if (messages[i].images.has_value()) {
messages[i].images.value().append(newImages);
} else {
messages[i].images = newImages;
}
LOG_MESSAGE(QString("Added %1 new image(s) to message").arg(newImages.size()));
}
break;
}
}
} else if (!imageFiles.isEmpty() && !provider->supportImage()) {
LOG_MESSAGE(QString("Provider %1 doesn't support images, %2 ignored")
.arg(provider->name(), QString::number(imageFiles.size())));
}
context.history = messages;
LLMCore::LLMConfig config;
@ -379,4 +447,86 @@ void ClientInterface::handleCleanAccumulatedData(const QString &requestId)
LOG_MESSAGE(QString("Cleared accumulated responses for continuation request %1").arg(requestId));
}
bool ClientInterface::isImageFile(const QString &filePath) const
{
static const QSet<QString> imageExtensions = {
"png", "jpg", "jpeg", "gif", "webp", "bmp", "svg"
};
QFileInfo fileInfo(filePath);
QString extension = fileInfo.suffix().toLower();
return imageExtensions.contains(extension);
}
QString ClientInterface::getMediaTypeForImage(const QString &filePath) const
{
static const QHash<QString, QString> mediaTypes = {
{"png", "image/png"},
{"jpg", "image/jpeg"},
{"jpeg", "image/jpeg"},
{"gif", "image/gif"},
{"webp", "image/webp"},
{"bmp", "image/bmp"},
{"svg", "image/svg+xml"}
};
QFileInfo fileInfo(filePath);
QString extension = fileInfo.suffix().toLower();
if (mediaTypes.contains(extension)) {
return mediaTypes[extension];
}
QMimeDatabase mimeDb;
QMimeType mimeType = mimeDb.mimeTypeForFile(filePath);
return mimeType.name();
}
QString ClientInterface::encodeImageToBase64(const QString &filePath) const
{
QFile file(filePath);
if (!file.open(QIODevice::ReadOnly)) {
LOG_MESSAGE(QString("Failed to open image file: %1").arg(filePath));
return QString();
}
QByteArray imageData = file.readAll();
file.close();
return imageData.toBase64();
}
QVector<LLMCore::ImageAttachment> ClientInterface::loadImagesFromStorage(const QList<ChatModel::ImageAttachment> &storedImages) const
{
QVector<LLMCore::ImageAttachment> apiImages;
for (const auto &storedImage : storedImages) {
QString base64Data = ChatSerializer::loadImageFromStorage(m_chatFilePath, storedImage.storedPath);
if (base64Data.isEmpty()) {
LOG_MESSAGE(QString("Warning: Failed to load image: %1").arg(storedImage.storedPath));
continue;
}
LLMCore::ImageAttachment apiImage;
apiImage.data = base64Data;
apiImage.mediaType = storedImage.mediaType;
apiImage.isUrl = false;
apiImages.append(apiImage);
}
return apiImages;
}
void ClientInterface::setChatFilePath(const QString &filePath)
{
m_chatFilePath = filePath;
}
QString ClientInterface::chatFilePath() const
{
return m_chatFilePath;
}
} // namespace QodeAssist::Chat