mirror/QodeAssist
1
0
Fork 0
mirror of https://github.com/Palm1r/QodeAssist.git synced 2025-11-22 02:22:44 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add image support for Claude, OpenAI and Google (#268)

Browse Source 
* feat: Add image support for Claude
* feat: Add images support for OpenAI
* feat: Add support images for google ai
* refactor: Separate ImageComponent
* feat: Add attach image button
* feat: Add support image for Mistral provider
* feat: Add support images for OpenAI compatible providers
* feat: Add support images for Ollama
This commit is contained in:
Petr Mironychev
2025-11-20 15:49:39 +01:00
committed by GitHub
parent ce9e2717d6
commit 55b6080273
41 changed files with 860 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ChatView/CMakeLists.txt
View File

@ -32,6 +32,7 @@ qt_add_qml_module(QodeAssistChatView
icons/close-light.svg
icons/link-file-light.svg
icons/link-file-dark.svg
icons/image-dark.svg
icons/load-chat-dark.svg
icons/save-chat-dark.svg
icons/clean-icon-dark.svg

2
ChatView/ChatData.hpp
View File

@ -26,7 +26,7 @@ namespace QodeAssist::Chat {
Q_NAMESPACE
QML_NAMED_ELEMENT(MessagePartType)
enum class MessagePartType { Code, Text };
enum class MessagePartType { Code, Text, Image };
Q_ENUM_NS(MessagePartType)
} // namespace QodeAssist::Chat

17
ChatView/ChatModel.cpp
View File

@ -84,6 +84,17 @@ QVariant ChatModel::data(const QModelIndex &index, int role) const
case Roles::IsRedacted: {
return message.isRedacted;
}
case Roles::Images: {
QVariantList imagesList;
for (const auto &image : message.images) {
QVariantMap imageMap;
imageMap["fileName"] = image.fileName;
imageMap["storedPath"] = image.storedPath;
imageMap["mediaType"] = image.mediaType;
imagesList.append(imageMap);
}
return imagesList;
}
default:
return QVariant();
}
@ -96,6 +107,7 @@ QHash<int, QByteArray> ChatModel::roleNames() const
roles[Roles::Content] = "content";
roles[Roles::Attachments] = "attachments";
roles[Roles::IsRedacted] = "isRedacted";
roles[Roles::Images] = "images";
return roles;
}
@ -103,7 +115,8 @@ void ChatModel::addMessage(
const QString &content,
ChatRole role,
const QString &id,
const QList<Context::ContentFile> &attachments)
const QList<Context::ContentFile> &attachments,
const QList<ImageAttachment> &images)
{
QString fullContent = content;
if (!attachments.isEmpty()) {
@ -119,11 +132,13 @@ void ChatModel::addMessage(
Message &lastMessage = m_messages.last();
lastMessage.content = content;
lastMessage.attachments = attachments;
lastMessage.images = images;
emit dataChanged(index(m_messages.size() - 1), index(m_messages.size() - 1));
} else {
beginInsertRows(QModelIndex(), m_messages.size(), m_messages.size());
Message newMessage{role, content, id};
newMessage.attachments = attachments;
newMessage.images = images;
m_messages.append(newMessage);
endInsertRows();

13
ChatView/ChatModel.hpp
View File

@ -40,9 +40,16 @@ public:
enum ChatRole { System, User, Assistant, Tool, FileEdit, Thinking };
Q_ENUM(ChatRole)
enum Roles { RoleType = Qt::UserRole, Content, Attachments, IsRedacted };
enum Roles { RoleType = Qt::UserRole, Content, Attachments, IsRedacted, Images };
Q_ENUM(Roles)
struct ImageAttachment
{
QString fileName; // Original filename
QString storedPath; // Path to stored image file (relative to chat folder)
QString mediaType; // MIME type
};
struct Message
{
ChatRole role;
@ -52,6 +59,7 @@ public:
QString signature = QString();
QList<Context::ContentFile> attachments;
QList<ImageAttachment> images;
};
explicit ChatModel(QObject *parent = nullptr);
@ -64,7 +72,8 @@ public:
const QString &content,
ChatRole role,
const QString &id,
const QList<Context::ContentFile> &attachments = {});
const QList<Context::ContentFile> &attachments = {},
const QList<ImageAttachment> &images = {});
Q_INVOKABLE void clear();
Q_INVOKABLE QList<MessagePart> processMessageContent(const QString &content) const;

158
ChatView/ChatRootView.cpp
View File

@ -242,6 +242,14 @@ void ChatRootView::sendMessage(const QString &message)
}
}
if (m_recentFilePath.isEmpty()) {
QString filePath = getAutosaveFilePath(message, m_attachmentFiles);
if (!filePath.isEmpty()) {
setRecentFilePath(filePath);
LOG_MESSAGE(QString("Set chat file path for new chat: %1").arg(filePath));
}
}
m_clientInterface->sendMessage(message, m_attachmentFiles, m_linkedFiles, m_isAgentMode);
clearAttachmentFiles();
setRequestProgressStatus(true);
@ -379,51 +387,22 @@ void ChatRootView::showLoadDialog()
QString ChatRootView::getSuggestedFileName() const
{
QStringList parts;
static const QRegularExpression saitizeSymbols = QRegularExpression("[\\/:*?\"<>|\\s]");
static const QRegularExpression underSymbols = QRegularExpression("_+");
QString shortMessage;
if (m_chatModel->rowCount() > 0) {
QString firstMessage
= m_chatModel->data(m_chatModel->index(0), ChatModel::Content).toString();
QString shortMessage = firstMessage.split('\n').first().simplified().left(30);
shortMessage = firstMessage.split('\n').first().simplified().left(30);
QString sanitizedMessage = shortMessage;
sanitizedMessage.replace(saitizeSymbols, "_");
sanitizedMessage.replace(underSymbols, "_");
sanitizedMessage = sanitizedMessage.trimmed();
if (!sanitizedMessage.isEmpty()) {
if (sanitizedMessage.startsWith('_')) {
sanitizedMessage.remove(0, 1);
}
if (sanitizedMessage.endsWith('_')) {
sanitizedMessage.chop(1);
}
QString targetDir = getChatsHistoryDir();
QString fullPath = QDir(targetDir).filePath(sanitizedMessage);
QFileInfo fileInfo(fullPath);
if (!fileInfo.exists() && QFileInfo(fileInfo.path()).isWritable()) {
parts << sanitizedMessage;
if (shortMessage.isEmpty()) {
QVariantList images = m_chatModel->data(m_chatModel->index(0), ChatModel::Images).toList();
if (!images.isEmpty()) {
shortMessage = "image_chat";
}
}
}
parts << QDateTime::currentDateTime().toString("yyyy-MM-dd_HH-mm");
QString fileName = parts.join("_");
QString fullPath = QDir(getChatsHistoryDir()).filePath(fileName);
QFileInfo finalCheck(fullPath);
if (fileName.isEmpty() || finalCheck.exists() || !QFileInfo(finalCheck.path()).isWritable()) {
fileName = QString("chat_%1").arg(QDateTime::currentDateTime().toString("yyyy-MM-dd_HH-mm"));
}
return fileName;
return generateChatFileName(shortMessage, getChatsHistoryDir());
}
void ChatRootView::autosave()
@ -453,6 +432,27 @@ QString ChatRootView::getAutosaveFilePath() const
return QDir(dir).filePath(getSuggestedFileName() + ".json");
}
QString ChatRootView::getAutosaveFilePath(const QString &firstMessage, const QStringList &attachments) const
{
if (!m_recentFilePath.isEmpty()) {
return m_recentFilePath;
}
QString dir = getChatsHistoryDir();
if (dir.isEmpty()) {
return QString();
}
QString shortMessage = firstMessage.split('\n').first().simplified().left(30);
if (shortMessage.isEmpty() && hasImageAttachments(attachments)) {
shortMessage = "image_chat";
}
QString fileName = generateChatFileName(shortMessage, dir);
return QDir(dir).filePath(fileName + ".json");
}
QStringList ChatRootView::attachmentFiles() const
{
return m_attachmentFiles;
@ -531,6 +531,33 @@ void ChatRootView::removeFileFromLinkList(int index)
}
}
void ChatRootView::showAddImageDialog()
{
QFileDialog dialog(nullptr, tr("Select Images to Attach"));
dialog.setFileMode(QFileDialog::ExistingFiles);
dialog.setNameFilter(tr("Images (*.png *.jpg *.jpeg *.gif *.bmp *.webp)"));
if (auto project = ProjectExplorer::ProjectManager::startupProject()) {
dialog.setDirectory(project->projectDirectory().toFSPathString());
}
if (dialog.exec() == QDialog::Accepted) {
QStringList newFilePaths = dialog.selectedFiles();
if (!newFilePaths.isEmpty()) {
bool filesAdded = false;
for (const QString &filePath : std::as_const(newFilePaths)) {
if (!m_attachmentFiles.contains(filePath)) {
m_attachmentFiles.append(filePath);
filesAdded = true;
}
}
if (filesAdded) {
emit attachmentFilesChanged();
}
}
}
}
void ChatRootView::calculateMessageTokensCount(const QString &message)
{
m_messageTokensCount = Context::TokenUtils::estimateTokens(message);
@ -665,10 +692,16 @@ QString ChatRootView::chatFileName() const
return QFileInfo(m_recentFilePath).baseName();
}
QString ChatRootView::chatFilePath() const
{
return m_recentFilePath;
}
void ChatRootView::setRecentFilePath(const QString &filePath)
{
if (m_recentFilePath != filePath) {
m_recentFilePath = filePath;
m_clientInterface->setChatFilePath(filePath);
emit chatFileNameChanged();
}
}
@ -1126,5 +1159,58 @@ bool ChatRootView::isThinkingSupport() const
return provider && provider->supportThinking();
}
QString ChatRootView::generateChatFileName(const QString &shortMessage, const QString &dir) const
{
static const QRegularExpression saitizeSymbols = QRegularExpression("[\\/:*?\"<>|\\s]");
static const QRegularExpression underSymbols = QRegularExpression("_+");
QStringList parts;
QString sanitizedMessage = shortMessage;
sanitizedMessage.replace(saitizeSymbols, "_");
sanitizedMessage.replace(underSymbols, "_");
sanitizedMessage = sanitizedMessage.trimmed();
if (!sanitizedMessage.isEmpty()) {
if (sanitizedMessage.startsWith('_')) {
sanitizedMessage.remove(0, 1);
}
if (sanitizedMessage.endsWith('_')) {
sanitizedMessage.chop(1);
}
QString fullPath = QDir(dir).filePath(sanitizedMessage);
QFileInfo fileInfo(fullPath);
if (!fileInfo.exists() && QFileInfo(fileInfo.path()).isWritable()) {
parts << sanitizedMessage;
}
}
parts << QDateTime::currentDateTime().toString("yyyy-MM-dd_HH-mm");
QString fileName = parts.join("_");
QString fullPath = QDir(dir).filePath(fileName);
QFileInfo finalCheck(fullPath);
if (fileName.isEmpty() || finalCheck.exists() || !QFileInfo(finalCheck.path()).isWritable()) {
fileName = QString("chat_%1").arg(QDateTime::currentDateTime().toString("yyyy-MM-dd_HH-mm"));
}
return fileName;
}
bool ChatRootView::hasImageAttachments(const QStringList &attachments) const
{
static const QSet<QString> imageExtensions = {
"png", "jpg", "jpeg", "gif", "webp", "bmp", "svg"
};
for (const QString &filePath : attachments) {
QFileInfo fileInfo(filePath);
if (imageExtensions.contains(fileInfo.suffix().toLower())) {
return true;
}
}
return false;
}
} // namespace QodeAssist::Chat

5
ChatView/ChatRootView.hpp
View File

@ -75,6 +75,7 @@ public:
void autosave();
QString getAutosaveFilePath() const;
QString getAutosaveFilePath(const QString &firstMessage, const QStringList &attachments) const;
QStringList attachmentFiles() const;
QStringList linkedFiles() const;
@ -83,6 +84,7 @@ public:
Q_INVOKABLE void removeFileFromAttachList(int index);
Q_INVOKABLE void showLinkFilesDialog();
Q_INVOKABLE void removeFileFromLinkList(int index);
Q_INVOKABLE void showAddImageDialog();
Q_INVOKABLE void calculateMessageTokensCount(const QString &message);
Q_INVOKABLE void setIsSyncOpenFiles(bool state);
Q_INVOKABLE void openChatHistoryFolder();
@ -98,6 +100,7 @@ public:
void onEditorCreated(Core::IEditor *editor, const Utils::FilePath &filePath);
QString chatFileName() const;
Q_INVOKABLE QString chatFilePath() const;
void setRecentFilePath(const QString &filePath);
bool shouldIgnoreFileForAttach(const Utils::FilePath &filePath);
@ -181,6 +184,8 @@ private:
void updateFileEditStatus(const QString &editId, const QString &status);
QString getChatsHistoryDir() const;
QString getSuggestedFileName() const;
QString generateChatFileName(const QString &shortMessage, const QString &dir) const;
bool hasImageAttachments(const QStringList &attachments) const;
ChatModel *m_chatModel;
LLMCore::PromptProviderChat m_promptProvider;

125
ChatView/ChatSerializer.cpp
View File

@ -20,10 +20,13 @@
#include "ChatSerializer.hpp"
#include "Logger.hpp"
#include <QBuffer>
#include <QDir>
#include <QFile>
#include <QFileInfo>
#include <QJsonArray>
#include <QJsonDocument>
#include <QUuid>
namespace QodeAssist::Chat {
@ -35,12 +38,20 @@ SerializationResult ChatSerializer::saveToFile(const ChatModel *model, const QSt
return {false, "Failed to create directory structure"};
}
QString imagesFolder = getChatImagesFolder(filePath);
QDir dir;
if (!dir.exists(imagesFolder)) {
if (!dir.mkpath(imagesFolder)) {
LOG_MESSAGE(QString("Warning: Failed to create images folder: %1").arg(imagesFolder));
}
}
QFile file(filePath);
if (!file.open(QIODevice::WriteOnly)) {
return {false, QString("Failed to open file for writing: %1").arg(filePath)};
}
QJsonObject root = serializeChat(model);
QJsonObject root = serializeChat(model, filePath);
QJsonDocument doc(root);
if (file.write(doc.toJson(QJsonDocument::Indented)) == -1) {
@ -70,14 +81,14 @@ SerializationResult ChatSerializer::loadFromFile(ChatModel *model, const QString
return {false, QString("Unsupported version: %1").arg(version)};
}
if (!deserializeChat(model, root)) {
if (!deserializeChat(model, root, filePath)) {
return {false, "Failed to deserialize chat data"};
}
return {true, QString()};
}
QJsonObject ChatSerializer::serializeMessage(const ChatModel::Message &message)
QJsonObject ChatSerializer::serializeMessage(const ChatModel::Message &message, const QString &chatFilePath)
{
QJsonObject messageObj;
messageObj["role"] = static_cast<int>(message.role);
@ -87,10 +98,23 @@ QJsonObject ChatSerializer::serializeMessage(const ChatModel::Message &message)
if (!message.signature.isEmpty()) {
messageObj["signature"] = message.signature;
}
if (!message.images.isEmpty()) {
QJsonArray imagesArray;
for (const auto &image : message.images) {
QJsonObject imageObj;
imageObj["fileName"] = image.fileName;
imageObj["storedPath"] = image.storedPath;
imageObj["mediaType"] = image.mediaType;
imagesArray.append(imageObj);
}
messageObj["images"] = imagesArray;
}
return messageObj;
}
ChatModel::Message ChatSerializer::deserializeMessage(const QJsonObject &json)
ChatModel::Message ChatSerializer::deserializeMessage(const QJsonObject &json, const QString &chatFilePath)
{
ChatModel::Message message;
message.role = static_cast<ChatModel::ChatRole>(json["role"].toInt());
@ -98,14 +122,27 @@ ChatModel::Message ChatSerializer::deserializeMessage(const QJsonObject &json)
message.id = json["id"].toString();
message.isRedacted = json["isRedacted"].toBool(false);
message.signature = json["signature"].toString();
if (json.contains("images")) {
QJsonArray imagesArray = json["images"].toArray();
for (const auto &imageValue : imagesArray) {
QJsonObject imageObj = imageValue.toObject();
ChatModel::ImageAttachment image;
image.fileName = imageObj["fileName"].toString();
image.storedPath = imageObj["storedPath"].toString();
image.mediaType = imageObj["mediaType"].toString();
message.images.append(image);
}
}
return message;
}
QJsonObject ChatSerializer::serializeChat(const ChatModel *model)
QJsonObject ChatSerializer::serializeChat(const ChatModel *model, const QString &chatFilePath)
{
QJsonArray messagesArray;
for (const auto &message : model->getChatHistory()) {
messagesArray.append(serializeMessage(message));
messagesArray.append(serializeMessage(message, chatFilePath));
}
QJsonObject root;
@ -115,14 +152,14 @@ QJsonObject ChatSerializer::serializeChat(const ChatModel *model)
return root;
}
bool ChatSerializer::deserializeChat(ChatModel *model, const QJsonObject &json)
bool ChatSerializer::deserializeChat(ChatModel *model, const QJsonObject &json, const QString &chatFilePath)
{
QJsonArray messagesArray = json["messages"].toArray();
QVector<ChatModel::Message> messages;
messages.reserve(messagesArray.size());
for (const auto &messageValue : messagesArray) {
messages.append(deserializeMessage(messageValue.toObject()));
messages.append(deserializeMessage(messageValue.toObject(), chatFilePath));
}
model->clear();
@ -130,7 +167,8 @@ bool ChatSerializer::deserializeChat(ChatModel *model, const QJsonObject &json)
model->setLoadingFromHistory(true);
for (const auto &message : messages) {
model->addMessage(message.content, message.role, message.id);
model->addMessage(message.content, message.role, message.id, message.attachments, message.images);
LOG_MESSAGE(QString("Loaded message with %1 image(s)").arg(message.images.size()));
}
model->setLoadingFromHistory(false);
@ -150,4 +188,73 @@ bool ChatSerializer::validateVersion(const QString &version)
return version == VERSION;
}
QString ChatSerializer::getChatImagesFolder(const QString &chatFilePath)
{
QFileInfo fileInfo(chatFilePath);
QString baseName = fileInfo.completeBaseName();
QString dirPath = fileInfo.absolutePath();
return QDir(dirPath).filePath(baseName + "_images");
}
bool ChatSerializer::saveImageToStorage(const QString &chatFilePath,
const QString &fileName,
const QString &base64Data,
QString &storedPath)
{
QString imagesFolder = getChatImagesFolder(chatFilePath);
QDir dir;
if (!dir.exists(imagesFolder)) {
if (!dir.mkpath(imagesFolder)) {
LOG_MESSAGE(QString("Failed to create images folder: %1").arg(imagesFolder));
return false;
}
}
QFileInfo originalFileInfo(fileName);
QString extension = originalFileInfo.suffix();
QString baseName = originalFileInfo.completeBaseName();
QString uniqueName = QString("%1_%2.%3")
.arg(baseName)
.arg(QUuid::createUuid().toString(QUuid::WithoutBraces).left(8))
.arg(extension);
QString fullPath = QDir(imagesFolder).filePath(uniqueName);
QByteArray imageData = QByteArray::fromBase64(base64Data.toUtf8());
QFile file(fullPath);
if (!file.open(QIODevice::WriteOnly)) {
LOG_MESSAGE(QString("Failed to open file for writing: %1").arg(fullPath));
return false;
}
if (file.write(imageData) == -1) {
LOG_MESSAGE(QString("Failed to write image data: %1").arg(file.errorString()));
return false;
}
file.close();
storedPath = uniqueName;
LOG_MESSAGE(QString("Saved image: %1 to %2").arg(fileName, fullPath));
return true;
}
QString ChatSerializer::loadImageFromStorage(const QString &chatFilePath, const QString &storedPath)
{
QString imagesFolder = getChatImagesFolder(chatFilePath);
QString fullPath = QDir(imagesFolder).filePath(storedPath);
QFile file(fullPath);
if (!file.open(QIODevice::ReadOnly)) {
LOG_MESSAGE(QString("Failed to open image file: %1").arg(fullPath));
return QString();
}
QByteArray imageData = file.readAll();
file.close();
return imageData.toBase64();
}
} // namespace QodeAssist::Chat

16
ChatView/ChatSerializer.hpp
View File

@ -40,10 +40,18 @@ public:
static SerializationResult loadFromFile(ChatModel *model, const QString &filePath);
// Public for testing purposes
static QJsonObject serializeMessage(const ChatModel::Message &message);
static ChatModel::Message deserializeMessage(const QJsonObject &json);
static QJsonObject serializeChat(const ChatModel *model);
static bool deserializeChat(ChatModel *model, const QJsonObject &json);
static QJsonObject serializeMessage(const ChatModel::Message &message, const QString &chatFilePath);
static ChatModel::Message deserializeMessage(const QJsonObject &json, const QString &chatFilePath);
static QJsonObject serializeChat(const ChatModel *model, const QString &chatFilePath);
static bool deserializeChat(ChatModel *model, const QJsonObject &json, const QString &chatFilePath);
// Image management
static QString getChatImagesFolder(const QString &chatFilePath);
static bool saveImageToStorage(const QString &chatFilePath,
const QString &fileName,
const QString &base64Data,
QString &storedPath);
static QString loadImageFromStorage(const QString &chatFilePath, const QString &storedPath);
private:
static const QString VERSION;

154
ChatView/ClientInterface.cpp
View File

@ -20,9 +20,12 @@
#include "ClientInterface.hpp"
#include <texteditor/textdocument.h>
#include <QFile>
#include <QFileInfo>
#include <QImageReader>
#include <QJsonArray>
#include <QJsonDocument>
#include <QMimeDatabase>
#include <QUuid>
#include <coreplugin/editormanager/editormanager.h>
@ -36,6 +39,7 @@
#include <texteditor/texteditor.h>
#include "ChatAssistantSettings.hpp"
#include "ChatSerializer.hpp"
#include "GeneralSettings.hpp"
#include "ToolsSettings.hpp"
#include "Logger.hpp"
@ -70,8 +74,44 @@ void ClientInterface::sendMessage(
Context::ChangesManager::instance().archiveAllNonArchivedEdits();
auto attachFiles = m_contextManager->getContentFiles(attachments);
m_chatModel->addMessage(message, ChatModel::ChatRole::User, "", attachFiles);
QList<QString> imageFiles;
QList<QString> textFiles;
for (const QString &filePath : attachments) {
if (isImageFile(filePath)) {
imageFiles.append(filePath);
} else {
textFiles.append(filePath);
}
}
auto attachFiles = m_contextManager->getContentFiles(textFiles);
QList<ChatModel::ImageAttachment> imageAttachments;
if (!imageFiles.isEmpty() && !m_chatFilePath.isEmpty()) {
for (const QString &imagePath : imageFiles) {
QString base64Data = encodeImageToBase64(imagePath);
if (base64Data.isEmpty()) {
continue;
}
QString storedPath;
QFileInfo fileInfo(imagePath);
if (ChatSerializer::saveImageToStorage(m_chatFilePath, fileInfo.fileName(), base64Data, storedPath)) {
ChatModel::ImageAttachment imageAttachment;
imageAttachment.fileName = fileInfo.fileName();
imageAttachment.storedPath = storedPath;
imageAttachment.mediaType = getMediaTypeForImage(imagePath);
imageAttachments.append(imageAttachment);
LOG_MESSAGE(QString("Stored image %1 as %2").arg(fileInfo.fileName(), storedPath));
}
}
} else if (!imageFiles.isEmpty()) {
LOG_MESSAGE(QString("Warning: Chat file path not set, cannot save %1 image(s)").arg(imageFiles.size()));
}
m_chatModel->addMessage(message, ChatModel::ChatRole::User, "", attachFiles, imageAttachments);
auto &chatAssistantSettings = Settings::chatAssistantSettings();
@ -133,8 +173,36 @@ void ClientInterface::sendMessage(
apiMessage.isRedacted = msg.isRedacted;
apiMessage.signature = msg.signature;
if (provider->supportImage() && !m_chatFilePath.isEmpty() && !msg.images.isEmpty()) {
auto apiImages = loadImagesFromStorage(msg.images);
if (!apiImages.isEmpty()) {
apiMessage.images = apiImages;
}
}
messages.append(apiMessage);
}
if (!imageAttachments.isEmpty() && provider->supportImage() && !messages.isEmpty()) {
for (int i = messages.size() - 1; i >= 0; --i) {
if (messages[i].role == "user") {
auto newImages = loadImagesFromStorage(imageAttachments);
if (!newImages.isEmpty()) {
if (messages[i].images.has_value()) {
messages[i].images.value().append(newImages);
} else {
messages[i].images = newImages;
}
LOG_MESSAGE(QString("Added %1 new image(s) to message").arg(newImages.size()));
}
break;
}
}
} else if (!imageFiles.isEmpty() && !provider->supportImage()) {
LOG_MESSAGE(QString("Provider %1 doesn't support images, %2 ignored")
.arg(provider->name(), QString::number(imageFiles.size())));
}
context.history = messages;
LLMCore::LLMConfig config;
@ -379,4 +447,86 @@ void ClientInterface::handleCleanAccumulatedData(const QString &requestId)
LOG_MESSAGE(QString("Cleared accumulated responses for continuation request %1").arg(requestId));
}
bool ClientInterface::isImageFile(const QString &filePath) const
{
static const QSet<QString> imageExtensions = {
"png", "jpg", "jpeg", "gif", "webp", "bmp", "svg"
};
QFileInfo fileInfo(filePath);
QString extension = fileInfo.suffix().toLower();
return imageExtensions.contains(extension);
}
QString ClientInterface::getMediaTypeForImage(const QString &filePath) const
{
static const QHash<QString, QString> mediaTypes = {
{"png", "image/png"},
{"jpg", "image/jpeg"},
{"jpeg", "image/jpeg"},
{"gif", "image/gif"},
{"webp", "image/webp"},
{"bmp", "image/bmp"},
{"svg", "image/svg+xml"}
};
QFileInfo fileInfo(filePath);
QString extension = fileInfo.suffix().toLower();
if (mediaTypes.contains(extension)) {
return mediaTypes[extension];
}
QMimeDatabase mimeDb;
QMimeType mimeType = mimeDb.mimeTypeForFile(filePath);
return mimeType.name();
}
QString ClientInterface::encodeImageToBase64(const QString &filePath) const
{
QFile file(filePath);
if (!file.open(QIODevice::ReadOnly)) {
LOG_MESSAGE(QString("Failed to open image file: %1").arg(filePath));
return QString();
}
QByteArray imageData = file.readAll();
file.close();
return imageData.toBase64();
}
QVector<LLMCore::ImageAttachment> ClientInterface::loadImagesFromStorage(const QList<ChatModel::ImageAttachment> &storedImages) const
{
QVector<LLMCore::ImageAttachment> apiImages;
for (const auto &storedImage : storedImages) {
QString base64Data = ChatSerializer::loadImageFromStorage(m_chatFilePath, storedImage.storedPath);
if (base64Data.isEmpty()) {
LOG_MESSAGE(QString("Warning: Failed to load image: %1").arg(storedImage.storedPath));
continue;
}
LLMCore::ImageAttachment apiImage;
apiImage.data = base64Data;
apiImage.mediaType = storedImage.mediaType;
apiImage.isUrl = false;
apiImages.append(apiImage);
}
return apiImages;
}
void ClientInterface::setChatFilePath(const QString &filePath)
{
m_chatFilePath = filePath;
}
QString ClientInterface::chatFilePath() const
{
return m_chatFilePath;
}
} // namespace QodeAssist::Chat

8
ChatView/ClientInterface.hpp
View File

@ -48,6 +48,9 @@ public:
void cancelRequest();
Context::ContextManager *contextManager() const;
void setChatFilePath(const QString &filePath);
QString chatFilePath() const;
signals:
void errorOccurred(const QString &error);
@ -65,6 +68,10 @@ private:
QString getCurrentFileContext() const;
QString getSystemPromptWithLinkedFiles(
const QString &basePrompt, const QList<QString> &linkedFiles) const;
bool isImageFile(const QString &filePath) const;
QString getMediaTypeForImage(const QString &filePath) const;
QString encodeImageToBase64(const QString &filePath) const;
QVector<LLMCore::ImageAttachment> loadImagesFromStorage(const QList<ChatModel::ImageAttachment> &storedImages) const;
struct RequestContext
{
@ -75,6 +82,7 @@ private:
LLMCore::IPromptProvider *m_promptProvider = nullptr;
ChatModel *m_chatModel;
Context::ContextManager *m_contextManager;
QString m_chatFilePath;
QHash<QString, RequestContext> m_activeRequests;
QHash<QString, QString> m_accumulatedResponses;

4
ChatView/MessagePart.hpp
View File

@ -32,11 +32,15 @@ class MessagePart
Q_PROPERTY(MessagePartType type MEMBER type CONSTANT FINAL)
Q_PROPERTY(QString text MEMBER text CONSTANT FINAL)
Q_PROPERTY(QString language MEMBER language CONSTANT FINAL)
Q_PROPERTY(QString imageData MEMBER imageData CONSTANT FINAL)
Q_PROPERTY(QString mediaType MEMBER mediaType CONSTANT FINAL)
QML_VALUE_TYPE(messagePart)
public:
MessagePartType type;
QString text;
QString language;
QString imageData; // Base64 data or URL
QString mediaType; // e.g., "image/png", "image/jpeg"
};
} // namespace QodeAssist::Chat

6
ChatView/icons/image-dark.svg Normal file
View File

@ -0,0 +1,6 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
<rect x="3" y="3" width="18" height="18" rx="2" ry="2"/>
<circle cx="8.5" cy="8.5" r="1.5"/>
<polyline points="21 15 16 10 5 21"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 304 B

3
ChatView/qml/RootItem.qml
View File

@ -170,6 +170,8 @@ ChatRootView {
width: parent.width
msgModel: root.chatModel.processMessageContent(model.content)
messageAttachments: model.attachments
messageImages: model.images
chatFilePath: root.chatFilePath()
isUserMessage: model.roleType === ChatModel.User
messageIndex: index
textFontFamily: root.textFontFamily
@ -394,6 +396,7 @@ ChatRootView {
onCheckedChanged: root.setIsSyncOpenFiles(bottomBar.syncOpenFiles.checked)
}
attachFiles.onClicked: root.showAttachFilesDialog()
attachImages.onClicked: root.showAddImageDialog()
linkFiles.onClicked: root.showLinkFilesDialog()
}
}

98
ChatView/qml/chatparts/ChatItem.qml
View File

@ -28,6 +28,8 @@ Rectangle {
property alias msgModel: msgCreator.model
property alias messageAttachments: attachmentsModel.model
property alias messageImages: imagesModel.model
property string chatFilePath: ""
property string textFontFamily: Qt.application.font.family
property string codeFontFamily: {
switch (Qt.platform.os) {
@ -140,6 +142,27 @@ Rectangle {
}
}
}
Flow {
id: imagesFlow
Layout.fillWidth: true
visible: imagesModel.model && imagesModel.model.length > 0
leftPadding: 10
rightPadding: 10
spacing: 10
Repeater {
id: imagesModel
delegate: ImageComponent {
required property int index
required property var modelData
itemData: modelData
}
}
}
}
Rectangle {
@ -215,4 +238,79 @@ Rectangle {
codeFontFamily: root.codeFontFamily
codeFontSize: root.codeFontSize
}
component ImageComponent : Rectangle {
required property var itemData
readonly property int maxImageWidth: Math.min(400, root.width - 40)
readonly property int maxImageHeight: 300
width: Math.min(imageDisplay.implicitWidth, maxImageWidth) + 16
height: imageDisplay.implicitHeight + fileNameText.implicitHeight + 16
radius: 4
color: palette.base
border.width: 1
border.color: palette.mid
ColumnLayout {
anchors.fill: parent
anchors.margins: 8
spacing: 4
Image {
id: imageDisplay
Layout.alignment: Qt.AlignHCenter
Layout.maximumWidth: parent.parent.maxImageWidth
Layout.maximumHeight: parent.parent.maxImageHeight
source: {
if (!itemData.storedPath || !root.chatFilePath) return "";
var fileInfo = chatFileInfo(root.chatFilePath);
var imagesFolder = fileInfo.dir + "/" + fileInfo.baseName + "_images";
return "file://" + imagesFolder + "/" + itemData.storedPath;
}
sourceSize.width: parent.parent.maxImageWidth
sourceSize.height: parent.parent.maxImageHeight
fillMode: Image.PreserveAspectFit
cache: true
asynchronous: true
smooth: true
mipmap: true
BusyIndicator {
anchors.centerIn: parent
running: imageDisplay.status === Image.Loading
visible: running
}
Text {
anchors.centerIn: parent
text: qsTr("Failed to load image")
visible: imageDisplay.status === Image.Error
color: palette.placeholderText
}
}
Text {
id: fileNameText
Layout.fillWidth: true
text: itemData.fileName || ""
color: palette.text
font.pointSize: root.textFontSize - 1
elide: Text.ElideMiddle
horizontalAlignment: Text.AlignHCenter
}
}
}
function chatFileInfo(filePath) {
if (!filePath) return {dir: "", baseName: ""};
var lastSlash = filePath.lastIndexOf("/");
var dir = lastSlash >= 0 ? filePath.substring(0, lastSlash) : "";
var fileName = lastSlash >= 0 ? filePath.substring(lastSlash + 1) : filePath;
var lastDot = fileName.lastIndexOf(".");
var baseName = lastDot >= 0 ? fileName.substring(0, lastDot) : fileName;
return {dir: dir, baseName: baseName};
}
}

15
ChatView/qml/controls/BottomBar.qml
View File

@ -29,9 +29,9 @@ Rectangle {
property alias sendButton: sendButtonId
property alias syncOpenFiles: syncOpenFilesId
property alias attachFiles: attachFilesId
property alias attachImages: attachImagesId
property alias linkFiles: linkFilesId
color: palette.window.hslLightness > 0.5 ?
Qt.darker(palette.window, 1.1) :
Qt.lighter(palette.window, 1.1)
@ -73,6 +73,19 @@ Rectangle {
ToolTip.text: qsTr("Attach file to message")
}
QoAButton {
id: attachImagesId
icon {
source: "qrc:/qt/qml/ChatView/icons/image-dark.svg"
height: 15
width: 15
}
ToolTip.visible: hovered
ToolTip.delay: 250
ToolTip.text: qsTr("Attach image to message")
}
QoAButton {
id: linkFilesId

48
llmcore/ContentBlocks.hpp
View File

@ -68,6 +68,54 @@ private:
QString m_text;
};
class ImageContent : public ContentBlock
{
Q_OBJECT
public:
enum class ImageSourceType { Base64, Url };
ImageContent(const QString &data, const QString &mediaType, ImageSourceType sourceType = ImageSourceType::Base64)
: ContentBlock()
, m_data(data)
, m_mediaType(mediaType)
, m_sourceType(sourceType)
{}
QString type() const override { return "image"; }
QString data() const { return m_data; }
QString mediaType() const { return m_mediaType; }
ImageSourceType sourceType() const { return m_sourceType; }
QJsonValue toJson(ProviderFormat format) const override
{
if (format == ProviderFormat::Claude) {
QJsonObject source;
if (m_sourceType == ImageSourceType::Base64) {
source["type"] = "base64";
source["media_type"] = m_mediaType;
source["data"] = m_data;
} else {
source["type"] = "url";
source["url"] = m_data;
}
return QJsonObject{{"type", "image"}, {"source", source}};
} else { // OpenAI format
QJsonObject imageUrl;
if (m_sourceType == ImageSourceType::Base64) {
imageUrl["url"] = QString("data:%1;base64,%2").arg(m_mediaType, m_data);
} else {
imageUrl["url"] = m_data;
}
return QJsonObject{{"type", "image_url"}, {"image_url", imageUrl}};
}
}
private:
QString m_data;
QString m_mediaType;
ImageSourceType m_sourceType;
};
class ToolUseContent : public ContentBlock
{
Q_OBJECT

10
llmcore/ContextData.hpp
View File

@ -24,6 +24,15 @@
namespace QodeAssist::LLMCore {
struct ImageAttachment
{
QString data; // Base64 encoded data or URL
QString mediaType; // e.g., "image/png", "image/jpeg", "image/webp", "image/gif"
bool isUrl = false; // true if data is URL, false if base64
bool operator==(const ImageAttachment &) const = default;
};
struct Message
{
QString role;
@ -31,6 +40,7 @@ struct Message
QString signature;
bool isThinking = false;
bool isRedacted = false;
std::optional<QVector<ImageAttachment>> images;
// clang-format off
bool operator==(const Message&) const = default;

1
llmcore/Provider.hpp
View File

@ -67,6 +67,7 @@ public:
virtual bool supportsTools() const { return false; };
virtual bool supportThinking() const { return false; };
virtual bool supportImage() const { return false; };
virtual void cancelRequest(const RequestID &requestId);

18
providers/ClaudeMessage.cpp
View File

@ -39,6 +39,24 @@ void ClaudeMessage::handleContentBlockStart(
if (blockType == "text") {
addCurrentContent<LLMCore::TextContent>();
} else if (blockType == "image") {
QJsonObject source = data["source"].toObject();
QString sourceType = source["type"].toString();
QString imageData;
QString mediaType;
LLMCore::ImageContent::ImageSourceType imgSourceType = LLMCore::ImageContent::ImageSourceType::Base64;
if (sourceType == "base64") {
imageData = source["data"].toString();
mediaType = source["media_type"].toString();
imgSourceType = LLMCore::ImageContent::ImageSourceType::Base64;
} else if (sourceType == "url") {
imageData = source["url"].toString();
imgSourceType = LLMCore::ImageContent::ImageSourceType::Url;
}
addCurrentContent<LLMCore::ImageContent>(imageData, mediaType, imgSourceType);
} else if (blockType == "tool_use") {
QString toolId = data["id"].toString();
QString toolName = data["name"].toString();

4
providers/ClaudeProvider.cpp
View File

@ -257,6 +257,10 @@ bool ClaudeProvider::supportThinking() const {
return true;
};
bool ClaudeProvider::supportImage() const {
return true;
};
void ClaudeProvider::cancelRequest(const LLMCore::RequestID &requestId)
{
LOG_MESSAGE(QString("ClaudeProvider: Cancelling request %1").arg(requestId));

1
providers/ClaudeProvider.hpp
View File

@ -55,6 +55,7 @@ public:
bool supportsTools() const override;
bool supportThinking() const override;
bool supportImage() const override;
void cancelRequest(const LLMCore::RequestID &requestId) override;
public slots:

5
providers/GoogleAIProvider.cpp
View File

@ -273,6 +273,11 @@ bool GoogleAIProvider::supportThinking() const
return true;
}
bool GoogleAIProvider::supportImage() const
{
return true;
}
void GoogleAIProvider::cancelRequest(const LLMCore::RequestID &requestId)
{
LOG_MESSAGE(QString("GoogleAIProvider: Cancelling request %1").arg(requestId));

1
providers/GoogleAIProvider.hpp
View File

@ -54,6 +54,7 @@ public:
bool supportsTools() const override;
bool supportThinking() const override;
bool supportImage() const override;
void cancelRequest(const LLMCore::RequestID &requestId) override;
public slots:

5
providers/LMStudioProvider.cpp
View File

@ -163,6 +163,11 @@ bool LMStudioProvider::supportsTools() const
return true;
}
bool LMStudioProvider::supportImage() const
{
return true;
}
void LMStudioProvider::cancelRequest(const LLMCore::RequestID &requestId)
{
LOG_MESSAGE(QString("LMStudioProvider: Cancelling request %1").arg(requestId));

1
providers/LMStudioProvider.hpp
View File

@ -53,6 +53,7 @@ public:
const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override;
bool supportsTools() const override;
bool supportImage() const override;
void cancelRequest(const LLMCore::RequestID &requestId) override;
public slots:

5
providers/LlamaCppProvider.cpp
View File

@ -206,6 +206,11 @@ bool LlamaCppProvider::supportsTools() const
return true;
}
bool LlamaCppProvider::supportImage() const
{
return true;
}
void LlamaCppProvider::cancelRequest(const LLMCore::RequestID &requestId)
{
LOG_MESSAGE(QString("LlamaCppProvider: Cancelling request %1").arg(requestId));

1
providers/LlamaCppProvider.hpp
View File

@ -53,6 +53,7 @@ public:
const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override;
bool supportsTools() const override;
bool supportImage() const override;
void cancelRequest(const LLMCore::RequestID &requestId) override;
public slots:

5
providers/MistralAIProvider.cpp
View File

@ -184,6 +184,11 @@ bool MistralAIProvider::supportsTools() const
return true;
}
bool MistralAIProvider::supportImage() const
{
return true;
}
void MistralAIProvider::cancelRequest(const LLMCore::RequestID &requestId)
{
LOG_MESSAGE(QString("MistralAIProvider: Cancelling request %1").arg(requestId));

1
providers/MistralAIProvider.hpp
View File

@ -53,6 +53,7 @@ public:
const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override;
bool supportsTools() const override;
bool supportImage() const override;
void cancelRequest(const LLMCore::RequestID &requestId) override;
public slots:

8
providers/OllamaProvider.cpp
View File

@ -168,6 +168,7 @@ QList<QString> OllamaProvider::validateRequest(const QJsonObject &request, LLMCo
{"prompt", {}},
{"suffix", {}},
{"system", {}},
{"images", QJsonArray{}},
{"options",
QJsonObject{
{"temperature", {}},
@ -182,7 +183,7 @@ QList<QString> OllamaProvider::validateRequest(const QJsonObject &request, LLMCo
{"keep_alive", {}},
{"model", {}},
{"stream", {}},
{"messages", QJsonArray{{QJsonObject{{"role", {}}, {"content", {}}}}}},
{"messages", QJsonArray{{QJsonObject{{"role", {}}, {"content", {}}, {"images", QJsonArray{}}}}}},
{"tools", QJsonArray{}},
{"options",
QJsonObject{
@ -241,6 +242,11 @@ bool OllamaProvider::supportsTools() const
return true;
}
bool OllamaProvider::supportImage() const
{
return true;
}
void OllamaProvider::cancelRequest(const LLMCore::RequestID &requestId)
{
LOG_MESSAGE(QString("OllamaProvider: Cancelling request %1").arg(requestId));

1
providers/OllamaProvider.hpp
View File

@ -54,6 +54,7 @@ public:
const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override;
bool supportsTools() const override;
bool supportImage() const override;
void cancelRequest(const LLMCore::RequestID &requestId) override;
public slots:

5
providers/OpenAICompatProvider.cpp
View File

@ -192,6 +192,11 @@ bool OpenAICompatProvider::supportsTools() const
return true;
}
bool OpenAICompatProvider::supportImage() const
{
return true;
}
void OpenAICompatProvider::cancelRequest(const LLMCore::RequestID &requestId)
{
LOG_MESSAGE(QString("OpenAICompatProvider: Cancelling request %1").arg(requestId));

1
providers/OpenAICompatProvider.hpp
View File

@ -53,6 +53,7 @@ public:
const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override;
bool supportsTools() const override;
bool supportImage() const override;
void cancelRequest(const LLMCore::RequestID &requestId) override;
public slots:

5
providers/OpenAIProvider.cpp
View File

@ -248,6 +248,11 @@ bool OpenAIProvider::supportsTools() const
return true;
}
bool OpenAIProvider::supportImage() const
{
return true;
}
void OpenAIProvider::cancelRequest(const LLMCore::RequestID &requestId)
{
LOG_MESSAGE(QString("OpenAIProvider: Cancelling request %1").arg(requestId));

1
providers/OpenAIProvider.hpp
View File

@ -53,6 +53,7 @@ public:
const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override;
bool supportsTools() const override;
bool supportImage() const override;
void cancelRequest(const LLMCore::RequestID &requestId) override;
public slots:

75
templates/Claude.hpp
View File

@ -41,34 +41,55 @@ public:
if (context.history) {
for (const auto &msg : context.history.value()) {
if (msg.role != "system") {
// Handle thinking blocks with structured content
if (msg.isThinking) {
// Create content array with thinking block
QJsonArray content;
QJsonObject thinkingBlock;
thinkingBlock["type"] = msg.isRedacted ? "redacted_thinking" : "thinking";
// Extract actual thinking text (remove display signature)
QString thinkingText = msg.content;
int signaturePos = thinkingText.indexOf("\n[Signature: ");
if (signaturePos != -1) {
thinkingText = thinkingText.left(signaturePos);
}
if (!msg.isRedacted) {
thinkingBlock["thinking"] = thinkingText;
}
if (!msg.signature.isEmpty()) {
thinkingBlock["signature"] = msg.signature;
}
content.append(thinkingBlock);
messages.append(QJsonObject{{"role", "assistant"}, {"content", content}});
} else {
// Normal message
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
if (msg.role == "system") continue;
if (msg.isThinking) {
QJsonArray content;
QJsonObject thinkingBlock;
thinkingBlock["type"] = msg.isRedacted ? "redacted_thinking" : "thinking";
QString thinkingText = msg.content;
int signaturePos = thinkingText.indexOf("\n[Signature: ");
if (signaturePos != -1) {
thinkingText = thinkingText.left(signaturePos);
}
if (!msg.isRedacted) {
thinkingBlock["thinking"] = thinkingText;
}
if (!msg.signature.isEmpty()) {
thinkingBlock["signature"] = msg.signature;
}
content.append(thinkingBlock);
messages.append(QJsonObject{{"role", "assistant"}, {"content", content}});
} else if (msg.images && !msg.images->isEmpty()) {
QJsonArray content;
if (!msg.content.isEmpty()) {
content.append(QJsonObject{{"type", "text"}, {"text", msg.content}});
}
for (const auto &image : msg.images.value()) {
QJsonObject imageBlock;
imageBlock["type"] = "image";
QJsonObject source;
if (image.isUrl) {
source["type"] = "url";
source["url"] = image.data;
} else {
source["type"] = "base64";
source["media_type"] = image.mediaType;
source["data"] = image.data;
}
imageBlock["source"] = source;
content.append(imageBlock);
}
messages.append(QJsonObject{{"role", msg.role}, {"content", content}});
} else {
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
}
}
}

24
templates/GoogleAI.hpp
View File

@ -46,7 +46,29 @@ public:
QJsonObject content;
QJsonArray parts;
parts.append(QJsonObject{{"text", msg.content}});
if (!msg.content.isEmpty()) {
parts.append(QJsonObject{{"text", msg.content}});
}
if (msg.images && !msg.images->isEmpty()) {
for (const auto &image : msg.images.value()) {
QJsonObject imagePart;
if (image.isUrl) {
QJsonObject fileData;
fileData["mime_type"] = image.mediaType;
fileData["file_uri"] = image.data;
imagePart["file_data"] = fileData;
} else {
QJsonObject inlineData;
inlineData["mime_type"] = image.mediaType;
inlineData["data"] = image.data;
imagePart["inline_data"] = inlineData;
}
parts.append(imagePart);
}
}
QString role = msg.role;
if (role == "assistant") {

28
templates/MistralAI.hpp
View File

@ -74,7 +74,31 @@ public:
if (context.history) {
for (const auto &msg : context.history.value()) {
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
if (msg.images && !msg.images->isEmpty()) {
QJsonArray content;
if (!msg.content.isEmpty()) {
content.append(QJsonObject{{"type", "text"}, {"text", msg.content}});
}
for (const auto &image : msg.images.value()) {
QJsonObject imageBlock;
imageBlock["type"] = "image_url";
QJsonObject imageUrl;
if (image.isUrl) {
imageUrl["url"] = image.data;
} else {
imageUrl["url"] = QString("data:%1;base64,%2").arg(image.mediaType, image.data);
}
imageBlock["image_url"] = imageUrl;
content.append(imageBlock);
}
messages.append(QJsonObject{{"role", msg.role}, {"content", content}});
} else {
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
}
}
}
@ -90,7 +114,7 @@ public:
" {\"role\": \"assistant\", \"content\": \"<assistant response>\"}\n"
" ]\n"
"}\n\n"
"Supports system messages and conversation history.";
"Supports system messages, conversation history, and images.";
}
bool isSupportProvider(LLMCore::ProviderID id) const override
{

19
templates/Ollama.hpp
View File

@ -76,7 +76,19 @@ public:
if (context.history) {
for (const auto &msg : context.history.value()) {
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
QJsonObject messageObj;
messageObj["role"] = msg.role;
messageObj["content"] = msg.content;
if (msg.images && !msg.images->isEmpty()) {
QJsonArray images;
for (const auto &image : msg.images.value()) {
images.append(image.data);
}
messageObj["images"] = images;
}
messages.append(messageObj);
}
}
@ -88,11 +100,12 @@ public:
"{\n"
" \"messages\": [\n"
" {\"role\": \"system\", \"content\": \"<system prompt>\"},\n"
" {\"role\": \"user\", \"content\": \"<user message>\"},\n"
" {\"role\": \"user\", \"content\": \"<user message>\", \"images\": [\"<base64>\"]},\n"
" {\"role\": \"assistant\", \"content\": \"<assistant response>\"}\n"
" ]\n"
"}\n\n"
"Recommended for Ollama models with chat capability.";
"Recommended for Ollama models with chat capability.\n"
"Supports images for multimodal models (e.g., llava).";
}
bool isSupportProvider(LLMCore::ProviderID id) const override
{

26
templates/OpenAI.hpp
View File

@ -42,7 +42,31 @@ public:
if (context.history) {
for (const auto &msg : context.history.value()) {
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
if (msg.images && !msg.images->isEmpty()) {
QJsonArray content;
if (!msg.content.isEmpty()) {
content.append(QJsonObject{{"type", "text"}, {"text", msg.content}});
}
for (const auto &image : msg.images.value()) {
QJsonObject imageBlock;
imageBlock["type"] = "image_url";
QJsonObject imageUrl;
if (image.isUrl) {
imageUrl["url"] = image.data;
} else {
imageUrl["url"] = QString("data:%1;base64,%2").arg(image.mediaType, image.data);
}
imageBlock["image_url"] = imageUrl;
content.append(imageBlock);
}
messages.append(QJsonObject{{"role", msg.role}, {"content", content}});
} else {
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
}
}
}

29
templates/OpenAICompatible.hpp
View File

@ -42,7 +42,31 @@ public:
if (context.history) {
for (const auto &msg : context.history.value()) {
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
if (msg.images && !msg.images->isEmpty()) {
QJsonArray content;
if (!msg.content.isEmpty()) {
content.append(QJsonObject{{"type", "text"}, {"text", msg.content}});
}
for (const auto &image : msg.images.value()) {
QJsonObject imageBlock;
imageBlock["type"] = "image_url";
QJsonObject imageUrl;
if (image.isUrl) {
imageUrl["url"] = image.data;
} else {
imageUrl["url"] = QString("data:%1;base64,%2").arg(image.mediaType, image.data);
}
imageBlock["image_url"] = imageUrl;
content.append(imageBlock);
}
messages.append(QJsonObject{{"role", msg.role}, {"content", content}});
} else {
messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}});
}
}
}
@ -58,7 +82,8 @@ public:
" {\"role\": \"assistant\", \"content\": \"<assistant response>\"}\n"
" ]\n"
"}\n\n"
"Works with any service implementing the OpenAI Chat API specification.";
"Works with any service implementing the OpenAI Chat API specification.\n"
"Supports images.";
}
bool isSupportProvider(LLMCore::ProviderID id) const override
{