diff --git a/ChatView/CMakeLists.txt b/ChatView/CMakeLists.txt index 8acc334..ae351ae 100644 --- a/ChatView/CMakeLists.txt +++ b/ChatView/CMakeLists.txt @@ -32,6 +32,7 @@ qt_add_qml_module(QodeAssistChatView icons/close-light.svg icons/link-file-light.svg icons/link-file-dark.svg + icons/image-dark.svg icons/load-chat-dark.svg icons/save-chat-dark.svg icons/clean-icon-dark.svg diff --git a/ChatView/ChatData.hpp b/ChatView/ChatData.hpp index fca7c36..ac306e9 100644 --- a/ChatView/ChatData.hpp +++ b/ChatView/ChatData.hpp @@ -26,7 +26,7 @@ namespace QodeAssist::Chat { Q_NAMESPACE QML_NAMED_ELEMENT(MessagePartType) -enum class MessagePartType { Code, Text }; +enum class MessagePartType { Code, Text, Image }; Q_ENUM_NS(MessagePartType) } // namespace QodeAssist::Chat diff --git a/ChatView/ChatModel.cpp b/ChatView/ChatModel.cpp index 059e0f2..ddc9e13 100644 --- a/ChatView/ChatModel.cpp +++ b/ChatView/ChatModel.cpp @@ -84,6 +84,17 @@ QVariant ChatModel::data(const QModelIndex &index, int role) const case Roles::IsRedacted: { return message.isRedacted; } + case Roles::Images: { + QVariantList imagesList; + for (const auto &image : message.images) { + QVariantMap imageMap; + imageMap["fileName"] = image.fileName; + imageMap["storedPath"] = image.storedPath; + imageMap["mediaType"] = image.mediaType; + imagesList.append(imageMap); + } + return imagesList; + } default: return QVariant(); } @@ -96,6 +107,7 @@ QHash ChatModel::roleNames() const roles[Roles::Content] = "content"; roles[Roles::Attachments] = "attachments"; roles[Roles::IsRedacted] = "isRedacted"; + roles[Roles::Images] = "images"; return roles; } @@ -103,7 +115,8 @@ void ChatModel::addMessage( const QString &content, ChatRole role, const QString &id, - const QList &attachments) + const QList &attachments, + const QList &images) { QString fullContent = content; if (!attachments.isEmpty()) { @@ -119,11 +132,13 @@ void ChatModel::addMessage( Message &lastMessage = m_messages.last(); lastMessage.content = content; lastMessage.attachments = attachments; + lastMessage.images = images; emit dataChanged(index(m_messages.size() - 1), index(m_messages.size() - 1)); } else { beginInsertRows(QModelIndex(), m_messages.size(), m_messages.size()); Message newMessage{role, content, id}; newMessage.attachments = attachments; + newMessage.images = images; m_messages.append(newMessage); endInsertRows(); diff --git a/ChatView/ChatModel.hpp b/ChatView/ChatModel.hpp index d27d4db..735cf50 100644 --- a/ChatView/ChatModel.hpp +++ b/ChatView/ChatModel.hpp @@ -40,9 +40,16 @@ public: enum ChatRole { System, User, Assistant, Tool, FileEdit, Thinking }; Q_ENUM(ChatRole) - enum Roles { RoleType = Qt::UserRole, Content, Attachments, IsRedacted }; + enum Roles { RoleType = Qt::UserRole, Content, Attachments, IsRedacted, Images }; Q_ENUM(Roles) + struct ImageAttachment + { + QString fileName; // Original filename + QString storedPath; // Path to stored image file (relative to chat folder) + QString mediaType; // MIME type + }; + struct Message { ChatRole role; @@ -52,6 +59,7 @@ public: QString signature = QString(); QList attachments; + QList images; }; explicit ChatModel(QObject *parent = nullptr); @@ -64,7 +72,8 @@ public: const QString &content, ChatRole role, const QString &id, - const QList &attachments = {}); + const QList &attachments = {}, + const QList &images = {}); Q_INVOKABLE void clear(); Q_INVOKABLE QList processMessageContent(const QString &content) const; diff --git a/ChatView/ChatRootView.cpp b/ChatView/ChatRootView.cpp index d2de37d..9508e36 100644 --- a/ChatView/ChatRootView.cpp +++ b/ChatView/ChatRootView.cpp @@ -242,6 +242,14 @@ void ChatRootView::sendMessage(const QString &message) } } + if (m_recentFilePath.isEmpty()) { + QString filePath = getAutosaveFilePath(message, m_attachmentFiles); + if (!filePath.isEmpty()) { + setRecentFilePath(filePath); + LOG_MESSAGE(QString("Set chat file path for new chat: %1").arg(filePath)); + } + } + m_clientInterface->sendMessage(message, m_attachmentFiles, m_linkedFiles, m_isAgentMode); clearAttachmentFiles(); setRequestProgressStatus(true); @@ -379,51 +387,22 @@ void ChatRootView::showLoadDialog() QString ChatRootView::getSuggestedFileName() const { - QStringList parts; - - static const QRegularExpression saitizeSymbols = QRegularExpression("[\\/:*?\"<>|\\s]"); - static const QRegularExpression underSymbols = QRegularExpression("_+"); + QString shortMessage; if (m_chatModel->rowCount() > 0) { QString firstMessage = m_chatModel->data(m_chatModel->index(0), ChatModel::Content).toString(); - QString shortMessage = firstMessage.split('\n').first().simplified().left(30); + shortMessage = firstMessage.split('\n').first().simplified().left(30); - QString sanitizedMessage = shortMessage; - sanitizedMessage.replace(saitizeSymbols, "_"); - sanitizedMessage.replace(underSymbols, "_"); - sanitizedMessage = sanitizedMessage.trimmed(); - - if (!sanitizedMessage.isEmpty()) { - if (sanitizedMessage.startsWith('_')) { - sanitizedMessage.remove(0, 1); - } - if (sanitizedMessage.endsWith('_')) { - sanitizedMessage.chop(1); - } - - QString targetDir = getChatsHistoryDir(); - QString fullPath = QDir(targetDir).filePath(sanitizedMessage); - - QFileInfo fileInfo(fullPath); - if (!fileInfo.exists() && QFileInfo(fileInfo.path()).isWritable()) { - parts << sanitizedMessage; + if (shortMessage.isEmpty()) { + QVariantList images = m_chatModel->data(m_chatModel->index(0), ChatModel::Images).toList(); + if (!images.isEmpty()) { + shortMessage = "image_chat"; } } } - parts << QDateTime::currentDateTime().toString("yyyy-MM-dd_HH-mm"); - - QString fileName = parts.join("_"); - - QString fullPath = QDir(getChatsHistoryDir()).filePath(fileName); - QFileInfo finalCheck(fullPath); - - if (fileName.isEmpty() || finalCheck.exists() || !QFileInfo(finalCheck.path()).isWritable()) { - fileName = QString("chat_%1").arg(QDateTime::currentDateTime().toString("yyyy-MM-dd_HH-mm")); - } - - return fileName; + return generateChatFileName(shortMessage, getChatsHistoryDir()); } void ChatRootView::autosave() @@ -453,6 +432,27 @@ QString ChatRootView::getAutosaveFilePath() const return QDir(dir).filePath(getSuggestedFileName() + ".json"); } +QString ChatRootView::getAutosaveFilePath(const QString &firstMessage, const QStringList &attachments) const +{ + if (!m_recentFilePath.isEmpty()) { + return m_recentFilePath; + } + + QString dir = getChatsHistoryDir(); + if (dir.isEmpty()) { + return QString(); + } + + QString shortMessage = firstMessage.split('\n').first().simplified().left(30); + + if (shortMessage.isEmpty() && hasImageAttachments(attachments)) { + shortMessage = "image_chat"; + } + + QString fileName = generateChatFileName(shortMessage, dir); + return QDir(dir).filePath(fileName + ".json"); +} + QStringList ChatRootView::attachmentFiles() const { return m_attachmentFiles; @@ -531,6 +531,33 @@ void ChatRootView::removeFileFromLinkList(int index) } } +void ChatRootView::showAddImageDialog() +{ + QFileDialog dialog(nullptr, tr("Select Images to Attach")); + dialog.setFileMode(QFileDialog::ExistingFiles); + dialog.setNameFilter(tr("Images (*.png *.jpg *.jpeg *.gif *.bmp *.webp)")); + + if (auto project = ProjectExplorer::ProjectManager::startupProject()) { + dialog.setDirectory(project->projectDirectory().toFSPathString()); + } + + if (dialog.exec() == QDialog::Accepted) { + QStringList newFilePaths = dialog.selectedFiles(); + if (!newFilePaths.isEmpty()) { + bool filesAdded = false; + for (const QString &filePath : std::as_const(newFilePaths)) { + if (!m_attachmentFiles.contains(filePath)) { + m_attachmentFiles.append(filePath); + filesAdded = true; + } + } + if (filesAdded) { + emit attachmentFilesChanged(); + } + } + } +} + void ChatRootView::calculateMessageTokensCount(const QString &message) { m_messageTokensCount = Context::TokenUtils::estimateTokens(message); @@ -665,10 +692,16 @@ QString ChatRootView::chatFileName() const return QFileInfo(m_recentFilePath).baseName(); } +QString ChatRootView::chatFilePath() const +{ + return m_recentFilePath; +} + void ChatRootView::setRecentFilePath(const QString &filePath) { if (m_recentFilePath != filePath) { m_recentFilePath = filePath; + m_clientInterface->setChatFilePath(filePath); emit chatFileNameChanged(); } } @@ -1126,5 +1159,58 @@ bool ChatRootView::isThinkingSupport() const return provider && provider->supportThinking(); } +QString ChatRootView::generateChatFileName(const QString &shortMessage, const QString &dir) const +{ + static const QRegularExpression saitizeSymbols = QRegularExpression("[\\/:*?\"<>|\\s]"); + static const QRegularExpression underSymbols = QRegularExpression("_+"); + + QStringList parts; + QString sanitizedMessage = shortMessage; + sanitizedMessage.replace(saitizeSymbols, "_"); + sanitizedMessage.replace(underSymbols, "_"); + sanitizedMessage = sanitizedMessage.trimmed(); + + if (!sanitizedMessage.isEmpty()) { + if (sanitizedMessage.startsWith('_')) { + sanitizedMessage.remove(0, 1); + } + if (sanitizedMessage.endsWith('_')) { + sanitizedMessage.chop(1); + } + + QString fullPath = QDir(dir).filePath(sanitizedMessage); + QFileInfo fileInfo(fullPath); + if (!fileInfo.exists() && QFileInfo(fileInfo.path()).isWritable()) { + parts << sanitizedMessage; + } + } + + parts << QDateTime::currentDateTime().toString("yyyy-MM-dd_HH-mm"); + + QString fileName = parts.join("_"); + QString fullPath = QDir(dir).filePath(fileName); + QFileInfo finalCheck(fullPath); + + if (fileName.isEmpty() || finalCheck.exists() || !QFileInfo(finalCheck.path()).isWritable()) { + fileName = QString("chat_%1").arg(QDateTime::currentDateTime().toString("yyyy-MM-dd_HH-mm")); + } + + return fileName; +} + +bool ChatRootView::hasImageAttachments(const QStringList &attachments) const +{ + static const QSet imageExtensions = { + "png", "jpg", "jpeg", "gif", "webp", "bmp", "svg" + }; + + for (const QString &filePath : attachments) { + QFileInfo fileInfo(filePath); + if (imageExtensions.contains(fileInfo.suffix().toLower())) { + return true; + } + } + return false; +} } // namespace QodeAssist::Chat diff --git a/ChatView/ChatRootView.hpp b/ChatView/ChatRootView.hpp index 0d79e90..7d45d8d 100644 --- a/ChatView/ChatRootView.hpp +++ b/ChatView/ChatRootView.hpp @@ -75,6 +75,7 @@ public: void autosave(); QString getAutosaveFilePath() const; + QString getAutosaveFilePath(const QString &firstMessage, const QStringList &attachments) const; QStringList attachmentFiles() const; QStringList linkedFiles() const; @@ -83,6 +84,7 @@ public: Q_INVOKABLE void removeFileFromAttachList(int index); Q_INVOKABLE void showLinkFilesDialog(); Q_INVOKABLE void removeFileFromLinkList(int index); + Q_INVOKABLE void showAddImageDialog(); Q_INVOKABLE void calculateMessageTokensCount(const QString &message); Q_INVOKABLE void setIsSyncOpenFiles(bool state); Q_INVOKABLE void openChatHistoryFolder(); @@ -98,6 +100,7 @@ public: void onEditorCreated(Core::IEditor *editor, const Utils::FilePath &filePath); QString chatFileName() const; + Q_INVOKABLE QString chatFilePath() const; void setRecentFilePath(const QString &filePath); bool shouldIgnoreFileForAttach(const Utils::FilePath &filePath); @@ -181,6 +184,8 @@ private: void updateFileEditStatus(const QString &editId, const QString &status); QString getChatsHistoryDir() const; QString getSuggestedFileName() const; + QString generateChatFileName(const QString &shortMessage, const QString &dir) const; + bool hasImageAttachments(const QStringList &attachments) const; ChatModel *m_chatModel; LLMCore::PromptProviderChat m_promptProvider; diff --git a/ChatView/ChatSerializer.cpp b/ChatView/ChatSerializer.cpp index 0ee125f..820944a 100644 --- a/ChatView/ChatSerializer.cpp +++ b/ChatView/ChatSerializer.cpp @@ -20,10 +20,13 @@ #include "ChatSerializer.hpp" #include "Logger.hpp" +#include #include #include #include +#include #include +#include namespace QodeAssist::Chat { @@ -35,12 +38,20 @@ SerializationResult ChatSerializer::saveToFile(const ChatModel *model, const QSt return {false, "Failed to create directory structure"}; } + QString imagesFolder = getChatImagesFolder(filePath); + QDir dir; + if (!dir.exists(imagesFolder)) { + if (!dir.mkpath(imagesFolder)) { + LOG_MESSAGE(QString("Warning: Failed to create images folder: %1").arg(imagesFolder)); + } + } + QFile file(filePath); if (!file.open(QIODevice::WriteOnly)) { return {false, QString("Failed to open file for writing: %1").arg(filePath)}; } - QJsonObject root = serializeChat(model); + QJsonObject root = serializeChat(model, filePath); QJsonDocument doc(root); if (file.write(doc.toJson(QJsonDocument::Indented)) == -1) { @@ -70,14 +81,14 @@ SerializationResult ChatSerializer::loadFromFile(ChatModel *model, const QString return {false, QString("Unsupported version: %1").arg(version)}; } - if (!deserializeChat(model, root)) { + if (!deserializeChat(model, root, filePath)) { return {false, "Failed to deserialize chat data"}; } return {true, QString()}; } -QJsonObject ChatSerializer::serializeMessage(const ChatModel::Message &message) +QJsonObject ChatSerializer::serializeMessage(const ChatModel::Message &message, const QString &chatFilePath) { QJsonObject messageObj; messageObj["role"] = static_cast(message.role); @@ -87,10 +98,23 @@ QJsonObject ChatSerializer::serializeMessage(const ChatModel::Message &message) if (!message.signature.isEmpty()) { messageObj["signature"] = message.signature; } + + if (!message.images.isEmpty()) { + QJsonArray imagesArray; + for (const auto &image : message.images) { + QJsonObject imageObj; + imageObj["fileName"] = image.fileName; + imageObj["storedPath"] = image.storedPath; + imageObj["mediaType"] = image.mediaType; + imagesArray.append(imageObj); + } + messageObj["images"] = imagesArray; + } + return messageObj; } -ChatModel::Message ChatSerializer::deserializeMessage(const QJsonObject &json) +ChatModel::Message ChatSerializer::deserializeMessage(const QJsonObject &json, const QString &chatFilePath) { ChatModel::Message message; message.role = static_cast(json["role"].toInt()); @@ -98,14 +122,27 @@ ChatModel::Message ChatSerializer::deserializeMessage(const QJsonObject &json) message.id = json["id"].toString(); message.isRedacted = json["isRedacted"].toBool(false); message.signature = json["signature"].toString(); + + if (json.contains("images")) { + QJsonArray imagesArray = json["images"].toArray(); + for (const auto &imageValue : imagesArray) { + QJsonObject imageObj = imageValue.toObject(); + ChatModel::ImageAttachment image; + image.fileName = imageObj["fileName"].toString(); + image.storedPath = imageObj["storedPath"].toString(); + image.mediaType = imageObj["mediaType"].toString(); + message.images.append(image); + } + } + return message; } -QJsonObject ChatSerializer::serializeChat(const ChatModel *model) +QJsonObject ChatSerializer::serializeChat(const ChatModel *model, const QString &chatFilePath) { QJsonArray messagesArray; for (const auto &message : model->getChatHistory()) { - messagesArray.append(serializeMessage(message)); + messagesArray.append(serializeMessage(message, chatFilePath)); } QJsonObject root; @@ -115,14 +152,14 @@ QJsonObject ChatSerializer::serializeChat(const ChatModel *model) return root; } -bool ChatSerializer::deserializeChat(ChatModel *model, const QJsonObject &json) +bool ChatSerializer::deserializeChat(ChatModel *model, const QJsonObject &json, const QString &chatFilePath) { QJsonArray messagesArray = json["messages"].toArray(); QVector messages; messages.reserve(messagesArray.size()); for (const auto &messageValue : messagesArray) { - messages.append(deserializeMessage(messageValue.toObject())); + messages.append(deserializeMessage(messageValue.toObject(), chatFilePath)); } model->clear(); @@ -130,7 +167,8 @@ bool ChatSerializer::deserializeChat(ChatModel *model, const QJsonObject &json) model->setLoadingFromHistory(true); for (const auto &message : messages) { - model->addMessage(message.content, message.role, message.id); + model->addMessage(message.content, message.role, message.id, message.attachments, message.images); + LOG_MESSAGE(QString("Loaded message with %1 image(s)").arg(message.images.size())); } model->setLoadingFromHistory(false); @@ -150,4 +188,73 @@ bool ChatSerializer::validateVersion(const QString &version) return version == VERSION; } +QString ChatSerializer::getChatImagesFolder(const QString &chatFilePath) +{ + QFileInfo fileInfo(chatFilePath); + QString baseName = fileInfo.completeBaseName(); + QString dirPath = fileInfo.absolutePath(); + return QDir(dirPath).filePath(baseName + "_images"); +} + +bool ChatSerializer::saveImageToStorage(const QString &chatFilePath, + const QString &fileName, + const QString &base64Data, + QString &storedPath) +{ + QString imagesFolder = getChatImagesFolder(chatFilePath); + QDir dir; + if (!dir.exists(imagesFolder)) { + if (!dir.mkpath(imagesFolder)) { + LOG_MESSAGE(QString("Failed to create images folder: %1").arg(imagesFolder)); + return false; + } + } + + QFileInfo originalFileInfo(fileName); + QString extension = originalFileInfo.suffix(); + QString baseName = originalFileInfo.completeBaseName(); + QString uniqueName = QString("%1_%2.%3") + .arg(baseName) + .arg(QUuid::createUuid().toString(QUuid::WithoutBraces).left(8)) + .arg(extension); + + QString fullPath = QDir(imagesFolder).filePath(uniqueName); + + QByteArray imageData = QByteArray::fromBase64(base64Data.toUtf8()); + QFile file(fullPath); + if (!file.open(QIODevice::WriteOnly)) { + LOG_MESSAGE(QString("Failed to open file for writing: %1").arg(fullPath)); + return false; + } + + if (file.write(imageData) == -1) { + LOG_MESSAGE(QString("Failed to write image data: %1").arg(file.errorString())); + return false; + } + + file.close(); + + storedPath = uniqueName; + LOG_MESSAGE(QString("Saved image: %1 to %2").arg(fileName, fullPath)); + + return true; +} + +QString ChatSerializer::loadImageFromStorage(const QString &chatFilePath, const QString &storedPath) +{ + QString imagesFolder = getChatImagesFolder(chatFilePath); + QString fullPath = QDir(imagesFolder).filePath(storedPath); + + QFile file(fullPath); + if (!file.open(QIODevice::ReadOnly)) { + LOG_MESSAGE(QString("Failed to open image file: %1").arg(fullPath)); + return QString(); + } + + QByteArray imageData = file.readAll(); + file.close(); + + return imageData.toBase64(); +} + } // namespace QodeAssist::Chat diff --git a/ChatView/ChatSerializer.hpp b/ChatView/ChatSerializer.hpp index 3721dbd..6a67ff1 100644 --- a/ChatView/ChatSerializer.hpp +++ b/ChatView/ChatSerializer.hpp @@ -40,10 +40,18 @@ public: static SerializationResult loadFromFile(ChatModel *model, const QString &filePath); // Public for testing purposes - static QJsonObject serializeMessage(const ChatModel::Message &message); - static ChatModel::Message deserializeMessage(const QJsonObject &json); - static QJsonObject serializeChat(const ChatModel *model); - static bool deserializeChat(ChatModel *model, const QJsonObject &json); + static QJsonObject serializeMessage(const ChatModel::Message &message, const QString &chatFilePath); + static ChatModel::Message deserializeMessage(const QJsonObject &json, const QString &chatFilePath); + static QJsonObject serializeChat(const ChatModel *model, const QString &chatFilePath); + static bool deserializeChat(ChatModel *model, const QJsonObject &json, const QString &chatFilePath); + + // Image management + static QString getChatImagesFolder(const QString &chatFilePath); + static bool saveImageToStorage(const QString &chatFilePath, + const QString &fileName, + const QString &base64Data, + QString &storedPath); + static QString loadImageFromStorage(const QString &chatFilePath, const QString &storedPath); private: static const QString VERSION; diff --git a/ChatView/ClientInterface.cpp b/ChatView/ClientInterface.cpp index 053fa63..14a5243 100644 --- a/ChatView/ClientInterface.cpp +++ b/ChatView/ClientInterface.cpp @@ -20,9 +20,12 @@ #include "ClientInterface.hpp" #include +#include #include +#include #include #include +#include #include #include @@ -36,6 +39,7 @@ #include #include "ChatAssistantSettings.hpp" +#include "ChatSerializer.hpp" #include "GeneralSettings.hpp" #include "ToolsSettings.hpp" #include "Logger.hpp" @@ -70,8 +74,44 @@ void ClientInterface::sendMessage( Context::ChangesManager::instance().archiveAllNonArchivedEdits(); - auto attachFiles = m_contextManager->getContentFiles(attachments); - m_chatModel->addMessage(message, ChatModel::ChatRole::User, "", attachFiles); + QList imageFiles; + QList textFiles; + + for (const QString &filePath : attachments) { + if (isImageFile(filePath)) { + imageFiles.append(filePath); + } else { + textFiles.append(filePath); + } + } + + auto attachFiles = m_contextManager->getContentFiles(textFiles); + + QList imageAttachments; + if (!imageFiles.isEmpty() && !m_chatFilePath.isEmpty()) { + for (const QString &imagePath : imageFiles) { + QString base64Data = encodeImageToBase64(imagePath); + if (base64Data.isEmpty()) { + continue; + } + + QString storedPath; + QFileInfo fileInfo(imagePath); + if (ChatSerializer::saveImageToStorage(m_chatFilePath, fileInfo.fileName(), base64Data, storedPath)) { + ChatModel::ImageAttachment imageAttachment; + imageAttachment.fileName = fileInfo.fileName(); + imageAttachment.storedPath = storedPath; + imageAttachment.mediaType = getMediaTypeForImage(imagePath); + imageAttachments.append(imageAttachment); + + LOG_MESSAGE(QString("Stored image %1 as %2").arg(fileInfo.fileName(), storedPath)); + } + } + } else if (!imageFiles.isEmpty()) { + LOG_MESSAGE(QString("Warning: Chat file path not set, cannot save %1 image(s)").arg(imageFiles.size())); + } + + m_chatModel->addMessage(message, ChatModel::ChatRole::User, "", attachFiles, imageAttachments); auto &chatAssistantSettings = Settings::chatAssistantSettings(); @@ -133,8 +173,36 @@ void ClientInterface::sendMessage( apiMessage.isRedacted = msg.isRedacted; apiMessage.signature = msg.signature; + if (provider->supportImage() && !m_chatFilePath.isEmpty() && !msg.images.isEmpty()) { + auto apiImages = loadImagesFromStorage(msg.images); + if (!apiImages.isEmpty()) { + apiMessage.images = apiImages; + } + } + messages.append(apiMessage); } + + if (!imageAttachments.isEmpty() && provider->supportImage() && !messages.isEmpty()) { + for (int i = messages.size() - 1; i >= 0; --i) { + if (messages[i].role == "user") { + auto newImages = loadImagesFromStorage(imageAttachments); + if (!newImages.isEmpty()) { + if (messages[i].images.has_value()) { + messages[i].images.value().append(newImages); + } else { + messages[i].images = newImages; + } + LOG_MESSAGE(QString("Added %1 new image(s) to message").arg(newImages.size())); + } + break; + } + } + } else if (!imageFiles.isEmpty() && !provider->supportImage()) { + LOG_MESSAGE(QString("Provider %1 doesn't support images, %2 ignored") + .arg(provider->name(), QString::number(imageFiles.size()))); + } + context.history = messages; LLMCore::LLMConfig config; @@ -379,4 +447,86 @@ void ClientInterface::handleCleanAccumulatedData(const QString &requestId) LOG_MESSAGE(QString("Cleared accumulated responses for continuation request %1").arg(requestId)); } +bool ClientInterface::isImageFile(const QString &filePath) const +{ + static const QSet imageExtensions = { + "png", "jpg", "jpeg", "gif", "webp", "bmp", "svg" + }; + + QFileInfo fileInfo(filePath); + QString extension = fileInfo.suffix().toLower(); + + return imageExtensions.contains(extension); +} + +QString ClientInterface::getMediaTypeForImage(const QString &filePath) const +{ + static const QHash mediaTypes = { + {"png", "image/png"}, + {"jpg", "image/jpeg"}, + {"jpeg", "image/jpeg"}, + {"gif", "image/gif"}, + {"webp", "image/webp"}, + {"bmp", "image/bmp"}, + {"svg", "image/svg+xml"} + }; + + QFileInfo fileInfo(filePath); + QString extension = fileInfo.suffix().toLower(); + + if (mediaTypes.contains(extension)) { + return mediaTypes[extension]; + } + + QMimeDatabase mimeDb; + QMimeType mimeType = mimeDb.mimeTypeForFile(filePath); + return mimeType.name(); +} + +QString ClientInterface::encodeImageToBase64(const QString &filePath) const +{ + QFile file(filePath); + if (!file.open(QIODevice::ReadOnly)) { + LOG_MESSAGE(QString("Failed to open image file: %1").arg(filePath)); + return QString(); + } + + QByteArray imageData = file.readAll(); + file.close(); + + return imageData.toBase64(); +} + +QVector ClientInterface::loadImagesFromStorage(const QList &storedImages) const +{ + QVector apiImages; + + for (const auto &storedImage : storedImages) { + QString base64Data = ChatSerializer::loadImageFromStorage(m_chatFilePath, storedImage.storedPath); + if (base64Data.isEmpty()) { + LOG_MESSAGE(QString("Warning: Failed to load image: %1").arg(storedImage.storedPath)); + continue; + } + + LLMCore::ImageAttachment apiImage; + apiImage.data = base64Data; + apiImage.mediaType = storedImage.mediaType; + apiImage.isUrl = false; + + apiImages.append(apiImage); + } + + return apiImages; +} + +void ClientInterface::setChatFilePath(const QString &filePath) +{ + m_chatFilePath = filePath; +} + +QString ClientInterface::chatFilePath() const +{ + return m_chatFilePath; +} + } // namespace QodeAssist::Chat diff --git a/ChatView/ClientInterface.hpp b/ChatView/ClientInterface.hpp index 361ea23..f944d42 100644 --- a/ChatView/ClientInterface.hpp +++ b/ChatView/ClientInterface.hpp @@ -48,6 +48,9 @@ public: void cancelRequest(); Context::ContextManager *contextManager() const; + + void setChatFilePath(const QString &filePath); + QString chatFilePath() const; signals: void errorOccurred(const QString &error); @@ -65,6 +68,10 @@ private: QString getCurrentFileContext() const; QString getSystemPromptWithLinkedFiles( const QString &basePrompt, const QList &linkedFiles) const; + bool isImageFile(const QString &filePath) const; + QString getMediaTypeForImage(const QString &filePath) const; + QString encodeImageToBase64(const QString &filePath) const; + QVector loadImagesFromStorage(const QList &storedImages) const; struct RequestContext { @@ -75,6 +82,7 @@ private: LLMCore::IPromptProvider *m_promptProvider = nullptr; ChatModel *m_chatModel; Context::ContextManager *m_contextManager; + QString m_chatFilePath; QHash m_activeRequests; QHash m_accumulatedResponses; diff --git a/ChatView/MessagePart.hpp b/ChatView/MessagePart.hpp index 13a6616..d714ae8 100644 --- a/ChatView/MessagePart.hpp +++ b/ChatView/MessagePart.hpp @@ -32,11 +32,15 @@ class MessagePart Q_PROPERTY(MessagePartType type MEMBER type CONSTANT FINAL) Q_PROPERTY(QString text MEMBER text CONSTANT FINAL) Q_PROPERTY(QString language MEMBER language CONSTANT FINAL) + Q_PROPERTY(QString imageData MEMBER imageData CONSTANT FINAL) + Q_PROPERTY(QString mediaType MEMBER mediaType CONSTANT FINAL) QML_VALUE_TYPE(messagePart) public: MessagePartType type; QString text; QString language; + QString imageData; // Base64 data or URL + QString mediaType; // e.g., "image/png", "image/jpeg" }; } // namespace QodeAssist::Chat diff --git a/ChatView/icons/image-dark.svg b/ChatView/icons/image-dark.svg new file mode 100644 index 0000000..c8c5fcc --- /dev/null +++ b/ChatView/icons/image-dark.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/ChatView/qml/RootItem.qml b/ChatView/qml/RootItem.qml index 421ea12..ce25326 100644 --- a/ChatView/qml/RootItem.qml +++ b/ChatView/qml/RootItem.qml @@ -170,6 +170,8 @@ ChatRootView { width: parent.width msgModel: root.chatModel.processMessageContent(model.content) messageAttachments: model.attachments + messageImages: model.images + chatFilePath: root.chatFilePath() isUserMessage: model.roleType === ChatModel.User messageIndex: index textFontFamily: root.textFontFamily @@ -394,6 +396,7 @@ ChatRootView { onCheckedChanged: root.setIsSyncOpenFiles(bottomBar.syncOpenFiles.checked) } attachFiles.onClicked: root.showAttachFilesDialog() + attachImages.onClicked: root.showAddImageDialog() linkFiles.onClicked: root.showLinkFilesDialog() } } diff --git a/ChatView/qml/chatparts/ChatItem.qml b/ChatView/qml/chatparts/ChatItem.qml index 389e2ef..988e690 100644 --- a/ChatView/qml/chatparts/ChatItem.qml +++ b/ChatView/qml/chatparts/ChatItem.qml @@ -28,6 +28,8 @@ Rectangle { property alias msgModel: msgCreator.model property alias messageAttachments: attachmentsModel.model + property alias messageImages: imagesModel.model + property string chatFilePath: "" property string textFontFamily: Qt.application.font.family property string codeFontFamily: { switch (Qt.platform.os) { @@ -140,6 +142,27 @@ Rectangle { } } } + + Flow { + id: imagesFlow + + Layout.fillWidth: true + visible: imagesModel.model && imagesModel.model.length > 0 + leftPadding: 10 + rightPadding: 10 + spacing: 10 + + Repeater { + id: imagesModel + + delegate: ImageComponent { + required property int index + required property var modelData + + itemData: modelData + } + } + } } Rectangle { @@ -215,4 +238,79 @@ Rectangle { codeFontFamily: root.codeFontFamily codeFontSize: root.codeFontSize } + + component ImageComponent : Rectangle { + required property var itemData + + readonly property int maxImageWidth: Math.min(400, root.width - 40) + readonly property int maxImageHeight: 300 + + width: Math.min(imageDisplay.implicitWidth, maxImageWidth) + 16 + height: imageDisplay.implicitHeight + fileNameText.implicitHeight + 16 + radius: 4 + color: palette.base + border.width: 1 + border.color: palette.mid + + ColumnLayout { + anchors.fill: parent + anchors.margins: 8 + spacing: 4 + + Image { + id: imageDisplay + Layout.alignment: Qt.AlignHCenter + Layout.maximumWidth: parent.parent.maxImageWidth + Layout.maximumHeight: parent.parent.maxImageHeight + + source: { + if (!itemData.storedPath || !root.chatFilePath) return ""; + var fileInfo = chatFileInfo(root.chatFilePath); + var imagesFolder = fileInfo.dir + "/" + fileInfo.baseName + "_images"; + return "file://" + imagesFolder + "/" + itemData.storedPath; + } + + sourceSize.width: parent.parent.maxImageWidth + sourceSize.height: parent.parent.maxImageHeight + fillMode: Image.PreserveAspectFit + cache: true + asynchronous: true + smooth: true + mipmap: true + + BusyIndicator { + anchors.centerIn: parent + running: imageDisplay.status === Image.Loading + visible: running + } + + Text { + anchors.centerIn: parent + text: qsTr("Failed to load image") + visible: imageDisplay.status === Image.Error + color: palette.placeholderText + } + } + + Text { + id: fileNameText + Layout.fillWidth: true + text: itemData.fileName || "" + color: palette.text + font.pointSize: root.textFontSize - 1 + elide: Text.ElideMiddle + horizontalAlignment: Text.AlignHCenter + } + } + } + + function chatFileInfo(filePath) { + if (!filePath) return {dir: "", baseName: ""}; + var lastSlash = filePath.lastIndexOf("/"); + var dir = lastSlash >= 0 ? filePath.substring(0, lastSlash) : ""; + var fileName = lastSlash >= 0 ? filePath.substring(lastSlash + 1) : filePath; + var lastDot = fileName.lastIndexOf("."); + var baseName = lastDot >= 0 ? fileName.substring(0, lastDot) : fileName; + return {dir: dir, baseName: baseName}; + } } diff --git a/ChatView/qml/controls/BottomBar.qml b/ChatView/qml/controls/BottomBar.qml index 33f6a20..e402600 100644 --- a/ChatView/qml/controls/BottomBar.qml +++ b/ChatView/qml/controls/BottomBar.qml @@ -29,9 +29,9 @@ Rectangle { property alias sendButton: sendButtonId property alias syncOpenFiles: syncOpenFilesId property alias attachFiles: attachFilesId + property alias attachImages: attachImagesId property alias linkFiles: linkFilesId - color: palette.window.hslLightness > 0.5 ? Qt.darker(palette.window, 1.1) : Qt.lighter(palette.window, 1.1) @@ -73,6 +73,19 @@ Rectangle { ToolTip.text: qsTr("Attach file to message") } + QoAButton { + id: attachImagesId + + icon { + source: "qrc:/qt/qml/ChatView/icons/image-dark.svg" + height: 15 + width: 15 + } + ToolTip.visible: hovered + ToolTip.delay: 250 + ToolTip.text: qsTr("Attach image to message") + } + QoAButton { id: linkFilesId diff --git a/llmcore/ContentBlocks.hpp b/llmcore/ContentBlocks.hpp index f9434b2..9286eea 100644 --- a/llmcore/ContentBlocks.hpp +++ b/llmcore/ContentBlocks.hpp @@ -68,6 +68,54 @@ private: QString m_text; }; +class ImageContent : public ContentBlock +{ + Q_OBJECT +public: + enum class ImageSourceType { Base64, Url }; + + ImageContent(const QString &data, const QString &mediaType, ImageSourceType sourceType = ImageSourceType::Base64) + : ContentBlock() + , m_data(data) + , m_mediaType(mediaType) + , m_sourceType(sourceType) + {} + + QString type() const override { return "image"; } + QString data() const { return m_data; } + QString mediaType() const { return m_mediaType; } + ImageSourceType sourceType() const { return m_sourceType; } + + QJsonValue toJson(ProviderFormat format) const override + { + if (format == ProviderFormat::Claude) { + QJsonObject source; + if (m_sourceType == ImageSourceType::Base64) { + source["type"] = "base64"; + source["media_type"] = m_mediaType; + source["data"] = m_data; + } else { + source["type"] = "url"; + source["url"] = m_data; + } + return QJsonObject{{"type", "image"}, {"source", source}}; + } else { // OpenAI format + QJsonObject imageUrl; + if (m_sourceType == ImageSourceType::Base64) { + imageUrl["url"] = QString("data:%1;base64,%2").arg(m_mediaType, m_data); + } else { + imageUrl["url"] = m_data; + } + return QJsonObject{{"type", "image_url"}, {"image_url", imageUrl}}; + } + } + +private: + QString m_data; + QString m_mediaType; + ImageSourceType m_sourceType; +}; + class ToolUseContent : public ContentBlock { Q_OBJECT diff --git a/llmcore/ContextData.hpp b/llmcore/ContextData.hpp index b174452..e95c135 100644 --- a/llmcore/ContextData.hpp +++ b/llmcore/ContextData.hpp @@ -24,6 +24,15 @@ namespace QodeAssist::LLMCore { +struct ImageAttachment +{ + QString data; // Base64 encoded data or URL + QString mediaType; // e.g., "image/png", "image/jpeg", "image/webp", "image/gif" + bool isUrl = false; // true if data is URL, false if base64 + + bool operator==(const ImageAttachment &) const = default; +}; + struct Message { QString role; @@ -31,6 +40,7 @@ struct Message QString signature; bool isThinking = false; bool isRedacted = false; + std::optional> images; // clang-format off bool operator==(const Message&) const = default; diff --git a/llmcore/Provider.hpp b/llmcore/Provider.hpp index 502ed61..5e50a9b 100644 --- a/llmcore/Provider.hpp +++ b/llmcore/Provider.hpp @@ -67,6 +67,7 @@ public: virtual bool supportsTools() const { return false; }; virtual bool supportThinking() const { return false; }; + virtual bool supportImage() const { return false; }; virtual void cancelRequest(const RequestID &requestId); diff --git a/providers/ClaudeMessage.cpp b/providers/ClaudeMessage.cpp index c2fde1b..29397a4 100644 --- a/providers/ClaudeMessage.cpp +++ b/providers/ClaudeMessage.cpp @@ -39,6 +39,24 @@ void ClaudeMessage::handleContentBlockStart( if (blockType == "text") { addCurrentContent(); + } else if (blockType == "image") { + QJsonObject source = data["source"].toObject(); + QString sourceType = source["type"].toString(); + QString imageData; + QString mediaType; + LLMCore::ImageContent::ImageSourceType imgSourceType = LLMCore::ImageContent::ImageSourceType::Base64; + + if (sourceType == "base64") { + imageData = source["data"].toString(); + mediaType = source["media_type"].toString(); + imgSourceType = LLMCore::ImageContent::ImageSourceType::Base64; + } else if (sourceType == "url") { + imageData = source["url"].toString(); + imgSourceType = LLMCore::ImageContent::ImageSourceType::Url; + } + + addCurrentContent(imageData, mediaType, imgSourceType); + } else if (blockType == "tool_use") { QString toolId = data["id"].toString(); QString toolName = data["name"].toString(); diff --git a/providers/ClaudeProvider.cpp b/providers/ClaudeProvider.cpp index dcfa6df..4aca5c8 100644 --- a/providers/ClaudeProvider.cpp +++ b/providers/ClaudeProvider.cpp @@ -257,6 +257,10 @@ bool ClaudeProvider::supportThinking() const { return true; }; +bool ClaudeProvider::supportImage() const { + return true; +}; + void ClaudeProvider::cancelRequest(const LLMCore::RequestID &requestId) { LOG_MESSAGE(QString("ClaudeProvider: Cancelling request %1").arg(requestId)); diff --git a/providers/ClaudeProvider.hpp b/providers/ClaudeProvider.hpp index a65d348..3802433 100644 --- a/providers/ClaudeProvider.hpp +++ b/providers/ClaudeProvider.hpp @@ -55,6 +55,7 @@ public: bool supportsTools() const override; bool supportThinking() const override; + bool supportImage() const override; void cancelRequest(const LLMCore::RequestID &requestId) override; public slots: diff --git a/providers/GoogleAIProvider.cpp b/providers/GoogleAIProvider.cpp index 2844a87..7d8a349 100644 --- a/providers/GoogleAIProvider.cpp +++ b/providers/GoogleAIProvider.cpp @@ -273,6 +273,11 @@ bool GoogleAIProvider::supportThinking() const return true; } +bool GoogleAIProvider::supportImage() const +{ + return true; +} + void GoogleAIProvider::cancelRequest(const LLMCore::RequestID &requestId) { LOG_MESSAGE(QString("GoogleAIProvider: Cancelling request %1").arg(requestId)); diff --git a/providers/GoogleAIProvider.hpp b/providers/GoogleAIProvider.hpp index 8795e91..3c0c080 100644 --- a/providers/GoogleAIProvider.hpp +++ b/providers/GoogleAIProvider.hpp @@ -54,6 +54,7 @@ public: bool supportsTools() const override; bool supportThinking() const override; + bool supportImage() const override; void cancelRequest(const LLMCore::RequestID &requestId) override; public slots: diff --git a/providers/LMStudioProvider.cpp b/providers/LMStudioProvider.cpp index 7c84b28..8092829 100644 --- a/providers/LMStudioProvider.cpp +++ b/providers/LMStudioProvider.cpp @@ -163,6 +163,11 @@ bool LMStudioProvider::supportsTools() const return true; } +bool LMStudioProvider::supportImage() const +{ + return true; +} + void LMStudioProvider::cancelRequest(const LLMCore::RequestID &requestId) { LOG_MESSAGE(QString("LMStudioProvider: Cancelling request %1").arg(requestId)); diff --git a/providers/LMStudioProvider.hpp b/providers/LMStudioProvider.hpp index f4d3e35..0d84846 100644 --- a/providers/LMStudioProvider.hpp +++ b/providers/LMStudioProvider.hpp @@ -53,6 +53,7 @@ public: const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override; bool supportsTools() const override; + bool supportImage() const override; void cancelRequest(const LLMCore::RequestID &requestId) override; public slots: diff --git a/providers/LlamaCppProvider.cpp b/providers/LlamaCppProvider.cpp index f974a4f..c15c357 100644 --- a/providers/LlamaCppProvider.cpp +++ b/providers/LlamaCppProvider.cpp @@ -206,6 +206,11 @@ bool LlamaCppProvider::supportsTools() const return true; } +bool LlamaCppProvider::supportImage() const +{ + return true; +} + void LlamaCppProvider::cancelRequest(const LLMCore::RequestID &requestId) { LOG_MESSAGE(QString("LlamaCppProvider: Cancelling request %1").arg(requestId)); diff --git a/providers/LlamaCppProvider.hpp b/providers/LlamaCppProvider.hpp index 2cb57c9..5635f87 100644 --- a/providers/LlamaCppProvider.hpp +++ b/providers/LlamaCppProvider.hpp @@ -53,6 +53,7 @@ public: const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override; bool supportsTools() const override; + bool supportImage() const override; void cancelRequest(const LLMCore::RequestID &requestId) override; public slots: diff --git a/providers/MistralAIProvider.cpp b/providers/MistralAIProvider.cpp index 177500f..200d52a 100644 --- a/providers/MistralAIProvider.cpp +++ b/providers/MistralAIProvider.cpp @@ -184,6 +184,11 @@ bool MistralAIProvider::supportsTools() const return true; } +bool MistralAIProvider::supportImage() const +{ + return true; +} + void MistralAIProvider::cancelRequest(const LLMCore::RequestID &requestId) { LOG_MESSAGE(QString("MistralAIProvider: Cancelling request %1").arg(requestId)); diff --git a/providers/MistralAIProvider.hpp b/providers/MistralAIProvider.hpp index 7ef1d90..7b92b4c 100644 --- a/providers/MistralAIProvider.hpp +++ b/providers/MistralAIProvider.hpp @@ -53,6 +53,7 @@ public: const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override; bool supportsTools() const override; + bool supportImage() const override; void cancelRequest(const LLMCore::RequestID &requestId) override; public slots: diff --git a/providers/OllamaProvider.cpp b/providers/OllamaProvider.cpp index 52ac5e5..2c49939 100644 --- a/providers/OllamaProvider.cpp +++ b/providers/OllamaProvider.cpp @@ -168,6 +168,7 @@ QList OllamaProvider::validateRequest(const QJsonObject &request, LLMCo {"prompt", {}}, {"suffix", {}}, {"system", {}}, + {"images", QJsonArray{}}, {"options", QJsonObject{ {"temperature", {}}, @@ -182,7 +183,7 @@ QList OllamaProvider::validateRequest(const QJsonObject &request, LLMCo {"keep_alive", {}}, {"model", {}}, {"stream", {}}, - {"messages", QJsonArray{{QJsonObject{{"role", {}}, {"content", {}}}}}}, + {"messages", QJsonArray{{QJsonObject{{"role", {}}, {"content", {}}, {"images", QJsonArray{}}}}}}, {"tools", QJsonArray{}}, {"options", QJsonObject{ @@ -241,6 +242,11 @@ bool OllamaProvider::supportsTools() const return true; } +bool OllamaProvider::supportImage() const +{ + return true; +} + void OllamaProvider::cancelRequest(const LLMCore::RequestID &requestId) { LOG_MESSAGE(QString("OllamaProvider: Cancelling request %1").arg(requestId)); diff --git a/providers/OllamaProvider.hpp b/providers/OllamaProvider.hpp index 62c4cdd..685b1d6 100644 --- a/providers/OllamaProvider.hpp +++ b/providers/OllamaProvider.hpp @@ -54,6 +54,7 @@ public: const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override; bool supportsTools() const override; + bool supportImage() const override; void cancelRequest(const LLMCore::RequestID &requestId) override; public slots: diff --git a/providers/OpenAICompatProvider.cpp b/providers/OpenAICompatProvider.cpp index 4b33867..6f486a1 100644 --- a/providers/OpenAICompatProvider.cpp +++ b/providers/OpenAICompatProvider.cpp @@ -192,6 +192,11 @@ bool OpenAICompatProvider::supportsTools() const return true; } +bool OpenAICompatProvider::supportImage() const +{ + return true; +} + void OpenAICompatProvider::cancelRequest(const LLMCore::RequestID &requestId) { LOG_MESSAGE(QString("OpenAICompatProvider: Cancelling request %1").arg(requestId)); diff --git a/providers/OpenAICompatProvider.hpp b/providers/OpenAICompatProvider.hpp index 79452e6..c8e4769 100644 --- a/providers/OpenAICompatProvider.hpp +++ b/providers/OpenAICompatProvider.hpp @@ -53,6 +53,7 @@ public: const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override; bool supportsTools() const override; + bool supportImage() const override; void cancelRequest(const LLMCore::RequestID &requestId) override; public slots: diff --git a/providers/OpenAIProvider.cpp b/providers/OpenAIProvider.cpp index 142ffaa..685ecee 100644 --- a/providers/OpenAIProvider.cpp +++ b/providers/OpenAIProvider.cpp @@ -248,6 +248,11 @@ bool OpenAIProvider::supportsTools() const return true; } +bool OpenAIProvider::supportImage() const +{ + return true; +} + void OpenAIProvider::cancelRequest(const LLMCore::RequestID &requestId) { LOG_MESSAGE(QString("OpenAIProvider: Cancelling request %1").arg(requestId)); diff --git a/providers/OpenAIProvider.hpp b/providers/OpenAIProvider.hpp index 7b37352..eafeae5 100644 --- a/providers/OpenAIProvider.hpp +++ b/providers/OpenAIProvider.hpp @@ -53,6 +53,7 @@ public: const LLMCore::RequestID &requestId, const QUrl &url, const QJsonObject &payload) override; bool supportsTools() const override; + bool supportImage() const override; void cancelRequest(const LLMCore::RequestID &requestId) override; public slots: diff --git a/templates/Claude.hpp b/templates/Claude.hpp index fb08e13..c2818d1 100644 --- a/templates/Claude.hpp +++ b/templates/Claude.hpp @@ -41,34 +41,55 @@ public: if (context.history) { for (const auto &msg : context.history.value()) { - if (msg.role != "system") { - // Handle thinking blocks with structured content - if (msg.isThinking) { - // Create content array with thinking block - QJsonArray content; - QJsonObject thinkingBlock; - thinkingBlock["type"] = msg.isRedacted ? "redacted_thinking" : "thinking"; - - // Extract actual thinking text (remove display signature) - QString thinkingText = msg.content; - int signaturePos = thinkingText.indexOf("\n[Signature: "); - if (signaturePos != -1) { - thinkingText = thinkingText.left(signaturePos); - } - - if (!msg.isRedacted) { - thinkingBlock["thinking"] = thinkingText; - } - if (!msg.signature.isEmpty()) { - thinkingBlock["signature"] = msg.signature; - } - content.append(thinkingBlock); - - messages.append(QJsonObject{{"role", "assistant"}, {"content", content}}); - } else { - // Normal message - messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); + if (msg.role == "system") continue; + + if (msg.isThinking) { + QJsonArray content; + QJsonObject thinkingBlock; + thinkingBlock["type"] = msg.isRedacted ? "redacted_thinking" : "thinking"; + + QString thinkingText = msg.content; + int signaturePos = thinkingText.indexOf("\n[Signature: "); + if (signaturePos != -1) { + thinkingText = thinkingText.left(signaturePos); } + + if (!msg.isRedacted) { + thinkingBlock["thinking"] = thinkingText; + } + if (!msg.signature.isEmpty()) { + thinkingBlock["signature"] = msg.signature; + } + content.append(thinkingBlock); + + messages.append(QJsonObject{{"role", "assistant"}, {"content", content}}); + } else if (msg.images && !msg.images->isEmpty()) { + QJsonArray content; + + if (!msg.content.isEmpty()) { + content.append(QJsonObject{{"type", "text"}, {"text", msg.content}}); + } + + for (const auto &image : msg.images.value()) { + QJsonObject imageBlock; + imageBlock["type"] = "image"; + + QJsonObject source; + if (image.isUrl) { + source["type"] = "url"; + source["url"] = image.data; + } else { + source["type"] = "base64"; + source["media_type"] = image.mediaType; + source["data"] = image.data; + } + imageBlock["source"] = source; + content.append(imageBlock); + } + + messages.append(QJsonObject{{"role", msg.role}, {"content", content}}); + } else { + messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); } } } diff --git a/templates/GoogleAI.hpp b/templates/GoogleAI.hpp index d265754..01d2bed 100644 --- a/templates/GoogleAI.hpp +++ b/templates/GoogleAI.hpp @@ -46,7 +46,29 @@ public: QJsonObject content; QJsonArray parts; - parts.append(QJsonObject{{"text", msg.content}}); + if (!msg.content.isEmpty()) { + parts.append(QJsonObject{{"text", msg.content}}); + } + + if (msg.images && !msg.images->isEmpty()) { + for (const auto &image : msg.images.value()) { + QJsonObject imagePart; + + if (image.isUrl) { + QJsonObject fileData; + fileData["mime_type"] = image.mediaType; + fileData["file_uri"] = image.data; + imagePart["file_data"] = fileData; + } else { + QJsonObject inlineData; + inlineData["mime_type"] = image.mediaType; + inlineData["data"] = image.data; + imagePart["inline_data"] = inlineData; + } + + parts.append(imagePart); + } + } QString role = msg.role; if (role == "assistant") { diff --git a/templates/MistralAI.hpp b/templates/MistralAI.hpp index f8f8eba..b9315ce 100644 --- a/templates/MistralAI.hpp +++ b/templates/MistralAI.hpp @@ -74,7 +74,31 @@ public: if (context.history) { for (const auto &msg : context.history.value()) { - messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); + if (msg.images && !msg.images->isEmpty()) { + QJsonArray content; + + if (!msg.content.isEmpty()) { + content.append(QJsonObject{{"type", "text"}, {"text", msg.content}}); + } + + for (const auto &image : msg.images.value()) { + QJsonObject imageBlock; + imageBlock["type"] = "image_url"; + + QJsonObject imageUrl; + if (image.isUrl) { + imageUrl["url"] = image.data; + } else { + imageUrl["url"] = QString("data:%1;base64,%2").arg(image.mediaType, image.data); + } + imageBlock["image_url"] = imageUrl; + content.append(imageBlock); + } + + messages.append(QJsonObject{{"role", msg.role}, {"content", content}}); + } else { + messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); + } } } @@ -90,7 +114,7 @@ public: " {\"role\": \"assistant\", \"content\": \"\"}\n" " ]\n" "}\n\n" - "Supports system messages and conversation history."; + "Supports system messages, conversation history, and images."; } bool isSupportProvider(LLMCore::ProviderID id) const override { diff --git a/templates/Ollama.hpp b/templates/Ollama.hpp index b9281ac..2890f2d 100644 --- a/templates/Ollama.hpp +++ b/templates/Ollama.hpp @@ -76,7 +76,19 @@ public: if (context.history) { for (const auto &msg : context.history.value()) { - messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); + QJsonObject messageObj; + messageObj["role"] = msg.role; + messageObj["content"] = msg.content; + + if (msg.images && !msg.images->isEmpty()) { + QJsonArray images; + for (const auto &image : msg.images.value()) { + images.append(image.data); + } + messageObj["images"] = images; + } + + messages.append(messageObj); } } @@ -88,11 +100,12 @@ public: "{\n" " \"messages\": [\n" " {\"role\": \"system\", \"content\": \"\"},\n" - " {\"role\": \"user\", \"content\": \"\"},\n" + " {\"role\": \"user\", \"content\": \"\", \"images\": [\"\"]},\n" " {\"role\": \"assistant\", \"content\": \"\"}\n" " ]\n" "}\n\n" - "Recommended for Ollama models with chat capability."; + "Recommended for Ollama models with chat capability.\n" + "Supports images for multimodal models (e.g., llava)."; } bool isSupportProvider(LLMCore::ProviderID id) const override { diff --git a/templates/OpenAI.hpp b/templates/OpenAI.hpp index 6b48fa9..0b9f7aa 100644 --- a/templates/OpenAI.hpp +++ b/templates/OpenAI.hpp @@ -42,7 +42,31 @@ public: if (context.history) { for (const auto &msg : context.history.value()) { - messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); + if (msg.images && !msg.images->isEmpty()) { + QJsonArray content; + + if (!msg.content.isEmpty()) { + content.append(QJsonObject{{"type", "text"}, {"text", msg.content}}); + } + + for (const auto &image : msg.images.value()) { + QJsonObject imageBlock; + imageBlock["type"] = "image_url"; + + QJsonObject imageUrl; + if (image.isUrl) { + imageUrl["url"] = image.data; + } else { + imageUrl["url"] = QString("data:%1;base64,%2").arg(image.mediaType, image.data); + } + imageBlock["image_url"] = imageUrl; + content.append(imageBlock); + } + + messages.append(QJsonObject{{"role", msg.role}, {"content", content}}); + } else { + messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); + } } } diff --git a/templates/OpenAICompatible.hpp b/templates/OpenAICompatible.hpp index be4bc3a..6484aa1 100644 --- a/templates/OpenAICompatible.hpp +++ b/templates/OpenAICompatible.hpp @@ -42,7 +42,31 @@ public: if (context.history) { for (const auto &msg : context.history.value()) { - messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); + if (msg.images && !msg.images->isEmpty()) { + QJsonArray content; + + if (!msg.content.isEmpty()) { + content.append(QJsonObject{{"type", "text"}, {"text", msg.content}}); + } + + for (const auto &image : msg.images.value()) { + QJsonObject imageBlock; + imageBlock["type"] = "image_url"; + + QJsonObject imageUrl; + if (image.isUrl) { + imageUrl["url"] = image.data; + } else { + imageUrl["url"] = QString("data:%1;base64,%2").arg(image.mediaType, image.data); + } + imageBlock["image_url"] = imageUrl; + content.append(imageBlock); + } + + messages.append(QJsonObject{{"role", msg.role}, {"content", content}}); + } else { + messages.append(QJsonObject{{"role", msg.role}, {"content", msg.content}}); + } } } @@ -58,7 +82,8 @@ public: " {\"role\": \"assistant\", \"content\": \"\"}\n" " ]\n" "}\n\n" - "Works with any service implementing the OpenAI Chat API specification."; + "Works with any service implementing the OpenAI Chat API specification.\n" + "Supports images."; } bool isSupportProvider(LLMCore::ProviderID id) const override {