From 862c2200690c3b17f6ff5320702b8bc78e6963be Mon Sep 17 00:00:00 2001 From: BEN ENGLISCH Date: Fri, 19 Nov 2021 23:06:38 -0600 Subject: [PATCH] Add libarchive decompression backend --- .gitignore | 3 + YACReader/YACReader.pro | 9 +- YACReaderLibrary/YACReaderLibrary.pro | 2 + YACReaderLibrary/library_creator.cpp | 2 +- YACReaderLibrary/main.cpp | 6 +- YACReaderLibrary/xml_info_library_scanner.cpp | 2 +- .../YACReaderLibraryServer.pro | 3 + compressed_archive/libarchive/README.txt | 24 +++ .../libarchive/compressed_archive.cpp | 155 ++++++++++++++++++ .../libarchive/compressed_archive.h | 44 +++++ .../libarchive/extract_delegate.h | 15 ++ .../libarchive/libarchive-wrapper.pri | 22 +++ config.pri | 2 +- .../compressed_archive_test.pro | 13 +- 14 files changed, 288 insertions(+), 14 deletions(-) create mode 100644 compressed_archive/libarchive/README.txt create mode 100644 compressed_archive/libarchive/compressed_archive.cpp create mode 100644 compressed_archive/libarchive/compressed_archive.h create mode 100644 compressed_archive/libarchive/extract_delegate.h create mode 100644 compressed_archive/libarchive/libarchive-wrapper.pri diff --git a/.gitignore b/.gitignore index 97951f8b..d8debe6c 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,6 @@ YACReaderLibrary/YACReaderLibrary\.xcodeproj/ .DS_Store compressed_archive/libp7zip c_x86_64.pch + +compile_commands.json +.ccls-cache diff --git a/YACReader/YACReader.pro b/YACReader/YACReader.pro index c8a21422..7e909ae3 100644 --- a/YACReader/YACReader.pro +++ b/YACReader/YACReader.pro @@ -156,13 +156,16 @@ SOURCES += ../common/comic.cpp \ } include(../custom_widgets/custom_widgets_yacreader.pri) -CONFIG(7zip){ + +CONFIG(7zip) { include(../compressed_archive/wrapper.pri) -} else:CONFIG(unarr){ +} else:CONFIG(unarr) { include(../compressed_archive/unarr/unarr-wrapper.pri) +} else:CONFIG(libarchive) { +include(../compressed_archive/libarchive/libarchive-wrapper.pri) } else { error(No compression backend specified. Did you mess with the build system?) - } +} include(../shortcuts_management/shortcuts_management.pri) RESOURCES += yacreader_images.qrc \ diff --git a/YACReaderLibrary/YACReaderLibrary.pro b/YACReaderLibrary/YACReaderLibrary.pro index 569c7812..29329c79 100644 --- a/YACReaderLibrary/YACReaderLibrary.pro +++ b/YACReaderLibrary/YACReaderLibrary.pro @@ -250,6 +250,8 @@ CONFIG(7zip){ include(../compressed_archive/wrapper.pri) } else:CONFIG(unarr) { include(../compressed_archive/unarr/unarr-wrapper.pri) +} else:CONFIG(libarchive) { +include(../compressed_archive/libarchive/libarchive-wrapper.pri) } else { error(No compression backend specified. Did you mess with the build system?) } diff --git a/YACReaderLibrary/library_creator.cpp b/YACReaderLibrary/library_creator.cpp index cc00c7f9..dfea43ab 100644 --- a/YACReaderLibrary/library_creator.cpp +++ b/YACReaderLibrary/library_creator.cpp @@ -108,7 +108,7 @@ void LibraryCreator::processLibrary(const QString &source, const QString &target void LibraryCreator::run() { stopRunning = false; -#ifndef use_unarr +#if !defined use_unarr && !defined use_libarchive // check for 7z lib #if defined Q_OS_UNIX && !defined Q_OS_MAC QLibrary *sevenzLib = new QLibrary(QString(LIBDIR) + "/p7zip/7z.so"); diff --git a/YACReaderLibrary/main.cpp b/YACReaderLibrary/main.cpp index b2f0d7f8..693f32a6 100644 --- a/YACReaderLibrary/main.cpp +++ b/YACReaderLibrary/main.cpp @@ -7,7 +7,7 @@ #include #include #include -#ifndef use_unarr +#if !defined use_unarr && !defined use_libarchive #include #endif #include @@ -40,7 +40,7 @@ void logSystemAndConfig() QLOG_INFO() << "OS:" << QSysInfo::prettyProductName() << "Version: " << QSysInfo::productVersion(); QLOG_INFO() << "Kernel:" << QSysInfo::kernelType() << QSysInfo::kernelVersion() << "Architecture:" << QSysInfo::currentCpuArchitecture(); -#ifndef use_unarr +#if !defined use_unarr && !defined use_libarchive #ifdef Q_OS_WIN if (QLibrary::isLibrary(QApplication::applicationDirPath() + "/utils/7z.dll")) #elif defined Q_OS_UNIX && !defined Q_OS_MAC @@ -51,6 +51,8 @@ void logSystemAndConfig() QLOG_INFO() << "7z : found"; else QLOG_ERROR() << "7z : not found"; +#elif defined use_libarchive + QLOG_INFO() << "using libarchive decompression backend"; #else // use_unarr QLOG_INFO() << "using unarr decompression backend"; #endif // use_unarr diff --git a/YACReaderLibrary/xml_info_library_scanner.cpp b/YACReaderLibrary/xml_info_library_scanner.cpp index 58cf9a85..223bc13c 100644 --- a/YACReaderLibrary/xml_info_library_scanner.cpp +++ b/YACReaderLibrary/xml_info_library_scanner.cpp @@ -28,7 +28,7 @@ void XMLInfoLibraryScanner::scanLibrary(const QString &source, const QString &ta void XMLInfoLibraryScanner::run() { -#ifndef use_unarr +#if !defined use_unarr && !defined use_libarchive // check for 7z lib #if defined Q_OS_UNIX && !defined Q_OS_MAC QLibrary *sevenzLib = new QLibrary(QString(LIBDIR) + "/p7zip/7z.so"); diff --git a/YACReaderLibraryServer/YACReaderLibraryServer.pro b/YACReaderLibraryServer/YACReaderLibraryServer.pro index 74308cf8..311c1764 100644 --- a/YACReaderLibraryServer/YACReaderLibraryServer.pro +++ b/YACReaderLibraryServer/YACReaderLibraryServer.pro @@ -94,10 +94,13 @@ SOURCES += ../YACReaderLibrary/library_creator.cpp \ libraries_updater.cpp include(../YACReaderLibrary/server/server.pri) + CONFIG(7zip) { include(../compressed_archive/wrapper.pri) } else:CONFIG(unarr) { include(../compressed_archive/unarr/unarr-wrapper.pri) +} else:CONFIG(libarchive) { +include(../compressed_archive/libarchive/libarchive-wrapper.pri) } else { error(No compression backend specified. Did you mess with the build system?) } diff --git a/compressed_archive/libarchive/README.txt b/compressed_archive/libarchive/README.txt new file mode 100644 index 00000000..1020b4cb --- /dev/null +++ b/compressed_archive/libarchive/README.txt @@ -0,0 +1,24 @@ +* Introduction + +TODO + +* Using + +qmake CONFIG+=libarchive + +* Supported Archives + +TODO + +* Limitations + +TODO + +v4 solid archives, slow 7z, etc. + +libarchive is stream based so it reads the archive in linear order. talk about +advantages of properly ordered archives (1.jpg..10.jpg vs 01.jpg..10.jpg) + +* Future Work + +fallback to unarr for v4 solid archives? diff --git a/compressed_archive/libarchive/compressed_archive.cpp b/compressed_archive/libarchive/compressed_archive.cpp new file mode 100644 index 00000000..ee2c317e --- /dev/null +++ b/compressed_archive/libarchive/compressed_archive.cpp @@ -0,0 +1,155 @@ +#include "compressed_archive.h" + +#define archive_error(msg) msg << ": [" << archive_errno(a) << "]" << archive_error_string(a) + +CompressedArchive::CompressedArchive(const QString &filePath, QObject *parent) + : QObject(parent), a(nullptr), num_entries(0), valid(false), idx(0), filename(filePath) +{ + if (!open_archive()) { + qWarning() << "error opening archive:" << filename; + return; + } + + archive_entry *entry; + int result; + while ((result = archive_read_next_header(a, &entry)) == ARCHIVE_OK) { + entries.append(archive_entry_pathname(entry)); + archive_read_data_skip(a); + idx++; + } + + num_entries = entries.size(); + + if (result != ARCHIVE_EOF) { + qDebug() << "finished reading archive with result of:" << result; + qWarning() << archive_error("error reading archive"); + } else if (num_entries == 0) { + qWarning() << "no entries read from archive."; + } else { + qDebug() << "# of pages in archive:" << num_entries; + valid = true; + } + + close_archive(); +} + +CompressedArchive::~CompressedArchive() +{ + close_archive(); +} + +bool CompressedArchive::open_archive() +{ + qDebug() << "opening archive:" << filename; + idx = 0; + + if (a != nullptr) { + close_archive(); + } + + a = archive_read_new(); + archive_read_support_format_all(a); + archive_read_support_filter_all(a); + + if (archive_read_open_filename(a, filename.toStdString().c_str(), 10240) != ARCHIVE_OK) { + qWarning() << archive_error("error opening archive"); + close_archive(); + return false; + } + + return true; +} + +void CompressedArchive::close_archive() +{ + qDebug() << "closing archive."; + archive_read_free(a); + a = nullptr; +} + +bool CompressedArchive::archive_seek(quint32 index) +{ + if (idx == index) { + return true; + } + + // libarchive uses a streaming architecture so we cannot read files after our current position. + // because of this, when we need to seek to an index before our current position, + // we must reopen the archive. + if (idx > index) { + qDebug() << "asked for index [" << index << "] less than position [" << idx << "]." + << "reopening archive."; + close_archive(); + + if (!open_archive()) { + return false; + } + } + + qDebug() << "current pos = [" << idx << "] seeking to [" << index << "]"; + + archive_entry *entry; + for (; idx < index; idx++) { + if (archive_read_next_header(a, &entry) != ARCHIVE_OK) { + qWarning() << archive_error("error reading header"); + return false; + } + + if (archive_read_data_skip(a) != ARCHIVE_OK) { + qWarning() << archive_error("error skipping data"); + return false; + } + } + return true; +} + +void CompressedArchive::getAllData(const QVector &indexes, + ExtractDelegate *delegate) +{ + qDebug() << "called getAllData: [" << indexes << "]"; + if (indexes.isEmpty()) + return; + + for (int i = 0; i < indexes.count(); i++) { + if (delegate == nullptr || delegate->isCancelled()) + return; + + quint32 index = indexes[i]; + QByteArray bytes = getRawDataAtIndex(index); + if (bytes.size() > 0) { + delegate->fileExtracted(index, bytes); + } else { + qWarning() << "getAllData error at index: [" << index << "]"; + delegate->unknownError(index); + return; + } + } +} + +QByteArray CompressedArchive::read_entry() +{ + QByteArray bytes; + archive_entry *entry; + + if (archive_read_next_header(a, &entry) == ARCHIVE_OK) { + int64_t size = archive_entry_size(entry); + bytes.resize(size); + archive_read_data(a, bytes.data(), size); + } else { + qWarning() << archive_error("error reading entry"); + } + + idx++; + return bytes; +} + +QByteArray CompressedArchive::getRawDataAtIndex(int index) +{ + QByteArray bytes; + if (archive_seek(index)) { + bytes = read_entry(); + } else { + qWarning() << "error reading data from archive. index:" << index; + } + return bytes; +} diff --git a/compressed_archive/libarchive/compressed_archive.h b/compressed_archive/libarchive/compressed_archive.h new file mode 100644 index 00000000..d7accc4f --- /dev/null +++ b/compressed_archive/libarchive/compressed_archive.h @@ -0,0 +1,44 @@ +#ifndef COMPRESSED_ARCHIVE_H +#define COMPRESSED_ARCHIVE_H + +#include "extract_delegate.h" + +#include +#include + +extern "C" { +#include +#include +} + +class CompressedArchive : public QObject +{ + Q_OBJECT +public: + explicit CompressedArchive(const QString &filePath, QObject *parent = nullptr); + ~CompressedArchive() override; + +public slots: + void getAllData(const QVector &indexes, ExtractDelegate *delegate = nullptr); + QByteArray getRawDataAtIndex(int index); + + int getNumFiles() { return num_entries; } + QList getFileNames() { return entries; } + bool isValid() { return valid; } + bool toolsLoaded() { return true; } + +private: + archive *a; + QStringList entries; + int num_entries; + bool valid; + quint32 idx; + QString filename; + + bool open_archive(); + void close_archive(); + bool archive_seek(quint32 index); + QByteArray read_entry(); +}; + +#endif // COMPRESSED_ARCHIVE_H diff --git a/compressed_archive/libarchive/extract_delegate.h b/compressed_archive/libarchive/extract_delegate.h new file mode 100644 index 00000000..af6820d1 --- /dev/null +++ b/compressed_archive/libarchive/extract_delegate.h @@ -0,0 +1,15 @@ +#ifndef EXTRACT_DELEGATE_H +#define EXTRACT_DELEGATE_H + +#include + +class ExtractDelegate +{ +public: + virtual void fileExtracted(int index, const QByteArray &rawData) = 0; + virtual void crcError(int index) = 0; + virtual void unknownError(int index) = 0; + virtual bool isCancelled() = 0; +}; + +#endif // EXTRACT_DELEGATE_H \ No newline at end of file diff --git a/compressed_archive/libarchive/libarchive-wrapper.pri b/compressed_archive/libarchive/libarchive-wrapper.pri new file mode 100644 index 00000000..79cf0211 --- /dev/null +++ b/compressed_archive/libarchive/libarchive-wrapper.pri @@ -0,0 +1,22 @@ +INCLUDEPATH += $$PWD +DEPENDPATH += $$PWD + +HEADERS += $$PWD/extract_delegate.h \ + $$PWD/compressed_archive.h + +SOURCES += $$PWD/compressed_archive.cpp + +if(mingw|unix):!contains(QT_CONFIG, no-pkg-config):packagesExist(libarchive) { + message(Using system provided libarchive installation found by pkg-config.) + CONFIG += link_pkgconfig + PKGCONFIG += libarchive + DEFINES += use_libarchive + } +else:unix:exists(/usr/include/archive.h) { + message(Using system provided libarchive installation.) + LIBS += -larchive + DEFINES += use_libarchive +} +else { + error(Missing dependency: libarchive decompression backend. Please install libarchive on your system) +} diff --git a/config.pri b/config.pri index 660f80c0..f725d5a6 100644 --- a/config.pri +++ b/config.pri @@ -56,7 +56,7 @@ CONFIG(no_opengl) { } # default value for comic archive decompression backend -unix:!macx:!CONFIG(unarr):!CONFIG(7zip) { +unix:!macx:!CONFIG(unarr):!CONFIG(7zip):!CONFIG(libarchive) { CONFIG += unarr } diff --git a/tests/compressed_archive_test/compressed_archive_test.pro b/tests/compressed_archive_test/compressed_archive_test.pro index 9fd3f52e..b4898014 100644 --- a/tests/compressed_archive_test/compressed_archive_test.pro +++ b/tests/compressed_archive_test/compressed_archive_test.pro @@ -15,11 +15,12 @@ win32 { CONFIG -= embed_manifest_exe } -!CONFIG(unarr){ - include(../../compressed_archive/wrapper.pri) +CONFIG(7zip) { +include(../../compressed_archive/wrapper.pri) +} else:CONFIG(unarr) { +include(../../compressed_archive/unarr/unarr-wrapper.pri) +} else:CONFIG(libarchive) { +include(../../compressed_archive/libarchive/libarchive-wrapper.pri) } else { - include(../../compressed_archive/unarr/unarr-wrapper.pri) +include(../../compressed_archive/wrapper.pri) } - - -