RIFF: support RF64 and BW64 (#1412)

RF64 and BW64 are the long forms of WAVE, used past 4 GB: each 32-bit
size field holds a 0xffffffff sentinel and the real sizes live in a
leading ds64 chunk. RIFF::WAV::File::isSupported() rejected them, but
FileRef reaches the class by extension for any .wav and RIFF::File::read()
never inspected the magic, so these files opened as valid.

updateGlobalSize() then wrote a real 32-bit total over the sentinel at
offset 4. Readers stop consulting ds64 once that field holds a number, so
a 4.8 GB file measured 0.005958 sec and 1144 audio bytes after a tag save
that returned true. Below 4 GB the tags were lost instead: the appended
LIST landed inside the region read() had clamped the sentinel data chunk
to, and a re-read found no properties.

Worse, the append offset is last.offset + last.size with size truncated to
0xffffffff, so on a long file the new chunk was spliced into the middle of
the audio — measured at offset 4294971392 on that same file, 4 GiB past
the data chunk's start, displacing everything after it.

Accept both magics, take the riff and data sizes from ds64, and write the
sentinel back on save. Writing it unconditionally is what the format
requires and also repairs a file an earlier version damaged: the same
4.8 GB file, clobbered and then saved through this path, read back at
25000.000000 sec and 4,800,000,000 bytes.

The ds64 table of additional oversized chunks is not parsed — the data
chunk has its own dedicated field and is the only one that is ever large —
so any chunk listed there stays on the clamping path added in #1329, which
this leaves untouched.

Chunk::size becomes offset_t so the append offset is computed correctly.
The struct is file-scope in rifffile.cpp and FilePrivate is only
forward-declared, so no protected signature changes and no ABI break.
chunkDataSize() still returns unsigned int, saturating rather than
truncating; a new chunkDataSize64() carries the real value to
WAV::Properties, which otherwise reports 0 s for a long file.

AIFF is big-endian and has no long form, so RIFF::File's only other
subclass cannot reach the new branch.

tests/data/rf64.wav is 9,680 bytes and built by construction, not by an
encoder; Core Audio reads it as RF64 at 0.050000 sec. The sentinels behave
identically at any size, so a small fixture covers the detection failure,
the sentinel overwrite and the repair.

taglib/riff/rifffile.cpp

@@ -43,7 +43,8 @@ struct Chunk
 {
   ByteVector   name;
   offset_t offset;
-  unsigned int size;
+  //! May exceed 32 bits for the "data" chunk of an RF64/BW64 file.
+  offset_t size;
   unsigned int padding;
 };
 
@@ -60,6 +61,13 @@ public:
   unsigned int size { 0 };
   offset_t sizeOffset { 0 };
 
+  //! An RF64 or BW64 file: the 32-bit size fields hold a 0xffffffff sentinel and
+  //! the real sizes live in a leading "ds64" chunk.
+  bool isLongForm { false };
+  //! Offset of the "ds64" chunk data, or 0 if the file has none.
+  offset_t ds64Offset { 0 };
+  offset_t dataSize64 { 0 };
+
   std::vector<Chunk> chunks;
 };
 
@@ -100,9 +108,14 @@ unsigned int RIFF::File::chunkCount() const
 }
 
 unsigned int RIFF::File::chunkDataSize(unsigned int i) const
+{
+  return static_cast<unsigned int>(std::min<offset_t>(chunkDataSize64(i), 0xffffffff));
+}
+
+offset_t RIFF::File::chunkDataSize64(unsigned int i) const
 {
   if(i >= d->chunks.size()) {
-    debug("RIFF::File::chunkDataSize() - Index out of range. Returning 0.");
+    debug("RIFF::File::chunkDataSize64() - Index out of range. Returning 0.");
     return 0;
   }
 
@@ -147,7 +160,10 @@ ByteVector RIFF::File::chunkData(unsigned int i)
   }
 
   seek(d->chunks[i].offset);
-  return readBlock(d->chunks[i].size);
+
+  // A ByteVector is limited to 32 bits. The only chunk that can be larger is the
+  // "data" chunk of an RF64 file, which no caller reads through this API.
+  return readBlock(static_cast<size_t>(std::min<offset_t>(d->chunks[i].size, 0xffffffff)));
 }
 
 void RIFF::File::setChunkData(unsigned int i, const ByteVector &data)
@@ -259,8 +275,8 @@ void RIFF::File::removeChunk(unsigned int i)
   auto it = d->chunks.begin();
   std::advance(it, i);
 
-  const unsigned int removeSize = it->size + it->padding + 8;
-  removeBlock(it->offset - 8, removeSize);
+  const offset_t removeSize = it->size + it->padding + 8;
+  removeBlock(it->offset - 8, static_cast<size_t>(removeSize));
   it = d->chunks.erase(it);
 
   while(it != d->chunks.end()) {
@@ -291,6 +307,15 @@ void RIFF::File::read()
 
   offset_t offset = tell();
 
+  // RF64 and BW64 are the long forms of WAVE, used past 4 GB: the 32-bit size fields
+  // hold a 0xffffffff sentinel and a leading "ds64" chunk carries the real sizes.
+  // Both are little-endian, so AIFF never takes this path.
+  if(!bigEndian) {
+    seek(offset);
+    const ByteVector magic = readBlock(4);
+    d->isLongForm = magic == "RF64" || magic == "BW64";
+  }
+
   offset += 4;
   d->sizeOffset = offset;
 
@@ -310,20 +335,36 @@ void RIFF::File::read()
 
     seek(offset);
     const ByteVector   chnkName = readBlock(4);
-    unsigned int chunkSize = readBlock(4).toUInt(bigEndian);
+    const unsigned int declaredSize = readBlock(4).toUInt(bigEndian);
 
     if(!isValidChunkName(chnkName)) {
       debug("RIFF::File::read() -- Chunk '" + chnkName + "' has invalid ID");
       break;
     }
 
-    if(static_cast<long long>(offset) + 8 + chunkSize > length()) {
+    // "ds64" is required to be the first chunk, so its sizes are known by the time
+    // the "data" chunk is reached. Only the four fixed fields are read; the table of
+    // additional oversized chunks that may follow them is not parsed, and any chunk
+    // listed there stays on the clamping path below.
+    if(d->isLongForm && chnkName == "ds64" && d->chunks.empty() && declaredSize >= 28) {
+      seek(offset + 8);
+      const ByteVector ds64 = readBlock(28);
+      d->ds64Offset = offset + 8;
+      d->dataSize64 = static_cast<offset_t>(ds64.toULongLong(8, bigEndian));
+    }
+
+    offset_t chunkSize = declaredSize;
+
+    if(d->isLongForm && chnkName == "data" && declaredSize == 0xffffffff && d->dataSize64 > 0)
+      chunkSize = d->dataSize64;
+
+    if(offset + 8 + chunkSize > length()) {
       // Clamp to available bytes rather than rejecting the chunk outright.
       // Some encoders write a correct data chunk but with a slightly too-large
       // declared size, or place the data chunk outside the declared RIFF boundary.
       // Lenient parsers (ffmpeg, QuickTime) handle this by clamping; we do the same.
       debug("RIFF::File::read() -- Chunk '" + chnkName + "' is truncated; clamping size to available bytes.");
-      chunkSize = static_cast<unsigned int>(length() - offset - 8);
+      chunkSize = length() - offset - 8;
     }
 
     Chunk chunk;
@@ -381,7 +422,26 @@ void RIFF::File::updateGlobalSize()
 
   const Chunk first = d->chunks.front();
   const Chunk last  = d->chunks.back();
-  d->size = static_cast<unsigned int>(last.offset + last.size + last.padding - first.offset + 12);
+  const offset_t totalSize = last.offset + last.size + last.padding - first.offset + 12;
+
+  if(d->isLongForm) {
+    // A long-form file always carries the sentinel here and its real size in "ds64"; any other
+    // value is malformed. Writing it unconditionally also repairs a file whose sentinel an
+    // older writer replaced with a real total, which past 4 GB is a truncated value that makes
+    // readers stop consulting "ds64" and believe it instead.
+    d->size = 0xffffffff;
+    insert(ByteVector::fromUInt(d->size, d->endianness == BigEndian), d->sizeOffset, 4);
+
+    // The "data" chunk's own size and "ds64"'s copy of it are left alone because no write path
+    // here changes the audio.
+    if(d->ds64Offset > 0)
+      insert(ByteVector::fromULongLong(totalSize, d->endianness == BigEndian),
+             d->ds64Offset, 8);
+
+    return;
+  }
+
+  d->size = static_cast<unsigned int>(totalSize);
 
   const ByteVector data = ByteVector::fromUInt(d->size, d->endianness == BigEndian);
   insert(data, d->sizeOffset, 4);

taglib/riff/rifffile.h

@@ -77,10 +77,18 @@ namespace TagLib {
       offset_t chunkOffset(unsigned int i) const;
 
       /*!
-       * \return The size of the chunk data.
+       * \return The size of the chunk data, saturated at 0xffffffff.
+       *
+       * \note The "data" chunk of an RF64 or BW64 file can be larger than this can
+       * express; use chunkDataSize64() where that matters.
        */
       unsigned int chunkDataSize(unsigned int i) const;
 
+      /*!
+       * \return The size of the chunk data, without a 32-bit limit.
+       */
+      offset_t chunkDataSize64(unsigned int i) const;
+
       /*!
        * \return The size of the padding after the chunk (can be either 0 or 1).
        */

taglib/riff/wav/wavfile.cpp

@@ -68,10 +68,12 @@ public:
 
 bool RIFF::WAV::File::isSupported(IOStream *stream)
 {
-  // A WAV file has to start with "RIFF????WAVE".
+  // A WAV file has to start with "RIFF????WAVE", or with the long-form "RF64" or
+  // "BW64" magic used past 4 GB.
 
   const ByteVector id = Utils::readHeader(stream, 12, false);
-  return id.startsWith("RIFF") && id.containsAt("WAVE", 8);
+  return (id.startsWith("RIFF") || id.startsWith("RF64") || id.startsWith("BW64")) &&
+         id.containsAt("WAVE", 8);
 }
 
 ////////////////////////////////////////////////////////////////////////////////

taglib/riff/wav/wavproperties.cpp

@@ -25,6 +25,8 @@
 
 #include "wavproperties.h"
 
+#include <algorithm>
+
 #include "tdebug.h"
 #include "wavfile.h"
 
@@ -108,7 +110,8 @@ int RIFF::WAV::Properties::format() const
 void RIFF::WAV::Properties::read(File *file)
 {
   ByteVector data;
-  unsigned int streamLength = 0;
+  // 64-bit: an RF64 "data" chunk does not fit in 32.
+  offset_t streamLength = 0;
   unsigned int totalSamples = 0;
 
   for(unsigned int i = 0; i < file->chunkCount(); ++i) {
@@ -120,7 +123,7 @@ void RIFF::WAV::Properties::read(File *file)
     }
     else if(name == "data") {
       if(streamLength == 0)
-        streamLength = file->chunkDataSize(i) + file->chunkPadding(i);
+        streamLength = file->chunkDataSize64(i) + file->chunkPadding(i);
       else
         debug("RIFF::WAV::Properties::read() - Duplicate 'data' chunk found.");
     }
@@ -162,17 +165,19 @@ void RIFF::WAV::Properties::read(File *file)
 
   if(d->format != FORMAT_PCM && (d->format != FORMAT_IEEE_FLOAT || totalSamples != 0))
     d->sampleFrames = totalSamples;
-  else if(d->channels > 0 && d->bitsPerSample > 0)
-    d->sampleFrames = streamLength / (d->channels * ((d->bitsPerSample + 7) / 8));
+  else if(d->channels > 0 && d->bitsPerSample > 0) {
+    const offset_t frames = streamLength / (d->channels * ((d->bitsPerSample + 7) / 8));
+    d->sampleFrames = static_cast<unsigned int>(std::min<offset_t>(frames, 0xffffffff));
+  }
 
   if(d->sampleFrames > 0 && d->sampleRate > 0) {
     const auto length = static_cast<double>(d->sampleFrames) * 1000.0 / d->sampleRate;
     d->length  = static_cast<int>(length + 0.5);
-    d->bitrate = static_cast<int>(streamLength * 8.0 / length + 0.5);
+    d->bitrate = static_cast<int>(static_cast<double>(streamLength) * 8.0 / length + 0.5);
   }
   else {
     if(const unsigned int byteRate = data.toUInt(8, false); byteRate > 0) {
-      d->length  = static_cast<int>(streamLength * 1000.0 / byteRate + 0.5);
+      d->length  = static_cast<int>(static_cast<double>(streamLength) * 1000.0 / byteRate + 0.5);
       d->bitrate = static_cast<int>(byteRate * 8.0 / 1000.0 + 0.5);
     }
   }

tests/test_wav.cpp

@@ -60,6 +60,10 @@ class TestWAV : public CppUnit::TestFixture
   CPPUNIT_TEST(testPCMWithFactChunk);
   CPPUNIT_TEST(testWaveFormatExtensible);
   CPPUNIT_TEST(testInvalidChunk);
+  CPPUNIT_TEST(testRF64IsSupported);
+  CPPUNIT_TEST(testRF64Properties);
+  CPPUNIT_TEST(testRF64Save);
+  CPPUNIT_TEST(testRF64SaveRepairsClobberedSize);
   CPPUNIT_TEST(testRIFFInfoProperties);
   CPPUNIT_TEST(testBEXTTag);
   CPPUNIT_TEST(testBEXTTagWithOtherTags);
@@ -408,6 +412,144 @@ public:
     }
   }
 
+  // rf64.wav is a 50 ms RF64: 0xffffffff sentinels in the 32-bit size fields at offset 4 and
+  // in the "data" chunk header, with the real sizes in a leading "ds64" chunk. That is what a
+  // WAVE file becomes past 4 GB; the sentinels behave the same at any size, so the fixture is
+  // small.
+
+  static void setMagic(const std::string &fileName, const ByteVector &magic)
+  {
+    FileStream stream(fileName.c_str());
+    stream.seek(0);
+    stream.writeBlock(magic);
+  }
+
+  void testRF64IsSupported()
+  {
+    ScopedFileCopy copy("rf64", ".wav");
+    {
+      FileStream stream(copy.fileName().c_str(), true);
+      CPPUNIT_ASSERT(RIFF::WAV::File::isSupported(&stream));
+    }
+    setMagic(copy.fileName(), "BW64");
+    {
+      FileStream stream(copy.fileName().c_str(), true);
+      CPPUNIT_ASSERT(RIFF::WAV::File::isSupported(&stream));
+    }
+    setMagic(copy.fileName(), "XX64");
+    {
+      FileStream stream(copy.fileName().c_str(), true);
+      CPPUNIT_ASSERT(!RIFF::WAV::File::isSupported(&stream));
+    }
+  }
+
+  void testRF64Properties()
+  {
+    ScopedFileCopy copy("rf64", ".wav");
+
+    // Bytes past the audio, so that clamping the sentinel to what is available gives a
+    // different answer from "ds64" and the test can tell which one was used.
+    {
+      FileStream stream(copy.fileName().c_str());
+      stream.seek(0, IOStream::End);
+      stream.writeBlock(ByteVector("junk", 4) + ByteVector::fromUInt(1000, false) +
+                        ByteVector(1000, '\0'));
+    }
+
+    RIFF::WAV::File f(copy.fileName().c_str());
+    CPPUNIT_ASSERT(f.isValid());
+    CPPUNIT_ASSERT_EQUAL(50, f.audioProperties()->lengthInMilliseconds());
+    CPPUNIT_ASSERT_EQUAL(48000, f.audioProperties()->sampleRate());
+    CPPUNIT_ASSERT_EQUAL(2, f.audioProperties()->channels());
+  }
+
+  void testRF64Save()
+  {
+    ScopedFileCopy copy("rf64", ".wav");
+
+    offset_t originalLength = 0;
+    {
+      FileStream stream(copy.fileName().c_str(), true);
+      originalLength = stream.length();
+    }
+
+    {
+      RIFF::WAV::File f(copy.fileName().c_str());
+      CPPUNIT_ASSERT(f.isValid());
+      PropertyMap properties;
+      properties["TITLE"] = StringList("Title");
+      properties["ARTIST"] = StringList("Artist");
+      CPPUNIT_ASSERT(f.setProperties(properties).isEmpty());
+      CPPUNIT_ASSERT(f.save());
+    }
+
+    {
+      RIFF::WAV::File f(copy.fileName().c_str());
+      const PropertyMap properties = f.properties();
+      CPPUNIT_ASSERT(properties.contains("TITLE"));
+      CPPUNIT_ASSERT(properties.contains("ARTIST"));
+      CPPUNIT_ASSERT_EQUAL(String("Title"), properties["TITLE"].front());
+      CPPUNIT_ASSERT_EQUAL(String("Artist"), properties["ARTIST"].front());
+      CPPUNIT_ASSERT_EQUAL(50, f.audioProperties()->lengthInMilliseconds());
+    }
+
+    {
+      FileStream stream(copy.fileName().c_str(), true);
+      const offset_t length = stream.length();
+      CPPUNIT_ASSERT(length > originalLength);
+
+      // The 32-bit field has to stay a sentinel: a real number there makes readers stop
+      // consulting "ds64", which past 4 GB is the only place the size fits.
+      stream.seek(4);
+      CPPUNIT_ASSERT_EQUAL(0xffffffffU, stream.readBlock(4).toUInt(false));
+
+      // "ds64" carries the real size, so it is what has to track the file's growth.
+      stream.seek(20);
+      CPPUNIT_ASSERT_EQUAL(static_cast<unsigned long long>(length - 8),
+                           stream.readBlock(8).toULongLong(false));
+
+      // The audio's own extent is untouched.
+      stream.seek(28);
+      CPPUNIT_ASSERT_EQUAL(9600ULL, stream.readBlock(8).toULongLong(false));
+    }
+  }
+
+  void testRF64SaveRepairsClobberedSize()
+  {
+    ScopedFileCopy copy("rf64", ".wav");
+
+    // A real total where the sentinel belongs, as an earlier version of this code left it. The
+    // value is malformed in a long-form file at any size, and past 4 GB it is also truncated,
+    // which is what makes readers report milliseconds for hours of audio.
+    {
+      FileStream stream(copy.fileName().c_str());
+      stream.seek(4);
+      stream.writeBlock(ByteVector::fromUInt(5230, false));
+    }
+
+    {
+      RIFF::WAV::File f(copy.fileName().c_str());
+      CPPUNIT_ASSERT(f.isValid());
+      f.InfoTag()->setTitle("Title");
+      CPPUNIT_ASSERT(f.save());
+    }
+
+    {
+      FileStream stream(copy.fileName().c_str(), true);
+      const offset_t length = stream.length();
+
+      stream.seek(4);
+      CPPUNIT_ASSERT_EQUAL(0xffffffffU, stream.readBlock(4).toUInt(false));
+
+      stream.seek(20);
+      CPPUNIT_ASSERT_EQUAL(static_cast<unsigned long long>(length - 8),
+                           stream.readBlock(8).toULongLong(false));
+
+      stream.seek(28);
+      CPPUNIT_ASSERT_EQUAL(9600ULL, stream.readBlock(8).toULongLong(false));
+    }
+  }
+
   void testRIFFInfoProperties()
   {
     PropertyMap tags;