RIFF: Read INFO text that isn't valid UTF-8 as Latin-1 when UTF-8 is assumed (#1480)

When setAssumedCodePage(65001) supplies the charset for an INFO tag
whose file declares none, a field whose bytes are not valid UTF-8 is
now read as ISO 8859-1, the RIFF default, instead of as an empty
string. A code page the file declares is never second-guessed, and a
handler installed with setStringHandler() still takes precedence.

taglib/riff/wav/infotag.cpp

@@ -28,6 +28,8 @@
 #include <memory>
 #include <utility>
 
+#include <utf8.h>
+
 #include "tcharset.h"
 #include "tbytevector.h"
 #include "tdebug.h"
@@ -102,6 +104,9 @@ public:
   //! the default.
   Charset::Type charset { Charset::Type::Latin1 };
 
+  //! Whether the charset comes from setAssumedCodePage() rather than the file.
+  bool charsetAssumed { false };
+
   String parseField(const ByteVector &data) const
   {
     // A handler the user installed with setStringHandler() is an explicit
@@ -109,6 +114,13 @@ public:
     // handler installed the field is decoded with the charset.
     if(stringHandler != &defaultStringHandler)
       return stringHandler->parse(data);
+
+    // An assumed UTF-8 is a hint: a field that is not valid UTF-8 contradicts
+    // it and falls back to the RIFF default rather than decoding as empty.
+    if(charsetAssumed && charset == Charset::Type::UTF8 &&
+       !utf8::is_valid(data.begin(), data.end()))
+      return decodeField(data, Charset::Type::Latin1);
+
     return decodeField(data, charset);
   }
 };
@@ -148,6 +160,7 @@ RIFF::Info::Tag::Tag(const ByteVector &data, unsigned int codePage) :
   d(std::make_unique<TagPrivate>())
 {
   d->charset = charsetForCodePage(codePage);
+  d->charsetAssumed = codePage == 0 && codePageAssumed != 0;
   parse(data);
 }
 

taglib/riff/wav/infotag.h

@@ -242,6 +242,10 @@ namespace TagLib {
        * a CSET chunk from one holding ISO 8859-1, because all three look the same on
        * disk. This is where the caller says which of them its files hold.
        *
+       * An assumed UTF-8 is still only a hint: a field whose bytes are not valid UTF-8
+       * is read as ISO 8859-1 instead. A code page the file declares is never second
+       * guessed this way.
+       *
        * \note This is a process wide setting, like setStringHandler(), and applies to
        * the tags read after the call. It never overrides a code page a file declares
        * itself, not even one this library has no table for.

tests/test_wav.cpp

@@ -397,6 +397,7 @@ class TestWAV : public CppUnit::TestFixture
   CPPUNIT_TEST(testInfoAssumedCodePageAcceptsUtf8);
   CPPUNIT_TEST(testInfoAssumedCodePageWindows1252);
   CPPUNIT_TEST(testInfoAssumedCodePageOnlyWhenUndeclared);
+  CPPUNIT_TEST(testInfoAssumedUtf8FallsBackToLatin1);
   CPPUNIT_TEST(testInfoAssumedCodePageRejectsUnsupported);
 #endif
   CPPUNIT_TEST_SUITE_END();
@@ -1111,6 +1112,33 @@ public:
     }
   }
 
+  // An assumed UTF-8 is a hint, and a field that is not valid UTF-8 contradicts
+  // it. Such a field is read as the RIFF default ISO 8859-1 instead of as empty,
+  // field by field, while a file that declares UTF-8 itself is taken at its word.
+  void testInfoAssumedUtf8FallsBackToLatin1()
+  {
+    const RestoreAssumedCodePage restore;
+
+    RIFF::Info::Tag::setAssumedCodePage(65001);
+
+    {
+      ByteVectorStream stream(riffWithInfo({{"IART", ByteVector("Caf\xC3\xA9", 5)},
+                                            {"ICOP", ByteVector("\xA9 2019 BOOM", 11)}}));
+      RIFF::WAV::File f(&stream);
+      CPPUNIT_ASSERT(f.isValid());
+      CPPUNIT_ASSERT_EQUAL(String(L"Caf\xE9"), f.InfoTag()->artist());
+      CPPUNIT_ASSERT_EQUAL(String(L"\xA9 2019 BOOM"), f.InfoTag()->fieldText("ICOP"));
+    }
+
+    {
+      ByteVectorStream stream(riffWithInfo({{"ICOP", ByteVector("\xA9 2019 BOOM", 11)}},
+                                           65001));
+      RIFF::WAV::File f(&stream);
+      CPPUNIT_ASSERT(f.isValid());
+      CPPUNIT_ASSERT(f.InfoTag()->fieldText("ICOP").isEmpty());
+    }
+  }
+
   // A code page this library cannot decode would quietly do nothing if it were
   // accepted as the assumption, so it is refused and the previous one stays.
   void testInfoAssumedCodePageRejectsUnsupported()