diff --git a/taglib/toolkit/tbytevector.cpp b/taglib/toolkit/tbytevector.cpp
index ad5d4523..45ad0317 100644
--- a/taglib/toolkit/tbytevector.cpp
+++ b/taglib/toolkit/tbytevector.cpp
@@ -703,15 +703,12 @@ ByteVector &ByteVector::resize(uint size, char padding)
   if(size != d->length) {
     detach();
 
-    const size_t bufferSize = d->data->data.size();
+    // Remove the excessive length of the internal buffer first to pad correctly.
+    // This doesn't reallocate the buffer, since std::vector::resize() doesn't
+    // reallocate the buffer when shrinking.
 
-    if(size > bufferSize - d->offset) {
-      d->data->data.resize(d->offset + size, padding);
-      ::memset(&*end(), padding, bufferSize - (d->length + d->offset));
-    }
-    else if(size > d->length) {
-      ::memset(&*end(), padding, size - d->length);
-    }
+    d->data->data.resize(d->offset + d->length);
+    d->data->data.resize(d->offset + size, padding);
 
     d->length = size;
   }