fix encoding fallback in Serializer.do_bytes

pdfplumber/convert.py

@@ -119,8 +119,8 @@ def do_bytes(self, obj: bytes) -> Optional[str]:
         for e in ENCODINGS_TO_TRY:
             try:
                 return obj.decode(e)
-            except UnicodeDecodeError:  # pragma: no cover
-                return None
+            except UnicodeDecodeError:
+                continue
         # If none of the decodings work, raise whatever error
         # decoding with utf-8 causes
         obj.decode(ENCODINGS_TO_TRY[0])  # pragma: no cover

tests/test_convert.py

@@ -173,6 +173,15 @@ def test_additional_attr_types(self):
             c = json.loads(pdf.to_json())
             assert len(c["pages"][0]["images"])
 
+    def test_serialize_bytes_encoding_fallback(self):
+        from pdfplumber.convert import Serializer
+
+        s = Serializer()
+        # Bytes that are not valid UTF-8 but decode under a later
+        # encoding in the fallback list must not be dropped to None.
+        assert s.do_bytes(b"\xff\xfe\x41") == "\xff\xfe\x41"
+        assert s.serialize(b"\xff\xfe\x41") == "\xff\xfe\x41"
+
     def test_csv(self):
         c = self.pdf.to_csv(precision=3)
         assert c.split("\r\n")[9] == (