Strip BOM before processing records

rispy/parser.py

@@ -199,14 +199,7 @@ def load(
     Returns:
         list: Returns list of RIS entries.
     """
-    c = file.read()
-
-    # Corrects for BOM in utf-8 encodings while keeping an 8-bit
-    # string representation
-    if len(c) > 3 and (c[0], c[1], c[2]) == ("\xef", "\xbb", "\xbf"):
-        c = c[3:]
-
-    return list(loads(c, mapping, implementation))
+    return list(loads(file.read(), mapping, implementation))
 
 
 def loads(
@@ -231,6 +224,9 @@ def loads(
         list: Returns list of RIS entries.
     """
 
+    # remove BOM if present
+    obj = obj.lstrip("\ufeff")
+
     filelines = obj.split("\n")
 
     implementation = RisImplementation(implementation)

tests/data/example_bom.ris

@@ -0,0 +1,4 @@
+TY  - JOUR
+DO  - 10.1186/s40981-020-0316-0
+ER  - 
+

tests/test_parser.py

@@ -141,6 +141,19 @@ def test_starting_newline():
     assert len(entries) == 1
 
 
+def test_strip_bom():
+    expected = {"type_of_reference": "JOUR", "doi": "10.1186/s40981-020-0316-0"}
+
+    filepath = DATA_DIR / "example_bom.ris"
+
+    # we properly decode the content of this file as UTF-8, but leave the BOM
+    with open(filepath, "r", encoding="utf-8") as f:
+        entries = rispy.load(f)
+
+    print(entries)
+    assert expected == entries[0]
+
+
 def test_wos_ris():
     fn = DATA_DIR / "example_wos.ris"
     with open(fn, "r") as f: