BUG: Fix to_json lines with escaped characters

Updates existing to_json methodology by adding is_escaping variable, which ensures escaped chars are handled correctly.

- Includes test for escaped characters in keys and values (i.e. columns and data).
- Includes bug fix in whatsnew
- Revised type of in_quotes and is_escaping to bint

xref #14693
xref #15096

Author: rouzazari

doc/source/whatsnew/v0.20.0.txt

@@ -351,6 +351,7 @@ Bug Fixes
 - Require at least 0.23 version of cython to avoid problems with character encodings (:issue:`14699`)
 - Bug in converting object elements of array-like objects to unsigned 64-bit integers (:issue:`4471`, :issue:`14982`)
 - Bug in ``pd.pivot_table()`` where no error was raised when values argument was not in the columns (:issue:`14938`)
+- Bug in ``.to_json()`` where lines=True and contents (keys or values) contain escaped characters (:issue:`15096`)
 
 
 

pandas/io/tests/json/test_pandas.py

@@ -972,6 +972,13 @@ def test_to_jsonl(self):
         self.assertEqual(result, expected)
         assert_frame_equal(pd.read_json(result, lines=True), df)
 
+        # GH15096
+        df = DataFrame([["foo\\", "bar"], ['foo"', "bar"]], columns=["a\\", 'b'])
+        result = df.to_json(orient="records", lines=True)
+        expected = '{"a\\\\":"foo\\\\","b":"bar"}\n{"a\\\\":"foo\\"","b":"bar"}'
+        self.assertEqual(result, expected)
+        assert_frame_equal(pd.read_json(result, lines=True), df)
+
     def test_latin_encoding(self):
         if compat.PY2:
             self.assertRaisesRegexp(

pandas/lib.pyx

@@ -1098,7 +1098,8 @@ def convert_json_to_lines(object arr):
     to quotes & brackets
     """
     cdef:
-        Py_ssize_t i = 0, num_open_brackets_seen = 0, in_quotes = 0, length
+        Py_ssize_t i = 0, num_open_brackets_seen = 0, length
+        bint in_quotes = 0, is_escaping = 0
         ndarray[uint8_t] narr
         unsigned char v, comma, left_bracket, right_brack, newline
 
@@ -1113,8 +1114,10 @@ def convert_json_to_lines(object arr):
     length = narr.shape[0]
     for i in range(length):
         v = narr[i]
-        if v == quote and i > 0 and narr[i - 1] != backslash:
+        if v == quote and i > 0 and not is_escaping:
             in_quotes = ~in_quotes
+        if v == backslash or is_escaping:
+            is_escaping = ~is_escaping
         if v == comma: # commas that should be \n
             if num_open_brackets_seen == 0 and not in_quotes:
                 narr[i] = newline