From 3202072ef65ab911f13d0871c304e4789a8b42ea Mon Sep 17 00:00:00 2001 From: iambus Date: Mon, 13 Dec 2021 17:02:34 +0800 Subject: [PATCH 1/3] fix panic: runtime error: slice bounds out of range --- diffmatchpatch/diff.go | 16 ++++------------ diffmatchpatch/patch_test.go | 25 +++++++++++++++++++++++++ diffmatchpatch/stringutil.go | 18 ++++-------------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/diffmatchpatch/diff.go b/diffmatchpatch/diff.go index 2a9f2dc..9b0e4ab 100644 --- a/diffmatchpatch/diff.go +++ b/diffmatchpatch/diff.go @@ -34,8 +34,6 @@ const ( DiffInsert Operation = 1 // DiffEqual item represents an equal diff. DiffEqual Operation = 0 - //IndexSeparator is used to seperate the array indexes in an index string - IndexSeparator = "," ) // Diff represents one diff operation @@ -406,17 +404,11 @@ func (dmp *DiffMatchPatch) DiffLinesToRunes(text1, text2 string) ([]rune, []rune func (dmp *DiffMatchPatch) DiffCharsToLines(diffs []Diff, lineArray []string) []Diff { hydrated := make([]Diff, 0, len(diffs)) for _, aDiff := range diffs { - chars := strings.Split(aDiff.Text, IndexSeparator) - text := make([]string, len(chars)) - - for i, r := range chars { - i1, err := strconv.Atoi(r) - if err == nil { - text[i] = lineArray[i1] - } + var sb strings.Builder + for _, i := range []rune(aDiff.Text) { + sb.WriteString(lineArray[i]) } - - aDiff.Text = strings.Join(text, "") + aDiff.Text = sb.String() hydrated = append(hydrated, aDiff) } return hydrated diff --git a/diffmatchpatch/patch_test.go b/diffmatchpatch/patch_test.go index b019f88..c564f8c 100644 --- a/diffmatchpatch/patch_test.go +++ b/diffmatchpatch/patch_test.go @@ -337,3 +337,28 @@ func TestPatchApply(t *testing.T) { assert.Equal(t, tc.ExpectedApplies, actualApplies, fmt.Sprintf("Test case #%d, %s", i, tc.Name)) } } + +func TestPatchMakeOutOfRangePanic(t *testing.T) { + text1 := ` + 1111111111111 000000 + ------------- ------ + xxxxxxxxxxxxx ------ + xxxxxxxxxxxxx ------ + xxxxxxxxxxxxx xxxxxx + xxxxxxxxxxxxx ...... + xxxxxxxxxxxxx 111111 + xxxxxxxxxxxxx ?????? + xxxxxxxxxxxxx 333333 + xxxxxxxxxxxxx 555555 + xxxxxxxxxx xxxxx + xxxxxxxxxx xxxxx + xxxxxxxxxx xxxxx + xxxxxxxxxx xxxxx +` + text2 := ` + 2222222222222 000000 + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx` + dmp := New() + patches := dmp.PatchMake(text1, text2) + assert.Equal(t, 6, len(patches), "TestPatchMakeOutOfRangePanic") +} diff --git a/diffmatchpatch/stringutil.go b/diffmatchpatch/stringutil.go index 44c4359..da0257c 100644 --- a/diffmatchpatch/stringutil.go +++ b/diffmatchpatch/stringutil.go @@ -9,7 +9,6 @@ package diffmatchpatch import ( - "strconv" "strings" "unicode/utf8" ) @@ -89,18 +88,9 @@ func runesIndex(r1, r2 []rune) int { } func intArrayToString(ns []uint32) string { - if len(ns) == 0 { - return "" + runes := make([]rune, len(ns)) + for i := 0; i < len(ns); i++ { + runes[i] = rune(ns[i]) } - - indexSeparator := IndexSeparator[0] - - // Appr. 3 chars per num plus the comma. - b := []byte{} - for _, n := range ns { - b = strconv.AppendInt(b, int64(n), 10) - b = append(b, indexSeparator) - } - b = b[:len(b)-1] - return string(b) + return string(runes) } From 3403a1642ff93712b699ebd52ecf3c53c2b08107 Mon Sep 17 00:00:00 2001 From: Boyu Guo Date: Fri, 23 Sep 2022 10:44:03 +0800 Subject: [PATCH 2/3] update tests --- diffmatchpatch/diff_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/diffmatchpatch/diff_test.go b/diffmatchpatch/diff_test.go index acb97e3..7c8c9a1 100644 --- a/diffmatchpatch/diff_test.go +++ b/diffmatchpatch/diff_test.go @@ -314,10 +314,10 @@ func TestDiffLinesToChars(t *testing.T) { dmp := New() for i, tc := range []TestCase{ - {"", "alpha\r\nbeta\r\n\r\n\r\n", "", "1,2,3,3", []string{"", "alpha\r\n", "beta\r\n", "\r\n"}}, - {"a", "b", "1", "2", []string{"", "a", "b"}}, + {"", "alpha\r\nbeta\r\n\r\n\r\n", "", "\x01\x02\x03\x03", []string{"", "alpha\r\n", "beta\r\n", "\r\n"}}, + {"a", "b", "\x01", "\x02", []string{"", "a", "b"}}, // Omit final newline. - {"alpha\nbeta\nalpha", "", "1,2,3", "", []string{"", "alpha\n", "beta\n", "alpha"}}, + {"alpha\nbeta\nalpha", "", "\x01\x02\x03", "", []string{"", "alpha\n", "beta\n", "alpha"}}, } { actualChars1, actualChars2, actualLines := dmp.DiffLinesToChars(tc.Text1, tc.Text2) assert.Equal(t, tc.ExpectedChars1, actualChars1, fmt.Sprintf("Test case #%d, %#v", i, tc)) @@ -330,14 +330,14 @@ func TestDiffLinesToChars(t *testing.T) { lineList := []string{ "", // Account for the initial empty element of the lines array. } - var charList []string + var charList []rune for x := 1; x < n+1; x++ { lineList = append(lineList, strconv.Itoa(x)+"\n") - charList = append(charList, strconv.Itoa(x)) + charList = append(charList, rune(x)) } lines := strings.Join(lineList, "") - chars := strings.Join(charList[:], ",") - assert.Equal(t, n, len(strings.Split(chars, ","))) + chars := string(charList) + assert.Equal(t, n, len(charList)) actualChars1, actualChars2, actualLines := dmp.DiffLinesToChars(lines, "") assert.Equal(t, chars, actualChars1) @@ -358,8 +358,8 @@ func TestDiffCharsToLines(t *testing.T) { for i, tc := range []TestCase{ { Diffs: []Diff{ - {DiffEqual, "1,2,1"}, - {DiffInsert, "2,1,2"}, + {DiffEqual, "\x01\x02\x01"}, + {DiffInsert, "\x02\x01\x02"}, }, Lines: []string{"", "alpha\n", "beta\n"}, @@ -378,13 +378,13 @@ func TestDiffCharsToLines(t *testing.T) { lineList := []string{ "", // Account for the initial empty element of the lines array. } - charList := []string{} + charList := []rune{} for x := 1; x <= n; x++ { lineList = append(lineList, strconv.Itoa(x)+"\n") - charList = append(charList, strconv.Itoa(x)) + charList = append(charList, rune(x)) } assert.Equal(t, n, len(charList)) - chars := strings.Join(charList[:], ",") + chars := string(charList) actual := dmp.DiffCharsToLines([]Diff{Diff{DiffDelete, chars}}, lineList) assert.Equal(t, []Diff{Diff{DiffDelete, strings.Join(lineList, "")}}, actual) From ff0d34ba8093cd989afb5ead57451452a16df340 Mon Sep 17 00:00:00 2001 From: Boyu Guo Date: Tue, 27 Sep 2022 01:17:47 +0800 Subject: [PATCH 3/3] skip invalid unicode code point to fix TestMassiveRuneDiffConversion --- diffmatchpatch/diff.go | 27 ++++++++++++++++----------- diffmatchpatch/diff_test.go | 12 ++++++------ diffmatchpatch/index.go | 32 ++++++++++++++++++++++++++++++++ diffmatchpatch/index_test.go | 16 ++++++++++++++++ diffmatchpatch/stringutil.go | 8 -------- 5 files changed, 70 insertions(+), 25 deletions(-) create mode 100644 diffmatchpatch/index.go create mode 100644 diffmatchpatch/index_test.go diff --git a/diffmatchpatch/diff.go b/diffmatchpatch/diff.go index 9b0e4ab..25cc51f 100644 --- a/diffmatchpatch/diff.go +++ b/diffmatchpatch/diff.go @@ -388,16 +388,21 @@ func (dmp *DiffMatchPatch) diffBisectSplit(runes1, runes2 []rune, x, y int, } // DiffLinesToChars splits two texts into a list of strings, and educes the texts to a string of hashes where each Unicode character represents one line. -// It's slightly faster to call DiffLinesToRunes first, followed by DiffMainRunes. func (dmp *DiffMatchPatch) DiffLinesToChars(text1, text2 string) (string, string, []string) { - chars1, chars2, lineArray := dmp.diffLinesToStrings(text1, text2) - return chars1, chars2, lineArray + chars1, chars2, lineArray := dmp.diffLinesToIndexes(text1, text2) + return indexesToString(chars1), indexesToString(chars2), lineArray } // DiffLinesToRunes splits two texts into a list of runes. func (dmp *DiffMatchPatch) DiffLinesToRunes(text1, text2 string) ([]rune, []rune, []string) { + chars1, chars2, lineArray := dmp.diffLinesToIndexes(text1, text2) + return []rune(indexesToString(chars1)), []rune(indexesToString(chars2)), lineArray +} + +// diffLinesToIndexes splits two texts into a list of indexes +func (dmp *DiffMatchPatch) diffLinesToIndexes(text1, text2 string) ([]index, []index, []string) { chars1, chars2, lineArray := dmp.diffLinesToStrings(text1, text2) - return []rune(chars1), []rune(chars2), lineArray + return chars1, chars2, lineArray } // DiffCharsToLines rehydrates the text in a diff from a string of line hashes to real lines of text. @@ -405,7 +410,7 @@ func (dmp *DiffMatchPatch) DiffCharsToLines(diffs []Diff, lineArray []string) [] hydrated := make([]Diff, 0, len(diffs)) for _, aDiff := range diffs { var sb strings.Builder - for _, i := range []rune(aDiff.Text) { + for _, i := range stringToIndex(aDiff.Text) { sb.WriteString(lineArray[i]) } aDiff.Text = sb.String() @@ -1301,7 +1306,7 @@ func (dmp *DiffMatchPatch) DiffFromDelta(text1 string, delta string) (diffs []Di } // diffLinesToStrings splits two texts into a list of strings. Each string represents one line. -func (dmp *DiffMatchPatch) diffLinesToStrings(text1, text2 string) (string, string, []string) { +func (dmp *DiffMatchPatch) diffLinesToStrings(text1, text2 string) ([]index, []index, []string) { // '\x00' is a valid character, but various debuggers don't like it. So we'll insert a junk entry to avoid generating a null character. lineArray := []string{""} // e.g. lineArray[4] == 'Hello\n' @@ -1309,16 +1314,16 @@ func (dmp *DiffMatchPatch) diffLinesToStrings(text1, text2 string) (string, stri strIndexArray1 := dmp.diffLinesToStringsMunge(text1, &lineArray) strIndexArray2 := dmp.diffLinesToStringsMunge(text2, &lineArray) - return intArrayToString(strIndexArray1), intArrayToString(strIndexArray2), lineArray + return strIndexArray1, strIndexArray2, lineArray } // diffLinesToStringsMunge splits a text into an array of strings, and reduces the texts to a []string. -func (dmp *DiffMatchPatch) diffLinesToStringsMunge(text string, lineArray *[]string) []uint32 { +func (dmp *DiffMatchPatch) diffLinesToStringsMunge(text string, lineArray *[]string) []index { // Walk the text, pulling out a substring for each line. text.split('\n') would would temporarily double our memory footprint. Modifying text would create many large strings to garbage collect. lineHash := map[string]int{} // e.g. lineHash['Hello\n'] == 4 lineStart := 0 lineEnd := -1 - strs := []uint32{} + strs := []index{} for lineEnd < len(text)-1 { lineEnd = indexOf(text, "\n", lineStart) @@ -1332,11 +1337,11 @@ func (dmp *DiffMatchPatch) diffLinesToStringsMunge(text string, lineArray *[]str lineValue, ok := lineHash[line] if ok { - strs = append(strs, uint32(lineValue)) + strs = append(strs, index(lineValue)) } else { *lineArray = append(*lineArray, line) lineHash[line] = len(*lineArray) - 1 - strs = append(strs, uint32(len(*lineArray)-1)) + strs = append(strs, index(len(*lineArray)-1)) } } diff --git a/diffmatchpatch/diff_test.go b/diffmatchpatch/diff_test.go index 7c8c9a1..7955b1f 100644 --- a/diffmatchpatch/diff_test.go +++ b/diffmatchpatch/diff_test.go @@ -330,13 +330,13 @@ func TestDiffLinesToChars(t *testing.T) { lineList := []string{ "", // Account for the initial empty element of the lines array. } - var charList []rune + var charList []index for x := 1; x < n+1; x++ { lineList = append(lineList, strconv.Itoa(x)+"\n") - charList = append(charList, rune(x)) + charList = append(charList, index(x)) } lines := strings.Join(lineList, "") - chars := string(charList) + chars := indexesToString(charList) assert.Equal(t, n, len(charList)) actualChars1, actualChars2, actualLines := dmp.DiffLinesToChars(lines, "") @@ -378,13 +378,13 @@ func TestDiffCharsToLines(t *testing.T) { lineList := []string{ "", // Account for the initial empty element of the lines array. } - charList := []rune{} + charList := []index{} for x := 1; x <= n; x++ { lineList = append(lineList, strconv.Itoa(x)+"\n") - charList = append(charList, rune(x)) + charList = append(charList, index(x)) } assert.Equal(t, n, len(charList)) - chars := string(charList) + chars := indexesToString(charList) actual := dmp.DiffCharsToLines([]Diff{Diff{DiffDelete, chars}}, lineList) assert.Equal(t, []Diff{Diff{DiffDelete, strings.Join(lineList, "")}}, actual) diff --git a/diffmatchpatch/index.go b/diffmatchpatch/index.go new file mode 100644 index 0000000..965a1c6 --- /dev/null +++ b/diffmatchpatch/index.go @@ -0,0 +1,32 @@ +package diffmatchpatch + +type index uint32 + +const runeSkipStart = 0xd800 +const runeSkipEnd = 0xdfff + 1 +const runeMax = 0x110000 // next invalid code point + +func stringToIndex(text string) []index { + runes := []rune(text) + indexes := make([]index, len(runes)) + for i, r := range runes { + if r < runeSkipEnd { + indexes[i] = index(r) + } else { + indexes[i] = index(r) - (runeSkipEnd - runeSkipStart) + } + } + return indexes +} + +func indexesToString(indexes []index) string { + runes := make([]rune, len(indexes)) + for i, index := range indexes { + if index < runeSkipStart { + runes[i] = rune(index) + } else { + runes[i] = rune(index + (runeSkipEnd - runeSkipStart)) + } + } + return string(runes) +} diff --git a/diffmatchpatch/index_test.go b/diffmatchpatch/index_test.go new file mode 100644 index 0000000..6f1d982 --- /dev/null +++ b/diffmatchpatch/index_test.go @@ -0,0 +1,16 @@ +package diffmatchpatch + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestIndexConversion(t *testing.T) { + n := runeMax - (runeSkipEnd - runeSkipStart) + indexes := make([]index, n) + for i := 0; i < n; i++ { + indexes[i] = index(i) + } + indexes2 := stringToIndex(indexesToString(indexes)) + assert.EqualValues(t, indexes, indexes2) +} diff --git a/diffmatchpatch/stringutil.go b/diffmatchpatch/stringutil.go index da0257c..265f29c 100644 --- a/diffmatchpatch/stringutil.go +++ b/diffmatchpatch/stringutil.go @@ -86,11 +86,3 @@ func runesIndex(r1, r2 []rune) int { } return -1 } - -func intArrayToString(ns []uint32) string { - runes := make([]rune, len(ns)) - for i := 0; i < len(ns); i++ { - runes[i] = rune(ns[i]) - } - return string(runes) -}