From 8df88ae25269ec9dd45924becb78c341bbe04492 Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Wed, 3 Jun 2020 22:03:40 -0600 Subject: [PATCH 01/16] attempt to add test of Chinese error message --- inst/tests/tests.Rraw | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index c71b27ca02..e644f83a42 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16910,3 +16910,14 @@ dt = data.table(time = 1:8, v = INT(5,7,6,1,8,4,2,3)) dt[time == 2L, v := 2L] dt[time == 7L, v := 7L] test(2140, dt[dt, on=.(time>time, v>v), .N, by=.EACHI], data.table(time=1:8, v=INT(5,2,6,1,8,4,7,3), N=INT(3,5,2,4,0,1,0,0))) + +# repeat of test 450 for #4402 +DT = data.table(a=1:3,b=4:6) +test(2141.1, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") +old = Sys.getenv("LANGUAGE") +Sys.setenv(LANGUAGE="zh_CN") +test(2141.2, rbind(DT,list(c=4L,a=7L)), error="第 2.*1.*c.*1") # arguments different order +Sys.setenv(LANGUAGE=old) +# this doesn't work for me and Chinese persists, hence putting this at the end of this script for now + + From 9b8e9683bd95189f5b70ec375179ad8b2bad2f9c Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Wed, 3 Jun 2020 22:18:50 -0600 Subject: [PATCH 02/16] removed chinese character for now, still test the changed argument order --- inst/tests/tests.Rraw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index e644f83a42..cc234de15d 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16916,7 +16916,7 @@ DT = data.table(a=1:3,b=4:6) test(2141.1, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") old = Sys.getenv("LANGUAGE") Sys.setenv(LANGUAGE="zh_CN") -test(2141.2, rbind(DT,list(c=4L,a=7L)), error="第 2.*1.*c.*1") # arguments different order +test(2141.2, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order Sys.setenv(LANGUAGE=old) # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now From 50bb912228f8b138a08fc9373309d5ab0c6c2b49 Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Wed, 3 Jun 2020 22:44:12 -0600 Subject: [PATCH 03/16] another attempt --- inst/tests/tests.Rraw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index cc234de15d..373ec7603a 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16915,7 +16915,7 @@ test(2140, dt[dt, on=.(time>time, v>v), .N, by=.EACHI], data.table(time=1:8, v=I DT = data.table(a=1:3,b=4:6) test(2141.1, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") old = Sys.getenv("LANGUAGE") -Sys.setenv(LANGUAGE="zh_CN") +Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "Chinese (Simplified)_China.936" else "zh_CN") test(2141.2, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order Sys.setenv(LANGUAGE=old) # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now From b4856837ecba109bb10447a1777fb7820d604728 Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Wed, 3 Jun 2020 23:19:09 -0600 Subject: [PATCH 04/16] more attempts --- inst/tests/tests.Rraw | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 373ec7603a..a611a1b9b0 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16917,7 +16917,22 @@ test(2141.1, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missi old = Sys.getenv("LANGUAGE") Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "Chinese (Simplified)_China.936" else "zh_CN") test(2141.2, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order +Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh-cn" else "zh_CN") +test(2141.3, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "chinese-simplified" else "zh_CN") +test(2141.4, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh-CN" else "zh_CN") +test(2141.5, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh_CN" else "zh_CN") +test(2141.6, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh" else "zh_CN") +test(2141.7, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "ZH" else "zh_CN") +test(2141.8, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh_cn" else "zh_CN") +test(2141.9, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") Sys.setenv(LANGUAGE=old) # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now + From 4e697d1dece70b14fb9826c832e3f3fe9cf5f89c Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Wed, 3 Jun 2020 23:47:55 -0600 Subject: [PATCH 05/16] more attempts --- inst/tests/tests.Rraw | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index a611a1b9b0..f3fdd1fc2f 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16913,24 +16913,21 @@ test(2140, dt[dt, on=.(time>time, v>v), .N, by=.EACHI], data.table(time=1:8, v=I # repeat of test 450 for #4402 DT = data.table(a=1:3,b=4:6) -test(2141.1, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") +test(2141.01, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") old = Sys.getenv("LANGUAGE") -Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "Chinese (Simplified)_China.936" else "zh_CN") -test(2141.2, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order -Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh-cn" else "zh_CN") -test(2141.3, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") -Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "chinese-simplified" else "zh_CN") -test(2141.4, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") -Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh-CN" else "zh_CN") -test(2141.5, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") -Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh_CN" else "zh_CN") -test(2141.6, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") -Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh" else "zh_CN") -test(2141.7, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") -Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "ZH" else "zh_CN") -test(2141.8, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") -Sys.setenv(LANGUAGE = if (.Platform$OS.type=="windows") "zh_cn" else "zh_CN") -test(2141.9, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +if (.Platform$OS.type=="windows") { + i = 2L + for (lang in c("Chinese_China.1252", "Chinese_China.936", "Chinese (Simplified)_China.936", "Chinese (Simplified)_China.1252", + "zh-cn", "zh-CN", "zh_CN", "zh", "ZH", "zh_cn", "Chinese - China", "Chinese_China", "Chinese-China")) { + Sys.setenv(LANGUAGE = lang) + Sys.setenv(LANG = lang) + test(2141+i/100, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order + i = i+1L + } +} else { + Sys.setenv(LANGUAGE = "zh_CN") + test(2141.02, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +} Sys.setenv(LANGUAGE=old) # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now From 326c0c5fd50f550e39b658e44ca6939335da50ca Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Thu, 4 Jun 2020 02:10:09 -0600 Subject: [PATCH 06/16] attempt to start R from R to set LANGUAGE before R starts on Windows --- inst/tests/tests.Rraw | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index f3fdd1fc2f..5fa2920327 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16926,10 +16926,12 @@ if (.Platform$OS.type=="windows") { } } else { Sys.setenv(LANGUAGE = "zh_CN") - test(2141.02, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") + test(2141.02, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # won't pass when run via cc though; TODO: turn off from cc() } Sys.setenv(LANGUAGE=old) # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now +print(.libPaths()) +test(2142, system2("R", input="q('no')", args="--no-save"), 0L) From b2433adff04aa3da3ebb8bd9afa109a7748a296a Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Thu, 4 Jun 2020 02:27:41 -0600 Subject: [PATCH 07/16] now try env= to system2 for LANGUAGE --- inst/tests/tests.Rraw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 5fa2920327..96067f17a9 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16932,6 +16932,6 @@ Sys.setenv(LANGUAGE=old) # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now print(.libPaths()) -test(2142, system2("R", input="q('no')", args="--no-save"), 0L) +test(2142, system2("R", input="q('no')", args="--no-save", env="LANGUAGE=zh_CN"), 0L) From d176990d977c988e5541e7a20a527ff178a42325 Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Thu, 4 Jun 2020 02:47:09 -0600 Subject: [PATCH 08/16] another attempt --- inst/tests/tests.Rraw | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 96067f17a9..c4418af179 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16915,13 +16915,19 @@ test(2140, dt[dt, on=.(time>time, v>v), .N, by=.EACHI], data.table(time=1:8, v=I DT = data.table(a=1:3,b=4:6) test(2141.01, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") old = Sys.getenv("LANGUAGE") +print(old) +print(.libPaths()) if (.Platform$OS.type=="windows") { i = 2L for (lang in c("Chinese_China.1252", "Chinese_China.936", "Chinese (Simplified)_China.936", "Chinese (Simplified)_China.1252", "zh-cn", "zh-CN", "zh_CN", "zh", "ZH", "zh_cn", "Chinese - China", "Chinese_China", "Chinese-China")) { - Sys.setenv(LANGUAGE = lang) - Sys.setenv(LANG = lang) - test(2141+i/100, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order + test(2141+i/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE='",lang,"'")), 0L) + test(2142+i/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL='",lang,"'")), 0L) + test(2143+i/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE=",lang)), 0L) + test(2144+i/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL=",lang)), 0L) + #Sys.setenv(LANGUAGE = lang) + #Sys.setenv(LANG = lang) + #test(2141+i/100, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order i = i+1L } } else { @@ -16931,7 +16937,7 @@ if (.Platform$OS.type=="windows") { Sys.setenv(LANGUAGE=old) # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now -print(.libPaths()) -test(2142, system2("R", input="q('no')", args="--no-save", env="LANGUAGE=zh_CN"), 0L) + + From 15b980256c3593ebc3e7f18c3f5586e94aef578e Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Thu, 4 Jun 2020 03:00:15 -0600 Subject: [PATCH 09/16] increasing test numbers --- inst/tests/tests.Rraw | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index c4418af179..71ed4b6574 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16913,26 +16913,27 @@ test(2140, dt[dt, on=.(time>time, v>v), .N, by=.EACHI], data.table(time=1:8, v=I # repeat of test 450 for #4402 DT = data.table(a=1:3,b=4:6) -test(2141.01, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") +test(2141, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") old = Sys.getenv("LANGUAGE") print(old) print(.libPaths()) if (.Platform$OS.type=="windows") { - i = 2L + i = 1L for (lang in c("Chinese_China.1252", "Chinese_China.936", "Chinese (Simplified)_China.936", "Chinese (Simplified)_China.1252", "zh-cn", "zh-CN", "zh_CN", "zh", "ZH", "zh_cn", "Chinese - China", "Chinese_China", "Chinese-China")) { - test(2141+i/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE='",lang,"'")), 0L) - test(2142+i/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL='",lang,"'")), 0L) - test(2143+i/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE=",lang)), 0L) - test(2144+i/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL=",lang)), 0L) + cat("Attempting LANGUAGE=",lang,"\n") + test(2142+(i+1L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE='",lang,"'")), 0L) + #test(2142+(i+2L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL='",lang,"'")), 0L) + test(2142+(i+3L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE=",lang)), 0L) + #test(2142+(i+4L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL=",lang)), 0L) #Sys.setenv(LANGUAGE = lang) #Sys.setenv(LANG = lang) #test(2141+i/100, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order - i = i+1L + i = i+4L } } else { Sys.setenv(LANGUAGE = "zh_CN") - test(2141.02, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # won't pass when run via cc though; TODO: turn off from cc() + test(2142, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # won't pass when run via cc though; TODO: turn off from cc() } Sys.setenv(LANGUAGE=old) # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now From 8982ab94f2d8bebc6857dfe688d2074fb7c27da3 Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Thu, 4 Jun 2020 03:31:21 -0600 Subject: [PATCH 10/16] more --- inst/tests/tests.Rraw | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 71ed4b6574..c8b88b42cc 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16919,13 +16919,13 @@ print(old) print(.libPaths()) if (.Platform$OS.type=="windows") { i = 1L - for (lang in c("Chinese_China.1252", "Chinese_China.936", "Chinese (Simplified)_China.936", "Chinese (Simplified)_China.1252", - "zh-cn", "zh-CN", "zh_CN", "zh", "ZH", "zh_cn", "Chinese - China", "Chinese_China", "Chinese-China")) { - cat("Attempting LANGUAGE=",lang,"\n") - test(2142+(i+1L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE='",lang,"'")), 0L) - #test(2142+(i+2L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL='",lang,"'")), 0L) + for (lang in c("Chinese_China.1252", "Chinese_China.936", + "zh-cn", "zh-CN", "zh_CN", "zh", "ZH", "zh_cn", "Chinese_China", "Chinese-China", + "chs", "chinese-simplified")) { + cat("Attempting LANGUAGE=",lang,"\n",sep="") test(2142+(i+3L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE=",lang)), 0L) - #test(2142+(i+4L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL=",lang)), 0L) + test(2142+(i+4L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_MESSAGES=",lang)), 0L) + test(2142+(i+4L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL=",lang)), 0L) #Sys.setenv(LANGUAGE = lang) #Sys.setenv(LANG = lang) #test(2141+i/100, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order From f25a3508706ef3fc2e69fb9d42b516b55333b490 Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Thu, 4 Jun 2020 03:35:03 -0600 Subject: [PATCH 11/16] more --- inst/tests/tests.Rraw | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index c8b88b42cc..c593a3a7a6 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16923,13 +16923,13 @@ if (.Platform$OS.type=="windows") { "zh-cn", "zh-CN", "zh_CN", "zh", "ZH", "zh_cn", "Chinese_China", "Chinese-China", "chs", "chinese-simplified")) { cat("Attempting LANGUAGE=",lang,"\n",sep="") - test(2142+(i+3L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE=",lang)), 0L) - test(2142+(i+4L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_MESSAGES=",lang)), 0L) - test(2142+(i+4L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL=",lang)), 0L) + test(2142+(i+1L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE=",lang)), 0L) + test(2142+(i+2L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_MESSAGES=",lang)), 0L) + test(2142+(i+3L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL=",lang)), 0L) #Sys.setenv(LANGUAGE = lang) #Sys.setenv(LANG = lang) #test(2141+i/100, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order - i = i+4L + i = i+3L } } else { Sys.setenv(LANGUAGE = "zh_CN") From 9beb371ddb85b4527bd6c398160abf33e7f15004 Mon Sep 17 00:00:00 2001 From: shrektan Date: Fri, 5 Jun 2020 01:03:36 +0800 Subject: [PATCH 12/16] try to display chinese error message, again --- inst/tests/tests.Rraw | 50 +++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index c593a3a7a6..e4da9ead2a 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16914,31 +16914,29 @@ test(2140, dt[dt, on=.(time>time, v>v), .N, by=.EACHI], data.table(time=1:8, v=I # repeat of test 450 for #4402 DT = data.table(a=1:3,b=4:6) test(2141, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") -old = Sys.getenv("LANGUAGE") -print(old) -print(.libPaths()) -if (.Platform$OS.type=="windows") { - i = 1L - for (lang in c("Chinese_China.1252", "Chinese_China.936", - "zh-cn", "zh-CN", "zh_CN", "zh", "ZH", "zh_cn", "Chinese_China", "Chinese-China", - "chs", "chinese-simplified")) { - cat("Attempting LANGUAGE=",lang,"\n",sep="") - test(2142+(i+1L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LANGUAGE=",lang)), 0L) - test(2142+(i+2L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_MESSAGES=",lang)), 0L) - test(2142+(i+3L)/100, system2("R", input="q('no')", args="--no-save", env=paste0("LC_ALL=",lang)), 0L) - #Sys.setenv(LANGUAGE = lang) - #Sys.setenv(LANG = lang) - #test(2141+i/100, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # arguments different order - i = i+3L +if (.Platform$OS.type=="windows") local({ + x = list( + LC_COLLATE = "Chinese (Simplified)_China.936", + LC_CTYPE = "Chinese (Simplified)_China.936", + LC_MONETARY = "Chinese (Simplified)_China.936", + LC_NUMERIC = "C", + LC_TIME = "Chinese (Simplified)_China.936" + ) + for (i in seq_along(x)) { + lc = names(x)[[i]] + old = Sys.getlocale(lc) + Sys.setlocale(lc, x[[i]]) + on.exit(Sys.setlocale(lc, old), add = TRUE) } -} else { - Sys.setenv(LANGUAGE = "zh_CN") - test(2142, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # won't pass when run via cc though; TODO: turn off from cc() -} -Sys.setenv(LANGUAGE=old) -# this doesn't work for me and Chinese persists, hence putting this at the end of this script for now - - - - + old = Sys.getenv('LANGUAGE') + Sys.setenv('LANGUAGE' = 'zh_CN') + on.exit({ + if (nzchar(old)) + Sys.setenv('LANGUAGE' = old) + else + Sys.unsetenv('LANGUAGE') + }, add = TRUE) + test(2142, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") +}) +# this doesn't work for me and Chinese persists, hence putting this at the end of this script for now From e4635fa868077de7a1c9bfed21910d5ceb396c6f Mon Sep 17 00:00:00 2001 From: shrektan Date: Fri, 5 Jun 2020 01:22:30 +0800 Subject: [PATCH 13/16] nice to have a Chinese message so that we know it works --- inst/tests/tests.Rraw | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index e4da9ead2a..0151595a77 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16936,6 +16936,9 @@ if (.Platform$OS.type=="windows") local({ else Sys.unsetenv('LANGUAGE') }, add = TRUE) + # to see if it's Chinese message that's printed + try(rbindlist(list(a = 1), list(b = 1, c = 2))) + # should trigger segfault here test(2142, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") }) From 7f78ad8680770fa0c4df703febd02e84cb3e288f Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Thu, 4 Jun 2020 15:48:11 -0600 Subject: [PATCH 14/16] see if we can compile & link to R's trio_vsnprintf; if so, will need to massage va_list next --- src/data.table.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data.table.h b/src/data.table.h index 1cf975e68b..cd58403d26 100644 --- a/src/data.table.h +++ b/src/data.table.h @@ -14,7 +14,7 @@ #include "types.h" #include "po.h" #ifdef WIN32 // positional specifiers (%n$) used in translations; #4402 -//# define snprintf _sprintf_p // the non-n one in Windows takes n anyway so there's no separate _snprintf_f +# define snprintf trio_vsnprintf // the non-n one in Windows takes n anyway so there's no separate _snprintf_f #endif #define sprintf USE_SNPRINTF_NOT_SPRINTF // prevent use of sprintf in data.table source; force us to use n always From da797d5dfdd1d754ebffea7cf444c7f3f7f1fdf5 Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Fri, 5 Jun 2020 00:35:47 -0600 Subject: [PATCH 15/16] created snprintf wrapper to support %n$ on Windows using C99-only features; no compile options or libraries needed --- inst/tests/tests.Rraw | 9 ++-- src/data.table.h | 6 ++- src/dt_stdio.h | 1 - src/init.c | 2 + src/snprintf.c | 103 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 116 insertions(+), 5 deletions(-) create mode 100644 src/snprintf.c diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 0151595a77..5d2d521473 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -25,6 +25,7 @@ if (exists("test.data.table", .GlobalEnv, inherits=FALSE)) { binary = data.table:::binary bmerge = data.table:::bmerge brackify = data.table:::brackify + Ctest_dt_win_snprintf = data.table:::Ctest_dt_win_snprintf chmatchdup = data.table:::chmatchdup compactprint = data.table:::compactprint cube.data.table = data.table:::cube.data.table @@ -16912,8 +16913,9 @@ dt[time == 7L, v := 7L] test(2140, dt[dt, on=.(time>time, v>v), .N, by=.EACHI], data.table(time=1:8, v=INT(5,2,6,1,8,4,7,3), N=INT(3,5,2,4,0,1,0,0))) # repeat of test 450 for #4402 +test(2141, .Call(Ctest_dt_win_snprintf), NULL) DT = data.table(a=1:3,b=4:6) -test(2141, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") +test(2142, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") if (.Platform$OS.type=="windows") local({ x = list( LC_COLLATE = "Chinese (Simplified)_China.936", @@ -16938,8 +16940,9 @@ if (.Platform$OS.type=="windows") local({ }, add = TRUE) # to see if it's Chinese message that's printed try(rbindlist(list(a = 1), list(b = 1, c = 2))) - # should trigger segfault here - test(2142, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") + # triggered segfault here in #4402, Windows-only under translation + test(2143, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # argument order changes under translation }) +test(2144, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") # test back to English # this doesn't work for me and Chinese persists, hence putting this at the end of this script for now diff --git a/src/data.table.h b/src/data.table.h index cd58403d26..aff0088ac0 100644 --- a/src/data.table.h +++ b/src/data.table.h @@ -14,7 +14,7 @@ #include "types.h" #include "po.h" #ifdef WIN32 // positional specifiers (%n$) used in translations; #4402 -# define snprintf trio_vsnprintf // the non-n one in Windows takes n anyway so there's no separate _snprintf_f +# define snprintf dt_win_snprintf // see our snprintf.c; tried and failed to link to _sprintf_p on Windows #endif #define sprintf USE_SNPRINTF_NOT_SPRINTF // prevent use of sprintf in data.table source; force us to use n always @@ -243,3 +243,7 @@ SEXP testMsgR(SEXP status, SEXP x, SEXP k); //fifelse.c SEXP fifelseR(SEXP l, SEXP a, SEXP b, SEXP na); SEXP fcaseR(SEXP na, SEXP rho, SEXP args); + +//snprintf.c +int dt_win_snprintf(char *dest, size_t n, const char *fmt, ...); + diff --git a/src/dt_stdio.h b/src/dt_stdio.h index f652da9805..4e69e0d87e 100644 --- a/src/dt_stdio.h +++ b/src/dt_stdio.h @@ -23,7 +23,6 @@ #define DT_STDIO_H #if defined(__MINGW32__) || (defined __MINGW64__) #define __USE_MINGW_ANSI_STDIO 1 - #define _XOPEN_SOURCE 1 #include #define PRId64 "lld" #define PRIu64 "llu" diff --git a/src/init.c b/src/init.c index 916db3ab57..d650a64661 100644 --- a/src/init.c +++ b/src/init.c @@ -119,6 +119,7 @@ SEXP lock(); SEXP unlock(); SEXP islockedR(); SEXP allNAR(); +SEXP test_dt_win_snprintf(); // .Externals SEXP fastmean(); @@ -211,6 +212,7 @@ R_CallMethodDef callMethods[] = { {"CfrollapplyR", (DL_FUNC) &frollapplyR, -1}, {"CtestMsgR", (DL_FUNC) &testMsgR, -1}, {"C_allNAR", (DL_FUNC) &allNAR, -1}, +{"Ctest_dt_win_snprintf", (DL_FUNC)&test_dt_win_snprintf, -1}, {NULL, NULL, 0} }; diff --git a/src/snprintf.c b/src/snprintf.c new file mode 100644 index 0000000000..d8d1c83370 --- /dev/null +++ b/src/snprintf.c @@ -0,0 +1,103 @@ +// For translations (#4402) we need positional specifiers (%n$), a non-C99 POSIX extension. +// On Linux and Mac, standard snprintf supports positional specifiers. +// On Windows, we tried many things but just couldn't achieve it. This may be why R uses +// a third party library, trio, on Windows. But R does not expose trio for use by packages. +// So ... +// Rather than require compile flags (such as _XOPEN_SOURCE or POSIX_C_SOURCE), or require +// linking to particular Windows libraries which may be fragile over time depending on +// user's environments, we use the standard C99 features here. To do so, we simulate +// positionals via format massage. Just on Windows, all snprintf calls are replaced with +// this dt_win_snprintf via a #define in data.table.h. The goal of this massage is to be +// as light and minimal as possible. +// In C is it is impossible, portably, to reorder a va_list (sadly). +// In C you must past the correct type to va_arg(), so even to navigate va_list you +// must parse and rely on fmt. But we don't want to reimplement all the types and modifiers. +// Hence, reordering the specifiers, passing the va_list to the library, and then +// putting the output strings into the desired order afterwards. +// NB: must be thread-safe + +#include "data.table.h" +#include +#undef snprintf // on Windows, just in this file, we do want to use the C library's snprintf + +int dt_win_snprintf(char *dest, size_t n, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + const char *ch = strstr(fmt, "%1$"); + if (ch==NULL) { + // no positionals present, just pass on to the C library vsnprintf as-is + int ans = vsnprintf(dest, n, fmt, ap); + va_end(ap); + return ans; + } + // Standards say that if one specifier uses position, they all must. Good. + // We will not allow repeats though; must be a permutation. + // As in C, there are few checks; wrong/mismatching positionals will be a crash. + // This is for messages/errors, so time should not be spent on a fast solution. + char *buff = (char *)malloc(n); // not R_alloc as we need to be thread-safe + if (!buff) error("Unable to allocate %d bytes for buffer in dt_win_snprintf", n); + int pos=1; + // Use dest as temp to write the reordered specifiers + char *ch2=dest; + #define NDELIM 2 + const char delim[NDELIM+1] = "\x7f\x7f"; // tokenize using 2 DELs + while (ch!=NULL) { // ch is resting on start of %pos$ in fmt + // Find end of %[parameter][flags][width][.precision][length]type + // https://en.wikipedia.org/wiki/Printf_format_string#Syntax + const char *start = strchr(ch, '$')+1; // look for $ since pos could be > 9 or potentially > 99 + const char *end = strpbrk(start,"diufFeEgGxXoscpaA"); // last character of specifier + *ch2++ = '%'; + strncpy(ch2, start, end-start+1); // write the specifer in order without the n$ part + ch2 += end-start+1; + strcpy(ch2, delim); // includes '\0' + ch2 += NDELIM; // now resting on the '\0' + char posstr[15]; // 15 to avoid C compiler warnings + snprintf(posstr, 15, "%%%d$", ++pos); // snprintf was #undef above, so this is the C library one + ch = strstr(fmt, posstr); + } + int narg = pos-1; + vsnprintf(buff, n, dest, ap); // dest used as tmp here, holds reordered specifiers same order as arglist + // All the hard formatting work and va_arg type navigation has now been done by the C library + // Now we just need to put the string results for each argument back into the desired positions + // First create lookups so we can loop through fmt once replacing the specifiers as they appear + const char *arg[narg]; + int len[narg]; + ch = buff; + for (int i=0; i'9') error("When positional %n$ is used, all specifiers must include positional"); + int pos = atoi(ch+1); + ch = strpbrk(ch,"diufFeEgGxXoscpaA")+1; // move to the end of the specifier + strncpy(ch2, arg[pos-1], len[pos-1]); // write the result of the appropriate argument + ch2 += len[pos-1]; + } + *ch2='\0'; + free(buff); + va_end(ap); + return ch2-dest; +} + +SEXP test_dt_win_snprintf() +{ + char buff[50]; + dt_win_snprintf(buff, 50, "No pos %d%%%d ok", 42, -84); + if (strcmp(buff, "No pos 42%-84 ok")) error("dt_win_snprintf test 1 failed: %s", buff); + dt_win_snprintf(buff, 50, "With pos %1$d%%%2$d ok", 42, -84); + if (strcmp(buff, "With pos 42%-84 ok")) error("dt_win_snprintf test 2 failed: %s", buff); + dt_win_snprintf(buff, 50, "With pos %2$d%%%1$d ok", 42, -84); + if (strcmp(buff, "With pos -84%42 ok")) error("dt_win_snprintf test 3 failed: %s", buff); + dt_win_snprintf(buff, 50, "%3$s %1$d %4$10s %2$03d$", -99, 12, "hello%2$d", "short"); + if (strcmp(buff, "hello%2$d -99 short 012$")) error("dt_win_snprintf test 4 failed: %s", buff); + return R_NilValue; +} + From 636843fdedef63bcc378d037d7355d30e5a3949e Mon Sep 17 00:00:00 2001 From: Matt Dowle Date: Fri, 5 Jun 2020 01:00:39 -0600 Subject: [PATCH 16/16] tidy --- inst/tests/tests.Rraw | 13 +++++++------ src/snprintf.c | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 5d2d521473..e28885fb4e 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -16938,11 +16938,12 @@ if (.Platform$OS.type=="windows") local({ else Sys.unsetenv('LANGUAGE') }, add = TRUE) - # to see if it's Chinese message that's printed - try(rbindlist(list(a = 1), list(b = 1, c = 2))) - # triggered segfault here in #4402, Windows-only under translation - test(2143, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") # argument order changes under translation + # triggered segfault here in #4402, Windows-only under translation. + # test that the argument order changes correctly (the 'item 2' moves to the beginning of the message) + # since the argument order changes in this example (and that was the crash) we don't need to test + # the display of the Chinese characters here. Thanks to @shrektan for all his help on this. + test(2143, rbind(DT,list(c=4L,a=7L)), error="2.*1.*c.*1") }) -test(2144, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") # test back to English +# test back to English (the argument order is back to 1,c,2,1) +test(2144, rbind(DT,list(c=4L,a=7L)), error="Column 1 ['c'] of item 2 is missing in item 1") -# this doesn't work for me and Chinese persists, hence putting this at the end of this script for now diff --git a/src/snprintf.c b/src/snprintf.c index d8d1c83370..44f7848d6a 100644 --- a/src/snprintf.c +++ b/src/snprintf.c @@ -9,7 +9,7 @@ // positionals via format massage. Just on Windows, all snprintf calls are replaced with // this dt_win_snprintf via a #define in data.table.h. The goal of this massage is to be // as light and minimal as possible. -// In C is it is impossible, portably, to reorder a va_list (sadly). +// In C it is not possible, portably, to reorder a va_list (sadly). // In C you must past the correct type to va_arg(), so even to navigate va_list you // must parse and rely on fmt. But we don't want to reimplement all the types and modifiers. // Hence, reordering the specifiers, passing the va_list to the library, and then @@ -57,7 +57,7 @@ int dt_win_snprintf(char *dest, size_t n, const char *fmt, ...) ch = strstr(fmt, posstr); } int narg = pos-1; - vsnprintf(buff, n, dest, ap); // dest used as tmp here, holds reordered specifiers same order as arglist + vsnprintf(buff, n, dest, ap); // dest used as tmp here, holds reordered specifiers same order as ap // All the hard formatting work and va_arg type navigation has now been done by the C library // Now we just need to put the string results for each argument back into the desired positions // First create lookups so we can loop through fmt once replacing the specifiers as they appear