From e586bed8de9f2d0267a9534d679c3114b6f404b9 Mon Sep 17 00:00:00 2001 From: Benjamin Schwendinger <52290390+ben-schwen@users.noreply.github.com> Date: Mon, 1 Nov 2021 15:27:41 +0100 Subject: [PATCH 1/8] add fix --- NEWS.md | 2 + R/data.table.R | 19 +++- inst/tests/programming.Rraw | 2 + inst/tests/tests.Rraw | 174 +++++++++++++++++++++++++++++++----- 4 files changed, 171 insertions(+), 26 deletions(-) diff --git a/NEWS.md b/NEWS.md index 5faf40723f..ff4c4a9596 100644 --- a/NEWS.md +++ b/NEWS.md @@ -207,6 +207,8 @@ # v1.14.4 0.4826 0.5586 0.6586 0.6329 0.7348 1.318 100 ``` +31. `:=` works now with GForce optimization, [#1414](https://github.com/Rdatatable/data.table/issues/1414). Thanks to Arun Srinivasan for reporting, and Benjamin Schwendinger for the PR. + ## BUG FIXES 1. `by=.EACHI` when `i` is keyed but `on=` different columns than `i`'s key could create an invalidly keyed result, [#4603](https://github.com/Rdatatable/data.table/issues/4603) [#4911](https://github.com/Rdatatable/data.table/issues/4911). Thanks to @myoung3 and @adamaltmejd for reporting, and @ColeMiller1 for the PR. An invalid key is where a `data.table` is marked as sorted by the key columns but the data is not sorted by those columns, leading to incorrect results from subsequent queries. diff --git a/R/data.table.R b/R/data.table.R index e020ea3e3d..5736b330ff 100644 --- a/R/data.table.R +++ b/R/data.table.R @@ -1729,13 +1729,15 @@ replace_dot_alias = function(e) { dotN = function(x) is.name(x) && x==".N" # For #334. TODO: Rprof() showed dotN() may be the culprit if iterated (#1470)?; avoid the == which converts each x to character? # FR #971, GForce kicks in on all subsets, no joins yet. Although joins could work with # nomatch=NULL even now.. but not switching it on yet, will deal it separately. - if (getOption("datatable.optimize")>=2L && !is.data.table(i) && !byjoin && length(f__) && !length(lhs)) { + if (getOption("datatable.optimize")>=2L && !is.data.table(i) && !byjoin && length(f__)) { if (!length(ansvars) && !use.I) { GForce = FALSE - if ( (is.name(jsub) && jsub==".N") || (jsub %iscall% 'list' && length(jsub)==2L && jsub[[2L]]==".N") ) { + if ( ((is.name(jsub) && jsub==".N") || (jsub %iscall% 'list' && length(jsub)==2L && jsub[[2L]]==".N")) && !length(lhs) ) { GForce = TRUE if (verbose) catf("GForce optimized j to '%s'\n",deparse(jsub, width.cutoff=200L, nlines=1L)) } + } else if (length(lhs) && is.symbol(jsub)) { # turn off GForce for the combination of := and .N + GForce = FALSE } else { # Apply GForce .gforce_ok = function(q) { @@ -1773,7 +1775,8 @@ replace_dot_alias = function(e) { # adding argument to ghead/gtail if none is supplied to g-optimized head/tail if (length(jsub) == 2L && jsub[[1L]] %chin% c("head", "tail")) jsub[["n"]] = 6L jsub[[1L]] = as.name(paste0("g", jsub[[1L]])) - if (length(jsub)==3L) jsub[[3L]] = eval(jsub[[3L]], parent.frame()) # tests 1187.3 & 1187.5 + # check if function arguments are symbols and if so evaluate them + if (length(jsub)>=3L && is.symbol(jsub[[3L]]) && exists(jsub[[3L]], parent.frame())) jsub[[3L]] = eval(jsub[[3L]], parent.frame()) # tests 1187.3 & 1187.5 & programming 101.17 } if (verbose) catf("GForce optimized j to '%s'\n", deparse(jsub, width.cutoff=200L, nlines=1L)) } else if (verbose) catf("GForce is on, left j unchanged\n"); @@ -1905,6 +1908,15 @@ replace_dot_alias = function(e) { # Grouping by by: i is by val, icols NULL, o__ may be subset of x, f__ points to o__ (or x if !length o__) # TO DO: setkey could mark the key whether it is unique or not. if (!is.null(lhs)) { + if (GForce) { # GForce should work with := #1414 + vlen = length(ans[[1L]]) + # replicate vals if GForce returns 1 value per group + jvals = if (vlen==length(len__)) lapply(ans[-1], rep.int, times=len__) else ans[-1] + jrows = if (!is.null(irows) && length(irows)!=length(o__)) irows else { if (length(o__)==0L) NULL else o__} + # unwrap single column jvals for assign + if (length(jvals)==1L) jvals = jvals[[1L]] + .Call(Cassign, x, jrows, lhs, newnames, jvals) + } if (any(names_x[cols] %chin% key(x))) setkey(x,NULL) # fixes #1479. Take care of secondary indices, TODO: cleaner way of doing this @@ -1925,6 +1937,7 @@ replace_dot_alias = function(e) { } else warningf("The setkey() normally performed by keyby= has been skipped (as if by= was used) because := is being used together with keyby= but the keyby= contains some expressions. To avoid this warning, use by= instead, or provide existing column names to keyby=.\n") } + if (GForce) return(suppPrint(x)) return(suppPrint(x)) } if (is.null(ans)) { diff --git a/inst/tests/programming.Rraw b/inst/tests/programming.Rraw index 429545dcb7..6b586ee6aa 100644 --- a/inst/tests/programming.Rraw +++ b/inst/tests/programming.Rraw @@ -536,7 +536,9 @@ ycols = paste0("y", yn) ydt = data.table(symbol = rep(1:3, each = 100)) ydt[, date := seq_len(.N), by = symbol] ydt[, ret := rnorm(.N)] +#options(datatable.optimize=1L) ydt[, (ycols) := shift(ret, yn, type = "lead"), by = symbol] +#options(datatable.optimize=2L) xdt = data.table(symbol = rep(1:2, each = 20)) xdt[, date := seq_len(.N), by = symbol] xdt[, `:=`(x1 = rnorm(.N), x2 = rnorm(.N))] diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index 6382a13a85..c05b63fb07 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -1808,25 +1808,41 @@ test(610.3, chorder(x), base::order(x)) test(610.4, unique(x[chgroup(x)]), unique(x)) # := by group +options(datatable.optimize=0L) +DT = data.table(a=1:3,b=(1:9)/10) +test(611.1, DT[,v:=sum(b),by=a], data.table(a=1:3,b=(1:9)/10,v=c(1.2,1.5,1.8))) +setkey(DT,a) +test(611.2, DT[,v:=min(b),by=a], data.table(a=1:3,b=(1:9)/10,v=(1:3)/10,key="a")) +# Combining := by group with i +test(611.3, DT[a>1,p:=sum(b)]$p, rep(c(NA,3.3),c(3,6))) +test(611.4, DT[a>1,q:=sum(b),by=a]$q, rep(c(NA,1.5,1.8),each=3)) +options(datatable.optimize=2L) DT = data.table(a=1:3,b=(1:9)/10) -test(611, DT[,v:=sum(b),by=a], data.table(a=1:3,b=(1:9)/10,v=c(1.2,1.5,1.8))) +test(612.1, DT[,v:=sum(b),by=a], data.table(a=1:3,b=(1:9)/10,v=c(1.2,1.5,1.8))) setkey(DT,a) -test(612, DT[,v:=min(b),by=a], data.table(a=1:3,b=(1:9)/10,v=(1:3)/10,key="a")) +test(612.2, DT[,v:=min(b),by=a], data.table(a=1:3,b=(1:9)/10,v=(1:3)/10,key="a")) +# Combining := by group with i +test(612.3, DT[a>1,p:=sum(b)]$p, rep(c(NA,3.3),c(3,6))) +test(612.4, DT[a>1,q:=sum(b),by=a]$q, rep(c(NA,1.5,1.8),each=3)) # Assign to subset ok (NA initialized in the other items) ok : test(613, DT[J(2),w:=8.3]$w, rep(c(NA,8.3,NA),each=3)) test(614, DT[J(3),x:=9L]$x, rep(c(NA_integer_,NA_integer_,9L),each=3)) test(615, DT[J(2),z:=list(list(c(10L,11L)))]$z, rep(list(NULL, 10:11, NULL),each=3)) -# Combining := by group with i -test(616, DT[a>1,p:=sum(b)]$p, rep(c(NA,3.3),c(3,6))) -test(617, DT[a>1,q:=sum(b),by=a]$q, rep(c(NA,1.5,1.8),each=3)) # Empty i clause, #2034. Thanks to Chris for testing, tests from him. Plus changes from #759 ans = copy(DT)[,r:=NA_real_] -test(618, copy(DT)[a>3,r:=sum(b)], ans) -test(619, copy(DT)[J(-1),r:=sum(b)], ans) -test(620.1, copy(DT)[NA,r:=sum(b)], ans) -test(620.2, copy(DT)[0,r:=sum(b)], ans) -test(620.3, copy(DT)[NULL,r:=sum(b)], null.data.table()) +options(datatable.optimize=0L) +test(618.1, copy(DT)[a>3,r:=sum(b)], ans) +test(618.2, copy(DT)[J(-1),r:=sum(b)], ans) +test(618.3, copy(DT)[NA,r:=sum(b)], ans) +test(618.4, copy(DT)[0,r:=sum(b)], ans) +test(618.5, copy(DT)[NULL,r:=sum(b)], null.data.table()) +options(datatable.optimize=2L) +test(619.1, copy(DT)[a>3,r:=sum(b)], ans) +test(619.2, copy(DT)[J(-1),r:=sum(b)], ans) +test(619.3, copy(DT)[NA,r:=sum(b)], ans) +test(619.4, copy(DT)[0,r:=sum(b)], ans) +test(619.5, copy(DT)[NULL,r:=sum(b)], null.data.table()) DT = data.table(x=letters, key="x") test(621, copy(DT)[J("bb"), x:="foo"], DT) # when no update, key should be retained @@ -1834,7 +1850,10 @@ test(622, copy(DT)[J("bb"), x:="foo",nomatch=0], DT, warning="ignoring nomatch") set.seed(2) DT = data.table(a=rnorm(5)*10, b=1:5) -test(623, DT[,s:=sum(b),by=round(a)%%2]$s, c(10L,5L,5L,10L,10L)) +options(datatable.optimize=0L) +test(623.1, copy(DT)[,s:=sum(b),by=round(a)%%2]$s, c(10L,5L,5L,10L,10L)) +options(datatable.optimize=2L) +test(623.2, copy(DT)[,s:=sum(b),by=round(a)%%2]$s, c(10L,5L,5L,10L,10L)) # Tests on POSIXct attributes @@ -1889,12 +1908,20 @@ test(635, names(DT[,list(x,y,a=y)]), c("x","y","a")) test(636, names(DT[,list(x,a=y)]), c("x","a")) # Test := by key, and that := to the key by key unsets the key. Make it non-trivial in size too. +options(datatable.optimize=0L) set.seed(1) DT = data.table(a=sample(1:100,1e6,replace=TRUE),b=sample(1:1000,1e6,replace=TRUE),key="a") -test(637, DT[,m:=sum(b),by=a][1:3], data.table(a=1L,b=c(156L,808L,848L),m=DT[J(1),sum(b)],key="a")) -test(638, key(DT[J(43L),a:=99L]), NULL) +test(637.1, DT[,m:=sum(b),by=a][1:3], data.table(a=1L,b=c(156L,808L,848L),m=DT[J(1),sum(b)],key="a")) +test(637.2, key(DT[J(43L),a:=99L]), NULL) setkey(DT,a) -test(639, key(DT[,a:=99L,by=a]), NULL) +test(637.3, key(DT[,a:=99L,by=a]), NULL) +options(datatable.optimize=2L) +set.seed(1) +DT = data.table(a=sample(1:100,1e6,replace=TRUE),b=sample(1:1000,1e6,replace=TRUE),key="a") +test(638.1, DT[,m:=sum(b),by=a][1:3], data.table(a=1L,b=c(156L,808L,848L),m=DT[J(1),sum(b)],key="a")) +test(638.2, key(DT[J(43L),a:=99L]), NULL) +setkey(DT,a) +test(638.3, key(DT[,a:=99L,by=a]), NULL) # Test printing is right aligned without quotes etc, and rownames are repeated ok for more than 20 rows DT=data.table(a=8:10,b=c("xy","x","xyz"),c=c(1.1,22.1,0)) @@ -1986,18 +2013,32 @@ test(668, DT[a<3,sum(b),by=eval(paste("a"))], DT[a<3,sum(b),by=a]) test(669, DT[a<3,sum(b),by=c(2)], error="must evaluate to 'character'") # Test := keyby does setkey, #2065 +options(datatable.optimize=0L) DT = data.table(x=1:2, y=1:6) ans = data.table(x=rep(1:2,each=3),y=c(1L,3L,5L,2L,4L,6L),z=rep(c(9L,12L),each=3),key="x") -test(670, DT[,z:=sum(y),keyby=x], ans) +test(670.1, DT[,z:=sum(y),keyby=x], ans) DT = data.table(x=1:2, y=1:6) -test(671, DT[,z:=sum(y),keyby="x"], ans) +test(670.2, DT[,z:=sum(y),keyby="x"], ans) DT = data.table(x=1:2, y=1:6) -test(672, DT[,z:=sum(y),keyby=x%%2], data.table(x=1:2,y=1:6,z=c(9L,12L)), +test(670.3, DT[,z:=sum(y),keyby=x%%2], data.table(x=1:2,y=1:6,z=c(9L,12L)), warning="The setkey() normally performed by keyby= has been skipped (as if by= was used) because := is being used together with keyby= but the keyby= contains some expressions. To avoid this warning, use by= instead, or provide existing column names to keyby=") DT = data.table(x=1:2, y=1:6) -test(673, DT[,z:=sum(y),by=x%%2], data.table(x=1:2,y=1:6,z=c(9L,12L))) +test(670.4, DT[,z:=sum(y),by=x%%2], data.table(x=1:2,y=1:6,z=c(9L,12L))) +DT = data.table(x=1:2, y=1:6) +test(670.5, DT[x>1,z:=sum(y),keyby=x], error=":= with keyby is only possible when i is not supplied since") +options(datatable.optimize=2L) +DT = data.table(x=1:2, y=1:6) +ans = data.table(x=rep(1:2,each=3),y=c(1L,3L,5L,2L,4L,6L),z=rep(c(9L,12L),each=3),key="x") +test(671.1, DT[,z:=sum(y),keyby=x], ans) +DT = data.table(x=1:2, y=1:6) +test(671.2, DT[,z:=sum(y),keyby="x"], ans) +DT = data.table(x=1:2, y=1:6) +test(671.3, DT[,z:=sum(y),keyby=x%%2], data.table(x=1:2,y=1:6,z=c(9L,12L)), + warning="The setkey() normally performed by keyby= has been skipped (as if by= was used) because := is being used together with keyby= but the keyby= contains some expressions. To avoid this warning, use by= instead, or provide existing column names to keyby=") DT = data.table(x=1:2, y=1:6) -test(674, DT[x>1,z:=sum(y),keyby=x], error=":= with keyby is only possible when i is not supplied since") +test(671.4, DT[,z:=sum(y),by=x%%2], data.table(x=1:2,y=1:6,z=c(9L,12L))) +DT = data.table(x=1:2, y=1:6) +test(671.5, DT[x>1,z:=sum(y),keyby=x], error=":= with keyby is only possible when i is not supplied since") # Test new .() DT = data.table(x=1:2, y=1:6, key="x") @@ -2176,13 +2217,22 @@ test(749, DT[,c("c","d","e"):=list(.N,sum(b),a*10L),by=a], data.table(a=rep(6:8, test(750, DT[a<8,`:=`(f=b+sum(d),g=.N),by=c][,6:7,with=FALSE], data.table(f=INT(2,12,13,NA,NA,NA),g=INT(1,2,2,NA,NA,NA))) # varname holding colnames, by group, linked from #2120. +options(datatable.optimize=0L) +DT = data.table(a=rep(1:3,1:3),b=1:6) +colname = "newcol" +test(751.1, DT[,(colname):=sum(b),by=a], data.table(a=rep(1:3,1:3),b=1:6,newcol=INT(1,5,5,15,15,15))) +options(datatable.optimize=2L) DT = data.table(a=rep(1:3,1:3),b=1:6) colname = "newcol" -test(751, DT[,(colname):=sum(b),by=a], data.table(a=rep(1:3,1:3),b=1:6,newcol=INT(1,5,5,15,15,15))) +test(751.2, DT[,(colname):=sum(b),by=a], data.table(a=rep(1:3,1:3),b=1:6,newcol=INT(1,5,5,15,15,15))) # Add tests for nested := in j by group, #1987 +options(datatable.optimize=0L) DT = data.table(a=rep(1:3,2:4),b=1:9) -test(752, DT[,head(.SD,2)[,new:=1:.N],by=a], data.table(a=rep(1:3,each=2),b=c(1:4,6:7),new=1:2)) +test(752.1, DT[,head(.SD,2)[,new:=1:.N],by=a], data.table(a=rep(1:3,each=2),b=c(1:4,6:7),new=1:2)) +options(datatable.optimize=2L) +DT = data.table(a=rep(1:3,2:4),b=1:9) +test(752.2, DT[,head(.SD,2)[,new:=1:.N],by=a], data.table(a=rep(1:3,each=2),b=c(1:4,6:7),new=1:2)) # Test duplicate() of recycled plonking RHS, #2298 DT = data.table(a=letters[3:1],x=1:3) @@ -3844,7 +3894,10 @@ test(1133.3, DT[, new := c(1,2), by=x], error="Supplied 2 items to be assigned test(1133.4, DT[, new := c(1L,2L), by=x], error="Supplied 2 items to be assigned to group 1 of size 5 in column 'new'") test(1133.5, DT, data.table(x=INT(1,1,1,1,1,2,2), new=99L)) test(1133.6, DT[, new := rep(-.GRP, .N), by=x], data.table(x=INT(1,1,1,1,1,2,2), new=INT(-1,-1,-1,-1,-1,-2,-2))) +options(datatable.optimize=0L) test(1133.7, DT[, new := .N, by=x], data.table(x=INT(1,1,1,1,1,2,2), new=INT(5,5,5,5,5,2,2))) +options(datatable.optimize=2L) +test(1133.75, DT[, new := .N, by=x], data.table(x=INT(1,1,1,1,1,2,2), new=INT(5,5,5,5,5,2,2))) # on a new column with warning on 2nd assign DT[,new:=NULL] test(1133.8, DT[, new := if (.GRP==1L) 7L else 3.4, by=x], data.table(x=INT(1,1,1,1,1,2,2), new=INT(7,7,7,7,7,3,3)), @@ -3935,8 +3988,10 @@ DT<-data.table(X=factor(2006:2012),Y=rep(1:7,2)) test(1143.2, DT[, Z:=paste(X,.N,sep=" - "), by=list(X)], data.table(X=factor(2006:2012),Y=rep(1:7,2), Z=paste(as.character(2006:2012), 2L, sep=" - "))) DT = data.table(x=as.POSIXct(c("2009-02-17 17:29:23.042", "2009-02-17 17:29:25.160")), y=c(1L,2L)) test(1143.3, DT[, list(lx=x[.N]), by=x], data.table(x=DT$x, lx=DT$x)) -ans = copy(DT) -test(1143.4, DT[,`:=`(lx=tail(x,1L)), by=y], ans[, lx := x]) +options(datatable.optimize=0L) +test(1143.4, copy(DT)[,`:=`(lx=tail(x,1L)), by=y], copy(DT)[, lx := x]) +options(datatable.optimize=2L) +test(1143.5, copy(DT)[,`:=`(lx=tail(x,1L)), by=y], copy(DT)[, lx := x]) # FR #2356 - retain names of named vector as column with keep.rownames=TRUE x <- 1:5 @@ -14206,9 +14261,14 @@ x <- as.array(1:5) test(1980, names(data.table(x)), "x") # crash when n="lead", #3354 +options(datatable.optimize=0L) DT = data.table( id = 1:5 , val = letters[1:5] ) test(1981.1, DT[, new_col := shift(val, "lead")], error="is.numeric(n) is not TRUE") test(1981.2, DT[, new_col := shift(val, NA_integer_)], error="Item 1 of n is NA") +options(datatable.optimize=Inf) +DT = data.table( id = 1:5 , val = letters[1:5] ) +test(1981.3, DT[, new_col := shift(val, "lead")], error="is.numeric(n) is not TRUE") +test(1981.4, DT[, new_col := shift(val, NA_integer_)], error="Item 1 of n is NA") # print of DT with many columns reordered them, #3306. DT = as.data.table(lapply(1:255, function(i)rep.int(i, 105L))) # 105 to be enough for 'top 5 ... bottom 5' to print @@ -18348,3 +18408,71 @@ test(2225.1, groupingsets(data.table(iris), j=sum(Sepal.Length), by=c('Sp'='Spec test(2225.2, groupingsets(data.table(iris), j=mean(Sepal.Length), by=c('Sp'='Species'), sets=list('Species')), groupingsets(data.table(iris), j=mean(Sepal.Length), by=c('Species'), sets=list('Species'))) + +# support := with GForce +options(datatable.optimize = 2L) +DT = data.table(a=1:3,b=(1:9)/10) +test(2226.01, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:3)/10), output="GForce TRUE") +# GForce returning full length +test(2226.02, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10), output="GForce TRUE") +# GForce neither returning 1 per group nor full length +test(2226.03, DT[, v := head(b, 2L), a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") +# compare to non GForce version +DT = data.table(a=1:3,b=(1:9)/10) +test(2226.04, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce TRUE") +test(2226.05, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce TRUE") + +# with key and grouping by key +DT = data.table(a=1:3,b=(1:9)/10, key="a") +test(2226.06, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:3)/10, key="a"), output="GForce TRUE") +test(2226.07, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10, key="a"), output="GForce TRUE") +test(2226.08, DT[, v := head(b, 2L), a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") +DT = data.table(a=1:3,b=(1:9)/10, key="a") +test(2226.09, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce TRUE") +test(2226.10, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce TRUE") + +# with key and grouping by nonkey +DT = data.table(a=1:3,b=(1:9)/10,c=(3:1),key="c") +test(2226.11, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:3)/10, key="c"), output="GForce TRUE") +test(2226.12, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:9)/10, key="c"), output="GForce TRUE") +test(2226.13, DT[, v := head(b, 2L), a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") +DT = data.table(a=1:3,b=(1:9)/10,c=(3:1),key="c") +test(2226.14, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce TRUE") +test(2226.15, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce TRUE") + +# with key and keyby by nonkey +DT = data.table(a=1:3,b=(1:9)/10,c=(3:1),key="c") +test(2226.16, copy(DT)[, v := min(b), keyby=a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:3)/10, key="a"), output="GForce TRUE") +test(2226.17, copy(DT)[, v := head(b, 3L), keyby=a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:9)/10, key="a"), output="GForce TRUE") +test(2226.18, copy(DT)[, v := head(b, 2L), keyby=a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") +DT = data.table(a=1:3,b=(1:9)/10,c=(3:1),key="c") +test(2226.19, copy(DT)[, v := min(b), keyby=a, verbose=TRUE], copy(DT)[, v := base::min(b), keyby=a], output="GForce TRUE") +test(2226.20, copy(DT)[, v := head(b, 3L), keyby=a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), keyby=a], output="GForce TRUE") +# with irows +DT = data.table(a=1:3,b=(1:9)/10) +test(2226.21, DT[a==2, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=c(NA,0.2,NA)), output="GForce TRUE") +test(2226.22, DT[a!=4, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10), output="GForce TRUE") +test(2226.23, DT[a!=4, v := head(b, 2L), a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") +DT = data.table(a=1:3,b=(1:9)/10) +test(2226.24, copy(DT)[a==2, v := min(b), a, verbose=TRUE], copy(DT)[a==2, v := base::min(b), a, ], output="GForce TRUE") +test(2226.25, copy(DT)[a!=4, v := head(b, 3L), a, verbose=TRUE], copy(DT)[a!=4, v := utils::head(b, 3L), a], output="GForce TRUE") + +# multiple assignments +DT = data.table(a=1:3,b=(1:9)/10) +test(2226.26, DT[, c("v1","v2") := .(min(b), max(b)), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v1=(1:3)/10, v2=(7:9)/10), output="GForce TRUE") +test(2226.27, DT[, c("v1","v2") := .(head(b,3L), tail(b,3L)), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v1=(1:9)/10, v2=(1:9)/10), output="GForce TRUE") +test(2226.28, DT[, c("v1","v2") := .(head(b,3L), tail(b,2L)), a], error="Supplied 6 items to be assigned to 9 items of column 'v2'.") +test(2226.29, DT[, c("v1","v2") := .(head(b,2L), tail(b,3L)), a], error="Supplied 6 items to be assigned to 9 items of column 'v1'.") +test(2226.30, DT[, c("v1","v2") := .(head(b,2L), tail(b,2L)), a], error="Supplied 6 items to be assigned to 9 items of column 'v1'.") +test(2226.31, DT[, c("v1","v2") := .(min(b), max(b)), a, verbose=TRUE], DT[, c("v1","v2") := .(base::min(b), base::max(b)), a ], output="GForce TRUE") +test(2226.32, DT[, c("v1","v2") := .(head(b,3L), tail(b,3L)), a, verbose=TRUE], DT[, c("v1","v2") := .(utils::head(b,3L), utils::tail(b,3L)), a], output="GForce TRUE") + +# gforce needs to evaluate variable arguments before calling C part (part of test 101.17 in programming.Rraw) +set.seed(108) +yn = c(1, 5, 10, 20) +ycols = paste0("y", yn) +ydt = data.table(symbol = rep(1:3, each = 100)) +ydt[, date := seq_len(.N), by = symbol] +ydt[, ret := rnorm(.N)] +f = shift +test(2226.33, copy(ydt)[, (ycols) := shift(ret, yn, type = "lead"), by = symbol, verbose=TRUE], copy(ydt)[, (ycols) := f(ret, yn, type = "lead"), by = symbol], output="GForce TRUE") From 3e7f4df4d2fa3112a8233311c38f5b99bae1d26d Mon Sep 17 00:00:00 2001 From: Benjamin Schwendinger <52290390+ben-schwen@users.noreply.github.com> Date: Mon, 1 Nov 2021 15:32:06 +0100 Subject: [PATCH 2/8] return invisible --- R/data.table.R | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/R/data.table.R b/R/data.table.R index 5736b330ff..859978623b 100644 --- a/R/data.table.R +++ b/R/data.table.R @@ -1775,7 +1775,7 @@ replace_dot_alias = function(e) { # adding argument to ghead/gtail if none is supplied to g-optimized head/tail if (length(jsub) == 2L && jsub[[1L]] %chin% c("head", "tail")) jsub[["n"]] = 6L jsub[[1L]] = as.name(paste0("g", jsub[[1L]])) - # check if function arguments are symbols and if so evaluate them + # check if function arguments are variables in parent frame and if so evaluate them for usage in gforce C if (length(jsub)>=3L && is.symbol(jsub[[3L]]) && exists(jsub[[3L]], parent.frame())) jsub[[3L]] = eval(jsub[[3L]], parent.frame()) # tests 1187.3 & 1187.5 & programming 101.17 } if (verbose) catf("GForce optimized j to '%s'\n", deparse(jsub, width.cutoff=200L, nlines=1L)) @@ -1916,6 +1916,7 @@ replace_dot_alias = function(e) { # unwrap single column jvals for assign if (length(jvals)==1L) jvals = jvals[[1L]] .Call(Cassign, x, jrows, lhs, newnames, jvals) + .global$print = address(x) } if (any(names_x[cols] %chin% key(x))) setkey(x,NULL) @@ -1937,7 +1938,7 @@ replace_dot_alias = function(e) { } else warningf("The setkey() normally performed by keyby= has been skipped (as if by= was used) because := is being used together with keyby= but the keyby= contains some expressions. To avoid this warning, use by= instead, or provide existing column names to keyby=.\n") } - if (GForce) return(suppPrint(x)) + if (GForce) return(invisible(x)) return(suppPrint(x)) } if (is.null(ans)) { From ca5125f6f39660d7c548c35bb95402bfcda1c2fa Mon Sep 17 00:00:00 2001 From: Benjamin Schwendinger <52290390+ben-schwen@users.noreply.github.com> Date: Mon, 1 Nov 2021 15:33:52 +0100 Subject: [PATCH 3/8] suppress print --- R/data.table.R | 1 - 1 file changed, 1 deletion(-) diff --git a/R/data.table.R b/R/data.table.R index 859978623b..97878f9c1f 100644 --- a/R/data.table.R +++ b/R/data.table.R @@ -1938,7 +1938,6 @@ replace_dot_alias = function(e) { } else warningf("The setkey() normally performed by keyby= has been skipped (as if by= was used) because := is being used together with keyby= but the keyby= contains some expressions. To avoid this warning, use by= instead, or provide existing column names to keyby=.\n") } - if (GForce) return(invisible(x)) return(suppPrint(x)) } if (is.null(ans)) { From 70b27ebc396696e56a5fa56119be056e5fae412f Mon Sep 17 00:00:00 2001 From: Benjamin Schwendinger <52290390+ben-schwen@users.noreply.github.com> Date: Mon, 1 Nov 2021 15:34:12 +0100 Subject: [PATCH 4/8] supp print2 --- R/data.table.R | 1 - 1 file changed, 1 deletion(-) diff --git a/R/data.table.R b/R/data.table.R index 97878f9c1f..6d31de870a 100644 --- a/R/data.table.R +++ b/R/data.table.R @@ -1916,7 +1916,6 @@ replace_dot_alias = function(e) { # unwrap single column jvals for assign if (length(jvals)==1L) jvals = jvals[[1L]] .Call(Cassign, x, jrows, lhs, newnames, jvals) - .global$print = address(x) } if (any(names_x[cols] %chin% key(x))) setkey(x,NULL) From a2b19b1a04291adbe289da90b6a0c46518437a3b Mon Sep 17 00:00:00 2001 From: Benjamin Schwendinger <52290390+ben-schwen@users.noreply.github.com> Date: Mon, 1 Nov 2021 15:41:39 +0100 Subject: [PATCH 5/8] remove debug lines from programming --- inst/tests/programming.Rraw | 2 -- 1 file changed, 2 deletions(-) diff --git a/inst/tests/programming.Rraw b/inst/tests/programming.Rraw index 6b586ee6aa..429545dcb7 100644 --- a/inst/tests/programming.Rraw +++ b/inst/tests/programming.Rraw @@ -536,9 +536,7 @@ ycols = paste0("y", yn) ydt = data.table(symbol = rep(1:3, each = 100)) ydt[, date := seq_len(.N), by = symbol] ydt[, ret := rnorm(.N)] -#options(datatable.optimize=1L) ydt[, (ycols) := shift(ret, yn, type = "lead"), by = symbol] -#options(datatable.optimize=2L) xdt = data.table(symbol = rep(1:2, each = 20)) xdt[, date := seq_len(.N), by = symbol] xdt[, `:=`(x1 = rnorm(.N), x2 = rnorm(.N))] From 9a417024a58ed5f54c78da1b8d099bb8fab418a5 Mon Sep 17 00:00:00 2001 From: Benjamin Schwendinger <52290390+ben-schwen@users.noreply.github.com> Date: Mon, 1 Nov 2021 15:42:57 +0100 Subject: [PATCH 6/8] add issues number to tests --- inst/tests/tests.Rraw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index c05b63fb07..bbdbf42d1d 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -18409,7 +18409,7 @@ test(2225.2, groupingsets(data.table(iris), j=mean(Sepal.Length), by=c('Sp'='Spe groupingsets(data.table(iris), j=mean(Sepal.Length), by=c('Species'), sets=list('Species'))) -# support := with GForce +# support := with GForce #1414 options(datatable.optimize = 2L) DT = data.table(a=1:3,b=(1:9)/10) test(2226.01, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:3)/10), output="GForce TRUE") From df96c30623b0b9b19b8c515bb16d18056121457e Mon Sep 17 00:00:00 2001 From: Benjamin Schwendinger <52290390+ben-schwen@users.noreply.github.com> Date: Mon, 1 Nov 2021 16:25:51 +0100 Subject: [PATCH 7/8] fix vignette error --- R/data.table.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/data.table.R b/R/data.table.R index 6d31de870a..f68eaf328f 100644 --- a/R/data.table.R +++ b/R/data.table.R @@ -1911,7 +1911,7 @@ replace_dot_alias = function(e) { if (GForce) { # GForce should work with := #1414 vlen = length(ans[[1L]]) # replicate vals if GForce returns 1 value per group - jvals = if (vlen==length(len__)) lapply(ans[-1], rep.int, times=len__) else ans[-1] + jvals = if (vlen==length(len__)) lapply(tail(ans, -length(g)), rep.int, times=len__) else tail(ans, -length(g)) jrows = if (!is.null(irows) && length(irows)!=length(o__)) irows else { if (length(o__)==0L) NULL else o__} # unwrap single column jvals for assign if (length(jvals)==1L) jvals = jvals[[1L]] From 6bcdacdc522e9f4822a3d7a920de8b1e7e33049d Mon Sep 17 00:00:00 2001 From: Benjamin Schwendinger <52290390+ben-schwen@users.noreply.github.com> Date: Mon, 1 Nov 2021 18:13:05 +0100 Subject: [PATCH 8/8] update tests --- R/data.table.R | 2 +- inst/tests/tests.Rraw | 50 +++++++++++++++++++++---------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/R/data.table.R b/R/data.table.R index f68eaf328f..5edd2de1eb 100644 --- a/R/data.table.R +++ b/R/data.table.R @@ -1769,7 +1769,7 @@ replace_dot_alias = function(e) { for (ii in seq_along(jsub)[-1L]) { if (dotN(jsub[[ii]])) next; # For #334 jsub[[ii]][[1L]] = as.name(paste0("g", jsub[[ii]][[1L]])) - if (length(jsub[[ii]])==3L) jsub[[ii]][[3L]] = eval(jsub[[ii]][[3L]], parent.frame()) # tests 1187.2 & 1187.4 + if (length(jsub[[ii]])>=3L && is.symbol(jsub[[ii]][[3L]]) && exists(jsub[[ii]][[3L]], parent.frame())) jsub[[ii]][[3L]] = eval(jsub[[ii]][[3L]], parent.frame()) # tests 1187.2 & 1187.4 } else { # adding argument to ghead/gtail if none is supplied to g-optimized head/tail diff --git a/inst/tests/tests.Rraw b/inst/tests/tests.Rraw index bbdbf42d1d..5bf13ea9eb 100644 --- a/inst/tests/tests.Rraw +++ b/inst/tests/tests.Rraw @@ -18412,60 +18412,60 @@ test(2225.2, groupingsets(data.table(iris), j=mean(Sepal.Length), by=c('Sp'='Spe # support := with GForce #1414 options(datatable.optimize = 2L) DT = data.table(a=1:3,b=(1:9)/10) -test(2226.01, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:3)/10), output="GForce TRUE") +test(2226.01, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:3)/10), output="GForce optimized j to") # GForce returning full length -test(2226.02, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10), output="GForce TRUE") +test(2226.02, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10), output="GForce optimized j to") # GForce neither returning 1 per group nor full length test(2226.03, DT[, v := head(b, 2L), a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") # compare to non GForce version DT = data.table(a=1:3,b=(1:9)/10) -test(2226.04, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce TRUE") -test(2226.05, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce TRUE") +test(2226.04, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce optimized j to") +test(2226.05, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce optimized j to") # with key and grouping by key DT = data.table(a=1:3,b=(1:9)/10, key="a") -test(2226.06, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:3)/10, key="a"), output="GForce TRUE") -test(2226.07, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10, key="a"), output="GForce TRUE") +test(2226.06, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:3)/10, key="a"), output="GForce optimized j to") +test(2226.07, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10, key="a"), output="GForce optimized j to") test(2226.08, DT[, v := head(b, 2L), a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") DT = data.table(a=1:3,b=(1:9)/10, key="a") -test(2226.09, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce TRUE") -test(2226.10, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce TRUE") +test(2226.09, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce optimized j to") +test(2226.10, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce optimized j to") # with key and grouping by nonkey DT = data.table(a=1:3,b=(1:9)/10,c=(3:1),key="c") -test(2226.11, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:3)/10, key="c"), output="GForce TRUE") -test(2226.12, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:9)/10, key="c"), output="GForce TRUE") +test(2226.11, DT[, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:3)/10, key="c"), output="GForce optimized j to") +test(2226.12, DT[, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:9)/10, key="c"), output="GForce optimized j to") test(2226.13, DT[, v := head(b, 2L), a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") DT = data.table(a=1:3,b=(1:9)/10,c=(3:1),key="c") -test(2226.14, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce TRUE") -test(2226.15, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce TRUE") +test(2226.14, copy(DT)[, v := min(b), a, verbose=TRUE], copy(DT)[, v := base::min(b), a, ], output="GForce optimized j to") +test(2226.15, copy(DT)[, v := head(b, 3L), a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), a], output="GForce optimized j to") # with key and keyby by nonkey DT = data.table(a=1:3,b=(1:9)/10,c=(3:1),key="c") -test(2226.16, copy(DT)[, v := min(b), keyby=a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:3)/10, key="a"), output="GForce TRUE") -test(2226.17, copy(DT)[, v := head(b, 3L), keyby=a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:9)/10, key="a"), output="GForce TRUE") +test(2226.16, copy(DT)[, v := min(b), keyby=a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:3)/10, key="a"), output="GForce optimized j to") +test(2226.17, copy(DT)[, v := head(b, 3L), keyby=a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, c=(3:1), v=(1:9)/10, key="a"), output="GForce optimized j to") test(2226.18, copy(DT)[, v := head(b, 2L), keyby=a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") DT = data.table(a=1:3,b=(1:9)/10,c=(3:1),key="c") -test(2226.19, copy(DT)[, v := min(b), keyby=a, verbose=TRUE], copy(DT)[, v := base::min(b), keyby=a], output="GForce TRUE") -test(2226.20, copy(DT)[, v := head(b, 3L), keyby=a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), keyby=a], output="GForce TRUE") +test(2226.19, copy(DT)[, v := min(b), keyby=a, verbose=TRUE], copy(DT)[, v := base::min(b), keyby=a], output="GForce optimized j to") +test(2226.20, copy(DT)[, v := head(b, 3L), keyby=a, verbose=TRUE], copy(DT)[, v := utils::head(b, 3L), keyby=a], output="GForce optimized j to") # with irows DT = data.table(a=1:3,b=(1:9)/10) -test(2226.21, DT[a==2, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=c(NA,0.2,NA)), output="GForce TRUE") -test(2226.22, DT[a!=4, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10), output="GForce TRUE") +test(2226.21, DT[a==2, v := min(b), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=c(NA,0.2,NA)), output="GForce optimized j to") +test(2226.22, DT[a!=4, v := head(b, 3L), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v=(1:9)/10), output="GForce optimized j to") test(2226.23, DT[a!=4, v := head(b, 2L), a], error="Supplied 6 items to be assigned to 9 items of column 'v'.") DT = data.table(a=1:3,b=(1:9)/10) -test(2226.24, copy(DT)[a==2, v := min(b), a, verbose=TRUE], copy(DT)[a==2, v := base::min(b), a, ], output="GForce TRUE") -test(2226.25, copy(DT)[a!=4, v := head(b, 3L), a, verbose=TRUE], copy(DT)[a!=4, v := utils::head(b, 3L), a], output="GForce TRUE") +test(2226.24, copy(DT)[a==2, v := min(b), a, verbose=TRUE], copy(DT)[a==2, v := base::min(b), a, ], output="GForce optimized j to") +test(2226.25, copy(DT)[a!=4, v := head(b, 3L), a, verbose=TRUE], copy(DT)[a!=4, v := utils::head(b, 3L), a], output="GForce optimized j to") # multiple assignments DT = data.table(a=1:3,b=(1:9)/10) -test(2226.26, DT[, c("v1","v2") := .(min(b), max(b)), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v1=(1:3)/10, v2=(7:9)/10), output="GForce TRUE") -test(2226.27, DT[, c("v1","v2") := .(head(b,3L), tail(b,3L)), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v1=(1:9)/10, v2=(1:9)/10), output="GForce TRUE") +test(2226.26, DT[, c("v1","v2") := .(min(b), max(b)), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v1=(1:3)/10, v2=(7:9)/10), output="GForce optimized j to") +test(2226.27, DT[, c("v1","v2") := .(head(b,3L), tail(b,3L)), a, verbose=TRUE], data.table(a=1:3, b=(1:9)/10, v1=(1:9)/10, v2=(1:9)/10), output="GForce optimized j to") test(2226.28, DT[, c("v1","v2") := .(head(b,3L), tail(b,2L)), a], error="Supplied 6 items to be assigned to 9 items of column 'v2'.") test(2226.29, DT[, c("v1","v2") := .(head(b,2L), tail(b,3L)), a], error="Supplied 6 items to be assigned to 9 items of column 'v1'.") test(2226.30, DT[, c("v1","v2") := .(head(b,2L), tail(b,2L)), a], error="Supplied 6 items to be assigned to 9 items of column 'v1'.") -test(2226.31, DT[, c("v1","v2") := .(min(b), max(b)), a, verbose=TRUE], DT[, c("v1","v2") := .(base::min(b), base::max(b)), a ], output="GForce TRUE") -test(2226.32, DT[, c("v1","v2") := .(head(b,3L), tail(b,3L)), a, verbose=TRUE], DT[, c("v1","v2") := .(utils::head(b,3L), utils::tail(b,3L)), a], output="GForce TRUE") +test(2226.31, DT[, c("v1","v2") := .(min(b), max(b)), a, verbose=TRUE], DT[, c("v1","v2") := .(base::min(b), base::max(b)), a ], output="GForce optimized j to") +test(2226.32, DT[, c("v1","v2") := .(head(b,3L), tail(b,3L)), a, verbose=TRUE], DT[, c("v1","v2") := .(utils::head(b,3L), utils::tail(b,3L)), a], output="GForce optimized j to") # gforce needs to evaluate variable arguments before calling C part (part of test 101.17 in programming.Rraw) set.seed(108) @@ -18475,4 +18475,4 @@ ydt = data.table(symbol = rep(1:3, each = 100)) ydt[, date := seq_len(.N), by = symbol] ydt[, ret := rnorm(.N)] f = shift -test(2226.33, copy(ydt)[, (ycols) := shift(ret, yn, type = "lead"), by = symbol, verbose=TRUE], copy(ydt)[, (ycols) := f(ret, yn, type = "lead"), by = symbol], output="GForce TRUE") +test(2226.33, copy(ydt)[, (ycols) := shift(ret, yn, type = "lead"), by = symbol, verbose=TRUE], copy(ydt)[, (ycols) := f(ret, yn, type = "lead"), by = symbol], output="GForce optimized j to")