Skip to content

add na.rm argument to first/last and gfirst/glast #4730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ben-schwen wants to merge 9 commits into from
Closed

add na.rm argument to first/last and gfirst/glast #4730

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2076608
first version
ben-schwen Sep 30, 2020
330dc9c
added testcases
ben-schwen Oct 1, 2020
9cf43b3
updated test cases
ben-schwen Oct 1, 2020
32123c2
Merge branch 'master' into last_first_narm
ben-schwen Oct 3, 2020
9c11e7f
fixed review remarks
ben-schwen Oct 15, 2020
6d29273
expanded lines for coverages
ben-schwen Oct 15, 2020
6d7b007
fixed coding style
ben-schwen Oct 15, 2020
0741e2f
added test for list columns
ben-schwen Oct 15, 2020
3db430c
exchanged complete.cases with na.omit to avoid #4744
ben-schwen Oct 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions R/data.table.R
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2849,8 +2849,8 @@ gfuns = c("[", "[[", "head", "tail", "first", "last", "sum", "mean", "prod",
`g[` = `g[[` = function(x, n) .Call(Cgnthvalue, x, as.integer(n)) # n is of length=1 here.
ghead = function(x, n) .Call(Cghead, x, as.integer(n)) # n is not used at the moment
gtail = function(x, n) .Call(Cgtail, x, as.integer(n)) # n is not used at the moment
gfirst = function(x) .Call(Cgfirst, x)
glast = function(x) .Call(Cglast, x)
gfirst = function(x, na.rm=FALSE) .Call(Cgfirst, x, na.rm)
glast = function(x, na.rm=FALSE) .Call(Cglast, x, na.rm)
gsum = function(x, na.rm=FALSE) .Call(Cgsum, x, na.rm, TRUE) # warnOverflow=TRUE, #986
gmean = function(x, na.rm=FALSE) .Call(Cgmean, x, na.rm)
gprod = function(x, na.rm=FALSE) .Call(Cgprod, x, na.rm)
Expand Down
35 changes: 33 additions & 2 deletions R/last.R
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,23 @@
# data.table defined last(x) with no arguments, just for last. If you need the last 10 then use tail(x,10).
# for xts class objects it will dispatch to xts::last
# reworked to avoid loading xts namespace (#3857) then again to fix dispatching of xts class (#4053)
last = function(x, n=1L, ...) {
last = function(x, n=1L, na.rm=FALSE, ...) {
verbose = isTRUE(getOption("datatable.verbose", FALSE))

stopifnot(isTRUEorFALSE(na.rm))

if (na.rm) {
if (is.vector(x) && length(x)) {
if (verbose)
cat("last: using last(x[!is.na(x)]): na.rm=T\n")
x = x[!is.na(x)]
} else if (is.data.frame(x)) {
if (verbose)
cat("last: using last(na.omit(x)): na.rm=T\n")
x = na.omit(x)
}
}

if (!inherits(x, "xts")) {
if (nargs()>1L) {
if ("package:xts" %chin% search()) {
Expand Down Expand Up @@ -42,8 +57,24 @@ last = function(x, n=1L, ...) {
}
}

first = function(x, n=1L, ...) {
first = function(x, n=1L, na.rm=FALSE, ...) {
verbose = isTRUE(getOption("datatable.verbose", FALSE))

stopifnot(isTRUEorFALSE(na.rm))

if (na.rm) {
if (is.vector(x) && length(x)) {
if (verbose)
cat("first: using first(x[!is.na(x)]): na.rm=T\n")
x = x[!is.na(x)]
} else if (is.data.frame(x)) {
if (verbose)
cat("first: using first(na.omit(x)): na.rm=T\n")
x = na.omit(x)
}
}


if (!inherits(x, "xts")) {
if (nargs()>1L) {
if ("package:xts" %chin% search()) {
Expand Down
51 changes: 45 additions & 6 deletions inst/tests/tests.Rraw
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16365,7 +16365,8 @@ xt = structure(
)
old = options(datatable.verbose=TRUE)
if (test_xts) {
test(2108.21, last(x, n=2L), 2:3, output="using xts::last: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.210, last(x, n=2L), 2:3, output="using xts::last: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.211, last(x, n=2L, na.rm=TRUE), 2:3, output="last: using last(x[!is.na(x)]): na.rm=T\nlast: using xts::last: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.22, last(y, n=2L), y[2:3], output="using xts::last: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.23, last(x, n=1L), 3L, output="using xts::last: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.24, last(y, n=1L), y[3L], output="using xts::last: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
Expand All @@ -16384,7 +16385,8 @@ if (test_xts) {
)
test(2108.25, last(xt), xt_last, output="using xts::last: is.xts(x)")
test(2108.26, last(xt, n=2L), xt_last2, output="using xts::last: is.xts(x)")
test(2108.31, first(x, n=2L), 1:2, output="using xts::first: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.310, first(x, n=2L), 1:2, output="using xts::first: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.311, first(x, n=2L, na.rm=TRUE), 1:2, output="first: using first(x[!is.na(x)]): na.rm=T\nfirst: using xts::first: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.32, first(y, n=2L), y[1:2], output="using xts::first: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.33, first(x, n=1L), 1L, output="using xts::first: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
test(2108.34, first(y, n=1L), y[1L], output="using xts::first: !is.xts(x) & nargs>1 & 'package:xts'%in%search()")
Expand All @@ -16403,13 +16405,15 @@ if (test_xts) {
test(2108.35, first(xt), xt_first, output="using xts::first: is.xts(x)")
test(2108.36, first(xt, n=2L), xt_first2, output="using xts::first: is.xts(x)")
} else {
test(2108.21, last(x, n=2L), 2:3, output="using utils::tail: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.210, last(x, n=2L), 2:3, output="using utils::tail: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.211, last(x, n=2L, na.rm=TRUE), 2:3, output="last: using last(x[!is.na(x)]): na.rm=T\nlast: using utils::tail: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.22, last(y, n=2L), y[2:3], output="using utils::tail: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.23, last(x, n=1L), 3L, output="using utils::tail: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.24, last(y, n=1L), y[3L], output="using utils::tail: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.25, last(xt), error="you should have 'xts' installed already")
test(2108.26, last(xt, n=2L), error="you should have 'xts' installed already")
test(2108.31, first(x, n=2L), 1:2, output="using utils::head: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.310, first(x, n=2L), 1:2, output="using utils::head: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.311, first(x, n=2L, na.rm=TRUE), 1:2, output="first: using first(x[!is.na(x)]): na.rm=T\nfirst: using utils::head: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.32, first(y, n=2L), y[1:2], output="using utils::head: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.33, first(x, n=1L), 1L, output="using utils::head: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
test(2108.34, first(y, n=1L), y[1L], output="using utils::head: !is.xts(x) & nargs>1 & !'package:xts'%in%search()")
Expand All @@ -16420,9 +16424,11 @@ test(2108.41, last(x), 3L, output="using 'x[[length(x)]]': !is.xts(x) & !nargs>1
test(2108.42, last(y), y[3L], output="using 'x[[length(x)]]': !is.xts(x) & !nargs>1 & is.null(dim(x))")
test(2108.51, first(x), 1L, output="using 'x[[1L]]': !is.xts(x) & !nargs>1 & is.null(dim(x))")
test(2108.52, first(y), y[1L], output="using 'x[[1L]]': !is.xts(x) & !nargs>1 & is.null(dim(x))")
test(2108.61, last(df), structure(list(a=2L, b=2L), row.names=2L, class="data.frame"), output="using 'x[nrow(x),]': !is.xts(x) & !nargs>1 & is.data.frame(x)")
test(2108.610, last(df), structure(list(a=2L, b=2L), row.names=2L, class="data.frame"), output="using 'x[nrow(x),]': !is.xts(x) & !nargs>1 & is.data.frame(x)")
test(2108.611, last(df, na.rm=TRUE), structure(list(a=2L, b=2L), row.names=2L, class="data.frame"), output="last: using last(na.omit(x)): na.rm=T\nlast: using xts::last: !is.xts(x) & nargs>1 & 'package:xts'%in%search()\n")
test(2108.62, last(dt), data.table(a=2L, b=2L), output="using 'x[nrow(x),]': !is.xts(x) & !nargs>1 & is.data.frame(x)")
test(2108.71, first(df), structure(list(a=1L, b=3L), row.names=1L, class="data.frame"), output="using 'x[1L,]': !is.xts(x) & !nargs>1 & is.data.frame(x)")
test(2108.710, first(df), structure(list(a=1L, b=3L), row.names=1L, class="data.frame"), output="using 'x[1L,]': !is.xts(x) & !nargs>1 & is.data.frame(x)")
test(2108.711, first(df, na.rm=TRUE), structure(list(a=1L, b=3L), row.names=1L, class="data.frame"), output="first: using first(na.omit(x)): na.rm=T\nfirst: using xts::first: !is.xts(x) & nargs>1 & 'package:xts'%in%search()\n")
test(2108.72, first(dt), data.table(a=1L, b=3L), output="using 'x[1L,]': !is.xts(x) & !nargs>1 & is.data.frame(x)")
# matrix/array utils::tail behavior is likely to change in future R, Michael is more in the topic
test(2108.81, last(mx), structure(c(3L, 6L, 9L), .Dim = c(1L, 3L), .Dimnames = list("[3,]", NULL)), output="using utils::tail: !is.xts(x) & !nargs>1 & !is.null(dim(x)) & !is.data.frame(x)")
Expand Down Expand Up @@ -17150,3 +17156,36 @@ test(2154.1, fread("0.0\n", colClasses="integer"), data.table(V1=0.0),
test(2154.2, fread("A\n0.0\n", colClasses="integer"), data.table(A=0.0),
warning="Attempt to override column 1 <<A>> of inherent type 'float64' down to 'int32' ignored.*please")

a = c(1L,1L,2L,2L,3L,3L,4L,4L)
bi = c(NA, NA, NA, 1L:4L, NA)
bs = c(NA, NA, NA, letters[1:4], NA)
DT = data.table(a,bi,bs)
i = c(1,1,2,2)
l = c(NA, TRUE, FALSE, NA)
c = c(1+1i, NA, 2+2i, NA)
n = c(sqrt(2), NA, NA, sqrt(5))
ll = list(NA, list(1,2), list(2,3), NA)
DT2 = data.table(i, l, c, n, ll)
test(2155.01, first(bi, na.rm=TRUE), 1L)
test(2155.02, last(bi, na.rm=TRUE), 4L)
test(2155.03, DT[, .(first(bi, na.rm=TRUE)), a], data.table(a=1L:4L, V1=c(NA,1L,2L,4L)))
test(2155.04, DT[, .(last(bi, na.rm=TRUE)), a], data.table(a=1L:4L, V1=c(NA,1L,3L,4L)))
test(2155.05, DT[, .(first(bs, na.rm=TRUE)), a], data.table(a=1L:4L, V1=c(NA,"a","b","d")))
test(2155.06, DT[, .(last(bs, na.rm=TRUE)), a], data.table(a=1L:4L, V1=c(NA,"a","c","d")))
test(2155.07, first(DT, na.rm=TRUE), data.table(a=2L, bi=1L, bs="a"))
test(2155.08, setDT(last(DT, na.rm=TRUE)), data.table(a=4L, bi=4L, bs="d")) #see #3047 why setDT
comp = c(NA, NA, NA, sapply(-1:-4, as.complex), NA)
DT = data.table(a, comp)
test(2155.09, DT[, .(first(comp, na.rm=TRUE)), a],
data.table(a=1L:4L, V1=c(NA_complex_,-1+0i,-2+0i,-4+0i)))
test(2155.10, DT[, .(last(comp, na.rm=TRUE)), a],
data.table(a=1L:4L, V1=c(NA_complex_,-1+0i,-3+0i,-4+0i)))
i = c(1,1,2,2)
l = c(NA, TRUE, FALSE, NA)
c = c(1+1i, NA, 2+2i, NA)
n = c(sqrt(2), NA, NA, sqrt(5))
ll = list(NULL, list(1), list(2), NULL)
DT2 = data.table(i, l, c, n, ll)
DT2_res = data.table(i=i[c(1,3)], l=l[c(2,3)], c=c[c(1,3)], n=n[c(1,4)], ll=ll[c(2,3)])
test(2155.11, DT2[, lapply(.SD, first, na.rm=T), i], DT2_res)
test(2155.12, DT2[, lapply(.SD, last, na.rm=T), i], DT2_res)
16 changes: 13 additions & 3 deletions man/last.Rd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,21 @@ or data.table. The main difference to head/tail is that the default for \code{n}
rather than 6.
}
\usage{
first(x, n=1L, \dots)
last(x, n=1L, \dots)
first(x, n=1L, na.rm=FALSE, \dots)
last(x, n=1L, na.rm=FALSE, \dots)
}
\arguments{
\item{x}{ A vector, list, data.frame or data.table. Otherwise the S3 method
of \code{xts::first} is deployed. }
\item{n}{ A numeric vector length 1. How many items to select. }
\item{na.rm}{ logical. Should missing values be ignored when selecting values? Defaults to \code{FALSE}.}
\item{\dots}{ Not applicable for \code{data.table} first/last. Any arguments here
are passed through to \code{xts}'s first/last. }
}
\value{
If no other arguments are supplied it depends on the type of \code{x}. The first/last item
of a vector or list. The first/last row of a \code{data.frame} or \code{data.table}.
of a vector or list. The first/last row of a \code{data.frame} or \code{data.table}. If \code{na.rm} is \code{TRUE}
the first/last row of a \code{data.frame} or \code{data.table} without missing values is selected.
For other types, or if any argument is supplied in addition to \code{x} (such as \code{n}, or
\code{keep} in \code{xts}) regardless of \code{x}'s type, then \code{xts::first}/
\code{xts::last} is called if \code{xts} has been loaded, otherwise \code{utils::head}/\code{utils::tail}.
Expand All @@ -34,5 +36,13 @@ first(x) # same as head(x, 1)
last(1:5) # [1] 5
x = data.table(x=1:5, y=6:10)
last(x) # same as tail(x, 1)

x = c(NA, 1, 2, NA)
first(x)
last(x)

x = data.table(x=c(NA, 1, 2, NA), y=1:4)
first(x)
last(x)
}
\keyword{ data }
Loading