diff --git a/.dev/CRAN_Release.cmd b/.dev/CRAN_Release.cmd index 313f04c5bb..f7f5ad7b9c 100644 --- a/.dev/CRAN_Release.cmd +++ b/.dev/CRAN_Release.cmd @@ -66,6 +66,17 @@ grep "PROTECT_PTR" ./src/*.c # No use of long long, instead use int64_t. TODO # grep "long long" ./src/*.c +// No use of llu, lld, zd or zu +grep -nE "(llu|lld|zd|zu)" src/*.[hc] +// Comment moved here from fread.c on 19 Nov 2019 +// [Moved from fread.c on 19 Nov 2019] On Windows variables of type `size_t` cannot be printed +// with "%zu" in the `snprintf()` function. For those variables we used to cast them into +// `unsigned long long int` before printing, and defined (llu) to make the cast shorter. +// We're now observing warnings from gcc-8 with -Wformat-extra-args, #4062. So +// now we're more strict and cast to [u]int64_t and use PRIu64/PRId64 from +// In many cases the format specifier is passed to our own macro (e.g. DTPRINT) or to Rprintf(), +// error() etc, and even if they don't call sprintf() now, they could in future. + # No tabs in C or R code (sorry, Richard Hendricks) grep -P "\t" ./R/*.R grep -P "\t" ./src/*.c diff --git a/NEWS.md b/NEWS.md index 0b346019b0..5d62dcf6f1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -16,6 +16,8 @@ 2. One test needed to be adjusted to pass R-devel (R 4.0.0) which will soon have reference counting turned on, [#4058](https://github.com/Rdatatable/data.table/issues/4058). This motivated early release to CRAN because every day CRAN tests every package using the previous day's changes in R-devel; a much valued feature of the R ecosystem. It helps R-core if packages can pass changes in R-devel as soon as possible. Thanks to Luke Tierney for the notice, and for implementing reference counting which we look forward to very much. +3. C internals have been standardized to use `PRI[u|d]64` to print `[u]int64_t`. This solves new warnings from `gcc-8` on Windows with `%lld`, [#4062](https://github.com/Rdatatable/data.table/issues/4062), in many cases already working around `snprintf` on Windows not supporting `%zu`. Release procedures have been augmented to prevent any internal use of `llu`, `lld`, `zu` or `zd`. + # data.table [v1.12.6](https://github.com/Rdatatable/data.table/milestone/18?closed=1) (18 Oct 2019) diff --git a/src/assign.c b/src/assign.c index 1d33f9c3c2..9478c8745b 100644 --- a/src/assign.c +++ b/src/assign.c @@ -864,7 +864,7 @@ const char *memrecycle(SEXP target, SEXP where, int start, int len, SEXP source, const char *sType = sourceIsI64 ? "integer64" : type2char(TYPEOF(source)); \ const char *tType = targetIsI64 ? "integer64" : type2char(TYPEOF(target)); \ int n = snprintf(memrecycle_message, MSGSIZE, \ - FMT" (type '%s') at RHS position %d "TO" when assigning to type '%s'", val, sType, i+1, tType); \ + "%"FMT" (type '%s') at RHS position %d "TO" when assigning to type '%s'", val, sType, i+1, tType); \ if (colnum>0 && n>0 && n255, "%d", "taken as 0") + case INTSXP: CHECK_RANGE(int, INTEGER, val<0 || val>255, "d", "taken as 0") case REALSXP: if (sourceIsI64) - CHECK_RANGE(long long, REAL, val<0 || val>255, "%lld", "taken as 0") - else CHECK_RANGE(double, REAL, !R_FINITE(val) || val<0.0 || val>256.0 || (int)val!=val, "%f", "either truncated (precision lost) or taken as 0") + CHECK_RANGE(int64_t, REAL, val<0 || val>255, PRId64, "taken as 0") + else CHECK_RANGE(double, REAL, !R_FINITE(val) || val<0.0 || val>256.0 || (int)val!=val, "f", "either truncated (precision lost) or taken as 0") } break; case INTSXP: if (TYPEOF(source)==REALSXP) { if (sourceIsI64) - CHECK_RANGE(long long, REAL, val!=NA_INTEGER64 && (val<=NA_INTEGER || val>INT_MAX), "%lld", "out-of-range (NA)") - else CHECK_RANGE(double, REAL, !ISNAN(val) && (!R_FINITE(val) || (int)val!=val), "%f", "truncated (precision lost)") + CHECK_RANGE(int64_t, REAL, val!=NA_INTEGER64 && (val<=NA_INTEGER || val>INT_MAX), PRId64, "out-of-range (NA)") + else CHECK_RANGE(double, REAL, !ISNAN(val) && (!R_FINITE(val) || (int)val!=val), "f", "truncated (precision lost)") } break; case REALSXP: if (targetIsI64 && isReal(source) && !sourceIsI64) { - CHECK_RANGE(double, REAL, !ISNAN(val) && (!R_FINITE(val) || (int)val!=val), "%f", "truncated (precision lost)") + CHECK_RANGE(double, REAL, !ISNAN(val) && (!R_FINITE(val) || (int)val!=val), "f", "truncated (precision lost)") } } } diff --git a/src/between.c b/src/between.c index d7de5c7175..1edf95a093 100644 --- a/src/between.c +++ b/src/between.c @@ -92,7 +92,7 @@ SEXP between(SEXP x, SEXP lower, SEXP upper, SEXP incbounds, SEXP NAboundsArg, S if (check) for (int i=0; iu) - error("Item %d of lower (%lld) is greater than item %d of upper (%lld)", (i&lowMask)+1, l, (i&uppMask)+1, u); + error("Item %d of lower (%"PRId64") is greater than item %d of upper (%"PRId64")", (i&lowMask)+1, l, (i&uppMask)+1, u); } if (NAbounds) { #pragma omp parallel for num_threads(getDTthreads()) diff --git a/src/chmatch.c b/src/chmatch.c index 4c7f6b8b9b..54d579a020 100644 --- a/src/chmatch.c +++ b/src/chmatch.c @@ -57,7 +57,7 @@ static SEXP chmatchMain(SEXP x, SEXP table, int nomatch, bool chin, bool chmatch // # nocov start for (int i=0; i // #include // the debugging machinery + breakpoint aidee // raise(SIGINT); -#include // for uint64_t rather than unsigned long long +#include // for uint64_t rather than unsigned long long +#include // for PRId64 and PRIu64 #include #include "myomp.h" #include "types.h" diff --git a/src/fifelse.c b/src/fifelse.c index f702f1c732..de53c9bd6a 100644 --- a/src/fifelse.c +++ b/src/fifelse.c @@ -35,9 +35,9 @@ SEXP fifelseR(SEXP l, SEXP a, SEXP b, SEXP na) { } if (len1!=1 && len1!=len0) - error("Length of 'yes' is %lld but must be 1 or length of 'test' (%lld).", len1, len0); + error("Length of 'yes' is %"PRId64" but must be 1 or length of 'test' (%"PRId64").", len1, len0); if (len2!=1 && len2!=len0) - error("Length of 'no' is %lld but must be 1 or length of 'test' (%lld).", len2, len0); + error("Length of 'no' is %"PRId64" but must be 1 or length of 'test' (%"PRId64").", len2, len0); const int64_t amask = len1>1 ? INT64_MAX : 0; // for scalar 'a' bitwise AND will reset iterator to first element: pa[i & amask] -> pa[0] const int64_t bmask = len2>1 ? INT64_MAX : 0; @@ -48,7 +48,7 @@ SEXP fifelseR(SEXP l, SEXP a, SEXP b, SEXP na) { bool nonna = !isNull(na); if (nonna) { if (xlength(na) != 1) - error("Length of 'na' is %lld but must be 1", xlength(na)); + error("Length of 'na' is %"PRId64" but must be 1", (int64_t)xlength(na)); SEXPTYPE tn = TYPEOF(na); if (tn == LGLSXP && LOGICAL(na)[0]==NA_LOGICAL) { nonna = false; diff --git a/src/forder.c b/src/forder.c index 2e86d7b84c..40821ee6b1 100644 --- a/src/forder.c +++ b/src/forder.c @@ -500,7 +500,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP sortGroupsArg, SEXP ascArg, S int keyAlloc = (ncol+n_cplx)*8 + 1; // +1 for NULL to mark end; calloc to initialize with NULLs key = calloc(keyAlloc, sizeof(uint8_t *)); // needs to be before loop because part II relies on part I, column-by-column. if (!key) - STOP("Unable to allocate %llu bytes of working memory", (unsigned long long)(keyAlloc*sizeof(uint8_t *))); // # nocov + STOP("Unable to allocate %"PRId64" bytes of working memory", (uint64_t)keyAlloc*sizeof(uint8_t *)); // # nocov nradix=0; // the current byte we're writing this column to; might be squashing into it (spare>0) int spare=0; // the amount of bits remaining on the right of the current nradix byte bool isReal=false; @@ -569,7 +569,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP sortGroupsArg, SEXP ascArg, S } uint64_t range = max-min+1 +1/*NA*/ +isReal*3/*NaN, -Inf, +Inf*/; - // Rprintf("range=%llu min=%llu max=%llu na_count==%d\n", range, min, max, na_count); + // Rprintf("range=%"PRIu64" min=%"PRIu64" max=%"PRIu64" na_count==%d\n", range, min, max, na_count); int maxBit=0; while (range) { maxBit++; range>>=1; } @@ -604,7 +604,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP sortGroupsArg, SEXP ascArg, S if (key[nradix+b]==NULL) { uint8_t *tt = calloc(nrow, sizeof(uint8_t)); // 0 initialize so that NA's can just skip (NA is always the 0 offset) if (!tt) - STOP("Unable to allocate %llu bytes of working memory", (unsigned long long)(nrow * sizeof(uint8_t))); // # nocov + STOP("Unable to allocate %"PRIu64" bytes of working memory", (uint64_t)nrow*sizeof(uint8_t)); // # nocov key[nradix+b] = tt; } } @@ -622,7 +622,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP sortGroupsArg, SEXP ascArg, S const uint64_t naval = ((nalast==1) == asc) ? max+1+isReal*2 : min-1-isReal*2; const uint64_t nanval = ((nalast==1) == asc) ? max+2 : min-2; // only used when isReal - // Rprintf("asc=%d min2=%llu max2=%llu naval==%llu nanval==%llu\n", asc, min2, max2, naval, nanval); + // Rprintf("asc=%d min2=%"PRIu64" max2=%"PRIu64" naval==%"PRIu64" nanval==%"PRIu64"\n", asc, min2, max2, naval, nanval); // several columns could squash into 1 byte. due to this bit squashing is why we deal // with asc|desc here, otherwise it could be done in the ugrp sorting by reversing the ugrp insert sort @@ -794,7 +794,7 @@ SEXP forder(SEXP DT, SEXP by, SEXP retGrpArg, SEXP sortGroupsArg, SEXP ascArg, S Rprintf("Timing block %2d%s = %8.3f %8d\n", i, (i>=17&&i<=19)?"(*)":" ", tblock[i], nblock[i]); } for (int i=0; i<=256; i++) { - if (stat[i]) Rprintf("stat[%03d]==%10zd\n", i, stat[i]); + if (stat[i]) Rprintf("stat[%03d]==%20"PRIu64"\n", i, (uint64_t)stat[i]); } } #endif diff --git a/src/fread.c b/src/fread.c index bb885eb088..7d6e4f0530 100644 --- a/src/fread.c +++ b/src/fread.c @@ -27,12 +27,6 @@ #include "fread.h" #include "freadLookups.h" -// On Windows variables of type `size_t` cannot be printed with "%zu" in the -// `snprintf()` function. For those variables we will cast them into -// `unsigned long long int` before printing; and this #define makes it -// slightly simpler. -#define llu unsigned long long int - // Private globals to save passing all of them through to highly iterated field processors static const char *sof, *eof; static char sep; @@ -405,7 +399,7 @@ static const char* filesize_to_str(size_t fsize) static char suffixes[NSUFFIXES] = {'T', 'G', 'M', 'K'}; static char output[BUFFSIZE]; static const char one_byte[] = "1 byte"; - llu lsize = (llu) fsize; + size_t lsize = fsize; for (int i = 0; i <= NSUFFIXES; i++) { int shift = (NSUFFIXES - i) * 10; if ((fsize >> shift) == 0) continue; @@ -415,18 +409,18 @@ static const char* filesize_to_str(size_t fsize) } if (ndigits == 0 || (fsize == (fsize >> shift << shift))) { if (i < NSUFFIXES) { - snprintf(output, BUFFSIZE, "%llu%cB (%llu bytes)", - lsize >> shift, suffixes[i], lsize); + snprintf(output, BUFFSIZE, "%"PRIu64"%cB (%"PRIu64" bytes)", + (uint64_t)(lsize >> shift), suffixes[i], (uint64_t)lsize); return output; } } else { - snprintf(output, BUFFSIZE, "%.*f%cB (%llu bytes)", - ndigits, (double)fsize / (1LL << shift), suffixes[i], lsize); + snprintf(output, BUFFSIZE, "%.*f%cB (%"PRIu64" bytes)", + ndigits, (double)fsize / (1LL << shift), suffixes[i], (uint64_t)lsize); return output; } } if (fsize == 1) return one_byte; - snprintf(output, BUFFSIZE, "%llu bytes", lsize); + snprintf(output, BUFFSIZE, "%"PRIu64" bytes", (uint64_t)lsize); return output; } @@ -1154,7 +1148,7 @@ int freadMain(freadMainArgs _args) { else DTPRINT(" None of the NAstrings look like numbers.\n"); } - if (args.skipNrow >= 0) DTPRINT(" skip num lines = %llu\n", (llu)args.skipNrow); + if (args.skipNrow >= 0) DTPRINT(" skip num lines = %"PRId64"\n", (int64_t)args.skipNrow); if (args.skipString) DTPRINT(" skip to string = <<%s>>\n", args.skipString); DTPRINT(" show progress = %d\n", args.showProgress); DTPRINT(" 0/1 column will be read as %s\n", args.logical01? "boolean" : "integer"); @@ -1379,8 +1373,8 @@ int freadMain(freadMainArgs _args) { pos = ch; ch = sof; while (ch= 0) { @@ -1392,8 +1386,8 @@ int freadMain(freadMainArgs _args) { row1line++; } } - if (ch > sof && verbose) DTPRINT(" Skipped to line %llu in the file", (llu)row1line); - if (ch>=eof) STOP("skip=%llu but the input only has %llu line%s", (llu)args.skipNrow, (llu)row1line, row1line>1?"s":""); + if (ch > sof && verbose) DTPRINT(" Skipped to line %"PRIu64" in the file", (uint64_t)row1line); + if (ch>=eof) STOP("skip=%"PRIu64" but the input only has %"PRIu64" line%s", (uint64_t)args.skipNrow, (uint64_t)row1line, row1line>1?"s":""); pos = ch; } @@ -1649,10 +1643,10 @@ int freadMain(freadMainArgs _args) { } if (verbose) { DTPRINT(" Number of sampling jump points = %d because ", nJumps); - if (nrowLimit allocnrow) STOP("Internal error: sampleLines(%llu) > allocnrow(%llu)", (llu)sampleLines, (llu)allocnrow); // # nocov + if (sampleLines > allocnrow) STOP("Internal error: sampleLines(%"PRIu64") > allocnrow(%"PRIu64")", (uint64_t)sampleLines, (uint64_t)allocnrow); // # nocov } } if (nrowLimit < allocnrow) { - if (verbose) DTPRINT(" Alloc limited to lower nrows=%llu passed in.\n", (llu)nrowLimit); + if (verbose) DTPRINT(" Alloc limited to lower nrows=%"PRIu64" passed in.\n", (uint64_t)nrowLimit); estnrow = allocnrow = nrowLimit; } } @@ -1934,8 +1928,8 @@ int freadMain(freadMainArgs _args) { //********************************************************************************************* if (verbose) { DTPRINT("[10] Allocate memory for the datatable\n"); - DTPRINT(" Allocating %d column slots (%d - %d dropped) with %llu rows\n", - ncol-ndrop, ncol, ndrop, (llu)allocnrow); + DTPRINT(" Allocating %d column slots (%d - %d dropped) with %"PRIu64" rows\n", + ncol-ndrop, ncol, ndrop, (uint64_t)allocnrow); } size_t DTbytes = allocateDT(type, size, ncol, ndrop, allocnrow); double tAlloc = wallclock(); @@ -1988,15 +1982,15 @@ int freadMain(freadMainArgs _args) { // should only engage when max_nrows is supplied, and supplied small too, so doesn't matter too much. if (initialBuffRows < 10) initialBuffRows = 10; - if (initialBuffRows > INT32_MAX) STOP("Buffer size %lld is too large\n", initialBuffRows); + if (initialBuffRows > INT32_MAX) STOP("Buffer size %"PRId64" is too large\n", (int64_t)initialBuffRows); nth = imin(nJumps, nth); if (verbose) DTPRINT("[11] Read the data\n"); read: // we'll return here to reread any columns with out-of-sample type exceptions, or dirty jumps restartTeam = false; - if (verbose) DTPRINT(" jumps=[%d..%d), chunk_size=%llu, total_size=%llu\n", - jump0, nJumps, (llu)chunkBytes, (llu)(eof-pos)); - ASSERT(allocnrow <= nrowLimit, "allocnrow(%llu) <= nrowLimit(%llu)", (llu)allocnrow, (llu)nrowLimit); + if (verbose) DTPRINT(" jumps=[%d..%d), chunk_size=%"PRIu64", total_size=%"PRIu64"\n", + jump0, nJumps, (uint64_t)chunkBytes, (uint64_t)(eof-pos)); + ASSERT(allocnrow <= nrowLimit, "allocnrow(%"PRIu64") <= nrowLimit(%"PRIu64")", (uint64_t)allocnrow, (uint64_t)nrowLimit); #pragma omp parallel num_threads(nth) { int me = omp_get_thread_num(); @@ -2214,10 +2208,10 @@ int freadMain(freadMainArgs _args) { if (verbose) { char temp[1001]; int len = snprintf(temp, 1000, - "Column %d (\"%.*s\") bumped from '%s' to '%s' due to <<%.*s>> on row %llu\n", + "Column %d (\"%.*s\") bumped from '%s' to '%s' due to <<%.*s>> on row %"PRIu64"\n", j+1, colNames[j].len, colNamesAnchor + colNames[j].off, typeName[abs(joldType)], typeName[abs(thisType)], - (int)(tch-fieldStart), fieldStart, (llu)(ctx.DTi+myNrow)); + (int)(tch-fieldStart), fieldStart, (uint64_t)(ctx.DTi+myNrow)); if (len > 1000) len = 1000; if (len > 0) { typeBumpMsg = (char*) realloc(typeBumpMsg, typeBumpMsgSize + (size_t)len + 1); @@ -2336,8 +2330,8 @@ int freadMain(freadMainArgs _args) { if (extraAllocRows) { allocnrow += extraAllocRows; if (allocnrow > nrowLimit) allocnrow = nrowLimit; - if (verbose) DTPRINT(" Too few rows allocated. Allocating additional %llu rows (now nrows=%llu) and continue reading from jump %d\n", - (llu)extraAllocRows, (llu)allocnrow, jump0); + if (verbose) DTPRINT(" Too few rows allocated. Allocating additional %"PRIu64" rows (now nrows=%"PRIu64") and continue reading from jump %d\n", + (uint64_t)extraAllocRows, (uint64_t)allocnrow, jump0); allocateDT(type, size, ncol, ncol - nStringCols - nNonStringCols, allocnrow); extraAllocRows = 0; goto read; @@ -2399,8 +2393,8 @@ int freadMain(freadMainArgs _args) { } double tTot = tReread-t0; // tReread==tRead when there was no reread - if (verbose) DTPRINT("Read %llu rows x %d columns from %s file in %02d:%06.3f wall clock time\n", - (llu)DTi, ncol-ndrop, filesize_to_str(fileSize), (int)tTot/60, fmod(tTot,60.0)); + if (verbose) DTPRINT("Read %"PRIu64" rows x %d columns from %s file in %02d:%06.3f wall clock time\n", + (uint64_t)DTi, ncol-ndrop, filesize_to_str(fileSize), (int)tTot/60, fmod(tTot,60.0)); //********************************************************************************************* // [12] Finalize the datatable @@ -2430,13 +2424,13 @@ int freadMain(freadMainArgs _args) { else { ch = headPos; int tt = countfields(&ch); - DTWARN("Stopped early on line %llu. Expected %d fields but found %d. Consider fill=TRUE and comment.char=. First discarded non-empty line: <<%s>>", - (llu)DTi+row1line, ncol, tt, strlim(skippedFooter,500)); + DTWARN("Stopped early on line %"PRIu64". Expected %d fields but found %d. Consider fill=TRUE and comment.char=. First discarded non-empty line: <<%s>>", + (uint64_t)DTi+row1line, ncol, tt, strlim(skippedFooter,500)); } } } if (quoteRuleBumpedCh!=NULL && quoteRuleBumpedCh>. If the fields are not quoted (e.g. field separator does not appear within any field), try quote=\"\" to avoid this warning.", (llu)quoteRuleBumpedLine, strlim(quoteRuleBumpedCh, 500)); + DTWARN("Found and resolved improper quoting out-of-sample. First healed line %"PRIu64": <<%s>>. If the fields are not quoted (e.g. field separator does not appear within any field), try quote=\"\" to avoid this warning.", (uint64_t)quoteRuleBumpedLine, strlim(quoteRuleBumpedCh, 500)); } if (verbose) { @@ -2446,10 +2440,10 @@ int freadMain(freadMainArgs _args) { DTPRINT("%8.3fs (%3.0f%%) sep=", tLayout-tMap, 100.0*(tLayout-tMap)/tTot); DTPRINT(sep=='\t' ? "'\\t'" : (sep=='\n' ? "'\\n'" : "'%c'"), sep); DTPRINT(" ncol=%d and header detection\n", ncol); - DTPRINT("%8.3fs (%3.0f%%) Column type detection using %llu sample rows\n", - tColType-tLayout, 100.0*(tColType-tLayout)/tTot, (llu)sampleLines); - DTPRINT("%8.3fs (%3.0f%%) Allocation of %llu rows x %d cols (%.3fGB) of which %llu (%3.0f%%) rows used\n", - tAlloc-tColType, 100.0*(tAlloc-tColType)/tTot, (llu)allocnrow, ncol, DTbytes/(1024.0*1024*1024), (llu)DTi, 100.0*DTi/allocnrow); + DTPRINT("%8.3fs (%3.0f%%) Column type detection using %"PRIu64" sample rows\n", + tColType-tLayout, 100.0*(tColType-tLayout)/tTot, (uint64_t)sampleLines); + DTPRINT("%8.3fs (%3.0f%%) Allocation of %"PRIu64" rows x %d cols (%.3fGB) of which %"PRIu64" (%3.0f%%) rows used\n", + tAlloc-tColType, 100.0*(tAlloc-tColType)/tTot, (uint64_t)allocnrow, ncol, DTbytes/(1024.0*1024*1024), (uint64_t)DTi, 100.0*DTi/allocnrow); thRead/=nth; thPush/=nth; double thWaiting = tReread-tAlloc-thRead-thPush; DTPRINT("%8.3fs (%3.0f%%) Reading %d chunks (%d swept) of %.3fMB (each chunk %d rows) using %d threads\n", diff --git a/src/fread.h b/src/fread.h index 8333bb53ef..e70dbbe42f 100644 --- a/src/fread.h +++ b/src/fread.h @@ -1,6 +1,7 @@ #ifndef dt_FREAD_H #define dt_FREAD_H #include // uint32_t +#include // PRId64 #include // size_t #include // bool #include "myomp.h" diff --git a/src/freadR.c b/src/freadR.c index 817805cfac..36e72ff968 100644 --- a/src/freadR.c +++ b/src/freadR.c @@ -572,7 +572,7 @@ void pushBuffer(ThreadLocalFreadParsingContext *ctx) if (thisSize == 4) { char *dest = (char *)DATAPTR(VECTOR_ELT(DT, resj)) + DTi*4; char *src4 = (char*)buff4 + off4; - // debug line for #3369 ... if (DTi>2638000) printf("freadR.c:460: thisSize==4, resj=%d, %zd, %d, %d, j=%d, done=%d\n", resj, DTi, off4, rowSize4, j, done); + // debug line for #3369 ... if (DTi>2638000) printf("freadR.c:460: thisSize==4, resj=%d, %"PRIu64", %d, %d, j=%d, done=%d\n", resj, (uint64_t)DTi, off4, rowSize4, j, done); for (int i=0; imessage[0]), 500, "%s: running for input length %llu, window %d, hasna %d, narm %d\n", __func__, (unsigned long long int)nx, k, hasna, (int)narm); + snprintf(end(ans->message[0]), 500, "%s: running for input length %"PRIu64", window %d, hasna %d, narm %d\n", __func__, (uint64_t)nx, k, hasna, (int)narm); long double w = 0.0; // sliding window aggregate bool truehasna = hasna>0; // flag to re-run with NA support if NAs detected if (!truehasna) { @@ -138,7 +138,7 @@ void frollmeanFast(double *x, uint64_t nx, ans_t *ans, int k, double fill, bool */ void frollmeanExact(double *x, uint64_t nx, ans_t *ans, int k, double fill, bool narm, int hasna, bool verbose) { if (verbose) - snprintf(end(ans->message[0]), 500, "%s: running in parallel for input length %llu, window %d, hasna %d, narm %d\n", __func__, (unsigned long long int)nx, k, hasna, (int)narm); + snprintf(end(ans->message[0]), 500, "%s: running in parallel for input length %"PRIu64", window %d, hasna %d, narm %d\n", __func__, (uint64_t)nx, k, hasna, (int)narm); for (int i=0; idbl_v[i] = fill; } @@ -252,7 +252,7 @@ void frollsum(unsigned int algo, double *x, uint64_t nx, ans_t *ans, int k, int } void frollsumFast(double *x, uint64_t nx, ans_t *ans, int k, double fill, bool narm, int hasna, bool verbose) { if (verbose) - snprintf(end(ans->message[0]), 500, "%s: running for input length %llu, window %d, hasna %d, narm %d\n", __func__, (unsigned long long int)nx, k, hasna, (int)narm); + snprintf(end(ans->message[0]), 500, "%s: running for input length %"PRIu64", window %d, hasna %d, narm %d\n", __func__, (uint64_t)nx, k, hasna, (int)narm); long double w = 0.0; bool truehasna = hasna>0; if (!truehasna) { @@ -336,7 +336,7 @@ void frollsumFast(double *x, uint64_t nx, ans_t *ans, int k, double fill, bool n } void frollsumExact(double *x, uint64_t nx, ans_t *ans, int k, double fill, bool narm, int hasna, bool verbose) { if (verbose) - snprintf(end(ans->message[0]), 500, "%s: running in parallel for input length %llu, window %d, hasna %d, narm %d\n", __func__, (unsigned long long int)nx, k, hasna, (int)narm); + snprintf(end(ans->message[0]), 500, "%s: running in parallel for input length %"PRIu64", window %d, hasna %d, narm %d\n", __func__, (uint64_t)nx, k, hasna, (int)narm); for (int i=0; idbl_v[i] = fill; } diff --git a/src/frolladaptive.c b/src/frolladaptive.c index 9dc5581a83..9451e70e22 100644 --- a/src/frolladaptive.c +++ b/src/frolladaptive.c @@ -30,7 +30,7 @@ void fadaptiverollmean(unsigned int algo, double *x, uint64_t nx, ans_t *ans, in */ void fadaptiverollmeanFast(double *x, uint64_t nx, ans_t *ans, int *k, double fill, bool narm, int hasna, bool verbose) { if (verbose) - snprintf(end(ans->message[0]), 500, "%s: running for input length %llu, hasna %d, narm %d\n", __func__, (unsigned long long int)nx, hasna, (int) narm); + snprintf(end(ans->message[0]), 500, "%s: running for input length %"PRIu64", hasna %d, narm %d\n", __func__, (uint64_t)nx, hasna, (int) narm); bool truehasna = hasna>0; // flag to re-run if NAs detected long double w = 0.0; double *cs = malloc(nx*sizeof(double)); // cumsum vector, same as double cs[nx] but no segfault @@ -115,7 +115,7 @@ void fadaptiverollmeanFast(double *x, uint64_t nx, ans_t *ans, int *k, double fi */ void fadaptiverollmeanExact(double *x, uint64_t nx, ans_t *ans, int *k, double fill, bool narm, int hasna, bool verbose) { if (verbose) - snprintf(end(ans->message[0]), 500, "%s: running in parallel for input length %llu, hasna %d, narm %d\n", __func__, (unsigned long long int)nx, hasna, (int) narm); + snprintf(end(ans->message[0]), 500, "%s: running in parallel for input length %"PRIu64", hasna %d, narm %d\n", __func__, (uint64_t)nx, hasna, (int) narm); bool truehasna = hasna>0; // flag to re-run if NAs detected if (!truehasna || !narm) { // narm=FALSE handled here as NAs properly propagated in exact algo #pragma omp parallel for num_threads(getDTthreads()) @@ -219,7 +219,7 @@ void fadaptiverollsum(unsigned int algo, double *x, uint64_t nx, ans_t *ans, int } void fadaptiverollsumFast(double *x, uint64_t nx, ans_t *ans, int *k, double fill, bool narm, int hasna, bool verbose) { if (verbose) - snprintf(end(ans->message[0]), 500, "%s: running for input length %llu, hasna %d, narm %d\n", __func__, (unsigned long long int)nx, hasna, (int) narm); + snprintf(end(ans->message[0]), 500, "%s: running for input length %"PRIu64", hasna %d, narm %d\n", __func__, (uint64_t)nx, hasna, (int) narm); bool truehasna = hasna>0; long double w = 0.0; double *cs = malloc(nx*sizeof(double)); @@ -299,7 +299,7 @@ void fadaptiverollsumFast(double *x, uint64_t nx, ans_t *ans, int *k, double fil } void fadaptiverollsumExact(double *x, uint64_t nx, ans_t *ans, int *k, double fill, bool narm, int hasna, bool verbose) { if (verbose) - snprintf(end(ans->message[0]), 500, "%s: running in parallel for input length %llu, hasna %d, narm %d\n", __func__, (unsigned long long int)nx, hasna, (int) narm); + snprintf(end(ans->message[0]), 500, "%s: running in parallel for input length %"PRIu64", hasna %d, narm %d\n", __func__, (uint64_t)nx, hasna, (int) narm); bool truehasna = hasna>0; if (!truehasna || !narm) { #pragma omp parallel for num_threads(getDTthreads()) diff --git a/src/fsort.c b/src/fsort.c index 2de548f9bd..9a71a302f6 100644 --- a/src/fsort.c +++ b/src/fsort.c @@ -124,7 +124,7 @@ SEXP fsort(SEXP x, SEXP verboseArg) { size_t batchSize = (xlength(x)-1)/nBatch + 1; if (batchSize < 1024) batchSize = 1024; // simple attempt to work reasonably for short vector. 1024*8 = 2 4kb pages nBatch = (xlength(x)-1)/batchSize + 1; - R_xlen_t lastBatchSize = xlength(x) - (nBatch-1)*batchSize; + size_t lastBatchSize = xlength(x) - (nBatch-1)*batchSize; // could be that lastBatchSize == batchSize when i) xlength(x) is multiple of nBatch // and ii) for small vectors with just one batch @@ -174,9 +174,10 @@ SEXP fsort(SEXP x, SEXP verboseArg) { // provided MSBsize>=9, each batch is a multiple of at least one 4k page, so no page overlap // TODO: change all calloc, malloc and free to Calloc and Free to be robust to error() and catch ooms. - if (verbose) Rprintf("counts is %dMB (%d pages per nBatch=%d, batchSize=%lld, lastBatchSize=%lld)\n", - nBatch*MSBsize*sizeof(R_xlen_t)/(1024*1024), nBatch*MSBsize*sizeof(R_xlen_t)/(4*1024*nBatch), - nBatch, batchSize, lastBatchSize); + if (verbose) Rprintf("counts is %dMB (%d pages per nBatch=%d, batchSize=%"PRIu64", lastBatchSize=%"PRIu64")\n", + (int)(nBatch*MSBsize*sizeof(R_xlen_t)/(1024*1024)), + (int)(nBatch*MSBsize*sizeof(R_xlen_t)/(4*1024*nBatch)), + nBatch, (uint64_t)batchSize, (uint64_t)lastBatchSize); t[3] = wallclock(); #pragma omp parallel for num_threads(nth) for (int batch=0; batch0 && msbCounts[order[MSBsize-1]] < 2) MSBsize--; diff --git a/src/fwrite.c b/src/fwrite.c index 603630e551..aa30ab8413 100644 --- a/src/fwrite.c +++ b/src/fwrite.c @@ -3,6 +3,7 @@ #include #include // true and false #include // INT32_MIN +#include // PRId64 #include // isfinite, isnan #include // abs #include // strlen, strerror @@ -228,7 +229,7 @@ void writeFloat64(double *col, int64_t row, char **pch) uint64_t l = y * SIZE_SF; // low magnitude mult 10^NUM_SF // l now contains NUM_SF+1 digits as integer where repeated /10 below is accurate - // if (verbose) Rprintf("\nTRACE: acc=%.20Le ; y=%.20Le ; l=%llu ; e=%d ", acc, y, l, exp); + // if (verbose) Rprintf("\nTRACE: acc=%.20Le ; y=%.20Le ; l=%"PRIu64" ; e=%d ", acc, y, l, exp); if (l%10 >= 5) l+=10; // use the last digit to round l /= 10; @@ -618,8 +619,8 @@ void fwriteMain(fwriteMainArgs args) DTPRINT("... "); for (int j=args.ncol-10; j= 2) { if (verbose && !hasPrinted) DTPRINT("\n"); - DTPRINT("\rWritten %.1f%% of %lld rows in %d secs using %d thread%s. " + DTPRINT("\rWritten %.1f%% of %"PRId64" rows in %d secs using %d thread%s. " "maxBuffUsed=%d%%. ETA %d secs. ", - (100.0*end)/args.nrow, (long long)args.nrow, (int)(now-startTime), nth, nth==1?"":"s", + (100.0*end)/args.nrow, (int64_t)args.nrow, (int)(now-startTime), nth, nth==1?"":"s", maxBuffUsedPC, ETA); // TODO: use progress() as in fread nextTime = now+1; diff --git a/src/init.c b/src/init.c index 7c8ec64e0b..518d1cc3ac 100644 --- a/src/init.c +++ b/src/init.c @@ -251,7 +251,7 @@ void attribute_visible R_init_datatable(DllInfo *info) // Variables rather than #define for NA_INT64 to ensure correct usage; i.e. not casted NA_INT64_LL = LLONG_MIN; NA_INT64_D = LLtoD(NA_INT64_LL); - if (NA_INT64_LL != DtoLL(NA_INT64_D)) error("Conversion of NA_INT64 via double failed %lld!=%lld", NA_INT64_LL, DtoLL(NA_INT64_D)); + if (NA_INT64_LL != DtoLL(NA_INT64_D)) error("Conversion of NA_INT64 via double failed %"PRId64"!=%"PRId64"", (int64_t)NA_INT64_LL, (int64_t)DtoLL(NA_INT64_D)); // LLONG_MIN when punned to double is the sign bit set and then all zeros in exponent and significand i.e. -0.0 // That's why we must never test for NA_INT64_D using == in double type. Must always DtoLL and compare long long types. // Assigning NA_INT64_D to a REAL is ok however. diff --git a/src/rbindlist.c b/src/rbindlist.c index f9bba8bb78..d85c9d916e 100644 --- a/src/rbindlist.c +++ b/src/rbindlist.c @@ -59,7 +59,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg) firstZeroCol+1, ch, firstZeroItem+1, numZero-1, numZero==2?"":"s"); } if (nrow==0 && ncol==0) return(R_NilValue); - if (nrow>INT32_MAX) error("Total rows in the list is %lld which is larger than the maximum number of rows, currently %d", nrow, INT32_MAX); + if (nrow>INT32_MAX) error("Total rows in the list is %"PRId64" which is larger than the maximum number of rows, currently %d", (int64_t)nrow, INT32_MAX); if (usenames==TRUE && !anyNames) error("use.names=TRUE but no item of input list has any names"); int *colMap=NULL; // maps each column in final result to the column of each list item @@ -68,7 +68,7 @@ SEXP rbindlist(SEXP l, SEXP usenamesArg, SEXP fillArg, SEXP idcolArg) // when use.names==NA we also proceed here as if use.names was TRUE to save new code and then check afterwards the map is 1:ncol for every item // first find number of unique column names present; i.e. length(unique(unlist(lapply(l,names)))) SEXP *uniq = (SEXP *)malloc(upperBoundUniqueNames * sizeof(SEXP)); // upperBoundUniqueNames was initialized with 1 to ensure this is defined (otherwise 0 when no item has names) - if (!uniq) error("Failed to allocate upper bound of %lld unique column names [sum(lapply(l,ncol))]", upperBoundUniqueNames); + if (!uniq) error("Failed to allocate upper bound of %"PRId64" unique column names [sum(lapply(l,ncol))]", (int64_t)upperBoundUniqueNames); savetl_init(); int nuniq=0; for (int i=0; incol) error("Item %d of cols is %d which is outside range of l [1,length(l)=%d]", i+1, elem, ncol); } for (int i=1; i