Add time extraction functions to project time as a dimension

docs/content/DimensionSpecs.md

@@ -1,78 +1,156 @@
 ---
 layout: doc_page
 ---
+
 # Transforming Dimension Values
-The following JSON fields can be used in a query to operate on dimension values. 
+
+The following JSON fields can be used in a query to operate on dimension values.
 
 ## DimensionSpec
 
 `DimensionSpec`s define how dimension values get transformed prior to aggregation.
 
-### DefaultDimensionSpec
+### Default DimensionSpec
 
 Returns dimension values as is and optionally renames the dimension.
 
 ```json
 { "type" : "default", "dimension" : <dimension>, "outputName": <output_name> }
 ```
 
-### ExtractionDimensionSpec
+### Extraction DimensionSpec
 
-Returns dimension values transformed using the given [DimExtractionFn](#toc_3)
+Returns dimension values transformed using the given [extraction function](#extraction-functions).
 
 ```json
 {
   "type" : "extraction",
   "dimension" : <dimension>,
   "outputName" :  <output_name>,
-  "dimExtractionFn" : <dim_extraction_fn>
+  "extractionFn" : <extraction_function>
 }
 ```
 
-## <a id="toc_3"></a>DimExtractionFn
+## Extraction Functions
+
+Extraction functions define the transformation applied to each dimension value.
 
-`DimExtractionFn`s define the transformation applied to each dimension value
+Transformations can be applied to both regular (string) dimensions, as well
+as the special `__time` dimension, which represents the current time bucket
+according to the query [aggregation granularity](Granularities.html).
 
-### RegexDimExtractionFn
+**Note**: for functions taking string values (such as regular expressions),
+`__time` dimension values will be formatted in [ISO-8601 format](https://en.wikipedia.org/wiki/ISO_8601)
+before getting passed to the extraction function.
 
-Returns the first group matched by the given regular expression. If there is no match it returns the dimension value as is.
+### Regular Expression Extraction Function
+
+Returns the first matching group for the given regular expression.
+If there is no match, it returns the dimension value as is.
 
 ```json
-{ "type" : "regex", "expr", <regular_expression> }
+{ "type" : "regex", "expr" : <regular_expression> }
 ```
 
-### PartialDimExtractionFn
+For example, using `"expr" : "(\\w\\w\\w).*"` will transform
+`'Monday'`, `'Tuesday'`, `'Wednesday'` into `'Mon'`, `'Tue'`, `'Wed'`.
+
+### Partial Extraction Function
 
-Returns the dimension value as is if there is a match, otherwise returns null.
+Returns the dimension value unchanged if the regular expression matches, otherwise returns null.
 
 ```json
-{ "type" : "partial", "expr", <regular_expression> }
+{ "type" : "partial", "expr" : <regular_expression> }
 ```
 
-### SearchQuerySpecDimExtractionFn
+### Search Query Extraction Function
 
-Returns the dimension value as is if the given [SearchQuerySpec](SearchQuerySpec.html) matches, otherwise returns null.
+Returns the dimension value unchanged if the given [`SearchQuerySpec`](SearchQuerySpec.html)
+matches, otherwise returns null.
 
 ```json
 { "type" : "searchQuery", "query" : <search_query_spec> }
 ```
 
-### TimeDimExtractionFn
+### Time Format Extraction Function
 
-Parses dimension values as timestamps using the given input format, and returns them formatted using the given output format. Time formats follow the [com.ibm.icu.text.SimpleDateFormat](http://icu-project.org/apiref/icu4j/com/ibm/icu/text/SimpleDateFormat.html) format
+Returns the dimension value formatted according to the given format string, time zone, and locale.
+
+For `__time` dimension values, this formats the time value bucketed by the
+[aggregation granularity](Granularities.html)
+
+For a regular dimension, it assumes the string is formatted in
+[ISO-8601 date and time format](https://en.wikipedia.org/wiki/ISO_8601).
+
+* `format` : date time format for the resulting dimension value, in [Joda Time DateTimeFormat](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html).
+* `locale` : locale (language and country) to use, given as a [IETF BCP 47 language tag](http://www.oracle.com/technetwork/java/javase/java8locales-2095355.html#util-text), e.g. `en-US`, `en-GB`, `fr-FR`, `fr-CA`, etc.
+* `timeZone` : time zone to use in [IANA tz database format](http://en.wikipedia.org/wiki/List_of_tz_database_time_zones), e.g. `Europe/Berlin` (this can possibly be different than the aggregation time-zone)
 
 ```json
-{ "type" : "time", "timeFormat" : <input_format>, "resultFormat" : <output_format> }
+{ "type" : "timeFormat",
+  "format" : <output_format>,
+  "timeZone" : <time_zone> (optional),
+  "locale" : <locale> (optional) }
 ```
 
-### JavascriptDimExtractionFn
+For example, the following dimension spec returns the day of the week for Montréal in French:
 
-Returns the dimension value as transformed by the given JavaScript function.
+```json
+{
+  "type" : "extraction",
+  "dimension" : "__time",
+  "outputName" :  "dayOfWeek",
+  "extractionFn" : {
+    "type" : "timeFormat",
+    "format" : "EEEE",
+    "timeZone" : "America/Montreal",
+    "locale" : "fr"
+  }
+}
+```
 
-Example
+### Time Parsing Extraction Function
+
+Parses dimension values as timestamps using the given input format,
+and returns them formatted using the given output format.
+
+Note, if you are working with the `__time` dimension, you should consider using the
+[time extraction function instead](#time-format-extraction-function) instead,
+which works on time value directly as opposed to string values.
+
+Time formats are described in the
+[SimpleDateFormat documentation](http://icu-project.org/apiref/icu4j/com/ibm/icu/text/SimpleDateFormat.html)
+
+```json
+{ "type" : "time",
+  "timeFormat" : <input_format>,
+  "resultFormat" : <output_format> }
+```
+
+
+### Javascript Extraction Function
+
+Returns the dimension value, as transformed by the given JavaScript function.
+
+For regular dimensions, the input value is passed as a string.
+
+For the `__time` dimension, the input value is passed as a number
+representing the number of milliseconds since January 1, 1970 UTC.
+
+Example for a regular dimension
 
 ```json
 {
   "type" : "javascript",
   "function" : "function(str) { return str.substr(0, 3); }"
 }
+```
+
+Example for the `__time` dimension:
+
+```json
+{
+  "type" : "javascript",
+  "function" : "function(t) { return 'Second ' + Math.floor((t % 60000) / 1000); }"
+}
+```

docs/content/Granularities.md

@@ -1,6 +1,7 @@
 ---
 layout: doc_page
 ---
+
 # Aggregation Granularity
 The granularity field determines how data gets bucketed across the time dimension, or how it gets aggregated by hour, day, minute, etc.
 

indexing-service/src/main/java/io/druid/indexing/firehose/IngestSegmentFirehoseFactory.java

@@ -256,7 +256,7 @@ public Sequence<InputRow> apply(final Cursor cursor)
 
                       final Map<String, DimensionSelector> dimSelectors = Maps.newHashMap();
                       for (String dim : dims) {
-                        final DimensionSelector dimSelector = cursor.makeDimensionSelector(dim);
+                        final DimensionSelector dimSelector = cursor.makeDimensionSelector(dim, null);
                         // dimSelector is null if the dimension is not present
                         if (dimSelector != null) {
                           dimSelectors.put(dim, dimSelector);

processing/src/main/java/io/druid/query/aggregation/cardinality/CardinalityAggregatorFactory.java

@@ -105,7 +105,7 @@ private List<DimensionSelector> makeDimensionSelectors(final ColumnSelectorFacto
               @Override
               public DimensionSelector apply(@Nullable String input)
               {
-                return columnFactory.makeDimensionSelector(input);
+                return columnFactory.makeDimensionSelector(input, null);
               }
             }
             ), Predicates.notNull()

processing/src/main/java/io/druid/query/dimension/DefaultDimensionSpec.java

@@ -20,7 +20,7 @@
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.metamx.common.StringUtils;
-import io.druid.query.extraction.DimExtractionFn;
+import io.druid.query.extraction.ExtractionFn;
 
 import java.nio.ByteBuffer;
 
@@ -59,7 +59,7 @@ public String getOutputName()
   }
 
   @Override
-  public DimExtractionFn getDimExtractionFn()
+  public ExtractionFn getExtractionFn()
   {
     return null;
   }

processing/src/main/java/io/druid/query/dimension/DimensionSpec.java

@@ -19,7 +19,7 @@
 
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import io.druid.query.extraction.DimExtractionFn;
+import io.druid.query.extraction.ExtractionFn;
 
 /**
  */
@@ -34,7 +34,7 @@ public interface DimensionSpec
 
   public String getOutputName();
 
-  public DimExtractionFn getDimExtractionFn();
+  public ExtractionFn getExtractionFn();
 
   public byte[] getCacheKey();
 

processing/src/main/java/io/druid/query/dimension/ExtractionDimensionSpec.java

@@ -19,8 +19,9 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
 import com.metamx.common.StringUtils;
-import io.druid.query.extraction.DimExtractionFn;
+import io.druid.query.extraction.ExtractionFn;
 
 import java.nio.ByteBuffer;
 
@@ -31,18 +32,23 @@ public class ExtractionDimensionSpec implements DimensionSpec
   private static final byte CACHE_TYPE_ID = 0x1;
 
   private final String dimension;
-  private final DimExtractionFn dimExtractionFn;
+  private final ExtractionFn extractionFn;
   private final String outputName;
 
   @JsonCreator
   public ExtractionDimensionSpec(
       @JsonProperty("dimension") String dimension,
       @JsonProperty("outputName") String outputName,
-      @JsonProperty("dimExtractionFn") DimExtractionFn dimExtractionFn
+      @JsonProperty("extractionFn") ExtractionFn extractionFn,
+      // for backwards compatibility
+      @Deprecated @JsonProperty("dimExtractionFn") ExtractionFn dimExtractionFn
   )
   {
+    Preconditions.checkNotNull(dimension, "dimension must not be null");
+    Preconditions.checkArgument(extractionFn != null || dimExtractionFn != null, "extractionFn must not be null");
+
     this.dimension = dimension;
-    this.dimExtractionFn = dimExtractionFn;
+    this.extractionFn = extractionFn != null ? extractionFn : dimExtractionFn;
 
     // Do null check for backwards compatibility
     this.outputName = outputName == null ? dimension : outputName;
@@ -64,16 +70,16 @@ public String getOutputName()
 
   @Override
   @JsonProperty
-  public DimExtractionFn getDimExtractionFn()
+  public ExtractionFn getExtractionFn()
   {
-    return dimExtractionFn;
+    return extractionFn;
   }
 
   @Override
   public byte[] getCacheKey()
   {
     byte[] dimensionBytes = StringUtils.toUtf8(dimension);
-    byte[] dimExtractionFnBytes = dimExtractionFn.getCacheKey();
+    byte[] dimExtractionFnBytes = extractionFn.getCacheKey();
 
     return ByteBuffer.allocate(1 + dimensionBytes.length + dimExtractionFnBytes.length)
                      .put(CACHE_TYPE_ID)
@@ -85,15 +91,15 @@ public byte[] getCacheKey()
   @Override
   public boolean preservesOrdering()
   {
-    return dimExtractionFn.preservesOrdering();
+    return extractionFn.preservesOrdering();
   }
 
   @Override
   public String toString()
   {
     return "ExtractionDimensionSpec{" +
            "dimension='" + dimension + '\'' +
-           ", dimExtractionFn=" + dimExtractionFn +
+           ", extractionFn=" + extractionFn +
            ", outputName='" + outputName + '\'' +
            '}';
   }
@@ -106,7 +112,7 @@ public boolean equals(Object o)
 
     ExtractionDimensionSpec that = (ExtractionDimensionSpec) o;
 
-    if (dimExtractionFn != null ? !dimExtractionFn.equals(that.dimExtractionFn) : that.dimExtractionFn != null)
+    if (extractionFn != null ? !extractionFn.equals(that.extractionFn) : that.extractionFn != null)
       return false;
     if (dimension != null ? !dimension.equals(that.dimension) : that.dimension != null) return false;
     if (outputName != null ? !outputName.equals(that.outputName) : that.outputName != null) return false;
@@ -118,7 +124,7 @@ public boolean equals(Object o)
   public int hashCode()
   {
     int result = dimension != null ? dimension.hashCode() : 0;
-    result = 31 * result + (dimExtractionFn != null ? dimExtractionFn.hashCode() : 0);
+    result = 31 * result + (extractionFn != null ? extractionFn.hashCode() : 0);
     result = 31 * result + (outputName != null ? outputName.hashCode() : 0);
     return result;
   }