Can not add a column when re-indexing a datasource using DruidInputSource

[suneet-s]

Affected Version

0.18

Description

See the following ingestionSpec that can be used in Integration tests. The transformed column "newPage" does not show up in the dimensions for the new re-indexed dataSource. To work around this, I fall back to use the ingestSegmentFirehose instead

{
    "type": "index",
    "spec": {
        "ioConfig": {
            "type": "index",
            "inputSource": {
                "type": "druid",
                "dataSource": "%%DATASOURCE%%",
                "interval": "2013-08-31/2013-09-01"
            }
        },
        "tuningConfig": {
            "type": "index",
            "partitionsSpec": {
                "type": "dynamic"
            }
        },
        "dataSchema": {
            "dataSource": "%%REINDEX_DATASOURCE%%",
            "granularitySpec": {
                "type": "uniform",
                "queryGranularity": "SECOND",
                "segmentGranularity": "DAY"
            },
            "timestampSpec": {
                "column": "__time",
                "format": "iso"
            },
            "dimensionsSpec": {
                "dimensions": [
                    {"type": "string", "name": "language", "createBitmapIndex": false},
                    "user",
                    "unpatrolled",
                    "page",
                    "newPage",
                    "anonymous",
                    "namespace",
                    "country",
                    "region",
                    "city"
                ],
                "dimensionExclusions" : ["robot", "continent"]
            },
            "transformSpec": {
                "transforms": [
                    {
                        "type": "expression",
                        "name": "newPage",
                        "expression": "\"page\""
                    },
                    {
                        "type": "expression",
                        "name": "one-plus-triple-added",
                        "expression": "\"triple-added\" + 1"
                    },
                    {
                        "type": "expression",
                        "name": "double-deleted",
                        "expression": "deleted * 2"
                    }
                ]
            },
            "metricsSpec": [
                {
                    "type": "doubleSum",
                    "name": "added",
                    "fieldName": "added"
                },
                {
                    "type": "doubleSum",
                    "name": "triple-added",
                    "fieldName": "triple-added"
                },
                {
                    "type": "doubleSum",
                    "name": "one-plus-triple-added",
                    "fieldName": "one-plus-triple-added"
                },
                {
                    "type": "doubleSum",
                    "name": "deleted",
                    "fieldName": "deleted"
                },
                {
                    "type": "doubleSum",
                    "name": "double-deleted",
                    "fieldName": "double-deleted"
                },
                {
                    "type": "doubleSum",
                    "name": "delta",
                    "fieldName": "delta"
                }
            ]
        }
    }
}