diff --git a/.gitignore b/.gitignore index 845a2ddc0072..c7f1290850b4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,6 @@ target .idea .project .settings/ -examples/rand/RealtimeNode.out -examples/twitter/RealtimeNode.out *.log *.DS_Store +_site diff --git a/client/pom.xml b/client/pom.xml deleted file mode 100644 index d7ce942253ad..000000000000 --- a/client/pom.xml +++ /dev/null @@ -1,243 +0,0 @@ - - - - - 4.0.0 - com.metamx.druid - druid-client - druid-client - druid-client - - - com.metamx - druid - 0.5.59-SNAPSHOT - - - - - com.metamx.druid - druid-common - ${project.parent.version} - - - com.metamx.druid - druid-indexing-common - ${project.parent.version} - - - - com.metamx - emitter - - - com.metamx - http-client - - - com.metamx - java-util - - - com.metamx - server-metrics - - - - com.davekoelle - alphanum - - - commons-codec - commons-codec - - - commons-httpclient - commons-httpclient - - - org.skife.config - config-magic - - - org.apache.curator - curator-client - - - org.apache.curator - curator-framework - - - org.apache.curator - curator-x-discovery - - - org.mortbay.jetty - jetty - - - org.mortbay.jetty - jetty-util - - - com.google.guava - guava - - - com.google.inject - guice - - - com.ibm.icu - icu4j - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - - - javax.inject - javax.inject - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - com.sun.jersey.contribs - jersey-guice - - - joda-time - joda-time - - - com.google.code.findbugs - jsr305 - - - com.google.inject.extensions - guice-servlet - - - log4j - log4j - - - org.jboss.netty - netty - 3.2.4.Final - - - javax.servlet - servlet-api - - - org.slf4j - slf4j-log4j12 - - - com.google.code.simple-spring-memcached - spymemcached - - - org.antlr - antlr4-runtime - - - commons-cli - commons-cli - - - com.metamx - bytebuffer-collections - - - net.jpountz.lz4 - lz4 - - - - - junit - junit - test - - - org.easymock - easymock - test - - - com.google.caliper - caliper - test - - - org.apache.curator - curator-test - test - - - - - - - - maven-jar-plugin - - - - test-jar - - - - - - - org.antlr - antlr4-maven-plugin - - - - antlr4 - - - - - - - diff --git a/client/src/main/java/com/metamx/druid/QueryableNode.java b/client/src/main/java/com/metamx/druid/QueryableNode.java deleted file mode 100644 index ca638f44727f..000000000000 --- a/client/src/main/java/com/metamx/druid/QueryableNode.java +++ /dev/null @@ -1,618 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.metamx.common.IAE; -import com.metamx.common.ISE; -import com.metamx.common.concurrent.ScheduledExecutorFactory; -import com.metamx.common.concurrent.ScheduledExecutors; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.lifecycle.LifecycleStart; -import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.common.logger.Logger; -import com.metamx.druid.client.BatchServerInventoryView; -import com.metamx.druid.client.DruidServerConfig; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.client.ServerInventoryView; -import com.metamx.druid.client.ServerInventoryViewConfig; -import com.metamx.druid.client.ServerView; -import com.metamx.druid.client.SingleServerInventoryView; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.coordination.AbstractDataSegmentAnnouncer; -import com.metamx.druid.coordination.BatchDataSegmentAnnouncer; -import com.metamx.druid.coordination.DataSegmentAnnouncer; -import com.metamx.druid.coordination.DruidServerMetadata; -import com.metamx.druid.coordination.MultipleDataSegmentAnnouncerDataSegmentAnnouncer; -import com.metamx.druid.coordination.SingleDataSegmentAnnouncer; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.http.NoopRequestLogger; -import com.metamx.druid.http.RequestLogger; -import com.metamx.druid.initialization.CuratorConfig; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.initialization.ServerConfig; -import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.utils.PropUtils; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.Emitters; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.HttpClientConfig; -import com.metamx.http.client.HttpClientInit; -import com.metamx.metrics.JvmMonitor; -import com.metamx.metrics.Monitor; -import com.metamx.metrics.MonitorScheduler; -import com.metamx.metrics.MonitorSchedulerConfig; -import com.metamx.metrics.SysMonitor; -import org.apache.curator.framework.CuratorFramework; -import org.joda.time.Duration; -import org.mortbay.jetty.Server; -import org.skife.config.ConfigurationObjectFactory; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -/** - */ -public abstract class QueryableNode extends RegisteringNode -{ - private final Logger log; - - private final Lifecycle lifecycle; - private final ObjectMapper jsonMapper; - private final ObjectMapper smileMapper; - private final Properties props; - private final ConfigurationObjectFactory configFactory; - private final String nodeType; - - private DruidServerMetadata druidServerMetadata = null; - private ServiceEmitter emitter = null; - private List monitors = null; - private Server server = null; - private CuratorFramework curator = null; - private DataSegmentAnnouncer announcer = null; - private ZkPathsConfig zkPaths = null; - private ScheduledExecutorFactory scheduledExecutorFactory = null; - private RequestLogger requestLogger = null; - private ServerInventoryView serverInventoryView = null; - private ServerView serverView = null; - private InventoryView inventoryView = null; - - private boolean initialized = false; - - public QueryableNode( - String nodeType, - Logger log, - Properties props, - Lifecycle lifecycle, - ObjectMapper jsonMapper, - ObjectMapper smileMapper, - ConfigurationObjectFactory configFactory - ) - { - super(Arrays.asList(jsonMapper, smileMapper)); - - this.log = log; - this.configFactory = configFactory; - this.props = props; - this.jsonMapper = jsonMapper; - this.lifecycle = lifecycle; - this.smileMapper = smileMapper; - - Preconditions.checkNotNull(props, "props"); - Preconditions.checkNotNull(lifecycle, "lifecycle"); - Preconditions.checkNotNull(jsonMapper, "jsonMapper"); - Preconditions.checkNotNull(smileMapper, "smileMapper"); - Preconditions.checkNotNull(configFactory, "configFactory"); - - Preconditions.checkState(smileMapper.getJsonFactory() instanceof SmileFactory, "smileMapper should use smile."); - this.nodeType = nodeType; - } - - public T setDruidServerMetadata(DruidServerMetadata druidServerMetadata) - { - checkFieldNotSetAndSet("druidServerMetadata", druidServerMetadata); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setCuratorFramework(CuratorFramework curator) - { - checkFieldNotSetAndSet("curator", curator); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setAnnouncer(DataSegmentAnnouncer announcer) - { - checkFieldNotSetAndSet("announcer", announcer); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setEmitter(ServiceEmitter emitter) - { - checkFieldNotSetAndSet("emitter", emitter); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setMonitors(List monitors) - { - checkFieldNotSetAndSet("monitors", monitors); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setServer(Server server) - { - checkFieldNotSetAndSet("server", server); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setZkPaths(ZkPathsConfig zkPaths) - { - checkFieldNotSetAndSet("zkPaths", zkPaths); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setScheduledExecutorFactory(ScheduledExecutorFactory factory) - { - checkFieldNotSetAndSet("scheduledExecutorFactory", factory); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setRequestLogger(RequestLogger requestLogger) - { - checkFieldNotSetAndSet("requestLogger", requestLogger); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setInventoryView(InventoryView inventoryView) - { - checkFieldNotSetAndSet("inventoryView", inventoryView); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setServerView(ServerView serverView) - { - checkFieldNotSetAndSet("serverView", serverView); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T registerJacksonSubtype(Class... clazzes) - { - jsonMapper.registerSubtypes(clazzes); - smileMapper.registerSubtypes(clazzes); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T registerJacksonSubtype(NamedType... namedTypes) - { - jsonMapper.registerSubtypes(namedTypes); - smileMapper.registerSubtypes(namedTypes); - return (T) this; - } - - public Lifecycle getLifecycle() - { - return lifecycle; - } - - public ObjectMapper getJsonMapper() - { - return jsonMapper; - } - - public ObjectMapper getSmileMapper() - { - return smileMapper; - } - - public Properties getProps() - { - return props; - } - - public ConfigurationObjectFactory getConfigFactory() - { - return configFactory; - } - - public DruidServerMetadata getDruidServerMetadata() - { - initializeDruidServerMetadata(); - return druidServerMetadata; - } - - public CuratorFramework getCuratorFramework() - { - initializeCuratorFramework(); - return curator; - } - - public DataSegmentAnnouncer getAnnouncer() - { - initializeAnnouncer(); - return announcer; - } - - public ServiceEmitter getEmitter() - { - initializeEmitter(); - return emitter; - } - - public List getMonitors() - { - initializeMonitors(); - return monitors; - } - - public Server getServer() - { - initializeServer(); - return server; - } - - public ZkPathsConfig getZkPaths() - { - initializeZkPaths(); - return zkPaths; - } - - public ScheduledExecutorFactory getScheduledExecutorFactory() - { - initializeScheduledExecutorFactory(); - return scheduledExecutorFactory; - } - - public RequestLogger getRequestLogger() - { - initializeRequestLogger(); - return requestLogger; - } - - public ServerView getServerView() - { - initializeServerView(); - return serverView; - } - - public InventoryView getInventoryView() - { - initializeInventoryView(); - return inventoryView; - } - - private void initializeDruidServerMetadata() - { - if (druidServerMetadata == null) { - final DruidServerConfig serverConfig = getConfigFactory().build(DruidServerConfig.class); - setDruidServerMetadata( - new DruidServerMetadata( - serverConfig.getServerName(), - serverConfig.getHost(), - serverConfig.getMaxSize(), - nodeType, - serverConfig.getTier() - ) - ); - } - } - - private void initializeServerView() - { - if (serverView == null) { - initializeServerInventoryView(); - serverView = serverInventoryView; - } - } - - private void initializeInventoryView() - { - if (inventoryView == null) { - initializeServerInventoryView(); - inventoryView = serverInventoryView; - } - } - - private void initializeServerInventoryView() - { - if (serverInventoryView == null) { - final ExecutorService exec = Executors.newFixedThreadPool( - 1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ServerInventoryView-%s").build() - ); - - final ServerInventoryViewConfig serverInventoryViewConfig = getConfigFactory().build(ServerInventoryViewConfig.class); - final String announcerType = serverInventoryViewConfig.getAnnouncerType(); - - if ("legacy".equalsIgnoreCase(announcerType)) { - serverInventoryView = new SingleServerInventoryView( - serverInventoryViewConfig, - getZkPaths(), - getCuratorFramework(), - exec, - getJsonMapper() - ); - } else if ("batch".equalsIgnoreCase(announcerType)) { - serverInventoryView = new BatchServerInventoryView( - serverInventoryViewConfig, - getZkPaths(), - getCuratorFramework(), - exec, - getJsonMapper() - ); - } else { - throw new IAE("Unknown type %s", announcerType); - } - lifecycle.addManagedInstance(serverInventoryView); - } - } - - private void initializeRequestLogger() - { - if (requestLogger == null) { - try { - final String loggingType = props.getProperty("druid.request.logging.type"); - if ("emitter".equals(loggingType)) { - setRequestLogger( - Initialization.makeEmittingRequestLogger( - getProps(), - getEmitter() - ) - ); - } else if ("file".equalsIgnoreCase(loggingType)) { - setRequestLogger( - Initialization.makeFileRequestLogger( - getJsonMapper(), - getScheduledExecutorFactory(), - getProps() - ) - ); - } else { - setRequestLogger(new NoopRequestLogger()); - } - } - catch (IOException e) { - throw Throwables.propagate(e); - } - lifecycle.addManagedInstance(requestLogger); - } - } - - private void initializeZkPaths() - { - if (zkPaths == null) { - setZkPaths(getConfigFactory().build(ZkPathsConfig.class)); - } - } - - private void initializeScheduledExecutorFactory() - { - if (scheduledExecutorFactory == null) { - setScheduledExecutorFactory(ScheduledExecutors.createFactory(getLifecycle())); - } - } - - private void initializeCuratorFramework() - { - if (curator == null) { - try { - setCuratorFramework(Initialization.makeCuratorFramework(configFactory.build(CuratorConfig.class), lifecycle)); - } - catch (IOException e) { - throw Throwables.propagate(e); - } - } - } - - private void initializeAnnouncer() - { - if (announcer == null) { - final Announcer announcer = new Announcer(getCuratorFramework(), Execs.singleThreaded("Announcer-%s")); - lifecycle.addManagedInstance(announcer); - - final ZkDataSegmentAnnouncerConfig config = getConfigFactory().build(ZkDataSegmentAnnouncerConfig.class); - final String announcerType = config.getAnnouncerType(); - - final DataSegmentAnnouncer dataSegmentAnnouncer; - if ("batch".equalsIgnoreCase(announcerType)) { - dataSegmentAnnouncer = new BatchDataSegmentAnnouncer( - getDruidServerMetadata(), - config, - announcer, - getJsonMapper() - ); - } else if ("legacy".equalsIgnoreCase(announcerType)) { - dataSegmentAnnouncer = new MultipleDataSegmentAnnouncerDataSegmentAnnouncer( - Arrays.asList( - new BatchDataSegmentAnnouncer( - getDruidServerMetadata(), - config, - announcer, - getJsonMapper() - ), - new SingleDataSegmentAnnouncer( - getDruidServerMetadata(), - getZkPaths(), - announcer, - getJsonMapper() - ) - ) - ); - } else { - throw new ISE("Unknown announcer type [%s]", announcerType); - } - - setAnnouncer(dataSegmentAnnouncer); - - lifecycle.addManagedInstance(getAnnouncer(), Lifecycle.Stage.LAST); - } - } - - private void initializeServer() - { - if (server == null) { - setServer(Initialization.makeJettyServer(configFactory.build(ServerConfig.class))); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - log.info("Starting Jetty"); - server.start(); - } - - @Override - public void stop() - { - log.info("Stopping Jetty"); - try { - server.stop(); - } - catch (Exception e) { - log.error(e, "Exception thrown while stopping Jetty"); - } - } - } - ); - } - } - - private void initializeMonitors() - { - if (monitors == null) { - List theMonitors = Lists.newArrayList(); - theMonitors.add(new JvmMonitor()); - if (Boolean.parseBoolean(props.getProperty("druid.monitoring.monitorSystem", "false"))) { - theMonitors.add(new SysMonitor()); - } - - setMonitors(theMonitors); - } - } - - private void initializeEmitter() - { - if (emitter == null) { - final HttpClientConfig.Builder configBuilder = HttpClientConfig.builder().withNumConnections(1); - - final String emitterTimeoutDuration = props.getProperty("druid.emitter.timeOut"); - if (emitterTimeoutDuration != null) { - configBuilder.withReadTimeout(new Duration(emitterTimeoutDuration)); - } - - final HttpClient httpClient = HttpClientInit.createClient(configBuilder.build(), lifecycle); - - setEmitter( - new ServiceEmitter( - PropUtils.getProperty(props, "druid.service"), - PropUtils.getProperty(props, "druid.host"), - Emitters.create(props, httpClient, jsonMapper, lifecycle) - ) - ); - } - EmittingLogger.registerEmitter(emitter); - } - - protected void init() throws Exception - { - doInit(); - initialized = true; - } - - protected abstract void doInit() throws Exception; - - @LifecycleStart - public synchronized void start() throws Exception - { - if (!initialized) { - init(); - } - - lifecycle.start(); - } - - @LifecycleStop - public synchronized void stop() - { - lifecycle.stop(); - } - - protected ScheduledExecutorService startMonitoring(List monitors) - { - final ScheduledExecutorService globalScheduledExec = getScheduledExecutorFactory().create(1, "Global--%d"); - final MonitorScheduler monitorScheduler = new MonitorScheduler( - getConfigFactory().build(MonitorSchedulerConfig.class), - globalScheduledExec, - getEmitter(), - monitors - ); - getLifecycle().addManagedInstance(monitorScheduler); - return globalScheduledExec; - } - - protected void checkFieldNotSetAndSet(String fieldName, Object value) - { - Class theClazz = this.getClass(); - while (theClazz != null && theClazz != Object.class) { - try { - final Field field = theClazz.getDeclaredField(fieldName); - field.setAccessible(true); - Preconditions.checkState(field.get(this) == null, "Cannot set %s once it has already been set.", fieldName); - - field.set(this, value); - return; - } - catch (NoSuchFieldException e) { - // Perhaps it is inherited? - theClazz = theClazz.getSuperclass(); - } - catch (IllegalAccessException e) { - throw Throwables.propagate(e); - } - } - - throw new ISE("Unknown field[%s] on class[%s]", fieldName, this.getClass()); - } -} diff --git a/client/src/main/java/com/metamx/druid/client/DataSegment.java b/client/src/main/java/com/metamx/druid/client/DataSegment.java deleted file mode 100644 index fec47128cf7e..000000000000 --- a/client/src/main/java/com/metamx/druid/client/DataSegment.java +++ /dev/null @@ -1,406 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.client; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.metamx.common.Granularity; -import com.metamx.druid.jackson.CommaListJoinDeserializer; -import com.metamx.druid.jackson.CommaListJoinSerializer; -import com.metamx.druid.query.segment.SegmentDescriptor; -import com.metamx.druid.shard.NoneShardSpec; -import com.metamx.druid.shard.ShardSpec; - -import org.joda.time.DateTime; -import org.joda.time.Interval; - -import javax.annotation.Nullable; -import java.util.Comparator; -import java.util.List; -import java.util.Map; - -/** - */ -public class DataSegment implements Comparable -{ - public static String delimiter = "_"; - private final Integer binaryVersion; - - public static String makeDataSegmentIdentifier( - String dataSource, - DateTime start, - DateTime end, - String version, - ShardSpec shardSpec - ) - { - StringBuilder sb = new StringBuilder(); - - sb.append(dataSource).append(delimiter) - .append(start).append(delimiter) - .append(end).append(delimiter) - .append(version); - - if (shardSpec.getPartitionNum() != 0) { - sb.append(delimiter).append(shardSpec.getPartitionNum()); - } - - return sb.toString(); - } - - private final String dataSource; - private final Interval interval; - private final String version; - private final Map loadSpec; - private final List dimensions; - private final List metrics; - private final ShardSpec shardSpec; - private final long size; - private final String identifier; - - @JsonCreator - public DataSegment( - @JsonProperty("dataSource") String dataSource, - @JsonProperty("interval") Interval interval, - @JsonProperty("version") String version, - @JsonProperty("loadSpec") Map loadSpec, - @JsonProperty("dimensions") @JsonDeserialize(using = CommaListJoinDeserializer.class) List dimensions, - @JsonProperty("metrics") @JsonDeserialize(using = CommaListJoinDeserializer.class) List metrics, - @JsonProperty("shardSpec") ShardSpec shardSpec, - @JsonProperty("binaryVersion") Integer binaryVersion, - @JsonProperty("size") long size - ) - { - final Predicate nonEmpty = new Predicate() - { - @Override - public boolean apply(@Nullable String input) - { - return input != null && !input.isEmpty(); - } - }; - - this.dataSource = dataSource.toLowerCase(); - this.interval = interval; - this.loadSpec = loadSpec; - this.version = version; - this.dimensions = dimensions == null - ? ImmutableList.of() - : ImmutableList.copyOf(Iterables.filter(dimensions, nonEmpty)); - this.metrics = metrics == null - ? ImmutableList.of() - : ImmutableList.copyOf(Iterables.filter(metrics, nonEmpty)); - this.shardSpec = (shardSpec == null) ? new NoneShardSpec() : shardSpec; - this.binaryVersion = binaryVersion; - this.size = size; - - this.identifier = makeDataSegmentIdentifier( - this.dataSource, - this.interval.getStart(), - this.interval.getEnd(), - this.version, - this.shardSpec - ); - } - - /** - * Get dataSource - * - * @return the dataSource - */ - @JsonProperty - public String getDataSource() - { - return dataSource; - } - - @JsonProperty - public Interval getInterval() - { - return interval; - } - - @JsonProperty - public Map getLoadSpec() - { - return loadSpec; - } - - @JsonProperty - public String getVersion() - { - return version; - } - - @JsonProperty - @JsonSerialize(using = CommaListJoinSerializer.class) - public List getDimensions() - { - return dimensions; - } - - @JsonProperty - @JsonSerialize(using = CommaListJoinSerializer.class) - public List getMetrics() - { - return metrics; - } - - @JsonProperty - public ShardSpec getShardSpec() - { - return shardSpec; - } - - @JsonProperty - public Integer getBinaryVersion() - { - return binaryVersion; - } - - @JsonProperty - public long getSize() - { - return size; - } - - @JsonProperty - public String getIdentifier() - { - return identifier; - } - - public SegmentDescriptor toDescriptor() - { - return new SegmentDescriptor(interval, version, shardSpec.getPartitionNum()); - } - - public DataSegment withLoadSpec(Map loadSpec) - { - return builder(this).loadSpec(loadSpec).build(); - } - - public DataSegment withDimensions(List dimensions) - { - return builder(this).dimensions(dimensions).build(); - } - - public DataSegment withSize(long size) - { - return builder(this).size(size).build(); - } - - public DataSegment withVersion(String version) - { - return builder(this).version(version).build(); - } - - public DataSegment withBinaryVersion(int binaryVersion) - { - return builder(this).binaryVersion(binaryVersion).build(); - } - - @Override - public int compareTo(DataSegment dataSegment) - { - return getIdentifier().compareTo(dataSegment.getIdentifier()); - } - - @Override - public boolean equals(Object o) - { - if (o instanceof DataSegment) { - return getIdentifier().equals(((DataSegment) o).getIdentifier()); - } - return false; - } - - @Override - public int hashCode() - { - return getIdentifier().hashCode(); - } - - @Override - public String toString() - { - return "DataSegment{" + - "size=" + size + - ", shardSpec=" + shardSpec + - ", metrics=" + metrics + - ", dimensions=" + dimensions + - ", version='" + version + '\'' + - ", loadSpec=" + loadSpec + - ", interval=" + interval + - ", dataSource='" + dataSource + '\'' + - ", binaryVersion='" + binaryVersion + '\'' + - '}'; - } - - public static Comparator bucketMonthComparator() - { - return new Comparator() - { - @Override - public int compare(DataSegment lhs, DataSegment rhs) - { - int retVal; - - DateTime lhsMonth = Granularity.MONTH.truncate(lhs.getInterval().getStart()); - DateTime rhsMonth = Granularity.MONTH.truncate(rhs.getInterval().getStart()); - - retVal = lhsMonth.compareTo(rhsMonth); - - if (retVal != 0) { - return retVal; - } - - return lhs.compareTo(rhs); - } - }; - } - - public static Builder builder() - { - return new Builder(); - } - - public static Builder builder(DataSegment segment) - { - return new Builder(segment); - } - - public static class Builder - { - private String dataSource; - private Interval interval; - private String version; - private Map loadSpec; - private List dimensions; - private List metrics; - private ShardSpec shardSpec; - private Integer binaryVersion; - private long size; - - public Builder() - { - this.loadSpec = ImmutableMap.of(); - this.dimensions = ImmutableList.of(); - this.metrics = ImmutableList.of(); - this.shardSpec = new NoneShardSpec(); - this.size = -1; - } - - public Builder(DataSegment segment) - { - this.dataSource = segment.getDataSource(); - this.interval = segment.getInterval(); - this.version = segment.getVersion(); - this.loadSpec = segment.getLoadSpec(); - this.dimensions = segment.getDimensions(); - this.metrics = segment.getMetrics(); - this.shardSpec = segment.getShardSpec(); - this.binaryVersion = segment.getBinaryVersion(); - this.size = segment.getSize(); - } - - public Builder dataSource(String dataSource) - { - this.dataSource = dataSource; - return this; - } - - public Builder interval(Interval interval) - { - this.interval = interval; - return this; - } - - public Builder version(String version) - { - this.version = version; - return this; - } - - public Builder loadSpec(Map loadSpec) - { - this.loadSpec = loadSpec; - return this; - } - - public Builder dimensions(List dimensions) - { - this.dimensions = dimensions; - return this; - } - - public Builder metrics(List metrics) - { - this.metrics = metrics; - return this; - } - - public Builder shardSpec(ShardSpec shardSpec) - { - this.shardSpec = shardSpec; - return this; - } - - public Builder binaryVersion(Integer binaryVersion) - { - this.binaryVersion = binaryVersion; - return this; - } - - public Builder size(long size) - { - this.size = size; - return this; - } - - public DataSegment build() - { - // Check stuff that goes into the identifier, at least. - Preconditions.checkNotNull(dataSource, "dataSource"); - Preconditions.checkNotNull(interval, "interval"); - Preconditions.checkNotNull(version, "version"); - Preconditions.checkNotNull(shardSpec, "shardSpec"); - - return new DataSegment( - dataSource, - interval, - version, - loadSpec, - dimensions, - metrics, - shardSpec, - binaryVersion, - size - ); - } - } -} diff --git a/client/src/main/java/com/metamx/druid/client/InventoryView.java b/client/src/main/java/com/metamx/druid/client/InventoryView.java deleted file mode 100644 index bbf7f6e46cca..000000000000 --- a/client/src/main/java/com/metamx/druid/client/InventoryView.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.metamx.druid.client; - -/** - */ -public interface InventoryView -{ - public DruidServer getInventoryValue(String string); - public Iterable getInventory(); -} diff --git a/client/src/main/java/com/metamx/druid/client/ServerInventoryViewConfig.java b/client/src/main/java/com/metamx/druid/client/ServerInventoryViewConfig.java deleted file mode 100644 index 6130a96a66cb..000000000000 --- a/client/src/main/java/com/metamx/druid/client/ServerInventoryViewConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.client; - -import org.skife.config.Config; -import org.skife.config.Default; - -/** - */ -public abstract class ServerInventoryViewConfig -{ - @Config("druid.master.removedSegmentLifetime") - @Default("1") - public abstract int getRemovedSegmentLifetime(); - - @Config("druid.announcer.type") - @Default("legacy") - public abstract String getAnnouncerType(); -} \ No newline at end of file diff --git a/client/src/main/java/com/metamx/druid/client/cache/CacheConfig.java b/client/src/main/java/com/metamx/druid/client/cache/CacheConfig.java deleted file mode 100644 index e970363e0e51..000000000000 --- a/client/src/main/java/com/metamx/druid/client/cache/CacheConfig.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.metamx.druid.client.cache; - -import org.skife.config.Config; -import org.skife.config.Default; - -public abstract class CacheConfig -{ - @Config("druid.bard.cache.type") - @Default("local") - public abstract String getType(); -} diff --git a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java b/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java deleted file mode 100644 index 2bd7b84e4cce..000000000000 --- a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCacheConfig.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.metamx.druid.client.cache; - -import org.skife.config.Config; -import org.skife.config.Default; - -public abstract class MemcachedCacheConfig -{ - @Config("${prefix}.expiration") - @Default("2592000") - public abstract int getExpiration(); - - @Config("${prefix}.timeout") - @Default("500") - public abstract int getTimeout(); - - @Config("${prefix}.hosts") - public abstract String getHosts(); - - @Config("${prefix}.maxObjectSize") - @Default("52428800") - public abstract int getMaxObjectSize(); - - @Config("${prefix}.memcachedPrefix") - @Default("druid") - public abstract String getMemcachedPrefix(); -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/AddressPortServiceInstanceFactory.java b/client/src/main/java/com/metamx/druid/curator/discovery/AddressPortServiceInstanceFactory.java deleted file mode 100644 index 3f4fa988655f..000000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/AddressPortServiceInstanceFactory.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.metamx.druid.curator.discovery; - -import com.google.common.base.Throwables; -import org.apache.curator.x.discovery.ServiceInstance; - -public class AddressPortServiceInstanceFactory implements ServiceInstanceFactory -{ - private final String address; - private final int port; - - public AddressPortServiceInstanceFactory(String address, int port) - { - this.address = address; - this.port = port; - } - - @Override - public ServiceInstance create(String service) - { - try { - return ServiceInstance.builder().name(service).address(address).port(port).build(); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/CuratorServiceAnnouncer.java b/client/src/main/java/com/metamx/druid/curator/discovery/CuratorServiceAnnouncer.java deleted file mode 100644 index a3156b3205d1..000000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/CuratorServiceAnnouncer.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.metamx.druid.curator.discovery; - -import com.google.common.collect.Maps; -import com.metamx.common.logger.Logger; -import org.apache.curator.x.discovery.ServiceDiscovery; -import org.apache.curator.x.discovery.ServiceInstance; - -import java.util.Map; - -/** - * Uses the Curator Service Discovery recipe to announce services. - */ -public class CuratorServiceAnnouncer implements ServiceAnnouncer -{ - private static final Logger log = new Logger(CuratorServiceAnnouncer.class); - - private final ServiceDiscovery discovery; - private final ServiceInstanceFactory instanceFactory; - private final Map> instanceMap = Maps.newHashMap(); - private final Object monitor = new Object(); - - public CuratorServiceAnnouncer( - ServiceDiscovery discovery, - ServiceInstanceFactory instanceFactory - ) - { - this.discovery = discovery; - this.instanceFactory = instanceFactory; - } - - @Override - public void announce(String service) throws Exception - { - final ServiceInstance instance; - - synchronized (monitor) { - if (instanceMap.containsKey(service)) { - log.warn("Ignoring request to announce service[%s]", service); - return; - } else { - instance = instanceFactory.create(service); - instanceMap.put(service, instance); - } - } - - try { - log.info("Announcing service[%s]", service); - discovery.registerService(instance); - } catch (Exception e) { - log.warn("Failed to announce service[%s]", service); - synchronized (monitor) { - instanceMap.remove(service); - } - } - } - - @Override - public void unannounce(String service) throws Exception - { - final ServiceInstance instance; - - synchronized (monitor) { - instance = instanceMap.get(service); - if (instance == null) { - log.warn("Ignoring request to unannounce service[%s]", service); - return; - } - } - - log.info("Unannouncing service[%s]", service); - try { - discovery.unregisterService(instance); - } catch (Exception e) { - log.warn(e, "Failed to unannounce service[%s]", service); - } finally { - synchronized (monitor) { - instanceMap.remove(service); - } - } - } -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/NoopServiceAnnouncer.java b/client/src/main/java/com/metamx/druid/curator/discovery/NoopServiceAnnouncer.java deleted file mode 100644 index 782739f7a24d..000000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/NoopServiceAnnouncer.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.metamx.druid.curator.discovery; - -/** - * Does nothing. - */ -public class NoopServiceAnnouncer implements ServiceAnnouncer -{ - @Override - public void unannounce(String service) - { - - } - - @Override - public void announce(String service) - { - - } -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/ServiceAnnouncer.java b/client/src/main/java/com/metamx/druid/curator/discovery/ServiceAnnouncer.java deleted file mode 100644 index 4e91122423a4..000000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/ServiceAnnouncer.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.metamx.druid.curator.discovery; - -/** - * Announces our ability to serve a particular function. Multiple users may announce the same service, in which - * case they are treated as interchangeable instances of that service. - */ -public interface ServiceAnnouncer -{ - public void announce(String service) throws Exception; - public void unannounce(String service) throws Exception; -} diff --git a/client/src/main/java/com/metamx/druid/curator/discovery/ServiceInstanceFactory.java b/client/src/main/java/com/metamx/druid/curator/discovery/ServiceInstanceFactory.java deleted file mode 100644 index 8f8cc19c0bd9..000000000000 --- a/client/src/main/java/com/metamx/druid/curator/discovery/ServiceInstanceFactory.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.metamx.druid.curator.discovery; - -import org.apache.curator.x.discovery.ServiceInstance; - -public interface ServiceInstanceFactory -{ - public ServiceInstance create(String service); -} diff --git a/client/src/main/java/com/metamx/druid/http/BrokerNode.java b/client/src/main/java/com/metamx/druid/http/BrokerNode.java deleted file mode 100644 index 75b69fe5d948..000000000000 --- a/client/src/main/java/com/metamx/druid/http/BrokerNode.java +++ /dev/null @@ -1,352 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.servlet.GuiceFilter; -import com.metamx.common.ISE; -import com.metamx.common.config.Config; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.QueryableNode; -import com.metamx.druid.client.BrokerServerView; -import com.metamx.druid.client.CachingClusteredClient; -import com.metamx.druid.client.cache.Cache; -import com.metamx.druid.client.cache.CacheConfig; -import com.metamx.druid.client.cache.CacheMonitor; -import com.metamx.druid.client.cache.MapCache; -import com.metamx.druid.client.cache.MapCacheConfig; -import com.metamx.druid.client.cache.MemcachedCache; -import com.metamx.druid.client.cache.MemcachedCacheConfig; -import com.metamx.druid.curator.discovery.ServiceAnnouncer; -import com.metamx.druid.curator.discovery.ServiceInstanceFactory; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.initialization.ServiceDiscoveryConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.query.ReflectionQueryToolChestWarehouse; -import com.metamx.druid.utils.PropUtils; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.HttpClientConfig; -import com.metamx.http.client.HttpClientInit; -import com.metamx.metrics.Monitor; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.x.discovery.ServiceDiscovery; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.ServletHolder; -import org.mortbay.servlet.GzipFilter; -import org.skife.config.ConfigurationObjectFactory; - -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - */ - -public class BrokerNode extends QueryableNode -{ - private static final Logger log = new Logger(BrokerNode.class); - - public static final String CACHE_TYPE_LOCAL = "local"; - public static final String CACHE_TYPE_MEMCACHED = "memcached"; - public static final String CACHE_PROPERTY_PREFIX = "druid.bard.cache"; - - private final List extraModules = Lists.newArrayList(); - private final List pathsForGuiceFilter = Lists.newArrayList(); - - private QueryToolChestWarehouse warehouse = null; - private HttpClient brokerHttpClient = null; - private Cache cache = null; - - private boolean useDiscovery = true; - - public static Builder builder() - { - return new Builder(); - } - - public BrokerNode( - Properties props, - Lifecycle lifecycle, - ObjectMapper jsonMapper, - ObjectMapper smileMapper, - ConfigurationObjectFactory configFactory - ) - { - super("broker", log, props, lifecycle, jsonMapper, smileMapper, configFactory); - } - - public QueryToolChestWarehouse getWarehouse() - { - initializeWarehouse(); - return warehouse; - } - - public BrokerNode setWarehouse(QueryToolChestWarehouse warehouse) - { - checkFieldNotSetAndSet("warehouse", warehouse); - return this; - } - - public HttpClient getBrokerHttpClient() - { - initializeBrokerHttpClient(); - return brokerHttpClient; - } - - public BrokerNode setBrokerHttpClient(HttpClient brokerHttpClient) - { - checkFieldNotSetAndSet("brokerHttpClient", brokerHttpClient); - return this; - } - - public Cache getCache() - { - initializeCacheBroker(); - return cache; - } - - public BrokerNode setCache(Cache cache) - { - checkFieldNotSetAndSet("cache", cache); - return this; - } - - public BrokerNode useDiscovery(boolean useDiscovery) - { - this.useDiscovery = useDiscovery; - return this; - } - - /** - * This method allows you to specify more Guice modules to use primarily for injected extra Jersey resources. - * I'd like to remove the Guice dependency for this, but I don't know how to set up Jersey without Guice... - * - * This is deprecated because at some point in the future, we will eliminate the Guice dependency and anything - * that uses this will break. Use at your own risk. - * - * @param module the module to register with Guice - * - * @return this - */ - @Deprecated - public BrokerNode addModule(Module module) - { - extraModules.add(module); - return this; - } - - /** - * This method is used to specify extra paths that the GuiceFilter should pay attention to. - * - * This is deprecated for the same reason that addModule is deprecated. - * - * @param path the path that the GuiceFilter should pay attention to. - * - * @return this - */ - @Deprecated - public BrokerNode addPathForGuiceFilter(String path) - { - pathsForGuiceFilter.add(path); - return this; - } - - @Override - protected void doInit() throws Exception - { - initializeWarehouse(); - initializeBrokerHttpClient(); - initializeCacheBroker(); - initializeDiscovery(); - - final Lifecycle lifecycle = getLifecycle(); - - final List monitors = getMonitors(); - monitors.add(new CacheMonitor(cache)); - startMonitoring(monitors); - - final ExecutorService viewExec = Executors.newFixedThreadPool( - 1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("BrokerServerView-%s").build() - ); - final BrokerServerView view = new BrokerServerView( - warehouse, getSmileMapper(), brokerHttpClient, getServerView(), viewExec - ); - - final CachingClusteredClient baseClient = new CachingClusteredClient(warehouse, view, cache, getSmileMapper()); - lifecycle.addManagedInstance(baseClient); - - final ClientQuerySegmentWalker texasRanger = new ClientQuerySegmentWalker(warehouse, getEmitter(), baseClient); - - List theModules = Lists.newArrayList(); - theModules.add(new ClientServletModule(texasRanger, getInventoryView(), getJsonMapper())); - theModules.addAll(extraModules); - - final Injector injector = Guice.createInjector(theModules); - final Context root = new Context(getServer(), "/", Context.SESSIONS); - root.addServlet(new ServletHolder(new StatusServlet()), "/status"); - root.addServlet( - new ServletHolder(new QueryServlet(getJsonMapper(), getSmileMapper(), texasRanger, getEmitter(), getRequestLogger())), - "/druid/v2/*" - ); - root.addFilter(GzipFilter.class, "/*", 0); - - root.addEventListener(new GuiceServletConfig(injector)); - root.addFilter(GuiceFilter.class, "/druid/v2/datasources/*", 0); - - for (String path : pathsForGuiceFilter) { - root.addFilter(GuiceFilter.class, path, 0); - } - } - - private void initializeDiscovery() throws Exception - { - if (useDiscovery) { - final Lifecycle lifecycle = getLifecycle(); - final ServiceDiscoveryConfig serviceDiscoveryConfig = getConfigFactory().build(ServiceDiscoveryConfig.class); - final CuratorFramework curatorFramework = Initialization.makeCuratorFramework( - serviceDiscoveryConfig, lifecycle - ); - final ServiceDiscovery serviceDiscovery = Initialization.makeServiceDiscoveryClient( - curatorFramework, serviceDiscoveryConfig, lifecycle - ); - final ServiceAnnouncer serviceAnnouncer = Initialization.makeServiceAnnouncer( - serviceDiscoveryConfig, serviceDiscovery - ); - Initialization.announceDefaultService(serviceDiscoveryConfig, serviceAnnouncer, lifecycle); - } - } - - private void initializeCacheBroker() - { - if (cache == null) { - String cacheType = getConfigFactory() - .build(CacheConfig.class) - .getType(); - - if (cacheType.equals(CACHE_TYPE_LOCAL)) { - setCache( - MapCache.create( - getConfigFactory().buildWithReplacements( - MapCacheConfig.class, - ImmutableMap.of("prefix", CACHE_PROPERTY_PREFIX) - ) - ) - ); - } else if (cacheType.equals(CACHE_TYPE_MEMCACHED)) { - setCache( - MemcachedCache.create( - getConfigFactory().buildWithReplacements( - MemcachedCacheConfig.class, - ImmutableMap.of("prefix", CACHE_PROPERTY_PREFIX) - ) - ) - ); - } else { - throw new ISE("Unknown cache type [%s]", cacheType); - } - } - } - - private void initializeBrokerHttpClient() - { - if (brokerHttpClient == null) { - setBrokerHttpClient( - HttpClientInit.createClient( - HttpClientConfig - .builder() - .withNumConnections(PropUtils.getPropertyAsInt(getProps(), "druid.client.http.connections")) - .build(), - getLifecycle() - ) - ); - } - } - - private void initializeWarehouse() - { - if (warehouse == null) { - setWarehouse(new ReflectionQueryToolChestWarehouse()); - } - } - - public static class Builder - { - private ObjectMapper jsonMapper = null; - private ObjectMapper smileMapper = null; - private Lifecycle lifecycle = null; - private Properties props = null; - private ConfigurationObjectFactory configFactory = null; - - public Builder withMappers(ObjectMapper jsonMapper, ObjectMapper smileMapper) - { - this.jsonMapper = jsonMapper; - this.smileMapper = smileMapper; - return this; - } - - public Builder withProps(Properties props) - { - this.props = props; - return this; - } - - public Builder withConfigFactory(ConfigurationObjectFactory configFactory) - { - this.configFactory = configFactory; - return this; - } - - public BrokerNode build() - { - if (jsonMapper == null && smileMapper == null) { - jsonMapper = new DefaultObjectMapper(); - smileMapper = new DefaultObjectMapper(new SmileFactory()); - smileMapper.getJsonFactory().setCodec(smileMapper); - } - else if (jsonMapper == null || smileMapper == null) { - throw new ISE("Only jsonMapper[%s] or smileMapper[%s] was set, must set neither or both.", jsonMapper, smileMapper); - } - - if (lifecycle == null) { - lifecycle = new Lifecycle(); - } - - if (props == null) { - props = Initialization.loadProperties(); - } - - if (configFactory == null) { - configFactory = Config.createFactory(props); - } - - return new BrokerNode(props, lifecycle, jsonMapper, smileMapper, configFactory); - } - } -} diff --git a/client/src/main/java/com/metamx/druid/http/ClientMain.java b/client/src/main/java/com/metamx/druid/http/ClientMain.java deleted file mode 100644 index 1beda5c405d4..000000000000 --- a/client/src/main/java/com/metamx/druid/http/ClientMain.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -/** - */ -@Deprecated -public class ClientMain -{ - public static void main(String[] args) throws Exception - { - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ClientMain!!!! PLZ Stop. Use BrokerMain instead."); - System.out.println("K thx bye."); - BrokerMain.main(args); - } -} diff --git a/client/src/main/java/com/metamx/druid/http/ClientServletModule.java b/client/src/main/java/com/metamx/druid/http/ClientServletModule.java deleted file mode 100644 index 8de99f594967..000000000000 --- a/client/src/main/java/com/metamx/druid/http/ClientServletModule.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; -import com.google.inject.Provides; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.sun.jersey.guice.JerseyServletModule; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; - -import javax.inject.Singleton; - -/** - */ -public class ClientServletModule extends JerseyServletModule -{ - private final QuerySegmentWalker texasRanger; - private final InventoryView serverInventoryView; - private final ObjectMapper jsonMapper; - - public ClientServletModule( - QuerySegmentWalker texasRanger, - InventoryView serverInventoryView, - ObjectMapper jsonMapper - ) - { - this.texasRanger = texasRanger; - this.serverInventoryView = serverInventoryView; - this.jsonMapper = jsonMapper; - } - - @Override - protected void configureServlets() - { - bind(ClientInfoResource.class); - bind(QuerySegmentWalker.class).toInstance(texasRanger); - bind(InventoryView.class).toInstance(serverInventoryView); - - serve("/*").with(GuiceContainer.class); - } - - @Provides - @Singleton - public JacksonJsonProvider getJacksonJsonProvider() - { - final JacksonJsonProvider provider = new JacksonJsonProvider(); - provider.setMapper(jsonMapper); - return provider; - } -} - diff --git a/client/src/main/java/com/metamx/druid/http/StatusServlet.java b/client/src/main/java/com/metamx/druid/http/StatusServlet.java deleted file mode 100644 index 95a40c543b8f..000000000000 --- a/client/src/main/java/com/metamx/druid/http/StatusServlet.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; - -/** - */ -public class StatusServlet extends HttpServlet -{ - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException - { - ByteArrayOutputStream retVal = new ByteArrayOutputStream(); - PrintWriter out = new PrintWriter(new OutputStreamWriter(retVal)); - - Runtime runtime = Runtime.getRuntime(); - long maxMemory = runtime.maxMemory(); - long totalMemory = runtime.totalMemory(); - long freeMemory = runtime.freeMemory(); - - out.printf("Max Memory:\t%,18d\t%1$d%n", maxMemory); - out.printf("Total Memory:\t%,18d\t%1$d%n", totalMemory); - out.printf("Free Memory:\t%,18d\t%1$d%n", freeMemory); - out.printf("Used Memory:\t%,18d\t%1$d%n", totalMemory - freeMemory); - - out.flush(); - - resp.setStatus(HttpServletResponse.SC_OK); - resp.setContentType("text/plain"); - resp.getOutputStream().write(retVal.toByteArray()); - } -} diff --git a/client/src/main/java/com/metamx/druid/initialization/Initialization.java b/client/src/main/java/com/metamx/druid/initialization/Initialization.java deleted file mode 100644 index de29eabd5502..000000000000 --- a/client/src/main/java/com/metamx/druid/initialization/Initialization.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.initialization; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.base.Throwables; -import com.google.common.io.Closeables; -import com.metamx.common.concurrent.ScheduledExecutorFactory; -import com.metamx.common.config.Config; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.PotentiallyGzippedCompressionProvider; -import com.metamx.druid.curator.discovery.AddressPortServiceInstanceFactory; -import com.metamx.druid.curator.discovery.CuratorServiceAnnouncer; -import com.metamx.druid.curator.discovery.ServiceAnnouncer; -import com.metamx.druid.curator.discovery.ServiceInstanceFactory; -import com.metamx.druid.http.EmittingRequestLogger; -import com.metamx.druid.http.FileRequestLogger; -import com.metamx.druid.http.RequestLogger; -import com.metamx.druid.utils.PropUtils; -import com.metamx.emitter.core.Emitter; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.BoundedExponentialBackoffRetry; -import org.apache.curator.x.discovery.ServiceDiscovery; -import org.apache.curator.x.discovery.ServiceDiscoveryBuilder; -import org.apache.curator.x.discovery.ServiceInstance; -import org.apache.curator.x.discovery.ServiceProvider; -import org.apache.zookeeper.data.Stat; -import org.mortbay.jetty.Connector; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.nio.SelectChannelConnector; -import org.mortbay.thread.QueuedThreadPool; -import org.skife.config.ConfigurationObjectFactory; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Properties; - -/** - */ -public class Initialization -{ - private static final Logger log = new Logger(Initialization.class); - - private static final String PROPERTIES_FILE = "runtime.properties"; - private static final Properties zkProps = new Properties(); - private static final Properties fileProps = new Properties(zkProps); - private static Properties props = null; - - /** - * Load properties. - * Properties are layered: - *

- * # stored in zookeeper - * # runtime.properties file, - * # cmdLine -D - *

- * command line overrides runtime.properties which overrides zookeeper - *

- * Idempotent. Thread-safe. Properties are only loaded once. - * If property druid.zk.service.host is not set then do not load properties from zookeeper. - * - * @return Properties ready to use. - */ - public synchronized static Properties loadProperties() - { - if (props != null) { - return props; - } - - // Note that zookeeper coordinates must be either in cmdLine or in runtime.properties - Properties sp = System.getProperties(); - - Properties tmp_props = new Properties(fileProps); // the head of the 3 level Properties chain - tmp_props.putAll(sp); - - final InputStream stream = ClassLoader.getSystemResourceAsStream(PROPERTIES_FILE); - if (stream == null) { - log.info("%s not found on classpath, relying only on system properties and zookeeper.", PROPERTIES_FILE); - } else { - log.info("Loading properties from %s", PROPERTIES_FILE); - try { - try { - fileProps.load(stream); - } - catch (IOException e) { - throw Throwables.propagate(e); - } - } - finally { - Closeables.closeQuietly(stream); - } - } - - // log properties from file; stringPropertyNames() would normally cascade down into the sub Properties objects, but - // zkProps (the parent level) is empty at this point so it will only log properties from runtime.properties - for (String prop : fileProps.stringPropertyNames()) { - log.info("Loaded(runtime.properties) Property[%s] as [%s]", prop, fileProps.getProperty(prop)); - } - - final String zkHostsProperty = "druid.zk.service.host"; - - if (tmp_props.getProperty(zkHostsProperty) != null) { - final ConfigurationObjectFactory factory = Config.createFactory(tmp_props); - - ZkPathsConfig config; - try { - config = factory.build(ZkPathsConfig.class); - } - catch (IllegalArgumentException e) { - log.warn(e, "Unable to build ZkPathsConfig. Cannot load properties from ZK."); - config = null; - } - - if (config != null) { - Lifecycle lifecycle = new Lifecycle(); - try { - CuratorFramework curator = makeCuratorFramework(factory.build(CuratorConfig.class), lifecycle); - - lifecycle.start(); - - final Stat stat = curator.checkExists().forPath(config.getPropertiesPath()); - if (stat != null) { - final byte[] data = curator.getData().forPath(config.getPropertiesPath()); - zkProps.load(new InputStreamReader(new ByteArrayInputStream(data), Charsets.UTF_8)); - } - - // log properties from zk - for (String prop : zkProps.stringPropertyNames()) { - log.info("Loaded(zk) Property[%s] as [%s]", prop, zkProps.getProperty(prop)); - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - finally { - lifecycle.stop(); - } - } - } else { - log.warn("property[%s] not set, skipping ZK-specified properties.", zkHostsProperty); - } - - props = tmp_props; - - return props; - } - - public static Server makeJettyServer(ServerConfig config) - { - final QueuedThreadPool threadPool = new QueuedThreadPool(); - threadPool.setMinThreads(config.getNumThreads()); - threadPool.setMaxThreads(config.getNumThreads()); - - final Server server = new Server(); - server.setThreadPool(threadPool); - - SelectChannelConnector connector = new SelectChannelConnector(); - connector.setPort(config.getPort()); - connector.setMaxIdleTime(config.getMaxIdleTimeMillis()); - connector.setStatsOn(true); - - server.setConnectors(new Connector[]{connector}); - - return server; - } - - public static CuratorFramework makeCuratorFramework( - CuratorConfig curatorConfig, - Lifecycle lifecycle - ) throws IOException - { - final CuratorFramework framework = - CuratorFrameworkFactory.builder() - .connectString(curatorConfig.getZkHosts()) - .sessionTimeoutMs(curatorConfig.getZkSessionTimeoutMs()) - .retryPolicy(new BoundedExponentialBackoffRetry(1000, 45000, 30)) - .compressionProvider(new PotentiallyGzippedCompressionProvider(curatorConfig.enableCompression())) - .build(); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - framework.start(); - } - - @Override - public void stop() - { - framework.close(); - } - } - ); - - return framework; - } - - public static ServiceDiscovery makeServiceDiscoveryClient( - CuratorFramework discoveryClient, - ServiceDiscoveryConfig config, - Lifecycle lifecycle - ) - throws Exception - { - final ServiceDiscovery serviceDiscovery = - ServiceDiscoveryBuilder.builder(Void.class) - .basePath(config.getDiscoveryPath()) - .client(discoveryClient) - .build(); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - serviceDiscovery.start(); - } - - @Override - public void stop() - { - try { - serviceDiscovery.close(); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - } - ); - - return serviceDiscovery; - } - - public static ServiceAnnouncer makeServiceAnnouncer( - ServiceDiscoveryConfig config, - ServiceDiscovery serviceDiscovery - ) - { - final ServiceInstanceFactory serviceInstanceFactory = makeServiceInstanceFactory(config); - return new CuratorServiceAnnouncer(serviceDiscovery, serviceInstanceFactory); - } - - public static void announceDefaultService( - final ServiceDiscoveryConfig config, - final ServiceAnnouncer serviceAnnouncer, - final Lifecycle lifecycle - ) throws Exception - { - final String service = config.getServiceName().replace('/', ':'); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - serviceAnnouncer.announce(service); - } - - @Override - public void stop() - { - try { - serviceAnnouncer.unannounce(service); - } - catch (Exception e) { - log.warn(e, "Failed to unannouce default service[%s]", service); - } - } - } - ); - } - - public static ServiceProvider makeServiceProvider( - String serviceName, - ServiceDiscovery serviceDiscovery, - Lifecycle lifecycle - ) - { - final ServiceProvider serviceProvider = serviceDiscovery.serviceProviderBuilder() - .serviceName(serviceName) - .build(); - - lifecycle.addHandler( - new Lifecycle.Handler() - { - @Override - public void start() throws Exception - { - serviceProvider.start(); - } - - @Override - public void stop() - { - try { - serviceProvider.close(); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - } - ); - - return serviceProvider; - } - - public static RequestLogger makeFileRequestLogger( - ObjectMapper objectMapper, - ScheduledExecutorFactory factory, - Properties props - ) throws IOException - { - return new FileRequestLogger( - objectMapper, - factory.create(1, "RequestLogger-%s"), - new File(PropUtils.getProperty(props, "druid.request.logging.dir")) - ); - } - - public static RequestLogger makeEmittingRequestLogger(Properties props, Emitter emitter) - { - return new EmittingRequestLogger( - PropUtils.getProperty(props, "druid.service"), - PropUtils.getProperty(props, "druid.host"), - emitter, - PropUtils.getProperty(props, "druid.request.logging.feed") - ); - } - - public static ServiceInstanceFactory makeServiceInstanceFactory(ServiceDiscoveryConfig config) - { - final String host = config.getHost(); - final String address; - final int colon = host.indexOf(':'); - if (colon < 0) { - address = host; - } else { - address = host.substring(0, colon); - } - - return new AddressPortServiceInstanceFactory(address, config.getPort()); - } -} diff --git a/client/src/main/java/com/metamx/druid/initialization/ZkDataSegmentAnnouncerConfig.java b/client/src/main/java/com/metamx/druid/initialization/ZkDataSegmentAnnouncerConfig.java deleted file mode 100644 index e14c65027eba..000000000000 --- a/client/src/main/java/com/metamx/druid/initialization/ZkDataSegmentAnnouncerConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.metamx.druid.initialization; - -import org.skife.config.Config; -import org.skife.config.Default; - -/** - */ -public abstract class ZkDataSegmentAnnouncerConfig extends ZkPathsConfig -{ - @Config("druid.zk.segmentsPerNode") - @Default("50") - public abstract int getSegmentsPerNode(); - - @Config("druid.zk.maxNumBytesPerNode") - @Default("512000") - public abstract long getMaxNumBytes(); - - @Config("druid.announcer.type") - @Default("legacy") - public abstract String getAnnouncerType(); -} diff --git a/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java b/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java deleted file mode 100644 index 3fa27b99ace9..000000000000 --- a/client/src/main/java/com/metamx/druid/query/group/having/AlwaysHavingSpec.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.metamx.druid.query.group.having; - -import com.metamx.druid.input.Row; - -/** - */ -public class AlwaysHavingSpec implements HavingSpec -{ - @Override - public boolean eval(Row row) - { - return true; - } -} diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java b/client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java deleted file mode 100644 index 3f383430b129..000000000000 --- a/client/src/main/java/com/metamx/druid/query/group/orderby/NoopLimitSpec.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.metamx.druid.query.group.orderby; - -import com.google.common.base.Function; -import com.google.common.base.Functions; -import com.metamx.common.guava.Sequence; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.aggregation.post.PostAggregator; -import com.metamx.druid.input.Row; -import com.metamx.druid.query.dimension.DimensionSpec; - -import java.util.List; - -/** - */ -public class NoopLimitSpec implements LimitSpec -{ - @Override - public Function, Sequence> build( - List dimensions, List aggs, List postAggs - ) - { - return Functions.identity(); - } - - @Override - public String toString() - { - return "NoopLimitSpec"; - } -} diff --git a/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java b/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java deleted file mode 100644 index 5b5a3f48cb60..000000000000 --- a/client/src/main/java/com/metamx/druid/query/group/orderby/OrderedPriorityQueueItems.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.metamx.druid.query.group.orderby; - -import com.google.common.collect.MinMaxPriorityQueue; - -import java.util.Iterator; - -/** - * Utility class that supports iterating a priority queue in sorted order. - */ -class OrderedPriorityQueueItems implements Iterable -{ - private MinMaxPriorityQueue rows; - - public OrderedPriorityQueueItems(MinMaxPriorityQueue rows) - { - this.rows = rows; - } - - @Override - public Iterator iterator() - { - return new Iterator() { - - @Override - public boolean hasNext() - { - return !rows.isEmpty(); - } - - @Override - public T next() - { - return rows.poll(); - } - - @Override - public void remove() - { - throw new UnsupportedOperationException("Can't remove any item from an intermediary heap for orderBy/limit"); - } - }; - } -} diff --git a/client/src/main/java/com/metamx/druid/query/segment/SegmentDescriptor.java b/client/src/main/java/com/metamx/druid/query/segment/SegmentDescriptor.java deleted file mode 100644 index 1e2ac83484a8..000000000000 --- a/client/src/main/java/com/metamx/druid/query/segment/SegmentDescriptor.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.query.segment; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import org.joda.time.Interval; - -/** -*/ -public class SegmentDescriptor -{ - private final Interval interval; - private final String version; - private final int partitionNumber; - - @JsonCreator - public SegmentDescriptor( - @JsonProperty("itvl") Interval interval, - @JsonProperty("ver") String version, - @JsonProperty("part") int partitionNumber) - { - this.interval = interval; - this.version = version; - this.partitionNumber = partitionNumber; - } - - @JsonProperty("itvl") - public Interval getInterval() - { - return interval; - } - - @JsonProperty("ver") - public String getVersion() - { - return version; - } - - @JsonProperty("part") - public int getPartitionNumber() - { - return partitionNumber; - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - SegmentDescriptor that = (SegmentDescriptor) o; - - if (partitionNumber != that.partitionNumber) { - return false; - } - if (interval != null ? !interval.equals(that.interval) : that.interval != null) { - return false; - } - if (version != null ? !version.equals(that.version) : that.version != null) { - return false; - } - - return true; - } - - @Override - public int hashCode() - { - int result = interval != null ? interval.hashCode() : 0; - result = 31 * result + (version != null ? version.hashCode() : 0); - result = 31 * result + partitionNumber; - return result; - } - - @Override - public String toString() - { - return "SegmentDescriptor{" + - "interval=" + interval + - ", version='" + version + '\'' + - ", partitionNumber=" + partitionNumber + - '}'; - } -} diff --git a/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java b/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java deleted file mode 100644 index 4d77a19086d6..000000000000 --- a/client/src/main/java/com/metamx/druid/shard/LinearShardSpec.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.metamx.druid.shard; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.partition.LinearPartitionChunk; -import com.metamx.druid.partition.PartitionChunk; - -import java.util.Map; - -public class LinearShardSpec implements ShardSpec { - private int partitionNum; - - public LinearShardSpec() { - this(-1); - } - - public LinearShardSpec(int partitionNum) { - this.partitionNum = partitionNum; - } - - @JsonProperty("partitionNum") - @Override - public int getPartitionNum() { - return partitionNum; - } - - @Override - public PartitionChunk createChunk(T obj) { - return new LinearPartitionChunk(partitionNum, obj); - } - - @Override - public boolean isInChunk(Map dimensions) { - return true; - } - - @Override - public boolean isInChunk(InputRow inputRow) { - return true; - } -} diff --git a/client/src/main/java/com/metamx/druid/shard/ShardSpec.java b/client/src/main/java/com/metamx/druid/shard/ShardSpec.java deleted file mode 100644 index b5b778283b54..000000000000 --- a/client/src/main/java/com/metamx/druid/shard/ShardSpec.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.shard; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.partition.PartitionChunk; - -import java.util.Map; - -/** - * A Marker interface that exists to combine ShardSpec objects together for Jackson - */ -@JsonTypeInfo(use= JsonTypeInfo.Id.NAME, property="type", include=JsonTypeInfo.As.PROPERTY) -@JsonSubTypes({ - @JsonSubTypes.Type(name="single", value=SingleDimensionShardSpec.class), - @JsonSubTypes.Type(name="none", value=NoneShardSpec.class), - @JsonSubTypes.Type(name="linear", value=LinearShardSpec.class), - @JsonSubTypes.Type(name="numbered", value=NumberedShardSpec.class) -}) -public interface ShardSpec -{ - public PartitionChunk createChunk(T obj); - public boolean isInChunk(Map dimensions); - public boolean isInChunk(InputRow inputRow); - public int getPartitionNum(); -} diff --git a/common/pom.xml b/common/pom.xml index 8d264bfb57e7..af29a79f989f 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -1,7 +1,7 @@ - - -DruidDataFlow - -1 - -REST query - -5 - -Realtime.working_set - -1->5 - - - query - -2 - -realtime_data_src - -2->5 - - -Realtime.Firehose - -4 - -indexed segments -blob_store (S3, HDFS) - -5->1 - - - results - -6 - -Realtime.spill_disk - -5->6 - - -Realtime.Firehose - -9 - -Realtime.local_disk - -5->9 - - -Realtime.Indexer - -10 - -metadata_store (mysql) - -5->10 - - -Realtime.MetadataUpdater - -6->9 - - -Realtime.Indexer - -7 - -HDFS_data_src - -11 - -HadoopDruidIndexer.working_set - -7->11 - - -HadoopDruidIndexer - -8 - -external_data_src - -12 - -IndexerService.working_set - -8->12 - - -IndexerService - -9->4 - - -Realtime.segmentPusher - -11->4 - - -HadoopDruidIndexer - -11->10 - - -HadoopDruidIndexer - -12->4 - - -IndexerService - -12->10 - - -IndexerService - - - diff --git a/doc/data_flow_simple.dot b/doc/data_flow_simple.dot deleted file mode 100644 index a30f8ad73de5..000000000000 --- a/doc/data_flow_simple.dot +++ /dev/null @@ -1,27 +0,0 @@ -digraph DruidDataFlow { - graph [bgcolor="#fffef5", clusterrank=global, rankdir=TB]; - node [color="#0a0701", fillcolor="#fdf4c6", fontname=Helvetica, shape=box, style=filled, label="\N"]; - edge [color="#377d18"]; - - 1 [label = "REST query"]; - 10 [label = "Broker.query_cache"]; - 2 [label = "realtime_data_src"]; - 3 [label = "Compute.disk_cache"]; - 4 [label = "indexed segments\nblob_store (S3, HDFS)"]; - 5 [label = "Realtime.working_set"]; - 6 [label = "Compute.working_set"]; - -2 -> 5 [label = "Realtime.Firehose"]; -5 -> 4 [label = "Realtime.segmentPusher"]; - -4 -> 3 [label = "Compute.load"]; -3 -> 6 [label = "Compute.map"]; - -5 -> 10 [label = " results "]; -6 -> 10 [label = " results "]; -10 -> 1 [label = " results "]; - -10 -> 5 [label = " query "]; -10 -> 6 [label = " query "]; -1 -> 10 [label = " query "]; -} diff --git a/doc/data_flow_simple.png b/doc/data_flow_simple.png deleted file mode 100644 index f397953b651a..000000000000 Binary files a/doc/data_flow_simple.png and /dev/null differ diff --git a/doc/data_flow_simple.svg b/doc/data_flow_simple.svg deleted file mode 100644 index 012a06c2977f..000000000000 --- a/doc/data_flow_simple.svg +++ /dev/null @@ -1,92 +0,0 @@ - - - - - -DruidDataFlow - -1 - -REST query - -10 - -Broker.query_cache - -1->10 - - - query - -10->1 - - - results - -5 - -Realtime.working_set - -10->5 - - - query - -6 - -Compute.working_set - -10->6 - - - query - -2 - -realtime_data_src - -2->5 - - -Realtime.Firehose - -3 - -Compute.disk_cache - -3->6 - - -Compute.map - -4 - -indexed segments -blob_store (S3, HDFS) - -4->3 - - -Compute.load - -5->10 - - - results - -5->4 - - -Realtime.segmentPusher - -6->10 - - - results - - - diff --git a/doc/publications/vldb/README.md b/doc/publications/vldb/README.md deleted file mode 100644 index 44025ab1dbf1..000000000000 --- a/doc/publications/vldb/README.md +++ /dev/null @@ -1,4 +0,0 @@ -Download [MacTeX](http://tug.org/mactex/) -```bash -make -``` diff --git a/doc/publications/vldb/druid.pdf b/doc/publications/vldb/druid.pdf deleted file mode 100644 index 3553c0c4b5be..000000000000 Binary files a/doc/publications/vldb/druid.pdf and /dev/null differ diff --git a/doc/segment_propagation.png b/doc/segment_propagation.png deleted file mode 100644 index e9d3e1a17a6c..000000000000 Binary files a/doc/segment_propagation.png and /dev/null differ diff --git a/doc/segmentation_propagation.txt b/doc/segmentation_propagation.txt deleted file mode 100644 index e70f4d37aa79..000000000000 --- a/doc/segmentation_propagation.txt +++ /dev/null @@ -1,13 +0,0 @@ -# http://www.websequencediagrams.com/ -title Segment Propagation - -Realtime->Deep Storage: push(segment) -Realtime->MySQL: write(metadata) -Master->+MySQL: segments? -MySQL-->-Master: metadata -note over Master: rules select segment owner -Master->Zookeeper: writes(ephemeral node) -Zookeeper->Compute: reads(ephemeral node) -Compute->+Deep Storage: pull(segment) -Deep Storage-->-Compute: transfer(segment) -Compute->Zookeeper: delete(ephemeral node) diff --git a/docs/Aggregations.md b/docs/Aggregations.md new file mode 100644 index 000000000000..37b99aeffc29 --- /dev/null +++ b/docs/Aggregations.md @@ -0,0 +1,90 @@ +--- +layout: default +--- +Aggregations are specifications of processing over metrics available in Druid. +Available aggregations are: + +### Sum aggregators + +#### `longSum` aggregator + +computes the sum of values as a 64-bit, signed integer + + { + "type" : "longSum", + "name" : , + "fieldName" : + } + +`name` – output name for the summed value +`fieldName` – name of the metric column to sum over + +#### `doubleSum` aggregator + +Computes the sum of values as 64-bit floating point value. Similar to `longSum` + + { + "type" : "doubleSum", + "name" : , + "fieldName" : + } + +### Count aggregator + +`count` computes the row count that match the filters + + { + "type" : "count", + "name" : , + } + +### Min / Max aggregators + +#### `min` aggregator + +`min` computes the minimum metric value + + { + "type" : "min", + "name" : , + "fieldName" : + } + +#### `max` aggregator + +`max` computes the maximum metric value + + { + "type" : "max", + "name" : , + "fieldName" : + } + +### JavaScript aggregator + +Computes an arbitrary JavaScript function over a set of columns (both metrics and dimensions). + +All JavaScript functions must return numerical values. + + { + "type": "javascript", + "name": "", + "fieldNames" : [ , , ... ], + "fnAggregate" : "function(current, column1, column2, ...) { + + return + }" + "fnCombine" : "function(partialA, partialB) { return ; }" + "fnReset" : "function() { return ; }" + } + +**Example** + + { + "type": "javascript", + "name": "sum(log(x)/y) + 10", + "fieldNames": ["x", "y"], + "fnAggregate" : "function(current, a, b) { return current + (Math.log(a) * b); }" + "fnCombine" : "function(partialA, partialB) { return partialA + partialB; }" + "fnReset" : "function() { return 10; }" + } \ No newline at end of file diff --git a/docs/Batch-ingestion.md b/docs/Batch-ingestion.md new file mode 100644 index 000000000000..42a42ac7b292 --- /dev/null +++ b/docs/Batch-ingestion.md @@ -0,0 +1,141 @@ +--- +layout: default +--- +Batch Data Ingestion +==================== + +There are two choices for batch data ingestion to your Druid cluster, you can use the [Indexing service](Indexing-service.html) or you can use the `HadoopDruidIndexerMain`. This page describes how to use the `HadoopDruidIndexerMain`. + +Which should I use? +------------------- + +The [Indexing service](Indexing-service.html) is a node that can run as part of your Druid cluster and can accomplish a number of different types of indexing tasks. Even if all you care about is batch indexing, it provides for the encapsulation of things like the Database that is used for segment metadata and other things, so that your indexing tasks do not need to include such information. Long-term, the indexing service is going to be the preferred method of ingesting data. + +The `HadoopDruidIndexerMain` runs hadoop jobs in order to separate and index data segments. It takes advantage of Hadoop as a job scheduling and distributed job execution platform. It is a simple method if you already have Hadoop running and don’t want to spend the time configuring and deploying the [Indexing service](Indexing service.html) just yet. + +HadoopDruidIndexer +------------------ + +Located at `com.metamx.druid.indexer.HadoopDruidIndexerMain` can be run like + + + java -cp hadoop_config_path:druid_indexer_selfcontained_jar_path com.metamx.druid.indexer.HadoopDruidIndexerMain + + +The interval is the [ISO8601 interval](http://en.wikipedia.org/wiki/ISO_8601#Time_intervals) of the data you are processing. The config\_file is a path to a file (the “specFile”) that contains JSON and an example looks like: + + + { + "dataSource": "the_data_source", + "timestampColumn": "ts", + "timestampFormat": "", + "dataSpec": { + "format": "", + "columns": ["ts", "column_1", "column_2", "column_3", "column_4", "column_5"], + "dimensions": ["column_1", "column_2", "column_3"] + }, + "granularitySpec": { + "type":"uniform", + "intervals":[""], + "gran":"day" + }, + "pathSpec": { "type": "granularity", + "dataGranularity": "hour", + "inputPath": "s3n://billy-bucket/the/data/is/here", + "filePattern": ".*" }, + "rollupSpec": { "aggs": [ + { "type": "count", "name":"event_count" }, + { "type": "doubleSum", "fieldName": "column_4", "name": "revenue" }, + { "type": "longSum", "fieldName" : "column_5", "name": "clicks" } + ], + "rollupGranularity": "minute"}, + "workingPath": "/tmp/path/on/hdfs", + "segmentOutputPath": "s3n://billy-bucket/the/segments/go/here", + "leaveIntermediate": "false", + "partitionsSpec": { + "targetPartitionSize": 5000000 + }, + "updaterJobSpec": { + "type":"db", + "connectURI":"jdbc:mysql://localhost:7980/test_db", + "user":"username", + "password":"passmeup", + "segmentTable":"segments" + } + } + + +### Hadoop indexer config + +|property|description|required?| +|--------|-----------|---------| +|dataSource|name of the dataSource the data will belong to|yes| +|timestampColumn|the column that is to be used as the timestamp column|yes| +|timestampFormat|the format of timestamps; auto = either iso or millis, Joda time formats:http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html|yes| +|dataSpec|a specification of the data format and an array that names all of the columns in the input data|yes| +|dimensions|the columns that are to be used as dimensions|yes| +|granularitySpec|the time granularity and interval to chunk segments up into|yes| +|pathSpec|a specification of where to pull the data in from|yes| +|rollupSpec|a specification of the rollup to perform while processing the data|yes| +|workingPath|the working path to use for intermediate results (results between Hadoop jobs)|yes| +|segmentOutputPath|the path to dump segments into|yes| +|leaveIntermediate|leave behind files in the workingPath when job completes or fails (debugging tool)|no| +|partitionsSpec|a specification of how to partition each time bucket into segments, absence of this property means no partitioning will occur|no| +|updaterJobSpec|a specification of how to update the metadata for the druid cluster these segments belong to|yes| +|registererers|a list of serde handler classnames|no| + +### Path specification + +There are multiple types of path specification: + +##### `granularity` + +Is a type of data loader that expects data to be laid out in a specific path format. Specifically, it expects it to be segregated by day in this directory format `y=XXXX/m=XX/d=XX/H=XX/M=XX/S=XX` (dates are represented by lowercase, time is represented by uppercase). + +|property|description|required?| +|--------|-----------|---------| +|dataGranularity|specifies the granularity to expect the data at, e.g. hour means to expect directories `y=XXXX/m=XX/d=XX/H=XX`|yes| +|inputPath|Base path to append the expected time path to|yes| +|filePattern|Pattern that files should match to be included|yes| + +For example, if the sample config were run with the interval 2012-06-01/2012-06-02, it would expect data at the paths + + s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=00 + s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=01 + ... + s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=23 + +### Rollup specification + +The indexing process has the ability to roll data up as it processes the incoming data. If data has already been summarized, summarizing it again will produce the same results so either way is not a problem. This specifies how that rollup should take place. + +|property|description|required?| +|--------|-----------|---------| +|aggs|specifies a list of aggregators to aggregate for each bucket (a bucket is defined by the tuple of the truncated timestamp and the dimensions). Aggregators available here are the same as available when querying.|yes| +|rollupGranularity|The granularity to use when truncating incoming timestamps for bucketization|yes| + +### Partitioning specification + +Segments are always partitioned based on timestamp (according to the granularitySpec) and may be further partitioned in some other way. For example, data for a day may be split by the dimension “last\_name” into two segments: one with all values from A-M and one with all values from N-Z. + +To use this option, the indexer must be given a target partition size. It can then find a good set of partition ranges on its own. + +|property|description|required?| +|--------|-----------|---------| +|targetPartitionSize|target number of rows to include in a partition, should be a number that targets segments of 700MB\~1GB.|yes| +|partitionDimension|the dimension to partition on. Leave blank to select a dimension automatically.|no| +|assumeGrouped|assume input data has already been grouped on time and dimensions. This is faster, but can choose suboptimal partitions if the assumption is violated.|no| + +### Updater job spec + +This is a specification of the properties that tell the job how to update metadata such that the Druid cluster will see the output segments and load them. + +|property|description|required?| +|--------|-----------|---------| +|type|“db” is the only value available|yes| +|connectURI|a valid JDBC url to MySQL|yes| +|user|username for db|yes| +|password|password for db|yes| +|segmentTable|table to use in DB|yes| + +These properties should parrot what you have configured for your [Master](Master.html). diff --git a/docs/Booting-a-production-cluster.md b/docs/Booting-a-production-cluster.md new file mode 100644 index 000000000000..f7e5444ab8eb --- /dev/null +++ b/docs/Booting-a-production-cluster.md @@ -0,0 +1,30 @@ +--- +layout: default +--- +# Booting a Single Node Cluster # + +[Loading Your Data](Loading-Your-Data.html) and [Querying Your Data](Querying-Your-Data.html) contain recipes to boot a small druid cluster on localhost. Here we will boot a small cluster on EC2. You can checkout the code, or download a tarball from [here](http://static.druid.io/artifacts/druid-services-0.5.51-SNAPSHOT-bin.tar.gz). + +The [ec2 run script](https://github.com/metamx/druid/blob/master/examples/bin/run_ec2.sh), run_ec2.sh, is located at 'examples/bin' if you have checked out the code, or at the root of the project if you've downloaded a tarball. The scripts rely on the [Amazon EC2 API Tools](http://aws.amazon.com/developertools/351), and you will need to set three environment variables: + +```bash +# Setup environment for ec2-api-tools +export EC2_HOME=/path/to/ec2-api-tools-1.6.7.4/ +export PATH=$PATH:$EC2_HOME/bin +export AWS_ACCESS_KEY= +export AWS_SECRET_KEY= +``` + +Then, booting an ec2 instance running one node of each type is as simple as running the script, run_ec2.sh :) + +# Apache Whirr # + +Apache Whirr is a set of libraries for launching cloud services. You can clone a version of Whirr that includes Druid as a service from git@github.com:rjurney/whirr.git: + +```bash +git clone git@github.com:rjurney/whirr.git +cd whirr +git checkout trunk +mvn clean install -Dmaven.test.failure.ignore=true -Dcheckstyle.skip +sp;bin/whirr launch-cluster --config recipes/druid.properties +``` \ No newline at end of file diff --git a/docs/Broker.md b/docs/Broker.md new file mode 100644 index 000000000000..fee33bedc835 --- /dev/null +++ b/docs/Broker.md @@ -0,0 +1,30 @@ +--- +layout: default +--- +Broker +====== + +The Broker is the node to route queries to if you want to run a distributed cluster. It understands the metadata published to ZooKeeper about what segments exist on what nodes and routes queries such that they hit the right nodes. This node also merges the result sets from all of the individual nodes together. + +Forwarding Queries +------------------ + +Most druid queries contain an interval object that indicates a span of time for which data is requested. Likewise, Druid [Segments](Segments.html) are partitioned to contain data for some interval of time and segments are distributed across a cluster. Consider a simple datasource with 7 segments where each segment contains data for a given day of the week. Any query issued to the datasource for more than one day of data will hit more than one segment. These segments will likely be distributed across multiple nodes, and hence, the query will likely hit multiple nodes. + +To determine which nodes to forward queries to, the Broker node first builds a view of the world from information in Zookeeper. Zookeeper maintains information about [Compute](Compute.html) and [Realtime](Realtime.html) nodes and the segments they are serving. For every datasource in Zookeeper, the Broker node builds a timeline of segments and the nodes that serve them. When queries are received for a specific datasource and interval, the Broker node performs a lookup into the timeline associated with the query datasource for the query interval and retrieves the nodes that contain data for the query. The Broker node then forwards down the query to the selected nodes. + +Caching +------- + +Broker nodes employ a distributed cache with a LRU cache invalidation strategy. The broker cache stores per segment results. The cache can be local to each broker node or shared across multiple nodes using an external distributed cache such as [memcached](http://memcached.org/). Each time a broker node receives a query, it first maps the query to a set of segments. A subset of these segment results may already exist in the cache and the results can be directly pulled from the cache. For any segment results that do not exist in the cache, the broker node will forward the query to the +compute nodes. Once the compute nodes return their results, the broker will store those results in the cache. Real-time segments are never cached and hence requests for real-time data will always be forwarded to real-time nodes. Real-time data is perpetually changing and caching the results would be unreliable. + +Running +------- + +Broker nodes can be run using the `com.metamx.druid.http.BrokerMain` class. + +Configuration +------------- + +See [Configuration](Configuration.html). diff --git a/docs/Build-from-source.md b/docs/Build-from-source.md new file mode 100644 index 000000000000..3f323259b808 --- /dev/null +++ b/docs/Build-from-source.md @@ -0,0 +1,24 @@ +--- +layout: default +--- +### Clone and Build from Source + +The other way to setup Druid is from source via git. To do so, run these commands: + +``` +git clone git@github.com:metamx/druid.git +cd druid +./build.sh +``` + +You should see a bunch of files: + +``` +DruidCorporateCLA.pdf README common examples indexer pom.xml server +DruidIndividualCLA.pdf build.sh doc group_by.body install publications services +LICENSE client eclipse_formatting.xml index-common merger realtime +``` + +You can find the example executables in the examples/bin directory: +* run_example_server.sh +* run_example_client.sh diff --git a/docs/Cluster-setup.md b/docs/Cluster-setup.md new file mode 100644 index 000000000000..b8281e99468c --- /dev/null +++ b/docs/Cluster-setup.md @@ -0,0 +1,114 @@ +--- +layout: default +--- +A Druid cluster consists of various node types that need to be set up depending on your use case. See our [Design](Design.html) docs for a description of the different node types. + +Setup Scripts +------------- + +One of our community members, [housejester](https://github.com/housejester/), contributed some scripts to help with setting up a cluster. Checkout the [github](https://github.com/housejester/druid-test-harness) and [wiki](https://github.com/housejester/druid-test-harness/wiki/Druid-Test-Harness). + +Minimum Physical Layout: Absolute Minimum +----------------------------------------- + +As a special case, the absolute minimum setup is one of the standalone examples for realtime ingestion and querying; see [Examples](Examples.html) that can easily run on one machine with one core and 1GB RAM. This layout can be set up to try some basic queries with Druid. + +Minimum Physical Layout: Experimental Testing with 4GB of RAM +------------------------------------------------------------- + +This layout can be used to load some data from deep storage onto a Druid compute node for the first time. A minimal physical layout for a 1 or 2 core machine with 4GB of RAM is: + +1. node1: [Master](Master.html) + metadata service + zookeeper + [Compute](Compute.html) +2. transient nodes: indexer + +This setup is only reasonable to prove that a configuration works. It would not be worthwhile to use this layout for performance measurement. + +Comfortable Physical Layout: Pilot Project with Multiple Machines +----------------------------------------------------------------- + +*The machine size “flavors” are using AWS/EC2 terminology for descriptive purposes only and is not meant to imply that AWS/EC2 is required or recommended. Another cloud provider or your own hardware can also work.* + +A minimal physical layout not constrained by cores that demonstrates parallel querying and realtime, using AWS-EC2 “small”/m1.small (one core, with 1.7GB of RAM) or larger, no realtime, is: + +1. node1: [Master](Master.html) (m1.small) +2. node2: metadata service (m1.small) +3. node3: zookeeper (m1.small) +4. node4: [Broker](Broker.html) (m1.small or m1.medium or m1.large) +5. node5: [Compute](Compute.html) (m1.small or m1.medium or m1.large) +6. node6: [Compute](Compute.html) (m1.small or m1.medium or m1.large) +7. node7: [Realtime](Realtime.html) (m1.small or m1.medium or m1.large) +8. transient nodes: indexer + +This layout naturally lends itself to adding more RAM and core to Compute nodes, and to adding many more Compute nodes. Depending on the actual load, the Master, metadata server, and Zookeeper might need to use larger machines. + +High Availability Physical Layout +--------------------------------- + +*The machine size “flavors” are using AWS/EC2 terminology for descriptive purposes only and is not meant to imply that AWS/EC2 is required or recommended. Another cloud provider or your own hardware can also work.* + +An HA layout allows full rolling restarts and heavy volume: + +1. node1: [Master](Master.html) (m1.small or m1.medium or m1.large) +2. node2: [Master](Master.html) (m1.small or m1.medium or m1.large) (backup) +3. node3: metadata service (c1.medium or m1.large) +4. node4: metadata service (c1.medium or m1.large) (backup) +5. node5: zookeeper (c1.medium) +6. node6: zookeeper (c1.medium) +7. node7: zookeeper (c1.medium) +8. node8: [Broker](Broker.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) +9. node9: [Broker](Broker.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) (backup) +10. node10: [Compute](Compute.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) +11. node11: [Compute](Compute.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) +12. node12: [Realtime](Realtime.html) (m1.small or m1.medium or m1.large or m2.xlarge or m2.2xlarge or m2.4xlarge) +13. transient nodes: indexer + +Sizing for Cores and RAM +------------------------ + +The Compute and Broker nodes will use as many cores as are available, depending on usage, so it is best to keep these on dedicated machines. The upper limit of effectively utilized cores is not well characterized yet and would depend on types of queries, query load, and the schema. Compute daemons should have a heap a size of at least 1GB per core for normal usage, but could be squeezed into a smaller heap for testing. Since in-memory caching is essential for good performance, even more RAM is better. Broker nodes will use RAM for caching, so they do more than just route queries. + +The effective utilization of cores by Zookeeper, MySQL, and Master nodes is likely to be between 1 and 2 for each process/daemon, so these could potentially share a machine with lots of cores. These daemons work with heap a size between 500MB and 1GB. + +Storage +------- + +Indexed segments should be kept in a permanent store accessible by all nodes like AWS S3 or HDFS or equivalent. Currently Druid supports S3, but this will be extended soon. + +Local disk (“ephemeral” on AWS EC2) for caching is recommended over network mounted storage (example of mounted: AWS EBS, Elastic Block Store) in order to avoid network delays during times of heavy usage. If your data center is suitably provisioned for networked storage, perhaps with separate LAN/NICs just for storage, then mounted might work fine. + +Setup +----- + +Setting up a cluster is essentially just firing up all of the nodes you want with the proper [configuration](configuration.html). One thing to be aware of is that there are a few properties in the configuration that potentially need to be set individually for each process: + + + druid.server.type=historical|realtime + druid.host=someHostOrIPaddrWithPort + druid.port=8080 + + +`druid.server.type` should be set to “historical” for your compute nodes and realtime for the realtime nodes. The master will only assign segments to a “historical” node and the broker has some intelligence around its ability to cache results when talking to a realtime node. This does not need to be set for the master or the broker. + +`druid.host` should be set to the hostname and port that can be used to talk to the given server process. Basically, someone should be able to send a request to http://\${druid.host}/ and actually talk to the process. + +`druid.port` should be set to the port that the server should listen on. In the vast majority of cases, this port should be the same as what is on `druid.host`. + +Build/Run +--------- + +The simplest way to build and run from the repository is to run `mvn package` from the base directory and then take `druid-services/target/druid-services-*-selfcontained.jar` and push that around to your machines; the jar does not need to be expanded, and since it contains the main() methods for each kind of service, it is **not** invoked with java ~~jar. It can be run from a normal java command-line by just including it on the classpath and then giving it the main class that you want to run. For example one instance of the Compute node/service can be started like this: +\ + +java~~Duser.timezone=UTC ~~Dfile.encoding=UTF-8~~cp compute/:druid-services/target/druid-services~~\*~~selfcontained.jar com.metamx.druid.http.ComputeMain + + + +The following table shows the possible services and fully qualified class for main(). + +|service|main class| +|-------|----------| +|[ Realtime ]( Realtime .html)|com.metamx.druid.realtime.RealtimeMain| +|[ Master ]( Master .html)|com.metamx.druid.http.MasterMain| +|[ Broker ]( Broker .html)|com.metamx.druid.http.BrokerMain| +|[ Compute ]( Compute .html)|com.metamx.druid.http.ComputeMain| + diff --git a/docs/Compute.md b/docs/Compute.md new file mode 100644 index 000000000000..e7df17ebbd5a --- /dev/null +++ b/docs/Compute.md @@ -0,0 +1,40 @@ +--- +layout: default +--- +Compute +======= + +Compute nodes are the work horses of a cluster. They load up historical segments and expose them for querying. + +Loading and Serving Segments +---------------------------- + +Each compute node maintains a constant connection to Zookeeper and watches a configurable set of Zookeeper paths for new segment information. Compute nodes do not communicate directly with each other or with the master nodes but instead rely on Zookeeper for coordination. + +The [Master](Master.html) node is responsible for assigning new segments to compute nodes. Assignment is done by creating an ephemeral Zookeeper entry under a load queue path associated with a compute node. For more information on how the master assigns segments to compute nodes, please see [Master](Master.html). + +When a compute node notices a new load queue entry in its load queue path, it will first check a local disk directory (cache) for the information about segment. If no information about the segment exists in the cache, the compute node will download metadata about the new segment to serve from Zookeeper. This metadata includes specifications about where the segment is located in deep storage and about how to decompress and process the segment. For more information about segment metadata and Druid segments in general, please see [Segments](Segments.html). Once a compute node completes processing a segment, the segment is announced in Zookeeper under a served segments path associated with the node. At this point, the segment is available for querying. + +Loading and Serving Segments From Cache +--------------------------------------- + +Recall that when a compute node notices a new segment entry in its load queue path, the compute node first checks a configurable cache directory on its local disk to see if the segment had been previously downloaded. If a local cache entry already exists, the compute node will directly read the segment binary files from disk and load the segment. + +The segment cache is also leveraged when a compute node is first started. On startup, a compute node will search through its cache directory and immediately load and serve all segments that are found. This feature allows compute nodes to be queried as soon they come online. + +Querying Segments +----------------- + +Please see [Querying](Querying.html) for more information on querying compute nodes. + +For every query that a compute node services, it will log the query and report metrics on the time taken to run the query. + +Running +------- + +Compute nodes can be run using the `com.metamx.druid.http.ComputeMain` class. + +Configuration +------------- + +See [Configuration](Configuration.html). diff --git a/docs/Concepts-and-Terminology.md b/docs/Concepts-and-Terminology.md new file mode 100644 index 000000000000..925941dd8c58 --- /dev/null +++ b/docs/Concepts-and-Terminology.md @@ -0,0 +1,15 @@ +--- +layout: default +--- +Concepts and Terminology +======================== + +- **Aggregators:** A mechanism for combining records during realtime incremental indexing, Hadoop batch indexing, and in queries. +- **DataSource:** A table-like view of data; specified in a “specFile” and in a query. +- **Granularity:** The time interval corresponding to aggregation by time. + - The *indexGranularity* setting in a schema is used to aggregate input (ingest) records within an interval into a single output (internal) record. + - The *segmentGranularity* is the interval specifying how internal records are stored together in a single file. + +- **Segment:** A collection of (internal) records that are stored and processed together. +- **Shard:** A unit of partitioning data across machine. TODO: clarify; by time or other dimensions? +- **specFile** is specification for services in JSON format; see [Realtime](Realtime.html) and [Batch-ingestion](Batch-ingestion.html) diff --git a/docs/Configuration.md b/docs/Configuration.md new file mode 100644 index 000000000000..4042d02d8258 --- /dev/null +++ b/docs/Configuration.md @@ -0,0 +1,162 @@ +--- +layout: default +--- +This describes the basic server configuration that is loaded by all the server processes; the same file is loaded by all. See also the json “specFile” descriptions in [Realtime](Realtime.html) and [Batch-ingestion](Batch-ingestion.html). + +JVM Configuration Best Practices +================================ + +There are three JVM parameters that we set on all of our processes: + +1. `-Duser.timezone=UTC` This sets the default timezone of the JVM to UTC. We always set this and do not test with other default timezones, so local timezones might work, but they also might uncover weird and interesting bugs +2. `-Dfile.encoding=UTF-8` This is similar to timezone, we test assuming UTF-8. Local encodings might work, but they also might result in weird and interesting bugs +3. `-Djava.io.tmpdir=` Various parts of the system that interact with the file system do it via temporary files, these files can get somewhat large. Many production systems are setup to have small (but fast) `/tmp` directories, these can be problematic with Druid so we recommend pointing the JVM’s tmp directory to something with a little more meat. + +Basic Service Configuration +=========================== + +Configuration of the various nodes is done via Java properties. These can either be provided as `-D` system properties on the java command line or they can be passed in via a file called `runtime.properties` that exists on the classpath. Note: as a future item, I’d like to consolidate all of the various configuration into a yaml/JSON based configuration files. + +The periodic time intervals (like “PT1M”) are [ISO8601 intervals](http://en.wikipedia.org/wiki/ISO_8601#Time_intervals) + +An example runtime.properties is as follows: + + + # S3 access + com.metamx.aws.accessKey= + com.metamx.aws.secretKey= + + # thread pool size for servicing queries + druid.client.http.connections=30 + + # JDBC connection string for metadata database + druid.database.connectURI= + druid.database.user=user + druid.database.password=password + # time between polling for metadata database + druid.database.poll.duration=PT1M + druid.database.segmentTable=prod_segments + + # Path on local FS for storage of segments; dir will be created if needed + druid.paths.indexCache=/tmp/druid/indexCache + # Path on local FS for storage of segment metadata; dir will be created if needed + druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache + + druid.request.logging.dir=/tmp/druid/log + + druid.server.maxSize=300000000000 + + # ZK quorum IPs + druid.zk.service.host= + # ZK path prefix for Druid-usage of zookeeper, Druid will create multiple paths underneath this znode + druid.zk.paths.base=/druid + # ZK path for discovery, the only path not to default to anything + druid.zk.paths.discoveryPath=/druid/discoveryPath + + # the host:port as advertised to clients + druid.host=someHostOrIPaddrWithPort + # the port on which to listen, this port should line up with the druid.host value + druid.port=8080 + + com.metamx.emitter.logging=true + com.metamx.emitter.logging.level=debug + + druid.processing.formatString=processing_%s + druid.processing.numThreads=3 + + + druid.computation.buffer.size=100000000 + + # S3 dest for realtime indexer + druid.pusher.s3.bucket= + druid.pusher.s3.baseKey= + + druid.bard.cache.sizeInBytes=40000000 + druid.master.merger.service=blah_blah + + +Configuration groupings +----------------------- + +### S3 Access + +These properties are for connecting with S3 and using it to pull down segments. In the future, we plan on being able to use other deep storage file systems as well, like HDFS. The file system is actually only accessed by the [Compute](Compute.html), [Realtime](Realtime.html) and [Indexing service](Indexing service.html) nodes. + +|Property|Description|Default| +|--------|-----------|-------| +|`com.metamx.aws.accessKey`|The access key to use to access S3.|none| +|`com.metamx.aws.secretKey`|The secret key to use to access S3.|none| +|`druid.pusher.s3.bucket`|The bucket to store segments, this is used by Realtime and the Indexing service.|none| +|`druid.pusher.s3.baseKey`|The base key to use when storing segments, this is used by Realtime and the Indexing service|none| + +### JDBC connection + +These properties specify the jdbc connection and other configuration around the “segments table” database. The only processes that connect to the DB with these properties are the [Master](Master.html) and [Indexing service](Indexing-service.html). This is tested on MySQL. + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.database.connectURI`|The jdbc connection uri|none| +|`druid.database.user`|The username to connect with|none| +|`druid.database.password`|The password to connect with|none| +|`druid.database.poll.duration`|The duration between polls the Master does for updates to the set of active segments. Generally defines the amount of lag time it can take for the master to notice new segments|PT1M| +|`druid.database.segmentTable`|The table to use to look for segments.|none| +|`druid.database.ruleTable`|The table to use to look for segment load/drop rules.|none| +|`druid.database.configTable`|The table to use to look for configs.|none| + +### Master properties + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.master.period`|The run period for the master. The master’s operates by maintaining the current state of the world in memory and periodically looking at the set of segments available and segments being served to make decisions about whether any changes need to be made to the data topology. This property sets the delay between each of these runs|PT60S| +|`druid.master.removedSegmentLifetime`|When a node disappears, the master can provide a grace period for how long it waits before deciding that the node really isn’t going to come back and it really should declare that all segments from that node are no longer available. This sets that grace period in number of runs of the master.|1| +|`druid.master.startDelay`|The operation of the Master works on the assumption that it has an up-to-date view of the state of the world when it runs, the current ZK interaction code, however, is written in a way that doesn’t allow the Master to know for a fact that it’s done loading the current state of the world. This delay is a hack to give it enough time to believe that it has all the data|PT600S| + +### Zk properties + +See [ZooKeeper](ZooKeeper.html) for a description of these properties. + +### Service properties + +These are properties that define various service/HTTP server aspects + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.client.http.connections`|Size of connection pool for the Broker to connect to compute nodes. If there are more queries than this number that all need to speak to the same node, then they will queue up.|none| +|`druid.paths.indexCache`|Segments assigned to a compute node are first stored on the local file system and then served by the compute node. This path defines where that local cache resides. Directory will be created if needed|none| +|`druid.paths.segmentInfoCache`|Compute nodes keep track of the segments they are serving so that when the process is restarted they can reload the same segments without waiting for the master to reassign. This path defines where this metadata is kept. Directory will be created if needed|none| +|`druid.http.numThreads`|The number of HTTP worker threads.|10| +|`druid.http.maxIdleTimeMillis`|The amount of time a connection can remain idle before it is terminated|300000 (5 min)| +|`druid.request.logging.dir`|Compute, Realtime and Broker nodes maintain request logs of all of the requests they get (interacton is via POST, so normal request logs don’t generally capture information about the actual query), this specifies the directory to store the request logs in|none| +|`druid.host`|The host for the current node. This is used to advertise the current processes location as reachable from another node and should generally be specified such that `http://${druid.host}/` could actually talk to this process|none| +|`druid.port`|This is the port to actually listen on; unless port mapping is used, this will be the same port as is on `druid.host`|none| +|`druid.processing.formatString`|Realtime and Compute nodes use this format string to name their processing threads.|none| +|`druid.processing.numThreads`|The number of processing threads to have available for parallel processing of segments. Our rule of thumb is `num_cores - 1`, this means that even under heavy load there will still be one core available to do background tasks like talking with ZK and pulling down segments.|none| +|`druid.computation.buffer.size`|This specifies a buffer size for the storage of intermediate results. The computation engine in both the Compute and Realtime nodes will use a scratch buffer of this size to do all of their intermediate computations off-heap. Larger values allow for more aggregations in a single pass over the data while smaller values can require more passes depending on the query that is being executed.|1073741824 (1GB)| +|`druid.service`|The name of the service. This is used as a dimension when emitting metrics and alerts to differentiate between the various services|none| +|`druid.bard.cache.sizeInBytes`|The Broker (called Bard internally) instance has the ability to store results of queries in an in-memory cache. This specifies the number of bytes to use for that cache|none| + +### Compute Properties + +These are properties that the compute nodes use + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.server.maxSize`|The maximum number of bytes worth of segment that the node wants assigned to it. This is not a limit that the compute nodes actually enforce, they just publish it to the master and trust the master to do the right thing|none| +|`druid.server.type`|Specifies the type of the node. This is published via ZK and depending on the value the node will be treated specially by the Master/Broker. Allowed values are “realtime” or “historical”. This is a configuration parameter because the plan is to allow for a more configurable cluster composition. At the current time, all realtime nodes should just be “realtime” and all compute nodes should just be “compute”|none| + +### Emitter Properties + +The Druid servers emit various metrics and alerts via something we call an [Emitter](Emitter.html). There are two emitter implementations included with the code, one that just logs to log4j and one that does POSTs of JSON events to a server. More information can be found on the [Emitter](Emitter.html) page. The properties for using the logging emitter are described below. + +|Property|Description|Default| +|--------|-----------|-------| +|`com.metamx.emitter.logging`|Set to “true” to use the logging emitter|none| +|`com.metamx.emitter.logging.level`|Sets the level to log at|debug| +|`com.metamx.emitter.logging.class`|Sets the class to log at|com.metamx.emiter.core.LoggingEmitter| + +### Realtime Properties + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.realtime.specFile`|The file with realtime specifications in it. See [Realtime](Realtime.html).|none| + diff --git a/docs/Contribute.md b/docs/Contribute.md new file mode 100644 index 000000000000..a853eb430d20 --- /dev/null +++ b/docs/Contribute.md @@ -0,0 +1,8 @@ +--- +layout: default +--- +If you are interested in contributing to the code, we accept [pull requests](https://help.github.com/articles/using-pull-requests). Note: we have only just completed decoupling our Metamarkets-specific code from the code base and we took some short-cuts in interface design to make it happen. So, there are a number of interfaces that exist right now which are likely to be in flux. If you are embedding Druid in your system, it will be safest for the time being to only extend/implement interfaces that this wiki describes, as those are intended as stable (unless otherwise mentioned). + +For issue tracking, we are using the github issue tracker. Please fill out an issue from the Issues tab on the github screen. + +We also have a [Libraries](Libraries.html) page that lists external libraries that people have created for working with Druid. diff --git a/docs/Deep-Storage.md b/docs/Deep-Storage.md new file mode 100644 index 000000000000..bd9a0ec8a661 --- /dev/null +++ b/docs/Deep-Storage.md @@ -0,0 +1,42 @@ +--- +layout: default +--- +Deep storage is where segments are stored. It is a storage mechanism that Druid does not provide. This deep storage infrastructure defines the level of durability of your data, as long as Druid nodes can see this storage infrastructure and get at the segments stored on it, you will not lose data no matter how many Druid nodes you lose. If segments disappear from this storage layer, then you will lose whatever data those segments represented. + +The currently supported types of deep storage follow. + +## S3-compatible + +S3-compatible deep storage is basically either S3 or something like riak-cs which exposes the same API as S3. This is the default deep storage implementation. + +S3 configuration parameters are + + com.metamx.aws.accessKey= + com.metamx.aws.secretKey= + druid.pusher.s3.bucket= + druid.pusher.s3.baseKey= + +## HDFS + +As of 0.4.0, HDFS can be used for storage of segments as well. + +In order to use hdfs for deep storage, you need to set the following configuration on your realtime nodes. + + druid.pusher.hdfs=true + druid.pusher.hdfs.storageDirectory= + +If you are using the Hadoop indexer, set your output directory to be a location on Hadoop and it will work + + +## Local Mount + +A local mount can be used for storage of segments as well. This allows you to use just your local file system or anything else that can be mount locally like NFS, Ceph, etc. + +In order to use a local mount for deep storage, you need to set the following configuration on your realtime nodes. + + druid.pusher.local=true + druid.pusher.local.storageDirectory= + +Note that you should generally set `druid.pusher.local.storageDirectory` to something different from `druid.paths.indexCache`. + +If you are using the Hadoop indexer in local mode, then just give it a local file as your output directory and it will work. \ No newline at end of file diff --git a/docs/Design.md b/docs/Design.md new file mode 100644 index 000000000000..25d69d95fe85 --- /dev/null +++ b/docs/Design.md @@ -0,0 +1,88 @@ +--- +layout: default +--- +For a comprehensive look at the architecture of Druid, read the [White Paper](http://static.druid.io/docs/druid.pdf). + +What is Druid? +============== + +Druid is a system built to allow fast (“real-time”) access to large sets of seldom-changing data. It was designed with the intent of being a service and maintaining 100% uptime in the face of code deployments, machine failures and other eventualities of a production system. It can be useful for back-office use cases as well, but design decisions were made explicitly targetting an always-up service. + +Druid currently allows for single-table queries in a similar manner to [Dremel](http://research.google.com/pubs/pub36632.html) and [PowerDrill](http://www.vldb.org/pvldb/vol5/p1436_alexanderhall_vldb2012.pdf). It adds to the mix + +1. columnar storage format for partially nested data structures +2. hierarchical query distribution with intermediate pruning +3. indexing for quick filtering +4. realtime ingestion (ingested data is immediately available for querying) +5. fault-tolerant distributed architecture that doesn’t lose data + +As far as a comparison of systems is concerned, Druid sits in between PowerDrill and Dremel on the spectrum of functionality. It implements almost everything Dremel offers (Dremel handles arbitrary nested data structures while Druid only allows for a single level of array-based nesting) and gets into some of the interesting data layout and compression methods from PowerDrill. + +Druid is a good fit for products that require real-time data ingestion of a single, large data stream. Especially if you are targetting no-downtime operation and are building your product on top of a time-oriented summarization of the incoming data stream. Druid is probably not the right solution if you care more about query flexibility and raw data access than query speed and no-downtime operation. When talking about query speed it is important to clarify what “fast” means, with Druid it is entirely within the realm of possibility (we have done it) to achieve queries that run in single-digit seconds across a 6TB data set. + +### Architecture + +Druid is architected as a grouping of systems each with a distinct role and together they form a working system. The name comes from the Druid class in many role-playing games: it is a shape-shifter, capable of taking many different forms to fulfill various different roles in a group. + +The node types that currently exist are: +\* **Compute** nodes are the workhorses that handle storage and querying on “historical” data (non-realtime) +\* **Realtime** nodes ingest data in real-time, they are in charge of listening to a stream of incoming data and making it available immediately inside the Druid system. As data they have ingested ages, they hand it off to the compute nodes. +\* **Master** nodes act as coordinators. They look over the grouping of computes and make sure that data is available, replicated and in a generally “optimal” configuration. +\* **Broker** nodes understand the topology of data across all of the other nodes in the cluster and re-write and route queries accordingly +\* **Indexer** nodes form a cluster of workers to load batch and real-time data into the system as well as allow for alterations to the data stored in the system (also known as the Indexing Service) + +This separation allows each node to only care about what it is best at. By separating Compute and Realtime, we separate the memory concerns of listening on a real-time stream of data and processing it for entry into the system. By separating the Master and Broker, we separate the needs for querying from the needs for maintaining “good” data distribution across the cluster. + +All nodes can be run in some highly available fashion. Either as symmetric peers in a share-nothing cluster or as hot-swap failover nodes. + +Aside from these nodes, there are 3 external dependencies to the system: + +1. A running [ZooKeeper](http://zookeeper.apache.org/) cluster for cluster service discovery and maintenance of current data topology +2. A MySQL instance for maintenance of metadata about the data segments that should be served by the system +3. A “deep storage” LOB store/file system to hold the stored segments + +### Data Storage + +Getting data into the Druid system requires an indexing process. This gives the system a chance to analyze the data, add indexing structures, compress and adjust the layout in an attempt to optimize query speed. A quick list of what happens to the data follows. + +- Converted to columnar format +- Indexed with bitmap indexes +- Compressed using various algorithms + - LZF (switching to Snappy is on the roadmap, not yet implemented) + - Dictionary encoding w/ id storage minimization + - Bitmap compression + - RLE (on the roadmap, but not yet implemented) + +The output of the indexing process is stored in a “deep storage” LOB store/file system ([Deep Storage](Deep Storage.html) for information about potential options). Data is then loaded by compute nodes by first downloading the data to their local disk and then memory mapping it before serving queries. + +If a compute node dies, it will no longer serve its segments, but given that the segments are still available on the “deep storage” any other node can simply download the segment and start serving it. This means that it is possible to actually remove all compute nodes from the cluster and then re-provision them without any data loss. It also means that if the “deep storage” is not available, the nodes can continue to serve the segments they have already pulled down (i.e. the cluster goes stale, not down). + +In order for a segment to exist inside of the cluster, an entry has to be added to a table in a MySQL instance. This entry is a self-describing bit of metadata about the segment, it includes things like the schema of the segment, the size, and the location on deep storage. These entries are what the Master uses to know what data **should** be available on the cluster. + +### Fault Tolerance + +- **Compute** As discussed above, if a compute node dies, another compute node can take its place and there is no fear of data loss +- **Master** Can be run in a hot fail-over configuration. If no masters are running, then changes to the data topology will stop happening (no new data and no data balancing decisions), but the system will continue to run. +- **Broker** Can be run in parallel or in hot fail-over. +- **Realtime** Depending on the semantics of the delivery stream, multiple of these can be run in parallel processing the exact same stream. They periodically checkpoint to disk and eventually push out to the Computes. Steps are taken to be able to recover from process death, but loss of access to the local disk can result in data loss if this is the only method of adding data to the system. +- **“deep storage” file system** If this is not available, new data will not be able to enter the cluster, but the cluster will continue operating as is. +- **MySQL** If this is not available, the master will be unable to find out about new segments in the system, but it will continue with its current view of the segments that should exist in the cluster. +- **ZooKeeper** If this is not available, data topology changes will not be able to be made, but the Brokers will maintain their most recent view of the data topology and continue serving requests accordingly. + +### Query processing + +A query first enters the Broker, where the broker will match the query with the data segments that are known to exist. It will then pick a set of machines that are serving those segments and rewrite the query for each server to specify the segment(s) targetted. The Compute/Realtime nodes will take in the query, process them and return results. The Broker then takes the results and merges them together to get the final answer, which it returns. In this way, the broker can prune all of the data that doesn’t match a query before ever even looking at a single row of data. + +For filters at a more granular level than what the Broker can prune based on, the indexing structures inside each segment allows the compute nodes to figure out which (if any) rows match the filter set before looking at any row of data. It can do all of the boolean algebra of the filter on the bitmap indices and never actually look directly at a row of data. + +Once it knows the rows that match the current query, it can access the columns it cares about for those rows directly without having to load data that it is just going to throw away. + +The following diagram shows the data flow for queries without showing batch indexing: + +![Simple Data Flow](https://raw.github.com/metamx/druid/master/doc/data_flow_simple.png "Simple Data Flow") + +### In-memory? + +Druid is not always and only in-memory. When we first built it, it is true that it was all in-memory all the time, but as time went on the price-performance tradeoff ended up swinging towards keeping all of our customers data in memory all the time a non-starter. We then added the ability to memory map data and allow the OS to handle paging data in and out of memory on demand. Our production cluster is primarily configured to operate with this memory mapping behavior and we are definitely over-subscribed in terms of memory available vs. data a node is serving. + +As you read some of the old blog posts or other literature about the project, you will see “in-memory” often touted as that is the history of where Druid came from, but the technical reality is that there is a spectrum of price vs. performance and being able to slide along it from all in-memory (high cost, great performance) to mostly on disk (low cost, low performance) is the important knob to be able to adjust. diff --git a/docs/Download.md b/docs/Download.md new file mode 100644 index 000000000000..1bf1352de58c --- /dev/null +++ b/docs/Download.md @@ -0,0 +1,14 @@ +--- +layout: default +--- +A version may be declared as a release candidate if it has been deployed to a sizable production cluster. Release candidates are declared as stable after we feel fairly confident there are no major bugs in the version. Check out the [Versioning](Versioning.html) section for how we describe software versions. + +Release Candidate +----------------- + +There is no release candidate at this time. + +Stable Release +-------------- + +The current stable is tagged at version [0.5.49](https://github.com/metamx/druid/tree/druid-0.5.49). diff --git a/docs/Druid-Personal-Demo-Cluster.md b/docs/Druid-Personal-Demo-Cluster.md new file mode 100644 index 000000000000..0ef9834f198e --- /dev/null +++ b/docs/Druid-Personal-Demo-Cluster.md @@ -0,0 +1,80 @@ +--- +layout: default +--- +# Druid Personal Demo Cluster (DPDC) + +Note, there are currently some issues with the CloudFormation. We are working through them and will update the documentation here when things work properly. In the meantime, the simplest way to get your feet wet with a cluster setup is to run through the instructions at [housejester/druid-test-harness](https://github.com/housejester/druid-test-harness), though it is based on an older version. If you just want to get a feel for the types of data and queries that you can issue, check out [Realtime Examples](Realtime-Examples.html) + +## Introduction +To make it easy for you to get started with Druid, we created an AWS (Amazon Web Services) [CloudFormation](http://aws.amazon.com/cloudformation/) Template that allows you to create a small pre-configured Druid cluster using your own AWS account. The cluster contains a pre-loaded sample workload, the Wikipedia edit stream, and a basic query interface that gets you familiar with Druid capabilities like drill-downs and filters. + + +This guide walks you through the steps to create the cluster and then how to create basic queries. (The cluster setup should take you about 15-20 minutes depending on AWS response times). + + +## What’s in this Druid Demo Cluster? + +1. A single "Master" node. This node co-locates the [Master](Master.html) process, the [Broker](Broker.html) process, Zookeeper, and the MySQL instance. You can read more about Druid architecture [Design](Design.html). + +1. Three compute nodes; these compute nodes, have been pre-configured to work with the Master node and should automatically load up the Wikipedia edit stream data (no specific setup is required). + +## Setup Instructions +1. Log in to your AWS account: Start by logging into the [Console page](https://console.aws.amazon.com) of your AWS account; if you don’t have one, follow this link to sign up for one [http://aws.amazon.com/](http://aws.amazon.com/). + +![AWS Console Page](images/demo/setup-01-console.png) + +1. If you have a [Key Pair](http://docs.aws.amazon.com/gettingstarted/latest/wah/getting-started-create-key-pair.html) already created you may skip this step. Note: this is required to create the demo cluster and is generally not used unless instances need to be accessed directly (e.g. via SSH). + + 1. Click **EC2** to go to the EC2 Dashboard. From there, click **Key Pairs** under Network & Security. + ![EC2 Dashboard](images/demo/setup-02a-keypair.png) + + 1. Click on the button **Create Key Pair**. A dialog box will appear prompting you to enter a Key Pair name (as long as you remember it, the name is arbitrary, for this example we entered `Druid`). Click **Create**. You will be prompted to download a .pam; store this file in a safe place. + ![Create Key Pair](images/demo/setup-02b-keypair.png) + +1. Unless you’re there already, go back to the Console page, or follow this link: https://console.aws.amazon.com. Click **CloudFormation** under Deployment & Management. +![CloudFormation](images/demo/setup-03-ec2.png) + +1. Click **Create New Stack**, which will bring up the **Create Stack** dialog. +![Create New Stack](images/demo/setup-04-newstack.png) + +1. Enter a **Stack Name** (it’s arbitrary, we chose, `DruidStack`). Click **Provide a Template URL** type in the following template URL: _**https://s3.amazonaws.com/cf-templates-jm2ikmzj3y6x-us-east-1/2013081cA9-Druid04012013.template**_. Press **Continue**, this will take you to the Create Stack dialog. +![Stack Name & URL](images/demo/setup-05-createstack.png) + +1. Enter `Druid` (or the Key Pair name you created in Step 2) in the **KeyPairName** field; click **Continue**. This should bring up another dialog prompting you to enter a **Key** and **Value**. +![Stack Parameters](images/demo/setup-06-parameters.png) + +1. While the inputs are arbitrary, it’s important to remember this information; we chose to enter `version` for **Key** and `1` for **Value**. Press **Continue** to bring up a confirmation dialog. +![Add Tags](images/demo/setup-07a-tags.png) + +1. Click **Continue** to start creating your Druid Demo environment (this will bring up another dialog box indicating your environment is being created; click **Close** to take you to a more detailed view of the Stack creation process). Note: depending on AWS, this step could take over 15 minutes – initialization continues even after the instances are created. (So yes, now would be a good time to grab that cup of coffee). +![Review](images/demo/setup-07b-review.png) +![Create Stack Complete](images/demo/setup-07c-complete.png) + +1. Click and expand the **Events** tab in the CloudFormation Stacks window to get a more detailed view of the Druid Demo Cluster setup. + +![CloudFormations](images/demo/setup-09-events.png) + +1. Get the IP address of your Druid Master Node: + 1. Go to the following URL: [https://console.aws.amazon.com/ec2](https://console.aws.amazon.com/ec2) + 1. Click **Instances** in the left pane – you should see something similar to the following figure. + 1. Select the **DruidMaster** instance + 1. Your IP address is right under the heading: **EC2 Instance: DruidMaster**. Select and copy that entire line, which ends with `amazonaws.com`. + +![EC2 Instances](images/demo/setup-10-ip.png) + +## Querying Data + +1. Use the following URL to bring up the Druid Demo Cluster query interface (replace **IPAddressDruidMaster** with the actual druid master IP Address): + +**`http://IPAddressDruidMaster:8082/druid/v3/demoServlet`** + +As you can see from the image below, there are default values in the Dimensions and Granularity fields. Clicking **Execute** will produce a basic query result. +![Demo Query Interface](images/demo/query-1.png) + +1. Note: when the Query is in running the **Execute** button will be disabled and read: **Fetching…** +![Demo Query](images/demo/query-2.png) + +1. You can add multiple Aggregation values, adjust Granularity, and Dimensions; query results will appear at the bottom of the window. + + +Enjoy! And for sure, please send along your comments and feedback or, aspirations on expanding and developing this demo. https://groups.google.com/d/forum/druid-development. Attention R users: we just open-sourced our R Druid connector: https://github.com/metamx/RDruid. diff --git a/docs/Druid-vs-Cassandra.md b/docs/Druid-vs-Cassandra.md new file mode 100644 index 000000000000..e191dde2af7d --- /dev/null +++ b/docs/Druid-vs-Cassandra.md @@ -0,0 +1,8 @@ +--- +layout: default +--- +We are not experts on Cassandra, if anything is incorrect about our portrayal, please let us know on the mailing list or via some other means. We will fix this page. + +Druid is highly optimized for scans and aggregations, it supports arbitrarily deep drill downs into data sets without the need to pre-compute, and it can ingest event streams in real-time and allow users to query events as they come in. Cassandra is a great key-value store and it has some features that allow you to use it to do more interesting things than what you can do with a pure key-value store. But, it is not built for the same use cases that Druid handles, namely regularly scanning over billions of entries per query. + +Furthermore, Druid is fully read-consistent. Druid breaks down a data set into immutable chunks known as segments. All replicants always present the exact same view for the piece of data they are holding and we don’t have to worry about data synchronization. The tradeoff is that Druid has limited semantics for write and update operations. Cassandra, similar to Amazon’s Dynamo, has an eventually consistent data model. Writes are always supported but updates to data may take some time before all replicas sync up (data reconciliation is done at read time). This model favors availability and scalability over consistency. \ No newline at end of file diff --git a/docs/Druid-vs-Hadoop.md b/docs/Druid-vs-Hadoop.md new file mode 100644 index 000000000000..37559b1da8f2 --- /dev/null +++ b/docs/Druid-vs-Hadoop.md @@ -0,0 +1,6 @@ +--- +layout: default +--- +Druid is a complementary addition to Hadoop. Hadoop is great at storing and making accessible large amounts of individually low-value data. Unfortunately, Hadoop is not great at providing query speed guarantees on top of that data, nor does it have very good operational characteristics for a customer-facing production system. Druid, on the other hand, excels at taking high-value summaries of the low-value data on Hadoop, making it available in a fast and always-on fashion, such that it could be exposed directly to a customer. + +Druid also requires some infrastructure to exist for “deep storage”. HDFS is one of the implemented options for this “deep storage”. diff --git a/docs/Druid-vs-Impala-or-Shark.md b/docs/Druid-vs-Impala-or-Shark.md new file mode 100644 index 000000000000..ee59b3def0c6 --- /dev/null +++ b/docs/Druid-vs-Impala-or-Shark.md @@ -0,0 +1,45 @@ +--- +layout: default +--- +The question of Druid versus Impala or Shark basically comes down to your product requirements and what the systems were designed to do. + +Druid was designed to + +1. be an always on service +1. ingest data in real-time +1. handle slice-n-dice style ad-hoc queries + +Impala and Shark's primary design concerns (as far as I am aware) were to replace Hadoop MapReduce with another, faster, query layer that is completely generic and plays well with the other ecosystem of Hadoop technologies. I will caveat this discussion with the statement that I am not an expert on Impala or Shark, nor am I intimately familiar with their roadmaps. If anything is incorrect on this page, I'd be happy to change it, please send a note to the mailing list. + +What does this mean? We can talk about it in terms of four general areas + +1. Fault Tolerance +1. Query Speed +1. Data Ingestion +1. Query Flexibility + +## Fault Tolerance + +Druid pulls segments down from [Deep Storage](Deep Storage.html) before serving queries on top of it. This means that for the data to exist in the Druid cluster, it must exist as a local copy on a historical node. If deep storage becomes unavailable for any reason, new segments will not be loaded into the system, but the cluster will continue to operate exactly as it was when the backing store disappeared. + +Impala and Shark, on the other hand, pull their data in from HDFS (or some other Hadoop FileSystem) in response to a query. This has implications for the operation of queries if you need to take HDFS down for a bit (say a software upgrade). It's possible that data that has been cached in the nodes is still available when the backing file system goes down, but I'm not sure. + +This is just one example, but Druid was built to continue operating in the face of failures of any one of its various pieces. The [Design](Design.html) describes these design decisions from the Druid side in more detail. + +## Query Speed + +Druid takes control of the data given to it, storing it in a column-oriented fashion, compressing it and adding indexing structures. All of which add to the speed at which queries can be processed. The column orientation means that we only look at the data that a query asks for in order to compute the answer. Compression increases the data storage capacity of RAM and allows us to fit more data into quickly accessible RAM. Indexing structures mean that as you add boolean filters to your queries, we do less processing and you get your result faster, whereas a lot of processing engines do *more* processing when filters are added. + +Impala/Shark can basically be thought of as daemon caching layers on top of HDFS. They are processes that stay on even if there is no query running (eliminating the JVM startup costs from Hadoop MapReduce) and they have facilities to cache data locally so that it can be accessed and updated quicker. But, I do not believe they go beyond caching capabilities to actually speed up queries. So, at the end of the day, they don't change the paradigm from a brute-force, scan-everything query processing paradigm. + +## Data Ingestion + +Druid is built to allow for real-time ingestion of data. You can ingest data and query it immediately upon ingestion, the latency between how quickly the event is reflected in the data is dominated by how long it takes to deliver the event to Druid. + +Impala/Shark, being based on data in HDFS or some other backing store, are limited in their data ingestion rates by the rate at which that backing store can make data available. Generally, the backing store is the biggest bottleneck for how quickly data can become available. + +## Query Flexibility + +Druid supports timeseries and groupBy style queries. It doesn't have support for joins, which makes it a lot less flexible for generic processing. + +Impala/Shark support SQL style queries with full joins. \ No newline at end of file diff --git a/docs/Druid-vs-redshift.md b/docs/Druid-vs-redshift.md new file mode 100644 index 000000000000..8469209b10b9 --- /dev/null +++ b/docs/Druid-vs-redshift.md @@ -0,0 +1,40 @@ +--- +layout: default +--- +###How does Druid compare to Redshift? + +In terms of drawing a differentiation, Redshift is essentially ParAccel (Actian) which Amazon is licensing. + +Aside from potential performance differences, there are some functional differences: + +###Real-time data ingestion + +Because Druid is optimized to provide insight against massive quantities of streaming data; it is able to load and aggregate data in real-time. + +Generally traditional data warehouses including column stores work only with batch ingestion and are not optimal for streaming data in regularly. + +###Druid is a read oriented analytical data store + +It’s write semantics aren’t as fluid and does not support joins. ParAccel is a full database with SQL support including joins and insert/update statements. + +###Data distribution model + +Druid’s data distribution, is segment based which exists on highly available “deep” storage, like S3 or HDFS. Scaling up (or down) does not require massive copy actions or downtime; in fact, losing any number of compute nodes does not result in data loss because new compute nodes can always be brought up by reading data from “deep” storage. + +To contrast, ParAccel’s data distribution model is hash-based. Expanding the cluster requires re-hashing the data across the nodes, making it difficult to perform without taking downtime. Amazon’s Redshift works around this issue with a multi-step process: + +* set cluster into read-only mode +* copy data from cluster to new cluster that exists in parallel +* redirect traffic to new cluster + +###Replication strategy + +Druid employs segment-level data distribution meaning that more nodes can be added and rebalanced without having to perform a staged swap. The replication strategy also makes all replicas available for querying. + +ParAccel’s hash-based distribution generally means that replication is conducted via hot spares. This puts a numerical limit on the number of nodes you can lose without losing data, and this replication strategy often does not allow the hot spare to help share query load. + +###Indexing strategy + +Along with column oriented structures, Druid uses indexing structures to speed up query execution when a filter is provided. Indexing structures do increase storage overhead (and make it more difficult to allow for mutation), but they can also significantly speed up queries. + +ParAccel does not appear to employ indexing strategies. \ No newline at end of file diff --git a/docs/Druid-vs-vertica.md b/docs/Druid-vs-vertica.md new file mode 100644 index 000000000000..535e5e06300f --- /dev/null +++ b/docs/Druid-vs-vertica.md @@ -0,0 +1,10 @@ +--- +layout: default +--- +How does Druid compare to Vertica? + +Vertica is similar to ParAccel/Redshift ([Druid-vs-Redshift](Druid-vs-Redshift.html)) described above in that it wasn’t built for real-time streaming data ingestion and it supports full SQL. + +The other big difference is that instead of employing indexing, Vertica tries to optimize processing by leveraging run-length encoding (RLE) and other compression techniques along with a “projection” system that creates materialized copies of the data in a different sort order (to maximize the effectiveness of RLE). + +We are unclear about how Vertica handles data distribution and replication, so we cannot speak to if/how Druid is different. diff --git a/docs/Examples.md b/docs/Examples.md new file mode 100644 index 000000000000..4207911464be --- /dev/null +++ b/docs/Examples.md @@ -0,0 +1,68 @@ +--- +layout: default +--- +Examples +======== + +The examples on this page are setup in order to give you a feel for what Druid does in practice. They are quick demos of Druid based on [RealtimeStandaloneMain](https://github.com/metamx/druid/blob/master/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java). While you wouldn’t run it this way in production you should be able to see how ingestion works and the kind of exploratory queries that are possible. Everything that can be done on your box here can be scaled out to 10’s of billions of events and terabytes of data per day in a production cluster while still giving the snappy responsive exploratory queries. + +Installing Standalone Druid +--------------------------- + +There are two options for installing standalone Druid. Building from source, and downloading the Druid Standalone Kit (DSK). + +### Building from source + +Clone Druid and build it: + + git clone https://github.com/metamx/druid.git druid + cd druid + git fetch --tags + git checkout druid-0.4.30 + ./build.sh + + +### Downloading the DSK (Druid Standalone Kit) + +[Download](http://static.druid.io/data/examples/druid-services-0.4.6.tar.gz) a stand-alone tarball and run it: + + + tar -xzf druid-services-0.X.X-SNAPSHOT-bin.tar.gz + cd druid-services-0.X.X-SNAPSHOT + + +Twitter Example +--------------- + +For a full tutorial based on the twitter example, check out this [Twitter Tutorial](Twitter-Tutorial.html). + +This Example uses a feature of Twitter that allows for sampling of it’s stream. We sample the Twitter stream via our [TwitterSpritzerFirehoseFactory](https://github.com/metamx/druid/blob/master/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java) class and use it to simulate the kinds of data you might ingest into Druid. Then, with the client part, the sample shows what kinds of analytics explorations you can do during and after the data is loaded. + +### What you’ll learn +\* See how large amounts of data gets ingested into Druid in real-time +\* Learn how to do fast, interactive, analytics queries on that real-time data + +### What you need +\* A build of standalone Druid with the Twitter example (see above) +\* A Twitter username and password. + +### What you’ll do + +See [Tutorial](Tutorial.html) + +Rand Example +------------ + +This uses `RandomFirehoseFactory` which emits a stream of random numbers (outColumn, a positive double) with timestamps along with an associated token (target). This provides a timeseries that requires no network access for demonstration, characterization, and testing. The generated tuples can be thought of as asynchronously produced triples (timestamp, outColumn, target) where the timestamp varies depending on speed of processing. + +In a terminal window, (NOTE: If you are using the cloned Github repository these scripts are in ./examples/bin) start the server with: + +`./run_example_server.sh` +`# type rand when prompted` + +In another terminal window: + +`./run_example_client.sh` +`# type rand when prompted` + +The result of the client query is in JSON format. The client makes a REST request using the program `curl` which is usually installed on Linux, Unix, and OSX by default. diff --git a/docs/Filters.md b/docs/Filters.md new file mode 100644 index 000000000000..41ae91f93e2f --- /dev/null +++ b/docs/Filters.md @@ -0,0 +1,91 @@ +--- +layout: default +--- +A filter is a JSON object indicating which rows of data should be included in the computation for a query. It’s essentially the equivalent of the WHERE clause in SQL. Druid supports the following types of filters. + +### Selector filter + +The simplest filter is a selector filter. The selector filter will match a specific dimension with a specific value. Selector filters can be used as the base filters for more complex Boolean expressions of filters. + +The grammar for a SELECTOR filter is as follows: + + "filter": { + "type": "selector", + "dimension": , + "value": + } + + +This is the equivalent of `WHERE = ''`. + +### Regular expression filter + +The regular expression filter is similar to the selector filter, but using regular expressions. It matches the specified dimension with the given pattern. The pattern can be any standard [Java regular expression](http://docs.oracle.com/javase/6/docs/api/java/util/regex/Pattern.html). + + "filter": { + "type": "regex", + "dimension": , + "pattern": + } + + +### Logical expression filters + +#### AND + +The grammar for an AND filter is as follows: + + "filter": { + "type": "and", + "fields": [, , ...] + } + + +The filters in fields can be any other filter defined on this page. + +#### OR + +The grammar for an OR filter is as follows: + + "filter": { + "type": "or", + "fields": [, , ...] + } + + +The filters in fields can be any other filter defined on this page. + +#### NOT + +The grammar for a NOT filter is as follows: + + "filter": { + "type": "not", + "field": + } + + +The filter specified at field can be any other filter defined on this page. + +### JavaScript filter + +The JavaScript filter matches a dimension against the specified JavaScript function predicate. The filter matches values for which the function returns true. + +The function takes a single argument, the dimension value, and returns either true or false. + + { + "type" : "javascript", + "dimension" : , + "function" : "function(value) { <...> }" + } + + +**Example** +The following matches any dimension values for the dimension `name` between `'bar'` and `'foo'` + + { + "type" : "javascript", + "dimension" : "name", + "function" : "function(x) { return(x >= 'bar' && x <= 'foo') }" + } + diff --git a/docs/Firehose.md b/docs/Firehose.md new file mode 100644 index 000000000000..92c5caa23866 --- /dev/null +++ b/docs/Firehose.md @@ -0,0 +1,52 @@ +--- +layout: default +--- +Firehoses describe the data stream source. They are pluggable and thus the configuration schema can and will vary based on the `type` of the firehose. + +|Field|Type|Description|Required| +|-----|----|-----------|--------| +|type|String|Specifies the type of firehose. Each value will have its own configuration schema, firehoses packaged with Druid are described [here](https://github.com/metamx/druid/wiki/Firehose#available-firehoses)|yes| + +We describe the configuration of the Kafka firehose from the example below, but check [here](https://github.com/metamx/druid/wiki/Firehose#available-firehoses) for more information about the various firehoses that are available in Druid. + +- `consumerProps` is a map of properties for the Kafka consumer. The JSON object is converted into a Properties object and passed along to the Kafka consumer. +- `feed` is the feed that the Kafka consumer should read from. +- `parser` represents a parser that knows how to convert from String representations into the required `InputRow` representation that Druid uses. This is a potentially reusable piece that can be found in many of the firehoses that are based on text streams. The spec in the example describes a JSON feed (new-line delimited objects), with a timestamp column called “timestamp” in ISO8601 format and that it should not include the dimension “value” when processing. More information about the options available for the parser are available [here](https://github.com/metamx/druid/wiki/Firehose#parsing-data). + +Available Firehoses +------------------- + +There are several firehoses readily available in Druid, some are meant for examples, others can be used directly in a production environment. + +#### KafkaFirehose + +This firehose acts as a Kafka consumer and ingests data from Kafka. + +#### StaticS3Firehose + +This firehose ingests events from a predefined list of S3 objects. + +#### TwitterSpritzerFirehose + +See [Examples](Examples.html). This firehose connects directly to the twitter spritzer data stream. + +#### RandomFirehose + +See [Examples](Examples.html). This firehose creates a stream of random numbers. + +#### RabbitMqFirehouse + +This firehose ingests events from a define rabbit-mq queue. + +Parsing Data +------------ + +There are several ways to parse data. + +#### StringInputRowParser + +This parser converts Strings. + +#### MapInputRowParser + +This parser converts flat, key/value pair maps. diff --git a/docs/Granularities.md b/docs/Granularities.md new file mode 100644 index 000000000000..0cb25a7a5dfe --- /dev/null +++ b/docs/Granularities.md @@ -0,0 +1,49 @@ +--- +layout: default +--- +The granularity field determines how data gets bucketed across the time dimension, i.e how it gets aggregated by hour, day, minute, etc. + +It can be specified either as a string for simple granularities or as an object for arbitrary granularities. + +### Simple Granularities + +Simple granularities are specified as a string and bucket timestamps by their UTC time (i.e. days start at 00:00 UTC). + +Supported granularity strings are: `all`, `none`, `minute`, `fifteen_minute`, `thirty_minute`, `hour` and `day` +\* **`all`** buckets everything into a single bucket +\* **`none`** does not bucket data (it actually uses the granularity of the index - minimum here is `none` which means millisecond granularity). Using `none` in a [timeseries query|TimeSeriesQuery](timeseries query|TimeSeriesQuery.html) is currently not recommended (the system will try to generate 0 values for all milliseconds that didn’t exist, which is often a lot). + +### Duration Granularities + +Duration granularities are specified as an exact duration in milliseconds and timestamps are returned as UTC. + +They also support specifying an optional origin, which defines where to start counting time buckets from (defaults to 1970-01-01T00:00:00Z). + + {"type": "duration", "duration": "7200000"} + +This chunks up every 2 hours. + + {"type": "duration", "duration": "3600000", "origin": "2012-01-01T00:30:00Z"} + +This chunks up every hour on the half-hour. + +### Period Granularities + +Period granularities are specified as arbitrary period combinations of years, months, weeks, hours, minutes and seconds (e.g. P2W, P3M, PT1H30M, PT0.750S) in ISO8601 format. + +They support specifying a time zone which determines where period boundaries start and also determines the timezone of the returned timestamps. + +By default years start on the first of January, months start on the first of the month and weeks start on Mondays unless an origin is specified. + +Time zone is optional (defaults to UTC) +Origin is optional (defaults to 1970-01-01T00:00:00 in the given time zone) + + {"type": "period", "period": "P2D", "timeZone": "America/Los_Angeles"} + +This will bucket by two day chunks in the Pacific timezone. + + {"type": "period", "period": "P3M", "timeZone": "America/Los_Angeles", "origin": "2012-02-01T00:00:00-08:00"} + +This will bucket by 3 month chunks in the Pacific timezone where the three-month quarters are defined as starting from February. + +Supported time zones: timezone support is provided by the [Joda Time library](http://www.joda.org), which uses the standard IANA time zones. [Joda Time supported timezones](http://joda-time.sourceforge.net/timezones.html) diff --git a/docs/GroupByQuery.md b/docs/GroupByQuery.md new file mode 100644 index 000000000000..01edc6bdc7e6 --- /dev/null +++ b/docs/GroupByQuery.md @@ -0,0 +1,134 @@ +--- +layout: default +--- +These types of queries take a groupBy query object and return an array of JSON objects where each object represents a grouping asked for by the query. + +An example groupBy query object is shown below: + +

+
+{
+ [queryType]() “groupBy”,
+ [dataSource]() “sample\_datasource”,
+ [granularity]() “day”,
+ [dimensions]() [“dim1”, “dim2”],
+ [limitSpec]() {
+ [type]() “default”,
+ [limit]() 5000,
+ [columns]() [“dim1”, “metric1”]
+ },
+ [filter]() {
+ [type]() “and”,
+ [fields]() [
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension1”,
+ [value]() “sample\_value1”
+ },
+ {
+ [type]() “or”,
+ [fields]() [
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension2”,
+ [value]() “sample\_value2”
+ },
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension3”,
+ [value]() “sample\_value3”
+ }
+ ]
+ }
+ ]
+ },
+ [aggregations]() [
+ {
+ [type]() “longSum”,
+ [name]() “sample\_name1”,
+ [fieldName]() “sample\_fieldName1”
+ },
+ {
+ [type]() “doubleSum”,
+ [name]() “sample\_name2”,
+ [fieldName]() “sample\_fieldName2”
+ }
+ ],
+ [postAggregations]() [
+ {
+ [type]() “arithmetic”,
+ [name]() “sample\_divide”,
+ [fn]() “/”,
+ [fields]() [
+ {
+ [type]() “fieldAccess”,
+ [name]() “sample\_name1”,
+ [fieldName]() “sample\_fieldName1”
+ },
+ {
+ [type]() “fieldAccess”,
+ [name]() “sample\_name2”,
+ [fieldName]() “sample\_fieldName2”
+ }
+ ]
+ }
+ ],
+ [intervals]() [
+ “2012-01-01T00:00:00.000/2012-01-03T00:00:00.000”
+ ],
+ [having]() {
+ [type]() “greaterThan”,
+ [aggregation]() “sample\_name1”,
+ [value]() 0
+ }
+}
+
+
+ + +There are 9 main parts to a groupBy query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “groupBy”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|dimensions|A JSON list of dimensions to do the groupBy over|yes| +|orderBy|See [OrderBy](OrderBy.html).|no| +|having|See [Having](Having.html).|no| +|granularity|Defines the granularity of the query. See [Granularities](Granularities.html)|yes| +|filter|See [Filters](Filters.html)|no| +|aggregations|See [Aggregations](Aggregations.html)|yes| +|postAggregations|See [Post Aggregations](Post-Aggregations.html)|no| +|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +To pull it all together, the above query would return *n\*m* data points, up to a maximum of 5000 points, where n is the cardinality of the “dim1” dimension, m is the cardinality of the “dim2” dimension, each day between 2012-01-01 and 2012-01-03, from the “sample\_datasource” table. Each data point contains the (long) sum of sample\_fieldName1 if the value of the data point is greater than 0, the (double) sum of sample\_fieldName2 and the (double) the result of sample\_fieldName1 divided by sample\_fieldName2 for the filter set for a particular grouping of “dim1” and “dim2”. The output looks like this: + +
+
+[ {
+ “version” : “v1”,
+ “timestamp” : “2012-01-01T00:00:00.000Z”,
+ “event” : {
+ “dim1” : ,
+ “dim2” : ,
+ “sample\_name1” : ,
+ “sample\_name2” :,
+ “sample\_divide” : 
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-01-01T00:00:00.000Z”,
+ “event” : {
+ “dim1” : ,
+ “dim2” : ,
+ “sample\_name1” : ,
+ “sample\_name2” :,
+ “sample\_divide” : 
+ }
+},
+…
+]
+
+
+ diff --git a/docs/Having.md b/docs/Having.md new file mode 100644 index 000000000000..62ab4644451d --- /dev/null +++ b/docs/Having.md @@ -0,0 +1,93 @@ +--- +layout: default +--- +A having clause is a JSON object identifying which rows from a groupBy query should be returned, by specifying conditions on aggregated values. + +It is essentially the equivalent of the HAVING clause in SQL. + +Druid supports the following types of having clauses. + +### Numeric filters + +The simplest having clause is a numeric filter. +Numeric filters can be used as the base filters for more complex boolean expressions of filters. + +#### Equal To + +The equalTo filter will match rows with a specific aggregate value. +The grammar for an `equalTo` filter is as follows: + + "having": { + "type": "equalTo", + "aggregation": , + "value": + } + + +This is the equivalent of `HAVING = `. + +#### Greater Than + +The greaterThan filter will match rows with aggregate values greater than the given value. +The grammar for a `greaterThan` filter is as follows: + + "having": { + "type": "greaterThan", + "aggregation": , + "value": + } + + +This is the equivalent of `HAVING > `. + +#### Less Than + +The lessThan filter will match rows with aggregate values less than the specified value. +The grammar for a `greaterThan` filter is as follows: + + "having": { + "type": "lessThan", + "aggregation": , + "value": + } + + +This is the equivalent of `HAVING < `. + +### Logical expression filters + +#### AND + +The grammar for an AND filter is as follows: + + "having": { + "type": "and", + "havingSpecs": [, , ...] + } + + +The having clauses in `havingSpecs` can be any other having clause defined on this page. + +#### OR + +The grammar for an OR filter is as follows: + + "having": { + "type": "or", + "havingSpecs": [, , ...] + } + + +The having clauses in `havingSpecs` can be any other having clause defined on this page. + +#### NOT + +The grammar for a NOT filter is as follows: + + "having": { + "type": "not", + "havingSpec": + } + + +The having clause specified at `havingSpec` can be any other having clause defined on this page. diff --git a/docs/Home.md b/docs/Home.md new file mode 100644 index 000000000000..8587aae4749f --- /dev/null +++ b/docs/Home.md @@ -0,0 +1,53 @@ +--- +layout: default +--- +Druid is an open-source analytics datastore designed for realtime, exploratory, queries on large-scale data sets (100’s of Billions entries, 100’s TB data). Druid provides for cost effective, always-on, realtime data ingestion and arbitrary data exploration. + +- Check out some [Examples](Examples.html) +- Try out Druid with our Getting Started [Tutorial](https://github.com/metamx/druid/wiki/Tutorial%3A-A-First-Look-at-Druid) +- Learn more by reading the [White Paper](http://static.druid.io/docs/druid.pdf) + +Why Druid? +---------- + +Druid was originally created to resolve query latency issues seen with trying to use Hadoop to power an interactive service. Hadoop has shown the world that it’s possible to house your data warehouse on commodity hardware for a fraction of the price of typical solutions. As people adopt Hadoop for their data warehousing needs, they find two things. + +1. They can now query all of their data in a fairly flexible manner and answer any question they have +2. The queries take a long time + +The first one is the joy that everyone feels the first time they get Hadoop running. The latter is what they realize after they have used Hadoop interactively for a while because Hadoop is optimized for throughput, not latency. Druid is a system that you can set up in your organization next to Hadoop. It provides the ability to access your data in an interactive slice-and-dice fashion. It trades off some query flexibility and takes over the storage format in order to provide the speed. + +Druid is especially useful if you are summarizing your data sets and then querying the summarizations. If you put your summarizations into Druid, you will get quick queryability out of a system that you can be confident will scale up as your data volumes increase. Deployments have scaled up to 2TB of data per hour at peak ingested and aggregated in real-time. + +We have more details about the general design of the system and why you might want to use it in our [White Paper](http://static.druid.io/docs/druid.pdf) or in our [Design](Design.html) doc. + +The data store world is vast, confusing and constantly in flux. This page is meant to help potential evaluators decide whether Druid is a good fit for the problem one needs to solve. If anything about it is incorrect please provide that feedback on the mailing list or via some other means, we will fix this page. + +#### When Druid? +\* You need to do interactive, fast, exploration of large amounts of data +\* You need analytics (not key value store) +\* You have a lot of data (10s of Billions of events added per day, 10s of TB of data added per day) +\* You want to do your analysis on data as it’s happening (realtime) +\* Your store needs to be always-on, 24x7x365 and years into the future. + +#### Not Druid? +\* The amount of data you have can easily be handled by MySql +\* Your querying for individual entries or doing lookups (Not Analytics) +\* Batch is good enough +\* Canned queries is good enough +\* Downtime is no big deal + +#### Druid vs… +\* [Druid-vs-Impala-or-Shark](Druid-vs-Impala-or-Shark.html) +\* [Druid-vs-Redshift](Druid-vs-Redshift.html) +\* [Druid-vs-Vertica](Druid-vs-Vertica.html) +\* [Druid-vs-Cassandra](Druid-vs-Cassandra.html) +\* [Druid-vs-Hadoop](Druid-vs-Hadoop.html) + +Key Features +------------ + +- **Designed for Analytics** - Druid is built for exploratory analytics for OLAP workflows (streamalytics). It supports a variety of filters, aggregators and query types and provides a framework for plugging in new functionality. Users have leveraged Druid’s infrastructure to develop features such as top K queries and histograms. +- **Interactive Queries** - Druid’s low latency data ingestion architecture allows events to be queried milliseconds after they are created. Druid’s query latency is optimized by only reading and scanning exactly what is needed. Aggregate and filter on data without sitting around waiting for results. +- **Highly Available** - Druid is used to back SaaS implementations that need to be up all the time. Your data is still available and queryable during system updates. Scale up or down without data loss. +- **Scalable** - Existing Druid deployments handle billions of events and terabytes of data per day. Druid is designed to be petabyte scale. diff --git a/docs/Indexing-Service.md b/docs/Indexing-Service.md new file mode 100644 index 000000000000..d878e1b4176b --- /dev/null +++ b/docs/Indexing-Service.md @@ -0,0 +1,193 @@ +--- +layout: default +--- +Disclaimer: We are still in the process of finalizing the indexing service and these configs are prone to change at any time. We will announce when we feel the indexing service and the configurations described are stable. + +The indexing service is a distributed task/job queue. It accepts requests in the form of [Tasks](Tasks.html) and executes those tasks across a set of worker nodes. Worker capacity can be automatically adjusted based on the number of tasks pending in the system. The indexing service is highly available, has built in retry logic, and can backup per task logs in deep storage. + +The indexing service is composed of two main components, a coordinator node that manages task distribution and worker capacity, and worker nodes that execute tasks in separate JVMs. + +Preamble +-------- + +The truth is, the indexing service is an experience that is difficult to characterize with words. When they asked me to write this preamble, I was taken aback. I wasn’t quite sure what exactly to write or how to describe this… entity. I accepted the job, as much for the challenge and inner growth as the money, and took to the mountains for reflection. Six months later, I knew I had it, I was done and had achieved the next euphoric victory in the continuous struggle that plagues my life. But, enough about me. This is about the indexing service. + +The indexing service is philosophical transcendence, an infallible truth that will shape your soul, mold your character, and define your reality. The indexing service is creating world peace, playing with puppies, unwrapping presents on Christmas morning, cradling a loved one, and beating Goro in Mortal Kombat for the first time. The indexing service is sustainable economic growth, global propensity, and a world of transparent financial transactions. The indexing service is a true belieber. The indexing service is panicking because you forgot you signed up for a course and the big exam is in a few minutes, only to wake up and realize it was all a dream. What is the indexing service? More like what isn’t the indexing service. The indexing service is here and it is ready, but are you? + +Indexer Coordinator Node +------------------------ + +The indexer coordinator node exposes HTTP endpoints where tasks can be submitted by posting a JSON blob to specific endpoints. It can be started by launching IndexerCoordinatorMain.java. The indexer coordinator node can operate in local mode or remote mode. In local mode, the coordinator and worker run on the same host and port. In remote mode, worker processes run on separate hosts and ports. + +Tasks can be submitted via POST requests to: + + http://:/druid/indexer/v1/task + +Tasks can cancelled via POST requests to: + + http://:/druid/indexer/v1/task/{taskId}/shutdown + +Issuing the cancel request once sends a graceful shutdown request. Graceful shutdowns may not stop a task right away, but instead issue a safe stop command at a point deemed least impactful to the system. Issuing the cancel request twice in succession will kill –9 the task. + +Task statuses can be retrieved via GET requests to: + + http://:/druid/indexer/v1/task/{taskId}/status + +Task segments can be retrieved via GET requests to: + + http://:/druid/indexer/v1/task/{taskId}/segments + +When a task is submitted, the coordinator creates a lock over the data source and interval of the task. The coordinator also stores the task in a MySQL database table. The database table is read at startup time to bootstrap any tasks that may have been submitted to the coordinator but may not yet have been executed. + +The coordinator also exposes a simple UI to show what tasks are currently running on what nodes at + + http://:/static/console.html + +#### Task Execution + +The coordinator retrieves worker setup metadata from the Druid [MySQL](MySQL.html) config table. This metadata contains information about the version of workers to create, the maximum and minimum number of workers in the cluster at one time, and additional information required to automatically create workers. + +Tasks are assigned to workers by creating entries under specific /tasks paths associated with a worker, similar to how the Druid master node assigns segments to compute nodes. See [Worker Configuration](Indexing-Service#configuration-1). Once a worker picks up a task, it deletes the task entry and announces a task status under a /status path associated with the worker. Tasks are submitted to a worker until the worker hits capacity. If all workers in a cluster are at capacity, the indexer coordinator node automatically creates new worker resources. + +#### Autoscaling + +The Autoscaling mechanisms currently in place are tightly coupled with our deployment infrastructure but the framework should be in place for other implementations. We are highly open to new implementations or extensions of the existing mechanisms. In our own deployments, worker nodes are Amazon AWS EC2 nodes and they are provisioned to register themselves in a [galaxy](https://github.com/ning/galaxy) environment. + +The Coordinator node controls the number of workers in the cluster according to a worker setup spec that is submitted via a POST request to the indexer at: + + http://:/druid/indexer/v1/worker/setup + +A sample worker setup spec is shown below: + + { + "minVersion":"some_version", + "minNumWorkers":"0", + "maxNumWorkers":"10", + "nodeData": { + "type":"ec2", + "amiId":"ami-someId", + "instanceType":"m1.xlarge", + "minInstances":"1", + "maxInstances":"1", + "securityGroupIds":["securityGroupIds"], + "keyName":"keyName" + }, + "userData":{ + "classType":"galaxy", + "env":"druid", + "version":"druid_version", + "type":"sample_cluster/worker" + } + } + + +Issuing a GET request at the same URL will return the current worker setup spec that is currently in place. The worker setup spec list above is just a sample and it is possible to write worker setup specs for other deployment environments. A description of the worker setup spec is shown below. + +|Property|Description|Default| +|--------|-----------|-------| +|`minVersion`|The coordinator only assigns tasks to workers with a version greater than the minVersion. If this is not specified, the minVersion will be the same as the coordinator version.|none| +|`minNumWorkers`|The minimum number of workers that can be in the cluster at any given time.|0| +|`maxNumWorkers`|The maximum number of workers that can be in the cluster at any given time.|0| +|`nodeData`|A JSON object that contains metadata about new nodes to create.|none| +|`userData`|A JSON object that contains metadata about how the node should register itself on startup. This data is sent with node creation requests.|none| + +For more information about configuring Auto-scaling, see [Auto-Scaling Configuration](https://github.com/metamx/druid/wiki/Indexing-Service#auto-scaling-configuration). + +#### Running + +Indexer Coordinator nodes can be run using the `com.metamx.druid.indexing.coordinator.http.IndexerCoordinatorMain` class. + +#### Configuration + +Indexer Coordinator nodes require [basic service configuration](https://github.com/metamx/druid/wiki/Configuration#basic-service-configuration). In addition, there are several extra configurations that are required. + + -Ddruid.zk.paths.indexer.announcementsPath=/druid/indexer/announcements + -Ddruid.zk.paths.indexer.leaderLatchPath=/druid/indexer/leaderLatchPath + -Ddruid.zk.paths.indexer.statusPath=/druid/indexer/status + -Ddruid.zk.paths.indexer.tasksPath=/druid/demo/indexer/tasks + + -Ddruid.indexer.runner=remote + -Ddruid.indexer.taskDir=/mnt/persistent/task/ + -Ddruid.indexer.configTable=sample_config + -Ddruid.indexer.workerSetupConfigName=worker_setup + -Ddruid.indexer.strategy=ec2 + -Ddruid.indexer.hadoopWorkingPath=/tmp/druid-indexing + -Ddruid.indexer.logs.s3bucket=some_bucket + -Ddruid.indexer.logs.s3prefix=some_prefix + +The indexing service requires some additional Zookeeper configs. + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.zk.paths.indexer.announcementsPath`|The base path where workers announce themselves.|none| +|`druid.zk.paths.indexer.leaderLatchPath`|The base that coordinator nodes use to determine a leader.|none| +|`druid.zk.paths.indexer.statusPath`|The base path where workers announce task statuses.|none| +|`druid.zk.paths.indexer.tasksPath`|The base path where the coordinator assigns new tasks.|none| + +There’s several additional configs that are required to run tasks. + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.indexer.runner`|Indicates whether tasks should be run locally or in a distributed environment. “local” or “remote”.|local| +|`druid.indexer.taskDir`|Intermediate temporary directory that tasks may use.|none| +|`druid.indexer.configTable`|The MySQL config table where misc configs live.|none| +|`druid.indexer.strategy`|The autoscaling strategy to use.|noop| +|`druid.indexer.hadoopWorkingPath`|Intermediate temporary hadoop working directory that certain index tasks may use.|none| +|`druid.indexer.logs.s3bucket`|S3 bucket to store logs.|none| +|`druid.indexer.logs.s3prefix`|S3 key prefix to store logs.|none| + +#### Console + +The indexer console can be used to view pending tasks, running tasks, available workers, and recent worker creation and termination. The console can be accessed at: + + http://:8080/static/console.html + +Worker Node +----------- + +The worker node executes submitted tasks. Workers run tasks in separate JVMs. + +#### Running + +Worker nodes can be run using the `com.metamx.druid.indexing.worker.http.WorkerMain` class. Worker nodes can automatically be created by the Indexer Coordinator as part of autoscaling. + +#### Configuration + +Worker nodes require [basic service configuration](https://github.com/metamx/druid/wiki/Configuration#basic-service-configuration). In addition, there are several extra configurations that are required. + + -Ddruid.worker.version=0 + -Ddruid.worker.capacity=3 + + -Ddruid.indexer.threads=3 + -Ddruid.indexer.taskDir=/mnt/persistent/task/ + -Ddruid.indexer.hadoopWorkingPath=/tmp/druid-indexing + + -Ddruid.worker.masterService=druid:sample_cluster:indexer + + -Ddruid.indexer.fork.hostpattern=:%d + -Ddruid.indexer.fork.startport=8080 + -Ddruid.indexer.fork.main=com.metamx.druid.indexing.worker.executor.ExecutorMain + -Ddruid.indexer.fork.opts="-server -Xmx1g -Xms1g -XX:NewSize=256m -XX:MaxNewSize=256m" + -Ddruid.indexer.fork.property.druid.service=druid/sample_cluster/executor + + # These configs are the same configs you would set for basic service configuration, just with a different prefix + -Ddruid.indexer.fork.property.druid.monitoring.monitorSystem=false + -Ddruid.indexer.fork.property.druid.computation.buffer.size=268435456 + -Ddruid.indexer.fork.property.druid.indexer.taskDir=/mnt/persistent/task/ + -Ddruid.indexer.fork.property.druid.processing.formatString=processing-%s + -Ddruid.indexer.fork.property.druid.processing.numThreads=1 + -Ddruid.indexer.fork.property.druid.server.maxSize=0 + -Ddruid.indexer.fork.property.druid.request.logging.dir=request_logs/ + +Many of the configurations for workers are similar to those for basic service configuration":https://github.com/metamx/druid/wiki/Configuration\#basic-service-configuration, but with a different config prefix. Below we describe the unique worker configs. + +|Property|Description|Default| +|--------|-----------|-------| +|`druid.worker.version`|Version identifier for the worker.|0| +|`druid.worker.capacity`|Maximum number of tasks the worker can accept.|1| +|`druid.indexer.threads`|Number of processing threads per worker.|1| +|`druid.worker.masterService`|Name of the indexer coordinator used for service discovery.|none| +|`druid.indexer.fork.hostpattern`|The format of the host name.|none| +|`druid.indexer.fork.startport`|Port in which child JVM starts from.|none| +|`druid.indexer.fork.opts`|JVM options for child JVMs.|none| + diff --git a/docs/Libraries.md b/docs/Libraries.md new file mode 100644 index 000000000000..0c57ffab3e86 --- /dev/null +++ b/docs/Libraries.md @@ -0,0 +1,23 @@ +--- +layout: default +--- +### R + +- [RDruid](https://github.com/metamx/RDruid) - Druid connector for R + +Community Libraries +------------------- + +Some great folks have written their own libraries to interact with Druid + +#### Ruby +\* [madvertise/ruby-druid](https://github.com/madvertise/ruby-druid) - A ruby client for Druid + +#### Python +\* [metamx/pydruid](https://github.com/metamx/pydruid) - A python client for Druid + +#### Helper Libraries + +- [madvertise/druid-dumbo](https://github.com/madvertise/druid-dumbo) - Scripts to help generate batch configs for the ingestion of data into Druid + +- [housejester/druid-test-harness](https://github.com/housejester/druid-test-harness) - A set of scripts to simplify standing up some servers and seeing how things work diff --git a/docs/Loading-Your-Data.md b/docs/Loading-Your-Data.md new file mode 100644 index 000000000000..2e27fad8303a --- /dev/null +++ b/docs/Loading-Your-Data.md @@ -0,0 +1,370 @@ +--- +layout: default +--- +Once you have a realtime node working, it is time to load your own data to see how Druid performs. + +Druid can ingest data in three ways: via Kafka and a realtime node, via the indexing service, and via the Hadoop batch loader. Data is ingested in realtime using a [Firehose](Firehose.html). + +## Create Config Directories ## +Each type of node needs its own config file and directory, so create them as subdirectories under the druid directory. +```bash +mkdir config +mkdir config/realtime +mkdir config/master +mkdir config/compute +mkdir config/broker +``` + +## Loading Data with Kafka ## + +[KafkaFirehoseFactory](https://github.com/metamx/druid/blob/master/realtime/src/main/java/com/metamx/druid/realtime/firehose/KafkaFirehoseFactory.java) is how druid communicates with Kafka. Using this [Firehose](Firehose.html) with the right configuration, we can import data into Druid in realtime without writing any code. To load data to a realtime node via Kafka, we'll first need to initialize Zookeeper and Kafka, and then configure and initialize a [Realtime](Realtime.html) node. + +### Booting Kafka ### + +Instructions for booting a Zookeeper and then Kafka cluster are available [here](http://kafka.apache.org/07/quickstart.html). + +1. Download Apache Kafka 0.7.2 from [http://kafka.apache.org/downloads.html](http://kafka.apache.org/downloads.html) +```bash +wget http://apache.spinellicreations.com/incubator/kafka/kafka-0.7.2-incubating/kafka-0.7.2-incubating-src.tgz +tar -xvzf kafka-0.7.2-incubating-src.tgz +cd kafka-0.7.2-incubating-src +``` +2. Build Kafka +```bash +./sbt update +./sbt package +``` +3. Boot Kafka +```bash +cat config/zookeeper.properties +bin/zookeeper-server-start.sh config/zookeeper.properties +# in a new console +bin/kafka-server-start.sh config/server.properties +``` +4. Launch the console producer (so you can type in JSON kafka messages in a bit) +```bash +bin/kafka-console-producer.sh --zookeeper localhost:2181 --topic druidtest +``` +### Launching a Realtime Node + +1. Create a valid configuration file similar to this called config/realtime/runtime.properties: +``` +druid.host=0.0.0.0:8080 +druid.port=8080 + +com.metamx.emitter.logging=true + +druid.processing.formatString=processing_%s +druid.processing.numThreads=1 +druid.processing.buffer.sizeBytes=10000000 + +#emitting, opaque marker +druid.service=example + +druid.request.logging.dir=/tmp/example/log +druid.realtime.specFile=realtime.spec +com.metamx.emitter.logging=true +com.metamx.emitter.logging.level=debug + +# below are dummy values when operating a realtime only node +druid.processing.numThreads=3 + +com.metamx.aws.accessKey=dummy_access_key +com.metamx.aws.secretKey=dummy_secret_key +druid.pusher.s3.bucket=dummy_s3_bucket + +druid.zk.service.host=localhost +druid.server.maxSize=300000000000 +druid.zk.paths.base=/druid +druid.database.segmentTable=prod_segments +druid.database.user=user +druid.database.password=diurd +druid.database.connectURI= +druid.host=127.0.0.1:8080 + +``` +2. Create a valid realtime configuration file similar to this called realtime.spec: +```json +[{ + "schema" : { "dataSource":"druidtest", + "aggregators":[ {"type":"count", "name":"impressions"}, + {"type":"doubleSum","name":"wp","fieldName":"wp"}], + "indexGranularity":"minute", + "shardSpec" : { "type": "none" } }, + "config" : { "maxRowsInMemory" : 500000, + "intermediatePersistPeriod" : "PT10m" }, + "firehose" : { "type" : "kafka-0.7.2", + "consumerProps" : { "zk.connect" : "localhost:2181", + "zk.connectiontimeout.ms" : "15000", + "zk.sessiontimeout.ms" : "15000", + "zk.synctime.ms" : "5000", + "groupid" : "topic-pixel-local", + "fetch.size" : "1048586", + "autooffset.reset" : "largest", + "autocommit.enable" : "false" }, + "feed" : "druidtest", + "parser" : { "timestampSpec" : { "column" : "utcdt", "format" : "iso" }, + "data" : { "format" : "json" }, + "dimensionExclusions" : ["wp"] } }, + "plumber" : { "type" : "realtime", + "windowPeriod" : "PT10m", + "segmentGranularity":"hour", + "basePersistDirectory" : "/tmp/realtime/basePersist", + "rejectionPolicy": {"type": "messageTime"} } + +}] +``` +3. Launch the realtime node +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-Ddruid.realtime.specFile=config/realtime/realtime.spec \ +-classpath lib/*:config/realtime com.metamx.druid.realtime.RealtimeMain +``` +4. Paste data into the Kafka console producer +```json +{"utcdt": "2010-01-01T01:01:01", "wp": 1000, "gender": "male", "age": 100} +{"utcdt": "2010-01-01T01:01:02", "wp": 2000, "gender": "female", "age": 50} +{"utcdt": "2010-01-01T01:01:03", "wp": 3000, "gender": "male", "age": 20} +{"utcdt": "2010-01-01T01:01:04", "wp": 4000, "gender": "female", "age": 30} +{"utcdt": "2010-01-01T01:01:05", "wp": 5000, "gender": "male", "age": 40} +``` +5. Watch the events as they are ingested by Druid's realtime node +```bash +... +2013-06-17 21:41:55,569 INFO [Global--0] com.metamx.emitter.core.LoggingEmitter - Event [{"feed":"metrics","timestamp":"2013-06-17T21:41:55.569Z","service":"example","host":"127.0.0.1","metric":"events/processed","value":5,"user2":"druidtest"}] +... +``` +6. In a new console, edit a file called query.body: +```json +{ + "queryType": "groupBy", + "dataSource": "druidtest", + "granularity": "all", + "dimensions": [], + "aggregations": [ + { "type": "count", "name": "rows" }, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], + "intervals": ["2010-01-01T00:00/2020-01-01T00"] +} +``` +7. Submit the query via curl +```bash +curl -X POST "http://localhost:8080/druid/v2/?pretty" \ +-H 'content-type: application/json' -d @query.body +``` +8. View Result! +```json +[ { + "timestamp" : "2010-01-01T01:01:00.000Z", + "result" : { + "imps" : 20, + "wp" : 60000.0, + "rows" : 5 + } +} ] +``` +Now you're ready for [Querying Your Data](Querying-Your-Data.html)! + +## Loading Data with the HadoopDruidIndexer ## + +Historical data can be loaded via a Hadoop job. + +The setup for a single node, 'standalone' Hadoop cluster is available at [http://hadoop.apache.org/docs/stable/single_node_setup.html](http://hadoop.apache.org/docs/stable/single_node_setup.html). + +### Setup MySQL ### +1. If you don't already have it, download MySQL Community Server here: [http://dev.mysql.com/downloads/mysql/](http://dev.mysql.com/downloads/mysql/) +2. Install MySQL +3. Create a druid user and database +```bash +mysql -u root +``` +```sql +GRANT ALL ON druid.* TO 'druid'@'localhost' IDENTIFIED BY 'diurd'; +CREATE database druid; +``` +The [Master](Master.html) node will create the tables it needs based on its configuration. + +### Make sure you have ZooKeeper Running ### + +Make sure that you have a zookeeper instance running. If you followed the instructions for Kafka, it is probably running. If you are unsure if you have zookeeper running, try running + +```bash +ps auxww | grep zoo | grep -v grep +``` + +If you get any result back, then zookeeper is most likely running. If you haven't setup Kafka or do not have zookeeper running, then you can download it and start it up with + +```bash +curl http://www.motorlogy.com/apache/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz -o zookeeper-3.4.5.tar.gz +tar xzf zookeeper-3.4.5.tar.gz +cd zookeeper-3.4.5 +cp conf/zoo_sample.cfg conf/zoo.cfg +./bin/zkServer.sh start +cd .. +``` + +### Launch a Master Node ### +If you've already setup a realtime node, be aware that although you can run multiple node types on one physical computer, you must assign them unique ports. Having used 8080 for the [Realtime](Realtime.html) node, we use 8081 for the [Master](Master.html). + +1. Setup a configuration file called config/master/runtime.properties similar to: +```bash +druid.host=0.0.0.0:8081 +druid.port=8081 + +com.metamx.emitter.logging=true + +druid.processing.formatString=processing_%s +druid.processing.numThreads=1 +druid.processing.buffer.sizeBytes=10000000 + +#emitting, opaque marker +druid.service=example + +druid.master.startDelay=PT60s +druid.request.logging.dir=/tmp/example/log +druid.realtime.specFile=realtime.spec +com.metamx.emitter.logging=true +com.metamx.emitter.logging.level=debug + +# below are dummy values when operating a realtime only node +druid.processing.numThreads=3 + +com.metamx.aws.accessKey=dummy_access_key +com.metamx.aws.secretKey=dummy_secret_key +druid.pusher.s3.bucket=dummy_s3_bucket + +druid.zk.service.host=localhost +druid.server.maxSize=300000000000 +druid.zk.paths.base=/druid +druid.database.segmentTable=prod_segments +druid.database.user=druid +druid.database.password=diurd +druid.database.connectURI=jdbc:mysql://localhost:3306/druid +druid.zk.paths.discoveryPath=/druid/discoveryPath +druid.database.ruleTable=rules +druid.database.configTable=config + +# Path on local FS for storage of segments; dir will be created if needed +druid.paths.indexCache=/tmp/druid/indexCache +# Path on local FS for storage of segment metadata; dir will be created if needed +druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache +``` +2. Launch the [Master](Master.html) node +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-classpath lib/*:config/master \ +com.metamx.druid.http.MasterMain +``` + +### Launch a Compute/Historical Node ### +1. Create a configuration file in config/compute/runtime.properties similar to: +```bash +druid.host=0.0.0.0:8082 +druid.port=8082 + +com.metamx.emitter.logging=true + +druid.processing.formatString=processing_%s +druid.processing.numThreads=1 +druid.processing.buffer.sizeBytes=10000000 + +#emitting, opaque marker +druid.service=example + +druid.request.logging.dir=/tmp/example/log +druid.realtime.specFile=realtime.spec +com.metamx.emitter.logging=true +com.metamx.emitter.logging.level=debug + +# below are dummy values when operating a realtime only node +druid.processing.numThreads=3 + +com.metamx.aws.accessKey=dummy_access_key +com.metamx.aws.secretKey=dummy_secret_key +druid.pusher.s3.bucket=dummy_s3_bucket + +druid.zk.service.host=localhost +druid.server.maxSize=300000000000 +druid.zk.paths.base=/druid +druid.database.segmentTable=prod_segments +druid.database.user=druid +druid.database.password=diurd +druid.database.connectURI=jdbc:mysql://localhost:3306/druid +druid.zk.paths.discoveryPath=/druid/discoveryPath +druid.database.ruleTable=rules +druid.database.configTable=config + +# Path on local FS for storage of segments; dir will be created if needed +druid.paths.indexCache=/tmp/druid/indexCache +# Path on local FS for storage of segment metadata; dir will be created if needed +druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache +# Setup local storage mode +druid.pusher.local.storageDirectory=/tmp/druid/localStorage +druid.pusher.local=true +``` +2. Launch the compute node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-classpath lib/*:config/compute \ +com.metamx.druid.http.ComputeMain +``` + +### Create a File of Records ### + +We can use the same records we have been, in a file called records.json: +```json +{"utcdt": "2010-01-01T01:01:01", "wp": 1000, "gender": "male", "age": 100} +{"utcdt": "2010-01-01T01:01:02", "wp": 2000, "gender": "female", "age": 50} +{"utcdt": "2010-01-01T01:01:03", "wp": 3000, "gender": "male", "age": 20} +{"utcdt": "2010-01-01T01:01:04", "wp": 4000, "gender": "female", "age": 30} +{"utcdt": "2010-01-01T01:01:05", "wp": 5000, "gender": "male", "age": 40} +``` + +### Run the Hadoop Job ### + +Now its time to run the Hadoop [Batch-ingestion](Batch-ingestion.html) job, HadoopDruidIndexer, which will fill a historical [Compute](Compute.html) node with data. First we'll need to configure the job. + +1. Create a config called batchConfig.json similar to: +```json +{ + "dataSource": "druidtest", + "timestampColumn": "utcdt", + "timestampFormat": "iso", + "dataSpec": { + "format": "json", + "dimensions": ["gender", "age"] + }, + "granularitySpec": { + "type":"uniform", + "intervals":["2010-01-01T01/PT1H"], + "gran":"hour" + }, + "pathSpec": { "type": "static", + "paths": "/Users/rjurney/Software/druid/records.json" }, + "rollupSpec": { "aggs":[ {"type":"count", "name":"impressions"}, + {"type":"doubleSum","name":"wp","fieldName":"wp"} + ], + "rollupGranularity": "minute"}, + "workingPath": "/tmp/working_path", + "segmentOutputPath": "/tmp/segments", + "leaveIntermediate": "false", + "partitionsSpec": { + "targetPartitionSize": 5000000 + }, + "updaterJobSpec": { + "type":"db", + "connectURI":"jdbc:mysql://localhost:3306/druid", + "user":"druid", + "password":"diurd", + "segmentTable":"prod_segments" + } +} +``` +2. Now run the job, with the config pointing at batchConfig.json: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=realtime.spec -classpath lib/* com.metamx.druid.indexer.HadoopDruidIndexerMain batchConfig.json +``` + +You can now move on to [Querying Your Data](Querying-Your-Data.html)! \ No newline at end of file diff --git a/docs/Master.md b/docs/Master.md new file mode 100644 index 000000000000..eb86a3e81fd9 --- /dev/null +++ b/docs/Master.md @@ -0,0 +1,106 @@ +--- +layout: default +--- +Master +====== + +The Druid master node is primarily responsible for segment management and distribution. More specifically, the Druid master node communicates to compute nodes to load or drop segments based on configurations. The Druid master is responsible for loading new segments, dropping outdated segments, managing segment replication, and balancing segment load. + +The Druid master runs periodically and the time between each run is a configurable parameter. Each time the Druid master runs, it assesses the current state of the cluster before deciding on the appropriate actions to take. Similar to the broker and compute nodes, the Druid master maintains a connection to a Zookeeper cluster for current cluster information. The master also maintains a connection to a database containing information about available segments and rules. Available segments are stored in a segment table and list all segments that should be loaded in the cluster. Rules are stored in a rule table and indicate how segments should be handled. + +Before any unassigned segments are serviced by compute nodes, the available compute nodes for each tier are first sorted in terms of capacity, with least capacity servers having the highest priority. Unassigned segments are always assigned to the nodes with least capacity to maintain a level of balance between nodes. The master does not directly communicate with a compute node when assigning it a new segment; instead the master creates some temporary information about the new segment under load queue path of the compute node. Once this request is seen, the compute node will load the segment and begin servicing it. + +Rules +----- + +Segments are loaded and dropped from the cluster based on a set of rules. Rules indicate how segments should be assigned to different compute node tiers and how many replicants of a segment should exist in each tier. Rules may also indicate when segments should be dropped entirely from the cluster. The master loads a set of rules from the database. Rules may be specific to a certain datasource and/or a default set of rules can be configured. Rules are read in order and hence the ordering of rules is important. The master will cycle through all available segments and match each segment with the first rule that applies. Each segment may only match a single rule + +For more information on rules, see [Rule Configuration](Rule-Configuration.html). + +Cleaning Up Segments +-------------------- + +Each run, the Druid master compares the list of available database segments in the database with the current segments in the cluster. Segments that are not in the database but are still being served in the cluster are flagged and appended to a removal list. Segments that are overshadowed (their versions are too old and their data has been replaced by newer segments) are also dropped. + +Segment Availability +-------------------- + +If a compute node restarts or becomes unavailable for any reason, the Druid master will notice a node has gone missing and treat all segments served by that node as being dropped. Given a sufficient period of time, the segments may be reassigned to other compute nodes in the cluster. However, each segment that is dropped is not immediately forgotten. Instead, there is a transitional data structure that stores all dropped segments with an associated lifetime. The lifetime represents a period of time in which the master will not reassign a dropped segment. Hence, if a compute node becomes unavailable and available again within a short period of time, the compute node will start up and serve segments from its cache without any those segments being reassigned across the cluster. + +Balancing Segment Load +---------------------- + +To ensure an even distribution of segments across compute nodes in the cluster, the master node will find the total size of all segments being served by every compute node each time the master runs. For every compute node tier in the cluster, the master node will determine the compute node with the highest utilization and the compute node with the lowest utilization. The percent difference in utilization between the two nodes is computed, and if the result exceeds a certain threshold, a number of segments will be moved from the highest utilized node to the lowest utilized node. There is a configurable limit on the number of segments that can be moved from one node to another each time the master runs. Segments to be moved are selected at random and only moved if the resulting utilization calculation indicates the percentage difference between the highest and lowest servers has decreased. + +HTTP Endpoints +-------------- + +The master node exposes several HTTP endpoints for interactions. + +### GET + +/info/master - returns the current true master of the cluster as a JSON object. E.g. A GET request to :8080/info/master will yield JSON of the form {[host]("IP"}) + +/info/cluster - returns JSON data about every node and segment in the cluster. E.g. A GET request to :8080/info/cluster will yield JSON data organized by nodes. Information about each node and each segment on each node will be returned. + +/info/servers (optional param ?full) - returns all segments in the cluster if the full flag is not set, otherwise returns full metadata about all servers in the cluster + +/info/servers/{serverName} - returns full metadata about a specific server + +/info/servers/{serverName}/segments (optional param ?full) - returns a list of all segments for a server if the full flag is not set, otherwise returns all segment metadata + +/info/servers/{serverName}/segments/{segmentId} - returns full metadata for a specific segment + +/info/segments (optional param ?full)- returns all segments in the cluster as a list if the full flag is not set, otherwise returns all metadata about segments in the cluster + +/info/segments/{segmentId} - returns full metadata for a specific segment + +/info/datasources (optional param ?full) - returns a list of datasources in the cluster if the full flag is not set, otherwise returns all the metadata for every datasource in the cluster + +/info/datasources/{dataSourceName} - returns full metadata for a datasource + +/info/datasources/{dataSourceName}/segments (optional param ?full) - returns a list of all segments for a datasource if the full flag is not set, otherwise returns full segment metadata for a datasource + +/info/datasources/{dataSourceName}/segments/{segmentId} - returns full segment metadata for a specific segment + +/info/rules - returns all rules for all data sources in the cluster including the default datasource. + +/info/rules/{dataSourceName} - returns all rules for a specified datasource + +### POST + +/info/rules/{dataSourceName} - POST with a list of rules in JSON form to update rules. + +The Master Console +------------------ + +The Druid master exposes a web GUI for displaying cluster information and rule configuration. After the master starts, the console can be accessed at http://HOST:PORT/static/. There exists a full cluster view, as well as views for individual compute nodes, datasources and segments themselves. Segment information can be displayed in raw JSON form or as part of a sortable and filterable table. + +The master console also exposes an interface to creating and editing rules. All valid datasources configured in the segment database, along with a default datasource, are available for configuration. Rules of different types can be added, deleted or edited. + +FAQ +--- + +1. **Do clients ever contact the master node?** + +The master is not involved in the lifecycle of a query. + +Compute nodes never directly contact the master node. The Druid master tells the compute nodes to load/drop data via Zookeeper, but the compute nodes are completely unaware of the master. + +Brokers also never contact the master. Brokers base their understanding of the data topology on metadata exposed by the compute nodes via ZK and are completely unaware of the master. + +2. **Does it matter if the master node starts up before or after other processes?** + +No. If the Druid master is not started up, no new segments will be loaded in the cluster and outdated segments will not be dropped. However, the master node can be started up at any time, and after a configurable delay, will start running master tasks. + +This also means that if you have a working cluster and all of your masters die, the cluster will continue to function, it just won’t experience any changes to its data topology. + +Running +------- + +Master nodes can be run using the `com.metamx.druid.http.MasterMain` class. + +Configuration +------------- + +See [Configuration](Configuration.html). diff --git a/docs/MySQL.md b/docs/MySQL.md new file mode 100644 index 000000000000..713ad0ab18d5 --- /dev/null +++ b/docs/MySQL.md @@ -0,0 +1,47 @@ +--- +layout: default +--- +MySQL is an external dependency of Druid. We use it to store various metadata about the system, but not to store the actual data. There are a number of tables used for various purposes described below. + +Segments Table +-------------- + +This is dictated by the `druid.database.segmentTable` property (Note that these properties are going to change in the next stable version after 0.4.12). + +This table stores metadata about the segments that are available in the system. The table is polled by the [Master](Master.html) to determine the set of segments that should be available for querying in the system. The table has two main functional columns, the other columns are for indexing purposes. + +The `used` column is a boolean “tombstone”. A 1 means that the segment should be “used” by the cluster (i.e. it should be loaded and available for requests). A 0 means that the segment should not be actively loaded into the cluster. We do this as a means of removing segments from the cluster without actually removing their metadata (which allows for simpler rolling back if that is ever an issue). + +The `payload` column stores a JSON blob that has all of the metadata for the segment (some of the data stored in this payload is redundant with some of the columns in the table, that is intentional). This looks something like + + { + "dataSource":"wikipedia", + "interval":"2012-05-23T00:00:00.000Z/2012-05-24T00:00:00.000Z", + "version":"2012-05-24T00:10:00.046Z", + "loadSpec":{"type":"s3_zip", + "bucket":"bucket_for_segment", + "key":"path/to/segment/on/s3"}, + "dimensions":"comma-delimited-list-of-dimension-names", + "metrics":"comma-delimited-list-of-metric-names", + "shardSpec":{"type":"none"}, + "binaryVersion":9, + "size":size_of_segment, + "identifier":"wikipedia_2012-05-23T00:00:00.000Z_2012-05-24T00:00:00.000Z_2012-05-23T00:10:00.046Z" + } + +Note that the format of this blob can and will change from time-to-time. + +Rule Table +---------- + +The rule table is used to store the various rules about where segments should land. These rules are used by the [Master](Master.html) when making segment (re-)allocation decisions about the cluster. + +Config Table +------------ + +The config table is used to store runtime configuration objects. We do not have many of these yet and we are not sure if we will keep this mechanism going forward, but it is the beginnings of a method of changing some configuration parameters across the cluster at runtime. + +Task-related Tables +------------------- + +There are also a number of tables created and used by the [Indexing Service](Indexing-Service.html) in the course of its work. diff --git a/docs/OrderBy.md b/docs/OrderBy.md new file mode 100644 index 000000000000..9dcffff7886a --- /dev/null +++ b/docs/OrderBy.md @@ -0,0 +1,27 @@ +--- +layout: default +--- +The orderBy field provides the functionality to sort and limit the set of results from a groupBy query. Available options are: + +### DefaultLimitSpec + +The default limit spec takes a limit and the list of columns to do an orderBy operation over. The grammar is: + + + { + "type" : "default", + "limit" : , + "columns" : [list of OrderByColumnSpec], + } + + +#### OrderByColumnSpec + +OrderByColumnSpecs indicate how to do order by operations. Each order by condition can be a String or a map of the following form: + + + { + "dimension" : "", + "direction" : "ASCENDING OR DESCENDING" + } + diff --git a/docs/Plumber.md b/docs/Plumber.md new file mode 100644 index 000000000000..b2123e943939 --- /dev/null +++ b/docs/Plumber.md @@ -0,0 +1,24 @@ +--- +layout: default +--- +The Plumber is the thing that handles generated segments both while they are being generated and when they are “done”. This is also technically a pluggable interface and there are multiple implementations, but there are a lot of details handled by the plumber such that it is expected that there will only be a few implementations and only more advanced third-parties will implement their own. See [here](https://github.com/metamx/druid/wiki/Plumber#available-plumbers) for a description of the plumbers included with Druid. + +|Field|Type|Description|Required| +|-----|----|-----------|--------| +|type|String|Specifies the type of plumber. Each value will have its own configuration schema, plumbers packaged with Druid are described [here](https://github.com/metamx/druid/wiki/Plumber#available-plumbers)|yes| + +We provide a brief description of the example to exemplify the types of things that are configured on the plumber. + +- `windowPeriod` is the amount of lag time to allow events. This is configured with a 10 minute window, meaning that any event more than 10 minutes ago will be thrown away and not included in the segment generated by the realtime server. +- `basePersistDirectory` is the directory to put things that need persistence. The plumber is responsible for the actual intermediate persists and this tells it where to store those persists. + +Available Plumbers +------------------ + +#### YeOldePlumber + +This plumber creates single historical segments. + +#### RealtimePlumber + +This plumber creates real-time/mutable segments. diff --git a/docs/Post-aggregations.md b/docs/Post-aggregations.md new file mode 100644 index 000000000000..2e11f98d0e03 --- /dev/null +++ b/docs/Post-aggregations.md @@ -0,0 +1,95 @@ +--- +layout: default +--- +Post-aggregations are specifications of processing that should happen on aggregated values as they come out of Druid. If you include a post aggregation as part of a query, make sure to include all aggregators the post-aggregator requires. + +There are several post-aggregators available. + +### Arithmetic post-aggregator + +The arithmetic post-aggregator applies the provided function to the given fields from left to right. The fields can be aggregators or other post aggregators. + +Supported functions are `+`, `-`, `*`, and `/` + +The grammar for an arithmetic post aggregation is: + + postAggregation : { + "type" : "arithmetic", + "name" : , + "fn" : , + "fields": [, , ...] + } + +### Field accessor post-aggregator + +This returns the value produced by the specified [aggregator|Aggregations](aggregator|Aggregations.html). + +`fieldName` refers to the output name of the aggregator given in the [aggregations|Aggregations](aggregations|Aggregations.html) portion of the query. + + field_accessor : { + "type" : "fieldAccess", + "fieldName" : + } + +### Constant post-aggregator + +The constant post-aggregator always returns the specified value. + + constant : { + "type" : "constant", + "name" : , + "value" : , + } + +### Example Usage + +In this example, let’s calculate a simple percentage using post aggregators. Let’s imagine our data set has a metric called “total”. + +The format of the query JSON is as follows: + + + { + ... + "aggregations" : [ + { + "type" : "count", + "name" : "rows" + }, + { + "type" : "doubleSum", + "name" : "tot", + "fieldName" : "total" + } + ], + "postAggregations" : { + "type" : "arithmetic", + "name" : "average", + "fn" : "*", + "fields" : [ + { + "type" : "arithmetic", + "name" : "div", + "fn" : "/", + "fields" : [ + { + "type" : "fieldAccess", + "name" : "tot", + "fieldName" : "tot" + }, + { + "type" : "fieldAccess", + "name" : "rows", + "fieldName" : "rows" + } + ] + }, + { + "type" : "constant", + "name": "const", + "value" : 100 + } + ] + } + ... + } + diff --git a/docs/Querying-your-data.md b/docs/Querying-your-data.md new file mode 100644 index 000000000000..dc3e04d645c6 --- /dev/null +++ b/docs/Querying-your-data.md @@ -0,0 +1,366 @@ +--- +layout: default +--- +# Setup # + +Before we start querying druid, we're going to finish setting up a complete cluster on localhost. In [Loading Your Data](Loading-Your-Data.html) we setup a [Realtime](Realtime.html), [Compute](Compute.html) and [Master](Master.html) node. If you've already completed that tutorial, you need only follow the directions for 'Booting a Broker Node'. + +## Booting a Broker Node ## + +1. Setup a config file at config/broker/runtime.properties that looks like this: +``` +druid.host=0.0.0.0:8083 +druid.port=8083 + +com.metamx.emitter.logging=true + +druid.processing.formatString=processing_%s +druid.processing.numThreads=1 +druid.processing.buffer.sizeBytes=10000000 + +#emitting, opaque marker +druid.service=example + +druid.request.logging.dir=/tmp/example/log +druid.realtime.specFile=realtime.spec +com.metamx.emitter.logging=true +com.metamx.emitter.logging.level=debug + +# below are dummy values when operating a realtime only node +druid.processing.numThreads=3 + +com.metamx.aws.accessKey=dummy_access_key +com.metamx.aws.secretKey=dummy_secret_key +druid.pusher.s3.bucket=dummy_s3_bucket + +druid.zk.service.host=localhost +druid.server.maxSize=300000000000 +druid.zk.paths.base=/druid +druid.database.segmentTable=prod_segments +druid.database.user=druid +druid.database.password=diurd +druid.database.connectURI=jdbc:mysql://localhost:3306/druid +druid.zk.paths.discoveryPath=/druid/discoveryPath +druid.database.ruleTable=rules +druid.database.configTable=config + +# Path on local FS for storage of segments; dir will be created if needed +druid.paths.indexCache=/tmp/druid/indexCache +# Path on local FS for storage of segment metadata; dir will be created if needed +druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache +druid.pusher.local.storageDirectory=/tmp/druid/localStorage +druid.pusher.local=true + +# thread pool size for servicing queries +druid.client.http.connections=30 +``` + +2. Run the broker node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-Ddruid.realtime.specFile=realtime.spec \ +-classpath services/target/druid-services-0.5.50-SNAPSHOT-selfcontained.jar:config/broker \ +com.metamx.druid.http.BrokerMain +``` + +## Booting a Master Node ## + +1. Setup a config file at config/master/runtime.properties that looks like this: [https://gist.github.com/rjurney/5818870](https://gist.github.com/rjurney/5818870) +2. Run the master node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-classpath services/target/druid-services-0.5.50-SNAPSHOT-selfcontained.jar:config/master \ +com.metamx.druid.http.MasterMain +``` + +## Booting a Realtime Node ## + +1. Setup a config file at config/realtime/runtime.properties that looks like this: [https://gist.github.com/rjurney/5818774](https://gist.github.com/rjurney/5818774) + +2. Setup a realtime.spec file like this: [https://gist.github.com/rjurney/5818779](https://gist.github.com/rjurney/5818779) +3. Run the realtime node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-Ddruid.realtime.specFile=realtime.spec \ +-classpath services/target/druid-services-0.5.50-SNAPSHOT-selfcontained.jar:config/realtime \ +com.metamx.druid.realtime.RealtimeMain +``` + +## Booting a Compute Node ## + +1. Setup a config file at config/compute/runtime.properties that looks like this: [https://gist.github.com/rjurney/5818885](https://gist.github.com/rjurney/5818885) +2. Run the compute node: +```bash +java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 \ +-classpath services/target/druid-services-0.5.50-SNAPSHOT-selfcontained.jar:config/compute \ +com.metamx.druid.http.ComputeMain +``` + +# Querying Your Data # + +Now that we have a complete cluster setup on localhost, we need to load data. To do so, refer to [Loading Your Data](Loading-Your-Data.html). Having done that, its time to query our data! For a complete specification of queries, see [Querying](Querying.html). + +## Querying Different Nodes ## + +As a shared-nothing system, there are three ways to query druid, against the [Realtime](Realtime.html), [Compute](Compute.html) or [Broker](Broker.html) node. Querying a Realtime node returns only realtime data, querying a compute node returns only historical segments. Querying the broker will query both realtime and compute segments and compose an overall result for the query. This is the normal mode of operation for queries in druid. + +### Construct a Query ### + +For constructing this query, see: Querying against the realtime.spec +```json +{ + "queryType": "groupBy", + "dataSource": "druidtest", + "granularity": "all", + "dimensions": [], + "aggregations": [ + {"type": "count", "name": "rows"}, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], + "intervals": ["2010-01-01T00:00/2020-01-01T00"] +} +``` + +### Querying the Realtime Node ### + +Run our query against port 8080: +```bash +curl -X POST "http://localhost:8080/druid/v2/?pretty" \ +-H 'content-type: application/json' -d @query.body +``` +See our result: +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 5, + "wp" : 15000.0, + "rows" : 5 + } +} ] +``` + +### Querying the Compute Node ### +Run the query against port 8082: +```bash +curl -X POST "http://localhost:8082/druid/v2/?pretty" \ +-H 'content-type: application/json' -d @query.body +``` +And get (similar to): +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 27, + "wp" : 77000.0, + "rows" : 9 + } +} ] +``` +### Querying both Nodes via the Broker ### +Run the query against port 8083: +```bash +curl -X POST "http://localhost:8083/druid/v2/?pretty" \ +-H 'content-type: application/json' -d @query.body +``` +And get: +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 5, + "wp" : 15000.0, + "rows" : 5 + } +} ] +``` + +Now that we know what nodes can be queried (although you should usually use the broker node), lets learn how to know what queries are available. + +## Querying Against the realtime.spec ## + +How are we to know what queries we can run? Although [Querying](Querying.html) is a helpful index, to get a handle on querying our data we need to look at our [Realtime](Realtime.html) node's realtime.spec file: + +```json +[{ + "schema" : { "dataSource":"druidtest", + "aggregators":[ {"type":"count", "name":"impressions"}, + {"type":"doubleSum","name":"wp","fieldName":"wp"}], + "indexGranularity":"minute", + "shardSpec" : { "type": "none" } }, + "config" : { "maxRowsInMemory" : 500000, + "intermediatePersistPeriod" : "PT10m" }, + "firehose" : { "type" : "kafka-0.7.2", + "consumerProps" : { "zk.connect" : "localhost:2181", + "zk.connectiontimeout.ms" : "15000", + "zk.sessiontimeout.ms" : "15000", + "zk.synctime.ms" : "5000", + "groupid" : "topic-pixel-local", + "fetch.size" : "1048586", + "autooffset.reset" : "largest", + "autocommit.enable" : "false" }, + "feed" : "druidtest", + "parser" : { "timestampSpec" : { "column" : "utcdt", "format" : "iso" }, + "data" : { "format" : "json" }, + "dimensionExclusions" : ["wp"] } }, + "plumber" : { "type" : "realtime", + "windowPeriod" : "PT10m", + "segmentGranularity":"hour", + "basePersistDirectory" : "/tmp/realtime/basePersist", + "rejectionPolicy": {"type": "messageTime"} } + +}] +``` + +### dataSource ### + +```json +"dataSource":"druidtest" +``` +Our dataSource tells us the name of the relation/table, or 'source of data', to query in both our realtime.spec and query.body! + +### aggregations ### + +Note the [Aggregations](Aggregations.html) in our query: + +```json + "aggregations": [ + {"type": "count", "name": "rows"}, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], +``` + +this matches up to the aggregators in the schema of our realtime.spec! + +```json +"aggregators":[ {"type":"count", "name":"impressions"}, + {"type":"doubleSum","name":"wp","fieldName":"wp"}], +``` + +### dimensions ### + +Lets look back at our actual records (from [Loading Your Data](Loading Your Data.html)): + +```json +{"utcdt": "2010-01-01T01:01:01", "wp": 1000, "gender": "male", "age": 100} +{"utcdt": "2010-01-01T01:01:02", "wp": 2000, "gender": "female", "age": 50} +{"utcdt": "2010-01-01T01:01:03", "wp": 3000, "gender": "male", "age": 20} +{"utcdt": "2010-01-01T01:01:04", "wp": 4000, "gender": "female", "age": 30} +{"utcdt": "2010-01-01T01:01:05", "wp": 5000, "gender": "male", "age": 40} +``` + +Note that we have two dimensions to our data, other than our primary metric, wp. They are 'gender' and 'age'. We can specify these in our query! Note that we have added a dimension: age, below. + +```json +{ + "queryType": "groupBy", + "dataSource": "druidtest", + "granularity": "all", + "dimensions": ["age"], + "aggregations": [ + {"type": "count", "name": "rows"}, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], + "intervals": ["2010-01-01T00:00/2020-01-01T00"] +} +``` + +Which gets us grouped data in return! + +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "100", + "wp" : 1000.0, + "rows" : 1 + } +}, { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "20", + "wp" : 3000.0, + "rows" : 1 + } +}, { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "30", + "wp" : 4000.0, + "rows" : 1 + } +}, { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "40", + "wp" : 5000.0, + "rows" : 1 + } +}, { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 1, + "age" : "50", + "wp" : 2000.0, + "rows" : 1 + } +} ] +``` + +### filtering ### + +Now that we've observed our dimensions, we can also filter: + +```json +{ + "queryType": "groupBy", + "dataSource": "druidtest", + "granularity": "all", + "filter": { + "type": "selector", + "dimension": "gender", + "value": "male" + }, + "aggregations": [ + {"type": "count", "name": "rows"}, + {"type": "longSum", "name": "imps", "fieldName": "impressions"}, + {"type": "doubleSum", "name": "wp", "fieldName": "wp"} + ], + "intervals": ["2010-01-01T00:00/2020-01-01T00"] +} +``` + +Which gets us just people aged 40: + +```json +[ { + "version" : "v1", + "timestamp" : "2010-01-01T00:00:00.000Z", + "event" : { + "imps" : 3, + "wp" : 9000.0, + "rows" : 3 + } +} ] +``` + +Check out [Filters](Filters.html) for more. + +## Learn More ## + +You can learn more about querying at [Querying](Querying.html)! Now check out [Booting a production cluster](Booting-a-production-cluster.html)! \ No newline at end of file diff --git a/docs/Querying.md b/docs/Querying.md new file mode 100644 index 000000000000..7e613a074ec3 --- /dev/null +++ b/docs/Querying.md @@ -0,0 +1,114 @@ +--- +layout: default +--- +Querying +======== + +Queries are made using an HTTP REST style request to a [Broker](Broker.html), [Compute](Compute.html), or [Realtime](Realtime.html) node. The query is expressed in JSON and each of these node types expose the same REST query interface. + +We start by describing an example query with additional comments that mention possible variations. Query operators are also summarized in a table below. + +Example Query “rand” +-------------------- + +Here is the query in the examples/rand subproject (file is query.body), followed by a commented version of the same. + +\`\`\`javascript +{ + [queryType]() “groupBy”, + [dataSource]() “randSeq”, + [granularity]() “all”, + [dimensions]() [], + [aggregations]() [ + { [type]() “count”, [name]() “rows” }, + { [type]() “doubleSum”, [fieldName]() “events”, [name]() “e” }, + { [type]() “doubleSum”, [fieldName]() “outColumn”, [name]() “randomNumberSum” } + ], + [postAggregations]() [{ + [type]() “arithmetic”, + [name]() “avg\_random”, + [fn]() “/”, + [fields]() [ + { [type]() “fieldAccess”, [fieldName]() “randomNumberSum” }, + { [type]() “fieldAccess”, [fieldName]() “rows” } + ] + }], + [intervals]() [“2012-10-01T00:00/2020-01-01T00”] +} +\`\`\` + +This query could be submitted via curl like so (assuming the query object is in a file “query.json”). + + curl -X POST "http://host:port/druid/v2/?pretty" -H 'content-type: application/json' -d @query.json + +The “pretty” query parameter gets the results formatted a bit nicer. + +Details of Example Query “rand” +------------------------------- + +The queryType JSON field identifies which kind of query operator is to be used, in this case it is groupBy, the most frequently used kind (which corresponds to an internal implementation class GroupByQuery registered as “groupBy”), and it has a set of required fields that are also part of this query. The queryType can also be “search” or “timeBoundary” which have similar or different required fields summarized below: +\`\`\`javascript +{ + [queryType]() “groupBy”, +\`\`\` +The dataSource JSON field shown next identifies where to apply the query. In this case, randSeq corresponds to the examples/rand/rand\_realtime.spec file schema: +\`\`\`javascript + [dataSource]() “randSeq”, +\`\`\` +The granularity JSON field specifies the bucket size for values. It could be a built-in time interval like “second”, “minute”, “fifteen\_minute”, “thirty\_minute”, “hour” or “day”. It can also be an expression like `{"type": "period", "period":"PT6m"}` meaning “6 minute buckets”. See [Granularities](Granularities.html) for more information on the different options for this field. In this example, it is set to the special value “all” which means [bucket all data points together into the same time bucket]() +\`\`\`javascript + [granularity]() “all”, +\`\`\` +The dimensions JSON field value is an array of zero or more fields as defined in the dataSource spec file or defined in the input records and carried forward. These are used to constrain the grouping. If empty, then one value per time granularity bucket is requested in the groupBy: +\`\`\`javascript + [dimensions]() [], +\`\`\` +A groupBy also requires the JSON field “aggregations” (See [Aggregations](Aggregations.html)), which are applied to the column specified by fieldName and the output of the aggregation will be named according to the value in the “name” field: +\`\`\`javascript + [aggregations]() [ + { [type]() “count”, [name]() “rows” }, + { [type]() “doubleSum”, [fieldName]() “events”, [name]() “e” }, + { [type]() “doubleSum”, [fieldName]() “outColumn”, [name]() “randomNumberSum” } + ], +\`\`\` +You can also specify postAggregations, which are applied after data has been aggregated for the current granularity and dimensions bucket. See [Post Aggregations](Post Aggregations.html) for a detailed description. In the rand example, an arithmetic type operation (division, as specified by “fn”) is performed with the result “name” of “avg\_random”. The “fields” field specifies the inputs from the aggregation stage to this expression. Note that identifiers corresponding to “name” JSON field inside the type “fieldAccess” are required but not used outside this expression, so they are prefixed with “dummy” for clarity: +\`\`\`javascript + [postAggregations]() [{ + [type]() “arithmetic”, + [name]() “avg\_random”, + [fn]() “/”, + [fields]() [ + { [type]() “fieldAccess”, [fieldName]() “randomNumberSum” }, + { [type]() “fieldAccess”, [fieldName]() “rows” } + ] + }], +\`\`\` +The time range(s) of the query; data outside the specified intervals will not be used; this example specifies from October 1, 2012 until January 1, 2020: +\`\`\`javascript + [intervals]() [“2012-10-01T00:00/2020-01-01T00”] +} +\`\`\` + +Query Operators +--------------- + +The following table summarizes query properties. + +|query types|property|description|required?| +|-----------|--------|-----------|---------| +|timeseries, groupBy, search, timeBoundary|dataSource|query is applied to this data source|yes| +|timeseries, groupBy, search|intervals|range of time series to include in query|yes| +|timeseries, groupBy, search, timeBoundary|context|This is a key-value map that can allow the query to alter some of the behavior of a query. It is primarily used for debugging, for example if you include `"bySegment":true` in the map, you will get results associated with the data segment they came from.|no| +|timeseries, groupBy, search|filter|Specifies the filter (the “WHERE” clause in SQL) for the query. See [Filters](Filters.html)|no| +|timeseries, groupBy, search|granularity|the timestamp granularity to bucket results into (i.e. “hour”). See [Granularities](Granularities.html) for more information.|no| +|groupBy|dimensions|constrains the groupings; if empty, then one value per time granularity bucket|yes| +|timeseries, groupBy|aggregations|aggregations that combine values in a bucket. See [Aggregations](Aggregations.html).|yes| +|timeseries, groupBy|postAggregations|aggregations of aggregations. See [Post Aggregations](Post Aggregations.html).|yes| +|search|limit|maximum number of results (default is 1000), a system-level maximum can also be set via `com.metamx.query.search.maxSearchLimit`|no| +|search|searchDimensions|Dimensions to apply the search query to. If not specified, it will search through all dimensions.|no| +|search|query|The query portion of the search query. This is essentially a predicate that specifies if something matches.|yes| + +Additional Information about Query Types +---------------------------------------- + +[TimeseriesQuery](TimeseriesQuery.html) diff --git a/docs/Realtime.md b/docs/Realtime.md new file mode 100644 index 000000000000..855607d7eb58 --- /dev/null +++ b/docs/Realtime.md @@ -0,0 +1,157 @@ +--- +layout: default +--- +Realtime +======== + +Realtime nodes provide a realtime index. Data indexed via these nodes is immediately available for querying. Realtime nodes will periodically build segments representing the data they’ve collected over some span of time and hand these segments off to [Compute](Compute.html) nodes. + +Running +------- + +Realtime nodes can be run using the `com.metamx.druid.realtime.RealtimeMain` class. + +Segment Propagation +------------------- + +The segment propagation diagram for real-time data ingestion can be seen below: + +![Segment Propagation](https://raw.github.com/metamx/druid/druid-0.5.4/doc/segment_propagation.png "Segment Propagation") + +Configuration +------------- + +Realtime nodes take a mix of base server configuration and spec files that describe how to connect, process and expose the realtime feed. See [Configuration](Configuration.html) for information about general server configuration. + +### Realtime “specFile” + +The property `druid.realtime.specFile` has the path of a file (absolute or relative path and file name) with realtime specifications in it. This “specFile” should be a JSON Array of JSON objects like the following: + + + [{ + "schema" : { "dataSource":"dataSourceName", + "aggregators":[ {"type":"count", "name":"events"}, + {"type":"doubleSum","name":"outColumn","fieldName":"inColumn"} ], + "indexGranularity":"minute", + "shardSpec" : { "type": "none" } }, + "config" : { "maxRowsInMemory" : 500000, + "intermediatePersistPeriod" : "PT10m" }, + "firehose" : { "type" : "kafka-0.7.2", + "consumerProps" : { "zk.connect" : "zk_connect_string", + "zk.connectiontimeout.ms" : "15000", + "zk.sessiontimeout.ms" : "15000", + "zk.synctime.ms" : "5000", + "groupid" : "consumer-group", + "fetch.size" : "1048586", + "autooffset.reset" : "largest", + "autocommit.enable" : "false" }, + "feed" : "your_kafka_topic", + "parser" : { "timestampSpec" : { "column" : "timestamp", "format" : "iso" }, + "data" : { "format" : "json" }, + "dimensionExclusions" : ["value"] } }, + "plumber" : { "type" : "realtime", + "windowPeriod" : "PT10m", + "segmentGranularity":"hour", + "basePersistDirectory" : "/tmp/realtime/basePersist" } + }] + + +This is a JSON Array so you can give more than one realtime stream to a given node. The number you can put in the same process depends on the exact configuration. In general, it is best to think of each realtime stream handler as requiring 2-threads: 1 thread for data consumption and aggregation, 1 thread for incremental persists and other background tasks. + +There are four parts to a realtime stream specification, `schema`, `config`, `firehose` and `plumber` which we will go into here. + +#### Schema + +This describes the data schema for the output Druid segment. More information about concepts in Druid and querying can be found at [Concepts-and-Terminology](Concepts-and-Terminology.html) and [Querying](Querying.html). + +|Field|Type|Description|Required| +|-----|----|-----------|--------| +|aggregators|Array of Objects|The list of aggregators to use to aggregate colliding rows together.|yes| +|dataSource|String|The name of the dataSource that the segment belongs to.|yes| +|indexGranularity|String|The granularity of the data inside the segment. E.g. a value of “minute” will mean that data is aggregated at minutely granularity. That is, if there are collisions in the tuple (minute(timestamp), dimensions), then it will aggregate values together using the aggregators instead of storing individual rows.|yes| +|segmentGranularity|String|The granularity of the segment as a whole. This is generally larger than the index granularity and describes the rate at which the realtime server will push segments out for historical servers to take over.|yes| +|shardSpec|Object|This describes the shard that is represented by this server. This must be specified properly in order to have multiple realtime nodes indexing the same data stream in a sharded fashion.|no| + +### Config + +This provides configuration for the data processing portion of the realtime stream processor. + +|Field|Type|Description|Required| +|-----|----|-----------|--------| +|intermediatePersistPeriod|ISO8601 Period String|The period that determines the rate at which intermediate persists occur. These persists determine how often commits happen against the incoming realtime stream. If the realtime data loading process is interrupted at time T, it should be restarted to re-read data that arrived at T minus this period.|yes| +|maxRowsInMemory|Number|The number of rows to aggregate before persisting. This number is the post-aggregation rows, so it is not equivalent to the number of input events, but the number of aggregated rows that those events result in. This is used to manage the required JVM heap size.|yes| + +### Firehose + +See [Firehose](Firehose.html). + +### Plumber + +See [Plumber](Plumber.html) + +Constraints +----------- + +The following tables summarizes constraints between settings in the spec file for the Realtime subsystem. + +|*. Name |*. Effect |*. Minimum |*. Recommended | +| windowPeriod| when reading an InputRow, events with timestamp older than now minus this window are discarded | time jitter tolerance | use this to reject outliers | +| segmentGranularity| time granularity (minute, hour, day, week, month) for loading data at query time | equal to indexGranularity| more than indexGranularity| +| indexGranularity| time granularity (minute, hour, day, week, month) of indexes | less than segmentGranularity| minute, hour, day, week, month | +| intermediatePersistPeriod| the max real time (ISO8601 Period) between flushes of InputRows from memory to disk | avoid excessive flushing | number of un-persisted rows in memory also constrained by maxRowsInMemory | +| maxRowsInMemory| the max number of InputRows to hold in memory before a flush to disk | number of un-persisted post-aggregation rows in memory is also constrained by intermediatePersistPeriod | use this to avoid running out of heap if too many rows in an intermediatePersistPeriod | + +The normal, expected use cases have the following overall constraints: `indexGranularity < intermediatePersistPeriod =< windowPeriod < segmentGranularity` + +If the RealtimeNode process runs out of heap, try adjusting druid.computation.buffer.size property which specifies a size in bytes that must fit into the heap. + +Requirements +------------ + +Realtime nodes currently require a Kafka cluster to sit in front of them and collect results. There’s more configuration required for these as well. + +Extending the code +------------------ + +Realtime integration is intended to be extended in two ways: + +1. Connect to data streams from varied systems ([Firehose](https://github.com/metamx/druid/blob/master/realtime/src/main/java/com/metamx/druid/realtime/FirehoseFactory.java)) +2. Adjust the publishing strategy to match your needs ([Plumber](https://github.com/metamx/druid/blob/master/realtime/src/main/java/com/metamx/druid/realtime/PlumberSchool.java)) + +The expectations are that the former will be very common and something that users of Druid will do on a fairly regular basis. Most users will probably never have to deal with the latter form of customization. Indeed, we hope that all potential use cases can be packaged up as part of Druid proper without requiring proprietary customization. + +Given those expectations, adding a firehose is straightforward and completely encapsulated inside of the interface. Adding a plumber is more involved and requires understanding of how the system works to get right, it’s not impossible, but it’s not intended that individuals new to Druid will be able to do it immediately. + +We will do our best to accept contributions from the community of new Firehoses and Plumbers, but we also understand the requirement for being able to plug in your own proprietary implementations. The model for doing this is by embedding the druid code in another project and writing your own `main()` method that initializes a RealtimeNode object and registers your proprietary objects with it. + + + public class MyRealtimeMain + { + private static final Logger log = new Logger(MyRealtimeMain.class); + + public static void main(String[] args) throws Exception + { + LogLevelAdjuster.register(); + + Lifecycle lifecycle = new Lifecycle(); + + lifecycle.addManagedInstance( + RealtimeNode.builder() + .build() + .registerJacksonSubtype(foo.bar.MyFirehose.class) + ); + + try { + lifecycle.start(); + } + catch (Throwable t) { + log.info(t, "Throwable caught at startup, committing seppuku"); + System.exit(2); + } + + lifecycle.join(); + } + } + + +Pluggable pieces of the system are either handled by a setter on the RealtimeNode object, or they are configuration driven and need to be setup to allow for [Jackson polymorphic deserialization](http://wiki.fasterxml.com/JacksonPolymorphicDeserialization) and registered via the relevant methods on the RealtimeNode object. diff --git a/docs/Rule-Configuration.md b/docs/Rule-Configuration.md new file mode 100644 index 000000000000..2695da646abb --- /dev/null +++ b/docs/Rule-Configuration.md @@ -0,0 +1,80 @@ +--- +layout: default +--- +Note: It is recommended that the master console is used to configure rules. However, the master node does have HTTP endpoints to programmatically configure rules. + +Load Rules +---------- + +Load rules indicate how many replicants of a segment should exist in a server tier. + +### Interval Load Rule + +Interval load rules are of the form: + + + { + "type" : "loadByInterval", + "interval" : "2012-01-01/2013-01-01", + "tier" : "hot" + } + + +type - this should always be “loadByInterval” +interval - A JSON Object representing ISO-8601 Intervals +tier - the configured compute node tier + +### Period Load Rule + +Period load rules are of the form: + + + { + "type" : "loadByInterval", + "period" : "P1M", + "tier" : "hot" + } + + +type - this should always be “loadByPeriod” +period - A JSON Object representing ISO-8601 Periods +tier - the configured compute node tier + +The interval of a segment will be compared against the specified period. The rule matches if the period overlaps the interval. + +Drop Rules +---------- + +Drop rules indicate when segments should be dropped from the cluster. + +### Interval Drop Rule + +Interval drop rules are of the form: + + + { + "type" : "dropByInterval", + "interval" : "2012-01-01/2013-01-01" + } + + +type - this should always be “dropByInterval” +interval - A JSON Object representing ISO-8601 Periods + +A segment is dropped if the interval contains the interval of the segment. + +### Period Drop Rule + +Period drop rules are of the form: + + + { + "type" : "dropByPeriod", + "period" : "P1M" + } + + +type - this should always be “dropByPeriod” +period - A JSON Object representing ISO-8601 Periods + +The interval of a segment will be compared against the specified period. The period is from some time in the past to the current time. The rule matches if the period contains the interval. diff --git a/docs/SearchQuery.md b/docs/SearchQuery.md new file mode 100644 index 000000000000..b206f652c6e0 --- /dev/null +++ b/docs/SearchQuery.md @@ -0,0 +1,71 @@ +--- +layout: default +--- +A search query returns dimension values that match the search specification. + + { + "queryType": "search", + "dataSource": "sample_datasource", + "granularity": "day", + "searchDimensions": [ + "dim1", + "dim2" + ], + "query": { + "type": "insensitive_contains", + "value": "Ke" + }, + "sort" : { + "type": "lexicographic" + }, + "intervals": [ + "2013-01-01T00:00:00.000/2013-01-03T00:00:00.000" + ] + } + + +There are several main parts to a search query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “search”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|granularity|Defines the granularity of the query. See [Granularities](Granularities.html)|yes| +|filter|See [Filters](Filters.html)|no| +|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| +|searchDimensions|The dimensions to run the search over. Excluding this means the search is run over all dimensions.|no| +|query|See [SearchQuerySpec](SearchQuerySpec.html).|yes| +|sort|How the results of the search should sorted. Two possible types here are “lexicographic” and “strlen”.|yes| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +The format of the result is: + + [ + { + "timestamp": "2012-01-01T00:00:00.000Z", + "result": [ + { + "dimension": "dim1", + "value": "Ke$ha" + }, + { + "dimension": "dim2", + "value": "Ke$haForPresident" + } + ] + }, + { + "timestamp": "2012-01-02T00:00:00.000Z", + "result": [ + { + "dimension": "dim1", + "value": "SomethingThatContainsKe" + }, + { + "dimension": "dim2", + "value": "SomethingElseThatContainsKe" + } + ] + } + ] + diff --git a/docs/SearchQuerySpec.md b/docs/SearchQuerySpec.md new file mode 100644 index 000000000000..9b9db04b8e67 --- /dev/null +++ b/docs/SearchQuerySpec.md @@ -0,0 +1,26 @@ +--- +layout: default +--- +Search query specs define how a “match” is defined between a search value and a dimension value. The available search query specs are: + +InsensitiveContainsSearchQuerySpec +---------------------------------- + +If any part of a dimension value contains the value specified in this search query spec, regardless of case, a “match” occurs. The grammar is: + + { + "type" : "insensitive_contains", + "value" : "some_value" + } + + +FragmentSearchQuerySpec +----------------------- + +If any part of a dimension value contains any of the values specified in this search query spec, regardless of case, a “match” occurs. The grammar is: + + { + "type" : "fragment", + "values" : ["fragment1", "fragment2"] + } + diff --git a/docs/SegmentMetadataQuery.md b/docs/SegmentMetadataQuery.md new file mode 100644 index 000000000000..0e6eefb78e1a --- /dev/null +++ b/docs/SegmentMetadataQuery.md @@ -0,0 +1,58 @@ +--- +layout: default +--- +Segment metadata queries return per segment information about: +\* Cardinality of all columns in the segment +\* Estimated byte size for the segment columns in TSV format +\* Interval the segment covers +\* Column type of all the columns in the segment +\* Estimated total segment byte size in TSV format +\* Segment id + + { + "queryType":"segmentMetadata", + "dataSource":"sample_datasource", + "intervals":["2013-01-01/2014-01-01"], + } + + +There are several main parts to a segment metadata query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “segmentMetadata”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| +|merge|Merge all individual segment metadata results into a single result|no| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +The format of the result is: + + [ { + "id" : "some_id", + "intervals" : [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ], + "columns" : { + "__time" : { + "type" : "LONG", + "size" : 407240380, + "cardinality" : null + }, + "dim1" : { + "type" : "STRING", + "size" : 100000, + "cardinality" : 1944 + }, + "dim2" : { + "type" : "STRING", + "size" : 100000, + "cardinality" : 1504 + }, + "metric1" : { + "type" : "FLOAT", + "size" : 100000, + "cardinality" : null + } + }, + "size" : 300000 + } ] + diff --git a/docs/Segments.md b/docs/Segments.md new file mode 100644 index 000000000000..5af50cd8e485 --- /dev/null +++ b/docs/Segments.md @@ -0,0 +1,63 @@ +--- +layout: default +--- +Segments +======== + +Segments are the fundamental structure to store data in Druid. [Compute](Compute.html) and [Realtime](Realtime.html) nodes load and serve segments for querying. To construct segments, Druid will always shard data by a time partition. Data may be further sharded based on dimension cardinality and row count. + +The latest Druid segment version is `v9`. + +Naming Convention +----------------- + +Identifiers for segments are typically constructed using the segment datasource, interval start time (in ISO 8601 format), interval end time (in ISO 8601 format), and a version. If data is additionally sharded beyond a time range, the segment identifier will also contain a partition number. + +An example segment identifier may be: +datasource\_intervalStart\_intervalEnd\_version\_partitionNum + +Segment Components +------------------ + +A segment is compromised of several files, listed below. + +### `version.bin` + +4 bytes representing the current segment version as an integer. E.g., for v9 segments, the version is 0x0, 0x0, 0x0, 0x9 + +### `meta.smoosh` + +A file with metadata (filenames and offsets) about the contents of the other `smoosh` files + +### `XXXXX.smoosh` + +There are some number of these files, which are concatenated binary data + +The `smoosh` files represent multiple files “smooshed” together in order to minimize the number of file descriptors that must be open to house the data. They are files of up to 2GB in size (to match the limit of a memory mapped ByteBuffer in Java). The `smoosh` files house individual files for each of the columns in the data as well as an `index.drd` file with extra metadata about the segment. + +There is also a special column called `__time` that refers to the time column of the segment. This will hopefully become less and less special as the code evolves, but for now it’s as special as my Mommy always told me I am. + +### `index.drd` + +The `index.drd` file houses 3 pieces of data in order + +1. The names of all of the columns of the data +2. The names of the “dimensions” of the data (these are the dictionary-encoded, string columns. This is here to support some legacy APIs and will be superfluous in the future) +3. The data interval represented by this segment stored as the start and end timestamps as longs + +Format of a column +------------------ + +Each column is stored as two parts: + +1. A Jackson-serialized ColumnDescriptor +2. The rest of the binary for the column + +A ColumnDescriptor is essentially an object that allows us to use jackson’s polymorphic deserialization to add new and interesting methods of serialization with minimal impact to the code. It consists of some metadata about the column (what type is it, is it multi-valued, etc.) and then a list of serde logic that can deserialize the rest of the binary. + +Sharding Data to Create Segments +-------------------------------- + +### Sharding Data by Dimension + +If the cumulative total number of rows for the different values of a given column exceed some configurable threshold, multiple segments representing the same time interval for the same datasource may be created. These segments will contain some partition number as part of their identifier. Sharding by dimension reduces some of the the costs associated with operations over high cardinality dimensions. diff --git a/docs/Spatial-Filters.md b/docs/Spatial-Filters.md new file mode 100644 index 000000000000..2ca83b9a3f97 --- /dev/null +++ b/docs/Spatial-Filters.md @@ -0,0 +1,35 @@ +--- +layout: default +--- +Note: This feature is highly experimental and only works with spatially indexed dimensions. + +The grammar for a spatial filter is as follows: + + + { + "dimension": "spatialDim", + "bound": { + "type": "rectangular", + "minCoords": [10.0, 20.0], + "maxCoords": [30.0, 40.0] + } + } + + +Bounds +------ + +### Rectangular + +|property|description|required?| +|--------|-----------|---------| +|minCoords|List of minimum dimension coordinates for coordinates [x, y, z, …]|yes| +|maxCoords|List of maximum dimension coordinates for coordinates [x, y, z, …]|yes| + +### Radius + +|property|description|required?| +|--------|-----------|---------| +|coords|Origin coordinates in the form [x, y, z, …]|yes| +|radius|The float radius value|yes| + diff --git a/docs/Spatial-Indexing.md b/docs/Spatial-Indexing.md new file mode 100644 index 000000000000..1df36593433d --- /dev/null +++ b/docs/Spatial-Indexing.md @@ -0,0 +1,26 @@ +--- +layout: default +--- +Note: This feature is highly experimental. + +In any of the data specs, there is now the option of providing spatial dimensions. For example, for a JSON data spec, spatial dimensions can be specified as follows: + + + { + "type": "JSON", + "dimensions": , + "spatialDimensions": [ + { + "dimName": "coordinates", + "dims": ["lat", "long"] + }, + ... + ] + } + + +|property|description|required?| +|--------|-----------|---------| +|dimName|The name of the spatial dimension. A spatial dimension may be constructed from multiple other dimensions or it may already exist as part of an event. If a spatial dimension already exists, it must be an array of dimension values.|yes| +|dims|A list of dimension names that comprise a spatial dimension.|no| + diff --git a/docs/Stand-Alone-With-Riak-CS.md b/docs/Stand-Alone-With-Riak-CS.md new file mode 100644 index 000000000000..d1dc8f780d83 --- /dev/null +++ b/docs/Stand-Alone-With-Riak-CS.md @@ -0,0 +1,226 @@ +--- +layout: default +--- +This page describes how to use Riak-CS for deep storage instead of S3. We are still setting up some of the peripheral stuff (file downloads, etc.). + +This guide provided by Pablo Nebrera, thanks! + +## The VMWare instance + +A VMWare [image](http://static.druid.io/artifacts/vmware/druid_riak.tgz) based on Druid 0.3.27.2 and built according to the instructions below has also been provided by Pablo Nebrera. + +The provided vmware machine has access with the following credentials: + + username: root + password: riakdruid + + +## The Setup + +We started with a minimal CentOS installation but you can use any other compatible installation. At the end of this setup you will one node that is running: + +1. A Kafka Broker +1. A single-node Zookeeper ensemble +1. A single-node Riak-CS cluster +1. A Druid [Master](Master.html) +1. A Druid [Broker](Broker.html) +1. A Druid [Compute](Compute.html) +1. A Druid [Realtime](Realtime.html) + +This just walks through getting the relevant software installed and running. You will then need to configure the [Realtime](Realtime.html) node to take in your data. + +### Configure System + +1. Install `CentOS-6.4-x86_64-minimal.iso` ("RedHat v6.4" is the name of the AWS AMI) or your favorite Linux OS (if you use a different OS, some of the installation instructions for peripheral services might differ, please adjust them according to the system you are using). The rest of these instructions assume that you have a running instance and are running as the root user. + +1. Configure the network. We used dhcp executing: + + dhclient eth0 + +1. Disable firewall for now + + service iptables stop + chkconfig iptables off + +1. Change the limits on the number of open files a process can have: + + cat >> /etc/security/limits.conf <<- _RBEOF_ + # ulimit settings for Riak CS + root soft nofile 65536 + root hard nofile 65536 + riak soft nofile 65536 + riak hard nofile 65536 + _RBEOF_ + + ulimit -n 65536 + +### Install base software packages + +1. Install necessary software with yum + + yum install -y java-1.7.0-openjdk-devel git wget mysql-server + +1. Install maven + + wget http://apache.rediris.es/maven/maven-3/3.0.5/binaries/apache-maven-3.0.5-bin.tar.gz + tar xzf apache-maven-3.0.5-bin.tar.gz -C /usr/local + pushd /usr/local + sudo ln -s apache-maven-3.0.5 maven + popd + echo 'export M2_HOME=/usr/local/maven' >> /etc/profile.d/maven.sh + echo 'export PATH=${M2_HOME}/bin:${PATH}' >> /etc/profile.d/maven.sh + source /etc/profile.d/maven.sh + +1. Install erlang + + wget http://binaries.erlang-solutions.com/rpm/centos/6/x86_64/esl-erlang-R15B01-1.x86_64.rpm + yum localinstall -y esl-erlang-R15B01-1.x86_64.rpm + +### Install Kafka And Zookeeper + +1. Install kafka and zookeeper: + + wget http://apache.rediris.es/incubator/kafka/kafka-0.7.2-incubating/kafka-0.7.2-incubating-src.tgz + tar zxvf kafka-0.7.2-incubating-src.tgz + pushd kafka-0.7.2-incubating-src/ + ./sbt update + ./sbt package + mkdir -p /var/lib/kafka + rsync -a * /var/lib/kafka/ + popd + +### Install Riak-CS + +1. Install s3cmd to manage riak s3 + + wget http://downloads.sourceforge.net/project/s3tools/s3cmd/1.5.0-alpha3/s3cmd-1.5.0-alpha3.tar.gz + tar xzvf s3cmd-1.5.0-alpha3.tar.gz + cd s3cmd-1.5.0-alpha3 + cp -r s3cmd S3 /usr/local/bin/ + +1. Install riak, riak-cs and stanchion. Note: riak-cs-control is optional + + wget http://s3.amazonaws.com/downloads.basho.com/riak/1.3/1.3.1/rhel/6/riak-1.3.1-1.el6.x86_64.rpm + wget http://s3.amazonaws.com/downloads.basho.com/riak-cs/1.3/1.3.1/rhel/6/riak-cs-1.3.1-1.el6.x86_64.rpm + wget http://s3.amazonaws.com/downloads.basho.com/stanchion/1.3/1.3.1/rhel/6/stanchion-1.3.1-1.el6.x86_64.rpm + wget http://s3.amazonaws.com/downloads.basho.com/riak-cs-control/1.0/1.0.0/rhel/6/riak-cs-control-1.0.0-1.el6.x86_64.rpm + yum localinstall -y riak-*.rpm stanchion-*.rpm + +### Install Druid + +1. Clone the git repository for druid, checkout a "stable" tag and build + + git clone https://github.com/metamx/druid.git druid + pushd druid + git checkout druid-0.4.12 + export LANGUAGE=C + export LC_MESSAGE=C + export LC_ALL=C + export LANG=en_US + ./build.sh + mkdir -p /var/lib/druid/app + cp ./services/target/druid-services-*-selfcontained.jar /var/lib/druid/app + ln -s /var/lib/druid/app/druid-services-*-selfcontained.jar /var/lib/druid/app/druid-services.jar + popd + + +### Configure stuff + +1. Add this line to /etc/hosts + + echo "127.0.0.1 s3.amazonaws.com bucket.s3.amazonaws.com `hostname`" >> /etc/hosts + + NOTE: the bucket name in this case is "bucket", but you might need to update it to your bucket name if you want to use a different bucket name. + +1. Download and extract run scripts and configuration files: + + wget http://static.druid.io/artifacts/scripts/druid_scripts_nebrera.tar / + pushd / + tar xvf ~/druid_scripts_nebrera.tar + popd + + +1. Start Riak in order to create a user: + + /etc/init.d/riak start + /etc/init.d/riak-cs start + /etc/init.d/stanchion start + + You can check riak status using: + + riak-admin member-status + + You should expect results like + + Attempting to restart script through sudo -H -u riak + ================================= Membership ================================== + Status Ring Pending Node + ------------------------------------------------------------------------------- + valid 100.0% -- 'riak@127.0.0.1' + ------------------------------------------------------------------------------- + Valid:1 / Leaving:0 / Exiting:0 / Joining:0 / Down:0 + + +1. Create riak-cs user and yoink out credentials. + + curl -H 'Content-Type: application/json' -X POST http://127.0.0.1:8088/riak-cs/user --data '{"email":"example@domain.com", "name":"admin"}' >> /tmp/riak_user.json + export RIAK_KEY_ID=`sed 's/^.*"key_id":"//' /tmp/riak_user.json | cut -d '"' -f 1` + export RIAK_KEY_SECRET=`sed 's/^.*"key_secret":"//' /tmp/riak_user.json | cut -d '"' -f 1` + sed -i "s/<%=[ ]*@key_id[ ]*%>/${RIAK_KEY_ID}/" /etc/riak-cs/app.config /etc/riak-cs-control/app.config /etc/stanchion/app.config /etc/druid/config.sh /etc/druid/base.properties /root/.s3cfg + sed -i "s/<%=[ ]*@key_secret[ ]*%>/${RIAK_KEY_SECRET}/" /etc/riak-cs/app.config /etc/riak-cs-control/app.config /etc/stanchion/app.config /etc/druid/config.sh /etc/druid/base.properties /root/.s3cfg + + This will store the result of creating the user into `/tmp/riak_user.json`. You can look at it if you are interested. It will look something like this + + {"email":"example@domain.com", + "display_name":"example", + "name":"admin", + "key_id":"DOXKZYR_QM2S-7HSKAEU", + "key_secret":"GtvVJow068RM-_viHIYR9DWMAXsFcL1SmjuNfA==", + "id":"4c5b5468c180f3efafd531b6cd8e2bb24371d99640aad5ced5fbbc0604fc473d", + "status":"enabled"} + +1. Stop riak-cs: + + /etc/init.d/riak-cs stop + /etc/init.d/stanchion stop + /etc/init.d/riak stop + +1. Disable anonymous user creation + + sed 's/{[ ]*anonymous_user_creation[ ]*,[ ]*true[ ]*}/{anonymous_user_creation, false}/' /etc/riak-cs/app.config |grep anonymous_user_creation + +1. Restart riak-cs services: + + /etc/init.d/riak start + /etc/init.d/riak-cs start + /etc/init.d/stanchion start + + +1. Create your bucket. The example name and in config files is "bucket" + + s3cmd mb s3://bucket + + You can verify that the bucket is created with: + + s3cmd ls + +1. Start MySQL server + + service mysqld start + chkconfig mysqld on + /usr/bin/mysqladmin -u root password 'riakdruid' + + NOTE: If you don't like "riakdruid" as your password, feel free to change it around. + NOTE: If you have used root user to connect to database. It should be changed by other user but I have used this one to simplify it + +1. Start zookeeper and kafka + + /etc/init.d/zookeeper start + /etc/init.d/kafka start + +1. Start druid + + /etc/init.d/druid_master start + /etc/init.d/druid_realtime start + /etc/init.d/druid_broker start + /etc/init.d/druid_compute start \ No newline at end of file diff --git a/docs/Support.md b/docs/Support.md new file mode 100644 index 000000000000..3dd512e050fb --- /dev/null +++ b/docs/Support.md @@ -0,0 +1,16 @@ +--- +layout: default +--- +Numerous backend engineers at [Metamarkets](http://www.metamarkets.com) work on Druid full-time. If you any questions about usage or code, feel free to contact any of us. + +Google Groups Mailing List +-------------------------- + +The best place for questions is through our mailing list: +[druid-development@googlegroups.com](mailto:druid-development@googlegroups.com) +[https://groups.google.com/d/forum/druid-development](https://groups.google.com/d/forum/druid-development) + +IRC +--- + +Several of us also hang out in the channel \#druid-dev on irc.freenode.net. diff --git a/docs/Tasks.md b/docs/Tasks.md new file mode 100644 index 000000000000..95341f581eca --- /dev/null +++ b/docs/Tasks.md @@ -0,0 +1,71 @@ +--- +layout: default +--- +Tasks are run on workers and always operate on a single datasource. Once an indexer coordinator node accepts a task, a lock is created for the datasource and interval specified in the task. Tasks do not need to explicitly release locks, they are released upon task completion. Tasks may potentially release locks early if they desire. Tasks ids are unique by naming them using UUIDs or the timestamp in which the task was created. Tasks are also part of a “task group”, which is a set of tasks that can share interval locks. + +There are several different types of tasks. + +Append Task +----------- + +Append tasks append a list of segments together into a single segment (one after the other). The grammar is: + + { + "id": , + "dataSource": , + "segments": + } + +Merge Task +---------- + +Merge tasks merge a list of segments together. Any common timestamps are merged. The grammar is: + + { + "id": , + "dataSource": , + "segments": + } + +Delete Task +----------- + +Delete tasks create empty segments with no data. The grammar is: + + { + "id": , + "dataSource": , + "segments": + } + +Kill Task +--------- + +Kill tasks delete all information about a segment and removes it from deep storage. Killable segments must be disabled (used==0) in the Druid segment table. The available grammar is: + + { + "id": , + "dataSource": , + "segments": + } + +Index Task +---------- + +Index Partitions Task +--------------------- + +Index Generator Task +-------------------- + +Index Hadoop Task +----------------- + +Index Realtime Task +------------------- + +Version Converter Task +---------------------- + +Version Converter SubTask +------------------------- diff --git a/docs/Thanks.md b/docs/Thanks.md new file mode 100644 index 000000000000..cb1c873cca0a --- /dev/null +++ b/docs/Thanks.md @@ -0,0 +1,11 @@ +--- +layout: default +--- +YourKit supports the Druid open source projects with its +full-featured Java Profiler. +YourKit, LLC is the creator of innovative and intelligent tools for profiling +Java and .NET applications. Take a look at YourKit's software products: +
YourKit Java +Profiler and +YourKit .NET +Profiler. \ No newline at end of file diff --git a/docs/TimeBoundaryQuery.md b/docs/TimeBoundaryQuery.md new file mode 100644 index 000000000000..bde4ca1c8126 --- /dev/null +++ b/docs/TimeBoundaryQuery.md @@ -0,0 +1,29 @@ +--- +layout: default +--- +Time boundary queries return the earliest and latest data points of a data set. The grammar is: + + { + "queryType" : "timeBoundary", + "dataSource": "sample_datasource" + } + + +There are 3 main parts to a time boundary query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “timeBoundary”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +The format of the result is: + + [ { + "timestamp" : "2013-05-09T18:24:00.000Z", + "result" : { + "minTime" : "2013-05-09T18:24:00.000Z", + "maxTime" : "2013-05-09T18:37:00.000Z" + } + } ] + diff --git a/docs/TimeseriesQuery.md b/docs/TimeseriesQuery.md new file mode 100644 index 000000000000..62ebcee59f11 --- /dev/null +++ b/docs/TimeseriesQuery.md @@ -0,0 +1,118 @@ +--- +layout: default +--- +Timeseries queries +================== + +These types of queries take a timeseries query object and return an array of JSON objects where each object represents a value asked for by the timeseries query. + +An example timeseries query object is shown below: + +
+
+{
+ [queryType]() “timeseries”,
+ [dataSource]() “sample\_datasource”,
+ [granularity]() “day”,
+ [filter]() {
+ [type]() “and”,
+ [fields]() [
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension1”,
+ [value]() “sample\_value1”
+ },
+ {
+ [type]() “or”,
+ [fields]() [
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension2”,
+ [value]() “sample\_value2”
+ },
+ {
+ [type]() “selector”,
+ [dimension]() “sample\_dimension3”,
+ [value]() “sample\_value3”
+ }
+ ]
+ }
+ ]
+ },
+ [aggregations]() [
+ {
+ [type]() “longSum”,
+ [name]() “sample\_name1”,
+ [fieldName]() “sample\_fieldName1”
+ },
+ {
+ [type]() “doubleSum”,
+ [name]() “sample\_name2”,
+ [fieldName]() “sample\_fieldName2”
+ }
+ ],
+ [postAggregations]() [
+ {
+ [type]() “arithmetic”,
+ [name]() “sample\_divide”,
+ [fn]() “/”,
+ [fields]() [
+ {
+ [type]() “fieldAccess”,
+ [name]() “sample\_name1”,
+ [fieldName]() “sample\_fieldName1”
+ },
+ {
+ [type]() “fieldAccess”,
+ [name]() “sample\_name2”,
+ [fieldName]() “sample\_fieldName2”
+ }
+ ]
+ }
+ ],
+ [intervals]() [
+ “2012-01-01T00:00:00.000/2012-01-03T00:00:00.000”
+ ]
+}
+
+
+ + +There are 7 main parts to a timeseries query: + +|property|description|required?| +|--------|-----------|---------| +|queryType|This String should always be “timeseries”; this is the first thing Druid looks at to figure out how to interpret the query|yes| +|dataSource|A String defining the data source to query, very similar to a table in a relational database|yes| +|granularity|Defines the granularity of the query. See [Granularities](Granularities.html)|yes| +|filter|See [Filters](Filters.html)|no| +|aggregations|See [Aggregations](Aggregations.html)|yes| +|postAggregations|See [Post Aggregations](Post-Aggregations.html)|no| +|intervals|A JSON Object representing ISO-8601 Intervals. This defines the time ranges to run the query over.|yes| +|context|An additional JSON Object which can be used to specify certain flags.|no| + +To pull it all together, the above query would return 2 data points, one for each day between 2012-01-01 and 2012-01-03, from the “sample\_datasource” table. Each data point would be the (long) sum of sample\_fieldName1, the (double) sum of sample\_fieldName2 and the (double) the result of sample\_fieldName1 divided by sample\_fieldName2 for the filter set. The output looks like this: + +
+
+[
+ {
+ [timestamp]() “2012-01-01T00:00:00.000Z”,
+ [result]() {
+ [sample\_name1]() ,
+ [sample\_name2]() ,
+ [sample\_divide]() 
+ }
+ },
+ {
+ [timestamp]() “2012-01-02T00:00:00.000Z”,
+ [result]() {
+ [sample\_name1]() ,
+ [sample\_name2]() ,
+ [sample\_divide]() 
+ }
+ }
+]
+
+
+ diff --git a/docs/Tutorial:-A-First-Look-at-Druid.md b/docs/Tutorial:-A-First-Look-at-Druid.md new file mode 100644 index 000000000000..987cf89fa28f --- /dev/null +++ b/docs/Tutorial:-A-First-Look-at-Druid.md @@ -0,0 +1,369 @@ +--- +layout: default +--- +Greetings! This tutorial will help clarify some core Druid concepts. We will use a realtime dataset and issue some basic Druid queries. If you are ready to explore Druid, and learn a thing or two, read on! + +About the data +-------------- + +The data source we’ll be working with is Wikipedia edits. Each time an edit is made in Wikipedia, an event gets pushed to an IRC channel associated with the language of the Wikipedia page. We scrape IRC channels for several different languages and load this data into Druid. + +Each event has a timestamp indicating the time of the edit (in UTC time), a list of dimensions indicating various metadata about the event (such as information about the user editing the page and where the user resides), and a list of metrics associated with the event (such as the number of characters added and deleted). + +Specifically. the data schema looks like so: + +Dimensions (things to filter on): +\`\`\`json +“page” +“language” +“user” +“unpatrolled” +“newPage” +“robot” +“anonymous” +“namespace” +“continent” +“country” +“region” +“city” +\`\`\` + +Metrics (things to aggregate over): +\`\`\`json +“count” +“added” +“delta” +“deleted” +\`\`\` + +These metrics track the number of characters added, deleted, and changed. + +Setting Up +---------- + +There are two ways to setup Druid: download a tarball, or [Build From Source](Build From Source.html). You only need to do one of these. + +### Download a Tarball + +We’ve built a tarball that contains everything you’ll need. You’ll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.5.54-bin.tar.gz) +Download this file to a directory of your choosing. + +You can extract the awesomeness within by issuing: + + tar -zxvf druid-services-*-bin.tar.gz + +Not too lost so far right? That’s great! If you cd into the directory: + + cd druid-services-0.5.54 + +You should see a bunch of files: +\* run\_example\_server.sh +\* run\_example\_client.sh +\* LICENSE, config, examples, lib directories + +Running Example Scripts +----------------------- + +Let’s start doing stuff. You can start a Druid [Realtime](Realtime.html) node by issuing: + + ./run_example_server.sh + +Select “wikipedia”. + +Once the node starts up you will see a bunch of logs about setting up properties and connecting to the data source. If everything was successful, you should see messages of the form shown below. + + + 2013-07-19 21:54:05,154 INFO [main] com.metamx.druid.realtime.RealtimeNode - Starting Jetty + 2013-07-19 21:54:05,154 INFO [main] org.mortbay.log - jetty-6.1.x + 2013-07-19 21:54:05,171 INFO [chief-wikipedia] com.metamx.druid.realtime.plumber.RealtimePlumberSchool - Expect to run at [2013-07-19T22:03:00.000Z] + 2013-07-19 21:54:05,246 INFO [main] org.mortbay.log - Started SelectChannelConnector@0.0.0.0:8083 + + +The Druid real time-node ingests events in an in-memory buffer. Periodically, these events will be persisted to disk. If you are interested in the details of our real-time architecture and why we persist indexes to disk, I suggest you read our [White Paper](http://static.druid.io/docs/druid.pdf). + +Okay, things are about to get real(~~time). To query the real-time node you’ve spun up, you can issue: +\./run\_example\_client.sh\ +Select “wikipedia” once again. This script issues ]s to the data we’ve been ingesting. The query looks like this: +\`\`\`json +{ + [queryType]("groupBy"), + [dataSource]("wikipedia"), + [granularity]("minute"), + [dimensions]([) + “page” + ], + [aggregations]([) + { + [type]("count"), + [name]("rows") + }, + { + [type]("longSum"), + [fieldName]("edit_count"), + [name]("count") + } + ], + [filter]({) + [type]("selector"), + [dimension]("namespace"), + [value]("article") + }, + [intervals]([) + “2013-06-01T00:00/2020-01-01T00” + ] +} +\`\`\` +This is a **groupBy** query, which you may be familiar with from SQL. We are grouping, or aggregating, via the **dimensions** field: . We are **filtering** via the **“namespace”** dimension, to only look at edits on **“articles”**. Our **aggregations** are what we are calculating: a count of the number of data rows, and a count of the number of edits that have occurred. +The result looks something like this: +\`\`\`json +[ + { + [version]() “v1”, + [timestamp]() “2013-09-04T21:44:00.000Z”, + [event]() { + [count]() 0, + [page]() “2013\\u201314\_Brentford\_F.C.*season", + [rows]() 1 + } + }, + { + [version]() "v1", + [timestamp]() "2013-09-04T21:44:00.000Z", + [event]() { + [count]() 0, + [page]() "8e*00e9tape\_du\_Tour\_de\_France\_2013”, + [rows]() 1 + } + }, + { + [version]() “v1”, + [timestamp]() “2013-09-04T21:44:00.000Z”, + [event]() { + [count]() 0, + [page]() “Agenda\_of\_the\_Tea\_Party\_movement”, + [rows]() 1 + } + }, +… +\`\`\` +This groupBy query is a bit complicated and we’ll return to it later. For the time being, just make sure you are getting some blocks of data back. If you are having problems, make sure you have [curl](http://curl.haxx.se/) installed. Control+C to break out of the client script. +h2. Querying Druid +In your favorite editor, create the file: +\time\_boundary\_query.body\ +Druid queries are JSON blobs which are relatively painless to create programmatically, but an absolute pain to write by hand. So anyway, we are going to create a Druid query by hand. Add the following to the file you just created: +\ +{ + [queryType]() “timeBoundary”, + [dataSource]() “wikipedia” +} +\ +The ] is one of the simplest Druid queries. To run the query, you can issue: +\ curl~~X POST ‘http://localhost:8083/druid/v2/?pretty’ ~~H ‘content-type: application/json’~~d ```` time_boundary_query.body + +We get something like this JSON back: + +```json +[ { + "timestamp" : "2013-09-04T21:44:00.000Z", + "result" : { + "minTime" : "2013-09-04T21:44:00.000Z", + "maxTime" : "2013-09-04T21:47:00.000Z" + } +} ] +``` +As you can probably tell, the result is indicating the maximum and minimum timestamps we've seen thus far (summarized to a minutely granularity). Let's explore a bit further. + +Return to your favorite editor and create the file: +
timeseries_query.body
+ +We are going to make a slightly more complicated query, the [TimeseriesQuery](TimeseriesQuery.html). Copy and paste the following into the file: +

+{
+    "queryType": "timeseries", 
+    "dataSource": "wikipedia", 
+    "intervals": [
+        "2010-01-01/2020-01-01"
+    ], 
+    "granularity": "all", 
+    "aggregations": [
+        {
+            "type": "longSum", 
+            "fieldName": "count",
+            "name": "edit_count"
+        }, 
+        {
+            "type": "doubleSum", 
+            "fieldName": "added", 
+            "name": "chars_added"
+        }
+    ]
+}
+
+ +You are probably wondering, what are these [Granularities](Granularities.html) and [Aggregations](Aggregations.html) things? What the query is doing is aggregating some metrics over some span of time. +To issue the query and get some results, run the following in your command line: +
curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json'  -d  ````timeseries\_query.body
+
+
+Once again, you should get a JSON blob of text back with your results, that looks something like this: + +\`\`\`json +[ { + “timestamp” : “2013-09-04T21:44:00.000Z”, + “result” : { + “chars\_added” : 312670.0, + “edit\_count” : 733 + } +} ] +\`\`\` + +If you issue the query again, you should notice your results updating. + +Right now all the results you are getting back are being aggregated into a single timestamp bucket. What if we wanted to see our aggregations on a per minute basis? What field can we change in the query to accomplish this? + +If you loudly exclaimed “we can change granularity to minute”, you are absolutely correct! We can specify different granularities to bucket our results, like so: + + + { + "queryType": "timeseries", + "dataSource": "wikipedia", + "intervals": [ + "2010-01-01/2020-01-01" + ], + "granularity": "minute", + "aggregations": [ + { + "type": "longSum", + "fieldName": "count", + "name": "edit_count" + }, + { + "type": "doubleSum", + "fieldName": "added", + "name": "chars_added" + } + ] + } + + +This gives us something like the following: + +\`\`\`json +[ + { + “timestamp” : “2013-09-04T21:44:00.000Z”, + “result” : { + “chars\_added” : 30665.0, + “edit\_count” : 128 + } + }, { + “timestamp” : “2013-09-04T21:45:00.000Z”, + “result” : { + “chars\_added” : 122637.0, + “edit\_count” : 167 + } + }, { + “timestamp” : “2013-09-04T21:46:00.000Z”, + “result” : { + “chars\_added” : 78938.0, + “edit\_count” : 159 + } + }, +… +\`\`\` + +Solving a Problem +----------------- + +One of Druid’s main powers is to provide answers to problems, so let’s pose a problem. What if we wanted to know what the top pages in the US are, ordered by the number of edits over the last few minutes you’ve been going through this tutorial? To solve this problem, we have to return to the query we introduced at the very beginning of this tutorial, the [GroupByQuery](GroupByQuery.html). It would be nice if we could group by results by dimension value and somehow sort those results… and it turns out we can! + +Let’s create the file: + + group_by_query.body + and put the following in there: +

+    {
+        "queryType": "groupBy", 
+        "dataSource": "wikipedia", 
+        "granularity": "all", 
+        "dimensions": [
+            "page"
+        ], 
+        "orderBy": {
+            "type": "default", 
+            "columns": [
+                {
+                    "dimension": "edit_count", 
+                    "direction": "DESCENDING"
+                }
+            ], 
+            "limit": 10
+        }, 
+        "aggregations": [
+            {
+                "type": "longSum", 
+                "fieldName": "count", 
+                "name": "edit_count"
+            }
+        ], 
+        "filter": {
+            "type": "selector", 
+            "dimension": "country", 
+            "value": "United States"
+        }, 
+        "intervals": [
+            "2012-10-01T00:00/2020-01-01T00"
+        ]
+    }
+    
+
+Woah! Our query just got a way more complicated. Now we have these [Filters](Filters.html) things and this [OrderBy](OrderBy.html) thing. Fear not, it turns out the new objects we’ve introduced to our query can help define the format of our results and provide an answer to our question.
+
+If you issue the query:
+
+    curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json'  -d @group_by_query.body
+
+You should see an answer to our question. As an example, some results are shown below:
+
+\`\`\`json
+[
+ {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “page” : “RTC\_Transit”,
+ “edit\_count” : 6
+ }
+ }, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “page” : “List\_of\_Deadly\_Women\_episodes”,
+ “edit\_count” : 4
+ }
+ }, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “page” : “User\_talk:David\_Biddulph”,
+ “edit\_count” : 4
+ }
+ },
+…
+\`\`\`
+
+Feel free to tweak other query parameters to answer other questions you may have about the data.
+
+Next Steps
+----------
+
+What to know even more information about the Druid Cluster? Check out [Tutorial: The Druid Cluster](Tutorial:-The-Druid-Cluster.html)
+
+Druid is even more fun if you load your own data into it! To learn how to load your data, see [Loading Your Data](Loading-Your-Data.html).
+
+Additional Information
+----------------------
+
+This tutorial is merely showcasing a small fraction of what Druid can do. If you are interested in more information about Druid, including setting up a more sophisticated Druid cluster, please read the other links in our wiki.
+
+And thus concludes our journey! Hopefully you learned a thing or two about Druid real-time ingestion, querying Druid, and how Druid can be used to solve problems. If you have additional questions, feel free to post in our [google groups page](http://www.groups.google.com/forum/#!forum/druid-development).
diff --git a/docs/Tutorial:-The-Druid-Cluster.md b/docs/Tutorial:-The-Druid-Cluster.md
new file mode 100644
index 000000000000..282ec9fa7f8d
--- /dev/null
+++ b/docs/Tutorial:-The-Druid-Cluster.md
@@ -0,0 +1,302 @@
+---
+layout: default
+---
+Welcome back! In our first [tutorial](https://github.com/metamx/druid/wiki/Tutorial%3A-A-First-Look-at-Druid), we introduced you to the most basic Druid setup: a single realtime node. We streamed in some data and queried it. Realtime nodes collect very recent data and periodically hand that data off to the rest of the Druid cluster. Some questions about the architecture must naturally come to mind. What does the rest of Druid cluster look like? How does Druid load available static data?
+
+This tutorial will hopefully answer these questions!
+
+In this tutorial, we will set up other types of Druid nodes as well as and external dependencies for a fully functional Druid cluster. The architecture of Druid is very much like the [Megazord](http://www.youtube.com/watch?v=7mQuHh1X4H4) from the popular 90s show Mighty Morphin' Power Rangers. Each Druid node has a specific purpose and the nodes come together to form a fully functional system.
+
+## Downloading Druid ##
+
+If you followed the first tutorial, you should already have Druid downloaded. If not, let's go back and do that first.
+
+You can download the latest version of druid [here](http://static.druid.io/artifacts/releases/druid-services-0.5.54-bin.tar.gz)
+
+and untar the contents within by issuing:
+```bash
+tar -zxvf druid-services-*-bin.tar.gz
+cd druid-services-*
+```
+
+You can also [Build From Source](Build-From-Source.html).
+
+## External Dependencies ##
+
+Druid requires 3 external dependencies. A "deep" storage that acts as a backup data repository, a relational database such as MySQL to hold configuration and metadata information, and [Apache Zookeeper](http://zookeeper.apache.org/) for coordination among different pieces of the cluster.
+
+For deep storage, we have made a public S3 bucket (static.druid.io) available where data for this particular tutorial can be downloaded. More on the data [later](https://github.com/metamx/druid/wiki/Tutorial-Part-2#the-data).
+
+### Setting up MySQL ###
+
+1. If you don't already have it, download MySQL Community Server here: [http://dev.mysql.com/downloads/mysql/](http://dev.mysql.com/downloads/mysql/)
+2. Install MySQL
+3. Create a druid user and database
+```bash
+mysql -u root
+```
+```sql
+GRANT ALL ON druid.* TO 'druid'@'localhost' IDENTIFIED BY 'diurd';
+CREATE database druid;
+```
+
+### Setting up Zookeeper ###
+```bash
+curl http://www.motorlogy.com/apache/zookeeper/zookeeper-3.4.5/zookeeper-3.4.5.tar.gz -o zookeeper-3.4.5.tar.gz
+tar xzf zookeeper-3.4.5.tar.gz
+cd zookeeper-3.4.5
+cp conf/zoo_sample.cfg conf/zoo.cfg
+./bin/zkServer.sh start
+cd ..
+```
+
+## The Data ##
+
+Similar to the first tutorial, the data we will be loading is based on edits that have occurred on Wikipedia. Every time someone edits a page in Wikipedia, metadata is generated about the editor and edited page. Druid collects each individual event and packages them together in a container known as a [segment](https://github.com/metamx/druid/wiki/Segments). Segments contain data over some span of time. We've prebuilt a segment for this tutorial and will cover making your own segments in other [pages](https://github.com/metamx/druid/wiki/Loading-Your-Data).The segment we are going to work with has the following format:
+
+Dimensions (things to filter on):
+```json
+"page"
+"language"
+"user"
+"unpatrolled"
+"newPage"
+"robot"
+"anonymous"
+"namespace"
+"continent"
+"country"
+"region"
+"city"
+```
+
+Metrics (things to aggregate over):
+```json
+"count"
+"added"
+"delta"
+"deleted"
+```
+
+## The Cluster ##
+
+Let's start up a few nodes and download our data. First things though, let's create a config directory where we will store configs for our various nodes:
+
+```
+mkdir config
+```
+
+If you are interested in learning more about Druid configuration files, check out this [link](https://github.com/metamx/druid/wiki/Configuration). Many aspects of Druid are customizable. For the purposes of this tutorial, we are going to use default values for most things.
+
+### Start a Master Node ###
+
+Master nodes are in charge of load assignment and distribution. Master nodes monitor the status of the cluster and command compute nodes to assign and drop segments.
+
+To create the master config file:
+
+```
+mkdir config/master
+```
+
+Under the directory we just created, create the file ```runtime.properties``` with the following contents:
+
+```
+druid.host=127.0.0.1:8082
+druid.port=8082
+druid.service=master
+
+# logging
+com.metamx.emitter.logging=true
+com.metamx.emitter.logging.level=info
+
+# zk
+druid.zk.service.host=localhost
+druid.zk.paths.base=/druid
+druid.zk.paths.discoveryPath=/druid/discoveryPath
+
+# aws (demo user)
+com.metamx.aws.accessKey=AKIAIMKECRUYKDQGR6YQ
+com.metamx.aws.secretKey=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
+
+# db
+druid.database.segmentTable=segments
+druid.database.user=druid
+druid.database.password=diurd
+druid.database.connectURI=jdbc:mysql://localhost:3306/druid
+druid.database.ruleTable=rules
+druid.database.configTable=config
+
+# master runtime configs
+druid.master.startDelay=PT60S
+```
+
+To start the master node:
+
+```bash
+java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/*:config/master com.metamx.druid.http.MasterMain
+```
+
+### Start a Compute Node ###
+
+Compute nodes are the workhorses of a cluster and are in charge of loading historical segments and making them available for queries. Our Wikipedia segment will be downloaded by a compute node.
+
+To create the compute config file:
+
+```
+mkdir config/compute
+```
+
+Under the directory we just created, create the file ```runtime.properties``` with the following contents:
+```
+druid.host=127.0.0.1:8081
+druid.port=8081
+druid.service=compute
+
+# logging
+com.metamx.emitter.logging=true
+com.metamx.emitter.logging.level=info
+
+# zk
+druid.zk.service.host=localhost
+druid.zk.paths.base=/druid
+druid.zk.paths.discoveryPath=/druid/discoveryPath
+
+# processing
+druid.processing.buffer.sizeBytes=10000000
+
+# aws (demo user)
+com.metamx.aws.accessKey=AKIAIMKECRUYKDQGR6YQ
+com.metamx.aws.secretKey=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
+
+# Path on local FS for storage of segments; dir will be created if needed
+druid.paths.indexCache=/tmp/druid/indexCache
+
+# Path on local FS for storage of segment metadata; dir will be created if needed
+druid.paths.segmentInfoCache=/tmp/druid/segmentInfoCache
+
+# server
+druid.server.maxSize=100000000
+```
+
+To start the compute node:
+
+```bash
+java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/*:config/compute com.metamx.druid.http.ComputeMain
+```
+
+### Start a Broker Node ###
+
+Broker nodes are responsible for figuring out which compute and/or realtime nodes correspond to which queries. They also merge partial results from these nodes in a scatter/gather fashion.
+
+To create the broker config file:
+
+```
+mkdir config/broker
+```
+
+Under the directory we just created, create the file ```runtime.properties``` with the following contents:
+
+```
+druid.host=127.0.0.1:8080
+druid.port=8080
+druid.service=broker
+
+# logging
+com.metamx.emitter.logging=true
+com.metamx.emitter.logging.level=info
+
+# zk
+druid.zk.service.host=localhost
+druid.zk.paths.base=/druid
+druid.zk.paths.discoveryPath=/druid/discoveryPath
+
+# thread pool size for servicing queries
+druid.client.http.connections=10
+```
+
+To start the broker node:
+
+```bash
+java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/*:config/broker com.metamx.druid.http.BrokerMain
+```
+
+## Loading the Data ##
+
+The MySQL dependency we introduced earlier on contains a 'segments' table that contains entries for segments that should be loaded into our cluster. The Druid master compares this table with segments that already exist in the cluster to determine what should be loaded and dropped. To load our wikipedia segment, we need to create an entry in our MySQL segment table.
+
+Usually, when new segments are created, these MySQL entries are created directly so you never have to do this by hand. For this tutorial, we can do this manually by going back into MySQL and issuing:
+
+```
+use druid;
+```
+
+``
+INSERT INTO segments (id, dataSource, created_date, start, end, partitioned, version, used, payload) VALUES ('wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z', 'wikipedia', '2013-08-08T21:26:23.799Z', '2013-08-01T00:00:00.000Z', '2013-08-02T00:00:00.000Z', '0', '2013-08-08T21:22:48.989Z', '1', '{\"dataSource\":\"wikipedia\",\"interval\":\"2013-08-01T00:00:00.000Z/2013-08-02T00:00:00.000Z\",\"version\":\"2013-08-08T21:22:48.989Z\",\"loadSpec\":{\"type\":\"s3_zip\",\"bucket\":\"static.druid.io\",\"key\":\"data/segments/wikipedia/20130801T000000.000Z_20130802T000000.000Z/2013-08-08T21_22_48.989Z/0/index.zip\"},\"dimensions\":\"dma_code,continent_code,geo,area_code,robot,country_name,network,city,namespace,anonymous,unpatrolled,page,postal_code,language,newpage,user,region_lookup\",\"metrics\":\"count,delta,variation,added,deleted\",\"shardSpec\":{\"type\":\"none\"},\"binaryVersion\":9,\"size\":24664730,\"identifier\":\"wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z\"}');
+``
+
+If you look in your master node logs, you should, after a maximum of a minute or so, see logs of the following form:
+
+```
+2013-08-08 22:48:41,967 INFO [main-EventThread] com.metamx.druid.master.LoadQueuePeon - Server[/druid/loadQueue/127.0.0.1:8081] done processing [/druid/loadQueue/127.0.0.1:8081/wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z]
+2013-08-08 22:48:41,969 INFO [ServerInventoryView-0] com.metamx.druid.client.SingleServerInventoryView - Server[127.0.0.1:8081] added segment[wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z]
+```
+
+When the segment completes downloading and ready for queries, you should see the following message on your compute node logs:
+
+```
+2013-08-08 22:48:41,959 INFO [ZkCoordinator-0] com.metamx.druid.coordination.BatchDataSegmentAnnouncer - Announcing segment[wikipedia_2013-08-01T00:00:00.000Z_2013-08-02T00:00:00.000Z_2013-08-08T21:22:48.989Z] at path[/druid/segments/127.0.0.1:8081/2013-08-08T22:48:41.959Z]
+```
+
+At this point, we can query the segment. For more information on querying, see this[link](https://github.com/metamx/druid/wiki/Querying).
+
+## Next Steps ##
+
+Now that you have an understanding of what the Druid clsuter looks like, why not load some of your own data?
+Check out the [Loading Your Own Data](https://github.com/metamx/druid/wiki/Loading-Your-Data) section for more info!
\ No newline at end of file
diff --git a/docs/Tutorial:-Webstream.md b/docs/Tutorial:-Webstream.md
new file mode 100644
index 000000000000..bfb7ed73bed3
--- /dev/null
+++ b/docs/Tutorial:-Webstream.md
@@ -0,0 +1,357 @@
+---
+layout: default
+---
+Greetings! This tutorial will help clarify some core Druid concepts. We will use a realtime dataset and issue some basic Druid queries. If you are ready to explore Druid, and learn a thing or two, read on!
+
+About the data
+--------------
+
+The data source we’ll be working with is the Bit.ly USA Government website statistics stream. You can see the stream [here](http://developer.usa.gov/1usagov), and read about the stream [here](http://www.usa.gov/About/developer-resources/1usagov.shtml) . This is a feed of json data that gets updated whenever anyone clicks a bit.ly shortened USA.gov website. A typical event might look something like this:
+\`\`\`json
+{
+ [user\_agent]() “Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)”,
+ [country]() “US”,
+ [known\_user]() 1,
+ [timezone]() “America/New\_York”,
+ [geo\_region]() “DC”,
+ [global\_bitly\_hash]() “17ctAFs”,
+ [encoding\_user\_bitly\_hash]() “17ctAFr”,
+ [encoding\_user\_login]() “senrubiopress”,
+ [aaccept\_language]() “en-US”,
+ [short\_url\_cname]() “1.usa.gov”,
+ [referring\_url]() “http://t.co/4Av4NUFAYq”,
+ [long\_url]() “http://www.rubio.senate.gov/public/index.cfm/fighting-for-florida?ID=c8357d12-9da8-4e9d-b00d-7168e1bf3599”,
+ [timestamp]() 1372190407,
+ [timestamp of time hash was created]() 1372190097,
+ [city]() “Washington”,
+ [latitude\_longitude]() [
+ 38.893299,
+ ~~77.014603
+ ]
+}
+\`\`\`
+The “known\_user” field is always 1 or 0. It is 1 if the user is known to the server, and 0 otherwise. We will use this field extensively in this demo.
+h2. Setting Up
+There are two ways to setup Druid: download a tarball, or ]. You only need to do one of these.
+h3. Download a Tarball
+We’ve built a tarball that contains everything you’ll need. You’ll find it [here](http://static.druid.io/artifacts/releases/druid-services-0.5.50-bin.tar.gz)
+Download this file to a directory of your choosing.
+You can extract the awesomeness within by issuing:
+\tar~~zxvf druid-services~~**~~bin.tar.gz\
+Not too lost so far right? That’s great! If you cd into the directory:
+\cd druid-services-0.5.50\
+You should see a bunch of files:
+\* run\_example\_server.sh
+\* run\_example\_client.sh
+\* LICENSE, config, examples, lib directories
+
+h2. Running Example Scripts
+Let’s start doing stuff. You can start a Druid ] node by issuing:
+\./run\_example\_server.sh\
+Select “webstream”.
+Once the node starts up you will see a bunch of logs about setting up properties and connecting to the data source. If everything was successful, you should see messages of the form shown below.
+\
+2013-07-19 21:54:05,154 INFO com.metamx.druid.realtime.RealtimeNode~~ Starting Jetty
+2013-07-19 21:54:05,154 INFO org.mortbay.log - jetty-6.1.x
+2013-07-19 21:54:05,171 INFO com.metamx.druid.realtime.plumber.RealtimePlumberSchool - Expect to run at
+2013-07-19 21:54:05,246 INFO org.mortbay.log - Started SelectChannelConnector@0.0.0.0:8083
+\
+The Druid real time-node ingests events in an in-memory buffer. Periodically, these events will be persisted to disk. If you are interested in the details of our real-time architecture and why we persist indexes to disk, I suggest you read our [White Paper](http://static.druid.io/docs/druid.pdf).
+Okay, things are about to get real. To query the real-time node you’ve spun up, you can issue:
+\./run\_example\_client.sh\
+Select “webstream” once again. This script issues ]s to the data we’ve been ingesting. The query looks like this:
+\`\`\`json
+{
+ [queryType]() “groupBy”,
+ [dataSource]() “webstream”,
+ [granularity]() “minute”,
+ [dimensions]() [
+ “timezone”
+ ],
+ [aggregations]() [
+ {
+ [type]() “count”,
+ [name]() “rows”
+ },
+ {
+ [type]() “doubleSum”,
+ [fieldName]() “known\_users”,
+ [name]() “known\_users”
+ }
+ ],
+ [filter]() {
+ [type]() “selector”,
+ [dimension]() “country”,
+ [value]() “US”
+ },
+ [intervals]() [
+ “2013-06-01T00:00/2020-01-01T00”
+ ]
+}
+\`\`\`
+This is a****groupBy**\* query, which you may be familiar with from SQL. We are grouping, or aggregating, via the **dimensions** field: . We are **filtering** via the **“country”** dimension, to only look at website hits in the US. Our **aggregations** are what we are calculating: a row count, and the sum of the number of known users in our data.
+The result looks something like this:
+\`\`\`json
+[
+ {
+ [version]() “v1”,
+ [timestamp]() “2013-07-18T19:39:00.000Z”,
+ [event]() {
+ [timezone]() “America/Chicago”,
+ [known\_users]() 10,
+ [rows]() 15
+ }
+ },
+ {
+ [version]() “v1”,
+ [timestamp]() “2013-07-18T19:39:00.000Z”,
+ [event]() {
+ [timezone]() “America/Los\_Angeles”,
+ [known\_users]() 0,
+ [rows]() 3
+ }
+ },
+…
+\`\`\`
+This groupBy query is a bit complicated and we’ll return to it later. For the time being, just make sure you are getting some blocks of data back. If you are having problems, make sure you have [curl](http://curl.haxx.se/) installed. Control+C to break out of the client script.
+h2. Querying Druid
+In your favorite editor, create the file:
+\time\_boundary\_query.body\
+Druid queries are JSON blobs which are relatively painless to create programmatically, but an absolute pain to write by hand. So anyway, we are going to create a Druid query by hand. Add the following to the file you just created:
+\
+{
+ [queryType]() “timeBoundary”,
+ [dataSource]() “webstream”
+}
+\
+The ] is one of the simplest Druid queries. To run the query, you can issue:
+\ curl~~X POST ‘http://localhost:8083/druid/v2/?pretty’ ~~H ‘content-type: application/json’~~d ```` time_boundary_query.body
+ +We get something like this JSON back: + +```json +[ + { + "timestamp": "2013-07-18T19:39:00.000Z", + "result": { + "minTime": "2013-07-18T19:39:00.000Z", + "maxTime": "2013-07-18T19:46:00.000Z" + } + } +] +``` +As you can probably tell, the result is indicating the maximum and minimum timestamps we've seen thus far (summarized to a minutely granularity). Let's explore a bit further. + +Return to your favorite editor and create the file: +
timeseries_query.body
+ +We are going to make a slightly more complicated query, the [TimeseriesQuery](TimeseriesQuery.html). Copy and paste the following into the file: +

+{
+    "queryType": "timeseries", 
+    "dataSource": "webstream", 
+    "intervals": [
+        "2010-01-01/2020-01-01"
+    ], 
+    "granularity": "all", 
+    "aggregations": [
+        {
+            "type": "count", 
+            "name": "rows"
+        }, 
+        {
+            "type": "doubleSum", 
+            "fieldName": "known_users", 
+            "name": "known_users"
+        }
+    ]
+}
+
+ +You are probably wondering, what are these [Granularities](Granularities.html) and [Aggregations](Aggregations.html) things? What the query is doing is aggregating some metrics over some span of time. +To issue the query and get some results, run the following in your command line: +
curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json'  -d  ````timeseries\_query.body
+
+
+Once again, you should get a JSON blob of text back with your results, that looks something like this: + +\`\`\`json +[ +{ + “timestamp” : “2013-07-18T19:39:00.000Z”, + “result” : { + “known\_users” : 787.0, + “rows” : 2004 + } +} +] +\`\`\` + +If you issue the query again, you should notice your results updating. + +Right now all the results you are getting back are being aggregated into a single timestamp bucket. What if we wanted to see our aggregations on a per minute basis? What field can we change in the query to accomplish this? + +If you loudly exclaimed “we can change granularity to minute”, you are absolutely correct! We can specify different granularities to bucket our results, like so: + + + { + "queryType": "timeseries", + "dataSource": "webstream", + "intervals": [ + "2010-01-01/2020-01-01" + ], + "granularity": "minute", + "aggregations": [ + { + "type": "count", + "name": "rows" + }, + { + "type": "doubleSum", + "fieldName": "known_users", + "name": "known_users" + } + ] + } + + +This gives us something like the following: + +\`\`\`json +[ + { + [timestamp]() “2013-07-18T19:39:00.000Z”, + [result]() { + [known\_users]() 33, + [rows]() 76 + } + }, + { + [timestamp]() “2013-07-18T19:40:00.000Z”, + [result]() { + [known\_users]() 105, + [rows]() 221 + } + }, + { + [timestamp]() “2013-07-18T19:41:00.000Z”, + [result]() { + [known\_users]() 53, + [rows]() 167 + } + }, +… +\`\`\` + +Solving a Problem +----------------- + +One of Druid’s main powers is to provide answers to problems, so let’s pose a problem. What if we wanted to know what the top states in the US are, ordered by the number of visits by known users over the last few minutes? To solve this problem, we have to return to the query we introduced at the very beginning of this tutorial, the [GroupByQuery](GroupByQuery.html). It would be nice if we could group by results by dimension value and somehow sort those results… and it turns out we can! + +Let’s create the file: + + group_by_query.body + and put the following in there: +

+    {
+        "queryType": "groupBy", 
+        "dataSource": "webstream", 
+        "granularity": "all", 
+        "dimensions": [
+            "geo_region"
+        ], 
+        "orderBy": {
+            "type": "default", 
+            "columns": [
+                {
+                    "dimension": "known_users", 
+                    "direction": "DESCENDING"
+                }
+            ], 
+            "limit": 10
+        }, 
+        "aggregations": [
+            {
+                "type": "count", 
+                "name": "rows"
+            }, 
+            {
+                "type": "doubleSum", 
+                "fieldName": "known_users", 
+                "name": "known_users"
+            }
+        ], 
+        "filter": {
+            "type": "selector", 
+            "dimension": "country", 
+            "value": "US"
+        }, 
+        "intervals": [
+            "2012-10-01T00:00/2020-01-01T00"
+        ]
+    }
+    
+
+Woah! Our query just got a way more complicated. Now we have these [Filters](Filters.html) things and this [OrderBy](OrderBy.html) thing. Fear not, it turns out the new objects we’ve introduced to our query can help define the format of our results and provide an answer to our question.
+
+If you issue the query:
+
+    curl -X POST 'http://localhost:8083/druid/v2/?pretty' -H 'content-type: application/json'  -d @group_by_query.body
+
+You should see an answer to our question. For my stream, it looks like this:
+
+\`\`\`json
+[
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [geo\_region]() “RI”,
+ [known\_users]() 359,
+ [rows]() 143
+ }
+ },
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [geo\_region]() “NY”,
+ [known\_users]() 187,
+ [rows]() 322
+ }
+ },
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [geo\_region]() “CA”,
+ [known\_users]() 145,
+ [rows]() 466
+ }
+ },
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [geo\_region]() “IL”,
+ [known\_users]() 121,
+ [rows]() 185
+ }
+ },
+…
+\`\`\`
+
+Feel free to tweak other query parameters to answer other questions you may have about the data.
+
+Next Steps
+----------
+
+What to know even more information about the Druid Cluster? Check out [Tutorial: The Druid Cluster](Tutorial:-The-Druid-Cluster.html)
+Druid is even more fun if you load your own data into it! To learn how to load your data, see [Loading Your Data](Loading-Your-Data.html).
+
+Additional Information
+----------------------
+
+This tutorial is merely showcasing a small fraction of what Druid can do. If you are interested in more information about Druid, including setting up a more sophisticated Druid cluster, please read the other links in our wiki.
+
+And thus concludes our journey! Hopefully you learned a thing or two about Druid real-time ingestion, querying Druid, and how Druid can be used to solve problems. If you have additional questions, feel free to post in our [google groups page](http://www.groups.google.com/forum/#!forum/druid-development).
diff --git a/docs/Twitter-Tutorial.md b/docs/Twitter-Tutorial.md
new file mode 100644
index 000000000000..dc2151ec3e45
--- /dev/null
+++ b/docs/Twitter-Tutorial.md
@@ -0,0 +1,329 @@
+---
+layout: default
+---
+Greetings! We see you’ve taken an interest in Druid. That’s awesome! Hopefully this tutorial will help clarify some core Druid concepts. We will go through one of the Real-time [Examples](Examples.html), and issue some basic Druid queries. The data source we’ll be working with is the [Twitter spritzer stream](https://dev.twitter.com/docs/streaming-apis/streams/public). If you are ready to explore Druid, brave its challenges, and maybe learn a thing or two, read on!
+
+Setting Up
+----------
+
+There are two ways to setup Druid: download a tarball, or build it from source.
+
+### Download a Tarball
+
+We’ve built a tarball that contains everything you’ll need. You’ll find it [here](http://static.druid.io/data/examples/druid-services-0.4.6.tar.gz).
+Download this bad boy to a directory of your choosing.
+
+You can extract the awesomeness within by issuing:
+
+    tar -zxvf druid-services-0.4.6.tar.gz
+
+Not too lost so far right? That’s great! If you cd into the directory:
+
+    cd druid-services-0.4.6-SNAPSHOT
+
+You should see a bunch of files:
+\* run\_example\_server.sh
+\* run\_example\_client.sh
+\* LICENSE, config, examples, lib directories
+
+### Clone and Build from Source
+
+The other way to setup Druid is from source via git. To do so, run these commands:
+
+\`\`\`
+git clone git@github.com:metamx/druid.git
+cd druid
+git checkout druid-0.4.32-branch
+./build.sh
+\`\`\`
+
+You should see a bunch of files:
+
+\`\`\`
+DruidCorporateCLA.pdf README common examples indexer pom.xml server
+DruidIndividualCLA.pdf build.sh doc group\_by.body install publications services
+LICENSE client eclipse\_formatting.xml index-common merger realtime
+\`\`\`
+
+You can find the example executables in the examples/bin directory:
+\* run\_example\_server.sh
+\* run\_example\_client.sh
+
+Running Example Scripts
+-----------------------
+
+Let’s start doing stuff. You can start a Druid [Realtime](Realtime.html) node by issuing:
+
+    ./run_example_server.sh
+
+Select “twitter”.
+
+You’ll need to register a new application with the twitter API, which only takes a minute. Go to [https://twitter.com/oauth\_clients/new](https://twitter.com/oauth_clients/new) and fill out the form and submit. Don’t worry, the home page and callback url can be anything. This will generate keys for the Twitter example application. Take note of the values for consumer key/secret and access token/secret.
+
+Enter your credentials when prompted.
+
+Once the node starts up you will see a bunch of logs about setting up properties and connecting to the data source. If everything was successful, you should see messages of the form shown below. If you see crazy exceptions, you probably typed in your login information incorrectly.
+
+    
+    2013-05-17 23:04:40,934 INFO [main] org.mortbay.log - Started SelectChannelConnector@0.0.0.0:8080
+    2013-05-17 23:04:40,935 INFO [main] com.metamx.common.lifecycle.Lifecycle$AnnotationBasedHandler - Invoking start method[public void com.metamx.druid.http.FileRequestLogger.start()] on object[com.metamx.druid.http.FileRequestLogger@42bb0406].
+    2013-05-17 23:04:41,578 INFO [Twitter Stream consumer-1[Establishing connection]] twitter4j.TwitterStreamImpl - Connection established.
+    2013-05-17 23:04:41,578 INFO [Twitter Stream consumer-1[Establishing connection]] druid.examples.twitter.TwitterSpritzerFirehoseFactory - Connected_to_Twitter
+    2013-05-17 23:04:41,578 INFO [Twitter Stream consumer-1[Establishing connection]] twitter4j.TwitterStreamImpl - Receiving status stream.
+    
+
+Periodically, you’ll also see messages of the form:
+
+    
+    2013-05-17 23:04:59,793 INFO [chief-twitterstream] druid.examples.twitter.TwitterSpritzerFirehoseFactory - nextRow() has returned 1,000 InputRows
+    
+
+These messages indicate you are ingesting events. The Druid real time-node ingests events in an in-memory buffer. Periodically, these events will be persisted to disk. Persisting to disk generates a whole bunch of logs:
+
+    
+    2013-05-17 23:06:40,918 INFO [chief-twitterstream] com.metamx.druid.realtime.plumber.RealtimePlumberSchool - Submitting persist runnable for dataSource[twitterstream]
+    2013-05-17 23:06:40,920 INFO [twitterstream-incremental-persist] com.metamx.druid.realtime.plumber.RealtimePlumberSchool - DataSource[twitterstream], Interval[2013-05-17T23:00:00.000Z/2013-05-18T00:00:00.000Z], persisting Hydrant[FireHydrant{index=com.metamx.druid.index.v1.IncrementalIndex@126212dd, queryable=com.metamx.druid.index.IncrementalIndexSegment@64c47498, count=0}]
+    2013-05-17 23:06:40,937 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Starting persist for interval[2013-05-17T23:00:00.000Z/2013-05-17T23:07:00.000Z], rows[4,666]
+    2013-05-17 23:06:41,039 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - outDir[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] completed index.drd in 11 millis.
+    2013-05-17 23:06:41,070 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - outDir[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] completed dim conversions in 31 millis.
+    2013-05-17 23:06:41,275 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.CompressedPools - Allocating new chunkEncoder[1]
+    2013-05-17 23:06:41,332 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - outDir[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] completed walk through of 4,666 rows in 262 millis.
+    2013-05-17 23:06:41,334 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Starting dimension[htags] with cardinality[634]
+    2013-05-17 23:06:41,381 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Completed dimension[htags] in 49 millis.
+    2013-05-17 23:06:41,382 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Starting dimension[lang] with cardinality[19]
+    2013-05-17 23:06:41,398 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Completed dimension[lang] in 17 millis.
+    2013-05-17 23:06:41,398 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Starting dimension[utc_offset] with cardinality[32]
+    2013-05-17 23:06:41,413 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - Completed dimension[utc_offset] in 15 millis.
+    2013-05-17 23:06:41,413 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexMerger - outDir[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] completed inverted.drd in 81 millis.
+    2013-05-17 23:06:41,425 INFO [twitterstream-incremental-persist] com.metamx.druid.index.v1.IndexIO$DefaultIndexIOHandler - Converting v8[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0/v8-tmp] to v9[/tmp/example/twitter_realtime/basePersist/twitterstream/2013-05-17T23:00:00.000Z_2013-05-18T00:00:00.000Z/0]
+    2013-05-17 23:06:41,426 INFO [twitterstream-incremental-persist] 
+    ... ETC
+    
+
+The logs are about building different columns, probably not the most exciting stuff (they might as well be in Vulcan) if are you learning about Druid for the first time. Nevertheless, if you are interested in the details of our real-time architecture and why we persist indexes to disk, I suggest you read our [White Paper](http://static.druid.io/docs/druid.pdf).
+
+Okay, things are about to get real (~~time). To query the real-time node you’ve spun up, you can issue:
+\./run\_example\_client.sh\
+Select “twitter” once again. This script issues ]s to the twitter data we’ve been ingesting. The query looks like this:
+\`\`\`json
+{
+ [queryType]() “groupBy”,
+ [dataSource]() “twitterstream”,
+ [granularity]() “all”,
+ [dimensions]() ,
+ [aggregations]([)
+ { [type]() “count”, [name]() “rows”},
+ { [type]() “doubleSum”, [fieldName]() “tweets”, [name]() “tweets”}
+ ],
+ [filter]() { [type]() “selector”, [dimension]() “lang”, [value]() “en” },
+ [intervals](["2012-10-01T00:00/2020-01-01T00"])
+}
+\`\`\`
+This is a **groupBy** query, which you may be familiar with from SQL. We are grouping, or aggregating, via the **dimensions** field: . We are **filtering** via the **“lang”** dimension, to only look at english tweets. Our **aggregations** are what we are calculating: a row count, and the sum of the tweets in our data.
+The result looks something like this:
+\`\`\`json
+[
+ {
+ [version]() “v1”,
+ [timestamp]() “2012-10-01T00:00:00.000Z”,
+ [event]() {
+ [utc\_offset]() “~~10800",
+ [tweets]() 90,
+ [lang]() "en",
+ [rows]() 81
+ }
+ },
+ {
+ [version]() "v1",
+ [timestamp]() "2012-10-01T00:00:00.000Z",
+ [event]() {
+ [utc\_offset]() "~~14400”,
+ [tweets]() 177,
+ [lang]() “en”,
+ [rows]() 154
+ }
+ },
+…
+\`\`\`
+This data, plotted in a time series/distribution, looks something like this:
+![Timezone / Tweets Scatter Plot](http://metamarkets.com/wp-content/uploads/2013/06/tweets_timezone_offset.png "Timezone / Tweets Scatter Plot")
+This groupBy query is a bit complicated and we’ll return to it later. For the time being, just make sure you are getting some blocks of data back. If you are having problems, make sure you have [curl](http://curl.haxx.se/) installed. Control+C to break out of the client script.
+h2. Querying Druid
+In your favorite editor, create the file:
+\time\_boundary\_query.body\
+Druid queries are JSON blobs which are relatively painless to create programmatically, but an absolute pain to write by hand. So anyway, we are going to create a Druid query by hand. Add the following to the file you just created:
+\
+\
+The ] is one of the simplest Druid queries. To run the query, you can issue:
+\ curl~~X POST ‘http://localhost:8080/druid/v2/?pretty’ ~~H ‘content-type: application/json’~~d ```` time_boundary_query.body
+ +We get something like this JSON back: + +```json +[ { + "timestamp" : "2013-06-10T19:09:00.000Z", + "result" : { + "minTime" : "2013-06-10T19:09:00.000Z", + "maxTime" : "2013-06-10T20:50:00.000Z" + } +} ] +``` +That's the result. What information do you think the result is conveying? +... +If you said the result is indicating the maximum and minimum timestamps we've seen thus far (summarized to a minutely granularity), you are absolutely correct. I can see you are a person legitimately interested in learning about Druid. Let's explore a bit further. + +Return to your favorite editor and create the file: +
timeseries_query.body
+ +We are going to make a slightly more complicated query, the [TimeseriesQuery](TimeseriesQuery.html). Copy and paste the following into the file: +
{
+  "queryType":"timeseries",
+  "dataSource":"twitterstream",
+  "intervals":["2010-01-01/2020-01-01"],
+  "granularity":"all",
+  "aggregations":[
+      { "type": "count", "name": "rows"},
+      { "type": "doubleSum", "fieldName": "tweets", "name": "tweets"}
+  ]
+}
+
+ +You are probably wondering, what are these [Granularities](Granularities.html) and [Aggregations](Aggregations.html) things? What the query is doing is aggregating some metrics over some span of time. +To issue the query and get some results, run the following in your command line: +
curl -X POST 'http://localhost:8080/druid/v2/?pretty' -H 'content-type: application/json'  -d  ````timeseries\_query.body
+
+
+Once again, you should get a JSON blob of text back with your results, that looks something like this: + +\`\`\`json +[ { + “timestamp” : “2013-06-10T19:09:00.000Z”, + “result” : { + “tweets” : 358562.0, + “rows” : 272271 + } +} ] +\`\`\` + +If you issue the query again, you should notice your results updating. + +Right now all the results you are getting back are being aggregated into a single timestamp bucket. What if we wanted to see our aggregations on a per minute basis? What field can we change in the query to accomplish this? + +If you loudly exclaimed “we can change granularity to minute”, you are absolutely correct again! We can specify different granularities to bucket our results, like so: + +\`\`\`json +{ + [queryType]("timeseries"), + [dataSource]("twitterstream"), + [intervals](["2010-01-01/2020-01-01"]), + [granularity]("minute"), + [aggregations]([) + { [type]() “count”, [name]() “rows”}, + { [type]() “doubleSum”, [fieldName]() “tweets”, [name]() “tweets”} + ] +} +\`\`\` + +This gives us something like the following: + +\`\`\`json +[ { + “timestamp” : “2013-06-10T19:09:00.000Z”, + “result” : { + “tweets” : 2650.0, + “rows” : 2120 + } +}, { + “timestamp” : “2013-06-10T19:10:00.000Z”, + “result” : { + “tweets” : 3401.0, + “rows” : 2609 + } +}, { + “timestamp” : “2013-06-10T19:11:00.000Z”, + “result” : { + “tweets” : 3472.0, + “rows” : 2610 + } +}, +… +\`\`\` + +Solving a Problem +----------------- + +One of Druid’s main powers (see what we did there?) is to provide answers to problems, so let’s pose a problem. What if we wanted to know what the top hash tags are, ordered by the number tweets, where the language is english, over the last few minutes you’ve been reading this tutorial? To solve this problem, we have to return to the query we introduced at the very beginning of this tutorial, the [GroupByQuery](GroupByQuery.html). It would be nice if we could group by results by dimension value and somehow sort those results… and it turns out we can! + +Let’s create the file: + + group_by_query.body + and put the following in there: +
{
+        "queryType": "groupBy",
+        "dataSource": "twitterstream",
+        "granularity": "all",
+        "dimensions": ["htags"],
+        "orderBy": {"type":"default", "columns":[{"dimension": "tweets", "direction":"DESCENDING"}], "limit":5},
+        "aggregations":[
+          { "type": "longSum", "fieldName": "tweets", "name": "tweets"}
+        ],
+        "filter": {"type": "selector", "dimension": "lang", "value": "en" },
+        "intervals":["2012-10-01T00:00/2020-01-01T00"]
+    }
+    
+
+Woah! Our query just got a way more complicated. Now we have these [Filters](Filters.html) things and this [OrderBy](OrderBy.html) thing. Fear not, it turns out the new objects we’ve introduced to our query can help define the format of our results and provide an answer to our question.
+
+If you issue the query:
+
+    curl -X POST 'http://localhost:8080/druid/v2/?pretty' -H 'content-type: application/json'  -d @group_by_query.body
+
+You should hopefully see an answer to our question. For my twitter stream, it looks like this:
+
+\`\`\`json
+[ {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 2660,
+ “htags” : “android”
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 1944,
+ “htags” : “E3”
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 1927,
+ “htags” : “15SueñosPendientes”
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 1717,
+ “htags” : “ipad”
+ }
+}, {
+ “version” : “v1”,
+ “timestamp” : “2012-10-01T00:00:00.000Z”,
+ “event” : {
+ “tweets” : 1515,
+ “htags” : “IDidntTextYouBackBecause”
+ }
+} ]
+\`\`\`
+
+Feel free to tweak other query parameters to answer other questions you may have about the data.
+
+Additional Information
+----------------------
+
+This tutorial is merely showcasing a small fraction of what Druid can do. Next, continue on to [Loading Your Data](Loading Your Data.html).
+
+And thus concludes our journey! Hopefully you learned a thing or two about Druid real-time ingestion, querying Druid, and how Druid can be used to solve problems. If you have additional questions, feel free to post in our [google groups page](http://www.groups.google.com/forum/#!forum/druid-development).
diff --git a/docs/Versioning.md b/docs/Versioning.md
new file mode 100644
index 000000000000..6b9e79fe9d3e
--- /dev/null
+++ b/docs/Versioning.md
@@ -0,0 +1,24 @@
+---
+layout: default
+---
+This page discusses how we do versioning and provides information on our stable releases.
+
+Versioning Strategy
+-------------------
+
+We generally follow [semantic versioning](http://semver.org/). The general idea is
+
+-   “Major” version (leftmost): backwards incompatible, no guarantees exist about APIs between the versions
+-   “Minor” version (middle number): you can move forward from a smaller number to a larger number, but moving backwards *might* be incompatible.
+-   “bug-fix” version (“patch” or the rightmost): Interchangeable. The higher the number, the more things are fixed (hopefully), but the programming interfaces are completely compatible and you should be able to just drop in a new jar and have it work.
+
+Note that this is defined in terms of programming API, **not** in terms of functionality. It is possible that a brand new awesome way of doing something is introduced in a “bug-fix” release version if it doesn’t add to the public API or change it.
+
+One exception for right now, while we are still in major version 0, we are considering the APIs to be in beta and are conflating “major” and “minor” so a minor version increase could be backwards incompatible for as long as we are at major version 0. These will be communicated via email on the group.
+
+For external deployments, we recommend running the stable release tag. Releases are considered stable after we have deployed them into our production environment and they have operated bug-free for some time.
+
+Tagging strategy
+----------------
+
+Tags of the codebase are equivalent to release candidates. We tag the code every time we want to take it through our release process, which includes some QA cycles and deployments. So, it is not safe to assume that a tag is a stable release, it is a solidification of the code as it goes through our production QA cycle and deployment. Tags will never change, but we often go through a number of iterations of tags before actually getting a stable release onto production. So, it is recommended that if you are not aware of what is on a tag, to stick to the stable releases listed on the [Download](Download.html) page.
diff --git a/docs/ZooKeeper.md b/docs/ZooKeeper.md
new file mode 100644
index 000000000000..d3e24e29ceb5
--- /dev/null
+++ b/docs/ZooKeeper.md
@@ -0,0 +1,66 @@
+---
+layout: default
+---
+Druid uses ZooKeeper (ZK) for management of current cluster state. The operations that happen over ZK are
+
+1.  [Master](Master.html) leader election
+2.  Segment “publishing” protocol from [Compute](Compute.html) and [Realtime](Realtime.html)
+3.  Segment load/drop protocol between [Master](Master.html) and [Compute](Compute.html)
+
+### Property Configuration
+
+ZooKeeper paths are set via the `runtime.properties` configuration file. Druid will automatically create paths that do not exist, so typos in config files is a very easy way to become split-brained.
+
+There is a prefix path that is required and can be used as the only (well, kinda, see the note below) path-related zookeeper configuration parameter (everything else will be a default based on the prefix):
+
+    druid.zk.paths.base
+
+You can also override each individual path (defaults are shown below):
+
+    druid.zk.paths.propertiesPath=${druid.zk.paths.base}/properties
+    druid.zk.paths.announcementsPath=${druid.zk.paths.base}/announcements
+    druid.zk.paths.servedSegmentsPath=${druid.zk.paths.base}/servedSegments
+    druid.zk.paths.loadQueuePath=${druid.zk.paths.base}/loadQueue
+    druid.zk.paths.masterPath=${druid.zk.paths.base}/master
+    druid.zk.paths.indexer.announcementsPath=${druid.zk.paths.base}/indexer/announcements
+    druid.zk.paths.indexer.tasksPath=${druid.zk.paths.base}/indexer/tasks
+    druid.zk.paths.indexer.statusPath=${druid.zk.paths.base}/indexer/status
+    druid.zk.paths.indexer.leaderLatchPath=${druid.zk.paths.base}/indexer/leaderLatchPath
+
+NOTE: We also use Curator’s service discovery module to expose some services via zookeeper. This also uses a zookeeper path, but this path is **not** affected by `druid.zk.paths.base` and **must** be specified separately. This property is
+
+    druid.zk.paths.discoveryPath
+
+### Master Leader Election
+
+We use the Curator LeadershipLatch recipe to do leader election at path
+
+    ${druid.zk.paths.masterPath}/_MASTER
+
+### Segment “publishing” protocol from Compute and Realtime
+
+The `announcementsPath` and `servedSegmentsPath` are used for this.
+
+All [Compute](Compute.html) and [Realtime](Realtime.html) nodes publish themselves on the `announcementsPath`, specifically, they will create an ephemeral znode at
+
+    ${druid.zk.paths.announcementsPath}/${druid.host}
+
+Which signifies that they exist. They will also subsequently create a permanent znode at
+
+    ${druid.zk.paths.servedSegmentsPath}/${druid.host}
+
+And as they load up segments, they will attach ephemeral znodes that look like
+
+    ${druid.zk.paths.servedSegmentsPath}/${druid.host}/_segment_identifier_
+
+Nodes like the [Master](Master.html) and [Broker](Broker.html) can then watch these paths to see which nodes are currently serving which segments.
+
+### Segment load/drop protocol between Master and Compute
+
+The `loadQueuePath` is used for this.
+
+When the [Master](Master.html) decides that a [Compute](Compute.html) node should load or drop a segment, it writes an ephemeral znode to
+
+    ${druid.zk.paths.loadQueuePath}/_host_of_compute_node/_segment_identifier
+
+This node will contain a payload that indicates to the Compute node what it should do with the given segment. When the Compute node is done with the work, it will delete the znode in order to signify to the Master that it is complete.
diff --git a/docs/_config.yml b/docs/_config.yml
new file mode 100644
index 000000000000..6609d8b3dea2
--- /dev/null
+++ b/docs/_config.yml
@@ -0,0 +1,5 @@
+name: Your New Jekyll Site
+pygments: true
+markdown: redcarpet
+redcarpet:
+  extensions: ["no_intra_emphasis", "fenced_code_blocks", "autolink", "tables", "with_toc_data"]
diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html
new file mode 100644
index 000000000000..12106274e0f6
--- /dev/null
+++ b/docs/_layouts/default.html
@@ -0,0 +1,147 @@
+
+
+  	
+    	
+    Druid | {{page.title}}	
+    
+    
+    
+    
+    
+    
+    
+    
+    
+  
+  	
+    
+ +
+
+
+ +
+
+ {% if page.id == 'home' %} +

Druid is open-source infrastructure for real²time exploratory analytics on large datasets.

+ + {% endif %} +
+ +
+ + {{ content }} + +
+
+ + + + + + diff --git a/docs/_layouts/docs.html b/docs/_layouts/docs.html new file mode 100644 index 000000000000..cb38f8c336e4 --- /dev/null +++ b/docs/_layouts/docs.html @@ -0,0 +1,8 @@ +--- +layout: default +--- +
+ + {{ content }} + +
diff --git a/docs/_layouts/page.html b/docs/_layouts/page.html new file mode 100644 index 000000000000..b5ebfb17ace0 --- /dev/null +++ b/docs/_layouts/page.html @@ -0,0 +1,11 @@ +--- +layout: default +--- + +
+
+ + {{ content }} + +
+
diff --git a/docs/_layouts/post.html b/docs/_layouts/post.html new file mode 100644 index 000000000000..affb233fd23a --- /dev/null +++ b/docs/_layouts/post.html @@ -0,0 +1,44 @@ +--- +layout: default +sectionid: blog +--- + +
+
+

Recent posts

+ +
+ +
+
+

+ {{ page.title }} + {{ page.date | date: "%B %e, %Y" }} · {{ page.author | upcase }} +

+ + {% if page.image %}{{ page.title }}{% endif %} + + {{ content }} + +
+ + + comments powered by Disqus + +
+
+
diff --git a/docs/_posts/2013-09-16-welcome-to-jekyll.markdown b/docs/_posts/2013-09-16-welcome-to-jekyll.markdown new file mode 100644 index 000000000000..c50a740d8472 --- /dev/null +++ b/docs/_posts/2013-09-16-welcome-to-jekyll.markdown @@ -0,0 +1,24 @@ +--- +layout: post +title: "Welcome to Jekyll!" +date: 2013-09-16 13:06:49 +categories: jekyll update +--- + +You'll find this post in your `_posts` directory - edit this post and re-build (or run with the `-w` switch) to see your changes! +To add new posts, simply add a file in the `_posts` directory that follows the convention: YYYY-MM-DD-name-of-post.ext. + +Jekyll also offers powerful support for code snippets: + +{% highlight ruby %} +def print_hi(name) + puts "Hi, #{name}" +end +print_hi('Tom') +#=> prints 'Hi, Tom' to STDOUT. +{% endhighlight %} + +Check out the [Jekyll docs][jekyll] for more info on how to get the most out of Jekyll. File all bugs/feature requests at [Jekyll's GitHub repo][jekyll-gh]. + +[jekyll-gh]: https://github.com/mojombo/jekyll +[jekyll]: http://jekyllrb.com diff --git a/docs/contents.md b/docs/contents.md new file mode 100644 index 000000000000..963f88926e1d --- /dev/null +++ b/docs/contents.md @@ -0,0 +1,71 @@ +--- +layout: default +--- +Contents +\* [Introduction|Home](Introduction|Home.html) +\* [Download](Download.html) +\* [Support](Support.html) +\* [Contribute](Contribute.html) +======================== + +Getting Started +\* [Tutorial: A First Look at Druid](Tutorial:-A-First-Look-at-Druid.html) +\* [Tutorial: The Druid Cluster](Tutorial:-The-Druid-Cluster.html) +\* [Loading Your Data](Loading-Your-Data.html) +\* [Querying Your Data](Querying-Your-Data.html) +\* [Booting a Production Cluster](Booting-a-Production-Cluster.html) +\* [Examples](Examples.html) +\* [Cluster Setup](Cluster-Setup.html) +\* [Configuration](Configuration.html) +-------------------------------------- + +Data Ingestion +\* [Realtime](Realtime.html) +\* [Batch|Batch Ingestion](Batch|Batch-Ingestion.html) +\* [Indexing Service](Indexing-Service.html) +---------------------------- + +Querying +\* [Querying](Querying.html) +**\* ] +**\* [Aggregations](Aggregations.html) +**\* ] +**\* [Granularities](Granularities.html) +\* Query Types +**\* ] +****\* ] +****\* ] +**\* [SearchQuery](SearchQuery.html) +**\* ] +** [SegmentMetadataQuery](SegmentMetadataQuery.html) +**\* ] +**\* [TimeseriesQuery](TimeseriesQuery.html) +--------------------------- + +Architecture +\* [Design](Design.html) +\* [Segments](Segments.html) +\* Node Types +**\* ] +**\* [Broker](Broker.html) +**\* ] +****\* ] +**\* [Realtime](Realtime.html) +**\* ] +**\* [Plumber](Plumber.html) +\* External Dependencies +**\* ] +**\* [MySQL](MySQL.html) +**\* ] +** [Concepts and Terminology](Concepts-and-Terminology.html) +------------------------------- + +Development +\* [Versioning](Versioning.html) +\* [Build From Source](Build-From-Source.html) +\* [Libraries](Libraries.html) +------------------------ + +Misc +\* [Thanks](Thanks.html) +------------- diff --git a/docs/css/bootstrap-responsive.css b/docs/css/bootstrap-responsive.css new file mode 100644 index 000000000000..9570d1c2e403 --- /dev/null +++ b/docs/css/bootstrap-responsive.css @@ -0,0 +1,1058 @@ +/*! + * Bootstrap Responsive v2.1.1 + * + * Copyright 2012 Twitter, Inc + * Licensed under the Apache License v2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Designed and built with all the love in the world @twitter by @mdo and @fat. + */ + +.clearfix { + *zoom: 1; +} + +.clearfix:before, +.clearfix:after { + display: table; + line-height: 0; + content: ""; +} + +.clearfix:after { + clear: both; +} + +.hide-text { + font: 0/0 a; + color: transparent; + text-shadow: none; + background-color: transparent; + border: 0; +} + +.input-block-level { + display: block; + width: 100%; + min-height: 30px; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} + +.hidden { + display: none; + visibility: hidden; +} + +.visible-phone { + display: none !important; +} + +.visible-tablet { + display: none !important; +} + +.hidden-desktop { + display: none !important; +} + +.visible-desktop { + display: inherit !important; +} + +@media (min-width: 768px) and (max-width: 979px) { + .hidden-desktop { + display: inherit !important; + } + .visible-desktop { + display: none !important ; + } + .visible-tablet { + display: inherit !important; + } + .hidden-tablet { + display: none !important; + } +} + +@media (max-width: 767px) { + .hidden-desktop { + display: inherit !important; + } + .visible-desktop { + display: none !important; + } + .visible-phone { + display: inherit !important; + } + .hidden-phone { + display: none !important; + } +} + +@media (min-width: 1200px) { + .row { + margin-left: -30px; + *zoom: 1; + } + .row:before, + .row:after { + display: table; + line-height: 0; + content: ""; + } + .row:after { + clear: both; + } + [class*="span"] { + float: left; + min-height: 1px; + margin-left: 30px; + } + .container, + .navbar-static-top .container, + .navbar-fixed-top .container, + .navbar-fixed-bottom .container { + width: 1170px; + } + .span12 { + width: 1170px; + } + .span11 { + width: 1070px; + } + .span10 { + width: 970px; + } + .span9 { + width: 870px; + } + .span8 { + width: 770px; + } + .span7 { + width: 670px; + } + .span6 { + width: 570px; + } + .span5 { + width: 470px; + } + .span4 { + width: 370px; + } + .span3 { + width: 270px; + } + .span2 { + width: 170px; + } + .span1 { + width: 70px; + } + .offset12 { + margin-left: 1230px; + } + .offset11 { + margin-left: 1130px; + } + .offset10 { + margin-left: 1030px; + } + .offset9 { + margin-left: 930px; + } + .offset8 { + margin-left: 830px; + } + .offset7 { + margin-left: 730px; + } + .offset6 { + margin-left: 630px; + } + .offset5 { + margin-left: 530px; + } + .offset4 { + margin-left: 430px; + } + .offset3 { + margin-left: 330px; + } + .offset2 { + margin-left: 230px; + } + .offset1 { + margin-left: 130px; + } + .row-fluid { + width: 100%; + *zoom: 1; + } + .row-fluid:before, + .row-fluid:after { + display: table; + line-height: 0; + content: ""; + } + .row-fluid:after { + clear: both; + } + .row-fluid [class*="span"] { + display: block; + float: left; + width: 100%; + min-height: 30px; + margin-left: 2.564102564102564%; + *margin-left: 2.5109110747408616%; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + } + .row-fluid [class*="span"]:first-child { + margin-left: 0; + } + .row-fluid .span12 { + width: 100%; + *width: 99.94680851063829%; + } + .row-fluid .span11 { + width: 91.45299145299145%; + *width: 91.39979996362975%; + } + .row-fluid .span10 { + width: 82.90598290598291%; + *width: 82.8527914166212%; + } + .row-fluid .span9 { + width: 74.35897435897436%; + *width: 74.30578286961266%; + } + .row-fluid .span8 { + width: 65.81196581196582%; + *width: 65.75877432260411%; + } + .row-fluid .span7 { + width: 57.26495726495726%; + *width: 57.21176577559556%; + } + .row-fluid .span6 { + width: 48.717948717948715%; + *width: 48.664757228587014%; + } + .row-fluid .span5 { + width: 40.17094017094017%; + *width: 40.11774868157847%; + } + .row-fluid .span4 { + width: 31.623931623931625%; + *width: 31.570740134569924%; + } + .row-fluid .span3 { + width: 23.076923076923077%; + *width: 23.023731587561375%; + } + .row-fluid .span2 { + width: 14.52991452991453%; + *width: 14.476723040552828%; + } + .row-fluid .span1 { + width: 5.982905982905983%; + *width: 5.929714493544281%; + } + .row-fluid .offset12 { + margin-left: 105.12820512820512%; + *margin-left: 105.02182214948171%; + } + .row-fluid .offset12:first-child { + margin-left: 102.56410256410257%; + *margin-left: 102.45771958537915%; + } + .row-fluid .offset11 { + margin-left: 96.58119658119658%; + *margin-left: 96.47481360247316%; + } + .row-fluid .offset11:first-child { + margin-left: 94.01709401709402%; + *margin-left: 93.91071103837061%; + } + .row-fluid .offset10 { + margin-left: 88.03418803418803%; + *margin-left: 87.92780505546462%; + } + .row-fluid .offset10:first-child { + margin-left: 85.47008547008548%; + *margin-left: 85.36370249136206%; + } + .row-fluid .offset9 { + margin-left: 79.48717948717949%; + *margin-left: 79.38079650845607%; + } + .row-fluid .offset9:first-child { + margin-left: 76.92307692307693%; + *margin-left: 76.81669394435352%; + } + .row-fluid .offset8 { + margin-left: 70.94017094017094%; + *margin-left: 70.83378796144753%; + } + .row-fluid .offset8:first-child { + margin-left: 68.37606837606839%; + *margin-left: 68.26968539734497%; + } + .row-fluid .offset7 { + margin-left: 62.393162393162385%; + *margin-left: 62.28677941443899%; + } + .row-fluid .offset7:first-child { + margin-left: 59.82905982905982%; + *margin-left: 59.72267685033642%; + } + .row-fluid .offset6 { + margin-left: 53.84615384615384%; + *margin-left: 53.739770867430444%; + } + .row-fluid .offset6:first-child { + margin-left: 51.28205128205128%; + *margin-left: 51.175668303327875%; + } + .row-fluid .offset5 { + margin-left: 45.299145299145295%; + *margin-left: 45.1927623204219%; + } + .row-fluid .offset5:first-child { + margin-left: 42.73504273504273%; + *margin-left: 42.62865975631933%; + } + .row-fluid .offset4 { + margin-left: 36.75213675213675%; + *margin-left: 36.645753773413354%; + } + .row-fluid .offset4:first-child { + margin-left: 34.18803418803419%; + *margin-left: 34.081651209310785%; + } + .row-fluid .offset3 { + margin-left: 28.205128205128204%; + *margin-left: 28.0987452264048%; + } + .row-fluid .offset3:first-child { + margin-left: 25.641025641025642%; + *margin-left: 25.53464266230224%; + } + .row-fluid .offset2 { + margin-left: 19.65811965811966%; + *margin-left: 19.551736679396257%; + } + .row-fluid .offset2:first-child { + margin-left: 17.094017094017094%; + *margin-left: 16.98763411529369%; + } + .row-fluid .offset1 { + margin-left: 11.11111111111111%; + *margin-left: 11.004728132387708%; + } + .row-fluid .offset1:first-child { + margin-left: 8.547008547008547%; + *margin-left: 8.440625568285142%; + } + input, + textarea, + .uneditable-input { + margin-left: 0; + } + .controls-row [class*="span"] + [class*="span"] { + margin-left: 30px; + } + input.span12, + textarea.span12, + .uneditable-input.span12 { + width: 1156px; + } + input.span11, + textarea.span11, + .uneditable-input.span11 { + width: 1056px; + } + input.span10, + textarea.span10, + .uneditable-input.span10 { + width: 956px; + } + input.span9, + textarea.span9, + .uneditable-input.span9 { + width: 856px; + } + input.span8, + textarea.span8, + .uneditable-input.span8 { + width: 756px; + } + input.span7, + textarea.span7, + .uneditable-input.span7 { + width: 656px; + } + input.span6, + textarea.span6, + .uneditable-input.span6 { + width: 556px; + } + input.span5, + textarea.span5, + .uneditable-input.span5 { + width: 456px; + } + input.span4, + textarea.span4, + .uneditable-input.span4 { + width: 356px; + } + input.span3, + textarea.span3, + .uneditable-input.span3 { + width: 256px; + } + input.span2, + textarea.span2, + .uneditable-input.span2 { + width: 156px; + } + input.span1, + textarea.span1, + .uneditable-input.span1 { + width: 56px; + } + .thumbnails { + margin-left: -30px; + } + .thumbnails > li { + margin-left: 30px; + } + .row-fluid .thumbnails { + margin-left: 0; + } +} + +@media (min-width: 768px) and (max-width: 979px) { + .row { + margin-left: -20px; + *zoom: 1; + } + .row:before, + .row:after { + display: table; + line-height: 0; + content: ""; + } + .row:after { + clear: both; + } + [class*="span"] { + float: left; + min-height: 1px; + margin-left: 20px; + } + .container, + .navbar-static-top .container, + .navbar-fixed-top .container, + .navbar-fixed-bottom .container { + width: 724px; + } + .span12 { + width: 724px; + } + .span11 { + width: 662px; + } + .span10 { + width: 600px; + } + .span9 { + width: 538px; + } + .span8 { + width: 476px; + } + .span7 { + width: 414px; + } + .span6 { + width: 352px; + } + .span5 { + width: 290px; + } + .span4 { + width: 228px; + } + .span3 { + width: 166px; + } + .span2 { + width: 104px; + } + .span1 { + width: 42px; + } + .offset12 { + margin-left: 764px; + } + .offset11 { + margin-left: 702px; + } + .offset10 { + margin-left: 640px; + } + .offset9 { + margin-left: 578px; + } + .offset8 { + margin-left: 516px; + } + .offset7 { + margin-left: 454px; + } + .offset6 { + margin-left: 392px; + } + .offset5 { + margin-left: 330px; + } + .offset4 { + margin-left: 268px; + } + .offset3 { + margin-left: 206px; + } + .offset2 { + margin-left: 144px; + } + .offset1 { + margin-left: 82px; + } + .row-fluid { + width: 100%; + *zoom: 1; + } + .row-fluid:before, + .row-fluid:after { + display: table; + line-height: 0; + content: ""; + } + .row-fluid:after { + clear: both; + } + .row-fluid [class*="span"] { + display: block; + float: left; + width: 100%; + min-height: 30px; + margin-left: 2.7624309392265194%; + *margin-left: 2.709239449864817%; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + } + .row-fluid [class*="span"]:first-child { + margin-left: 0; + } + .row-fluid .span12 { + width: 100%; + *width: 99.94680851063829%; + } + .row-fluid .span11 { + width: 91.43646408839778%; + *width: 91.38327259903608%; + } + .row-fluid .span10 { + width: 82.87292817679558%; + *width: 82.81973668743387%; + } + .row-fluid .span9 { + width: 74.30939226519337%; + *width: 74.25620077583166%; + } + .row-fluid .span8 { + width: 65.74585635359117%; + *width: 65.69266486422946%; + } + .row-fluid .span7 { + width: 57.18232044198895%; + *width: 57.12912895262725%; + } + .row-fluid .span6 { + width: 48.61878453038674%; + *width: 48.56559304102504%; + } + .row-fluid .span5 { + width: 40.05524861878453%; + *width: 40.00205712942283%; + } + .row-fluid .span4 { + width: 31.491712707182323%; + *width: 31.43852121782062%; + } + .row-fluid .span3 { + width: 22.92817679558011%; + *width: 22.87498530621841%; + } + .row-fluid .span2 { + width: 14.3646408839779%; + *width: 14.311449394616199%; + } + .row-fluid .span1 { + width: 5.801104972375691%; + *width: 5.747913483013988%; + } + .row-fluid .offset12 { + margin-left: 105.52486187845304%; + *margin-left: 105.41847889972962%; + } + .row-fluid .offset12:first-child { + margin-left: 102.76243093922652%; + *margin-left: 102.6560479605031%; + } + .row-fluid .offset11 { + margin-left: 96.96132596685082%; + *margin-left: 96.8549429881274%; + } + .row-fluid .offset11:first-child { + margin-left: 94.1988950276243%; + *margin-left: 94.09251204890089%; + } + .row-fluid .offset10 { + margin-left: 88.39779005524862%; + *margin-left: 88.2914070765252%; + } + .row-fluid .offset10:first-child { + margin-left: 85.6353591160221%; + *margin-left: 85.52897613729868%; + } + .row-fluid .offset9 { + margin-left: 79.8342541436464%; + *margin-left: 79.72787116492299%; + } + .row-fluid .offset9:first-child { + margin-left: 77.07182320441989%; + *margin-left: 76.96544022569647%; + } + .row-fluid .offset8 { + margin-left: 71.2707182320442%; + *margin-left: 71.16433525332079%; + } + .row-fluid .offset8:first-child { + margin-left: 68.50828729281768%; + *margin-left: 68.40190431409427%; + } + .row-fluid .offset7 { + margin-left: 62.70718232044199%; + *margin-left: 62.600799341718584%; + } + .row-fluid .offset7:first-child { + margin-left: 59.94475138121547%; + *margin-left: 59.838368402492065%; + } + .row-fluid .offset6 { + margin-left: 54.14364640883978%; + *margin-left: 54.037263430116376%; + } + .row-fluid .offset6:first-child { + margin-left: 51.38121546961326%; + *margin-left: 51.27483249088986%; + } + .row-fluid .offset5 { + margin-left: 45.58011049723757%; + *margin-left: 45.47372751851417%; + } + .row-fluid .offset5:first-child { + margin-left: 42.81767955801105%; + *margin-left: 42.71129657928765%; + } + .row-fluid .offset4 { + margin-left: 37.01657458563536%; + *margin-left: 36.91019160691196%; + } + .row-fluid .offset4:first-child { + margin-left: 34.25414364640884%; + *margin-left: 34.14776066768544%; + } + .row-fluid .offset3 { + margin-left: 28.45303867403315%; + *margin-left: 28.346655695309746%; + } + .row-fluid .offset3:first-child { + margin-left: 25.69060773480663%; + *margin-left: 25.584224756083227%; + } + .row-fluid .offset2 { + margin-left: 19.88950276243094%; + *margin-left: 19.783119783707537%; + } + .row-fluid .offset2:first-child { + margin-left: 17.12707182320442%; + *margin-left: 17.02068884448102%; + } + .row-fluid .offset1 { + margin-left: 11.32596685082873%; + *margin-left: 11.219583872105325%; + } + .row-fluid .offset1:first-child { + margin-left: 8.56353591160221%; + *margin-left: 8.457152932878806%; + } + input, + textarea, + .uneditable-input { + margin-left: 0; + } + .controls-row [class*="span"] + [class*="span"] { + margin-left: 20px; + } + input.span12, + textarea.span12, + .uneditable-input.span12 { + width: 710px; + } + input.span11, + textarea.span11, + .uneditable-input.span11 { + width: 648px; + } + input.span10, + textarea.span10, + .uneditable-input.span10 { + width: 586px; + } + input.span9, + textarea.span9, + .uneditable-input.span9 { + width: 524px; + } + input.span8, + textarea.span8, + .uneditable-input.span8 { + width: 462px; + } + input.span7, + textarea.span7, + .uneditable-input.span7 { + width: 400px; + } + input.span6, + textarea.span6, + .uneditable-input.span6 { + width: 338px; + } + input.span5, + textarea.span5, + .uneditable-input.span5 { + width: 276px; + } + input.span4, + textarea.span4, + .uneditable-input.span4 { + width: 214px; + } + input.span3, + textarea.span3, + .uneditable-input.span3 { + width: 152px; + } + input.span2, + textarea.span2, + .uneditable-input.span2 { + width: 90px; + } + input.span1, + textarea.span1, + .uneditable-input.span1 { + width: 28px; + } +} + +@media (max-width: 767px) { + body { + padding-right: 20px; + padding-left: 20px; + } + .navbar-fixed-top, + .navbar-fixed-bottom, + .navbar-static-top { + margin-right: -20px; + margin-left: -20px; + } + .container-fluid { + padding: 0; + } + .dl-horizontal dt { + float: none; + width: auto; + clear: none; + text-align: left; + } + .dl-horizontal dd { + margin-left: 0; + } + .container { + width: auto; + } + .row-fluid { + width: 100%; + } + .row, + .thumbnails { + margin-left: 0; + } + .thumbnails > li { + float: none; + margin-left: 0; + } + [class*="span"], + .row-fluid [class*="span"] { + display: block; + float: none; + width: auto; + margin-left: 0; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + } + .span12, + .row-fluid .span12 { + width: 100%; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + } + .input-large, + .input-xlarge, + .input-xxlarge, + input[class*="span"], + select[class*="span"], + textarea[class*="span"], + .uneditable-input { + display: block; + width: 100%; + min-height: 30px; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + } + .input-prepend input, + .input-append input, + .input-prepend input[class*="span"], + .input-append input[class*="span"] { + display: inline-block; + width: auto; + } + .controls-row [class*="span"] + [class*="span"] { + margin-left: 0; + } + .modal { + position: fixed; + top: 20px; + right: 20px; + left: 20px; + width: auto; + margin: 0; + } + .modal.fade.in { + top: auto; + } +} + +@media (max-width: 480px) { + .nav-collapse { + -webkit-transform: translate3d(0, 0, 0); + } + .page-header h1 small { + display: block; + line-height: 20px; + } + input[type="checkbox"], + input[type="radio"] { + border: 1px solid #ccc; + } + .form-horizontal .control-label { + float: none; + width: auto; + padding-top: 0; + text-align: left; + } + .form-horizontal .controls { + margin-left: 0; + } + .form-horizontal .control-list { + padding-top: 0; + } + .form-horizontal .form-actions { + padding-right: 10px; + padding-left: 10px; + } + .modal { + top: 10px; + right: 10px; + left: 10px; + } + .modal-header .close { + padding: 10px; + margin: -10px; + } + .carousel-caption { + position: static; + } +} + +@media (max-width: 979px) { + body { + padding-top: 0; + } + .navbar-fixed-top, + .navbar-fixed-bottom { + position: static; + } + .navbar-fixed-top { + margin-bottom: 20px; + } + .navbar-fixed-bottom { + margin-top: 20px; + } + .navbar-fixed-top .navbar-inner, + .navbar-fixed-bottom .navbar-inner { + padding: 5px; + } + .navbar .container { + width: auto; + padding: 0; + } + .navbar .brand { + padding-right: 10px; + padding-left: 10px; + margin: 0 0 0 -5px; + } + .nav-collapse { + clear: both; + } + .nav-collapse .nav { + float: none; + margin: 0 0 10px; + } + .nav-collapse .nav > li { + float: none; + } + .nav-collapse .nav > li > a { + margin-bottom: 2px; + } + .nav-collapse .nav > .divider-vertical { + display: none; + } + .nav-collapse .nav .nav-header { + color: #777777; + text-shadow: none; + } + .nav-collapse .nav > li > a, + .nav-collapse .dropdown-menu a { + padding: 9px 15px; + font-weight: bold; + color: #777777; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + } + .nav-collapse .btn { + padding: 4px 10px 4px; + font-weight: normal; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + } + .nav-collapse .dropdown-menu li + li a { + margin-bottom: 2px; + } + .nav-collapse .nav > li > a:hover, + .nav-collapse .dropdown-menu a:hover { + background-color: #f2f2f2; + } + .navbar-inverse .nav-collapse .nav > li > a:hover, + .navbar-inverse .nav-collapse .dropdown-menu a:hover { + background-color: #111111; + } + .nav-collapse.in .btn-group { + padding: 0; + margin-top: 5px; + } + .nav-collapse .dropdown-menu { + position: static; + top: auto; + left: auto; + display: block; + float: none; + max-width: none; + padding: 0; + margin: 0 15px; + background-color: transparent; + border: none; + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; + } + .nav-collapse .dropdown-menu:before, + .nav-collapse .dropdown-menu:after { + display: none; + } + .nav-collapse .dropdown-menu .divider { + display: none; + } + .nav-collapse .nav > li > .dropdown-menu:before, + .nav-collapse .nav > li > .dropdown-menu:after { + display: none; + } + .nav-collapse .navbar-form, + .nav-collapse .navbar-search { + float: none; + padding: 10px 15px; + margin: 10px 0; + border-top: 1px solid #f2f2f2; + border-bottom: 1px solid #f2f2f2; + -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); + -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.1); + } + .navbar-inverse .nav-collapse .navbar-form, + .navbar-inverse .nav-collapse .navbar-search { + border-top-color: #111111; + border-bottom-color: #111111; + } + .navbar .nav-collapse .nav.pull-right { + float: none; + margin-left: 0; + } + .nav-collapse, + .nav-collapse.collapse { + height: 0; + overflow: hidden; + } + .navbar .btn-navbar { + display: block; + } + .navbar-static .navbar-inner { + padding-right: 10px; + padding-left: 10px; + } +} + +@media (min-width: 980px) { + .nav-collapse.collapse { + height: auto !important; + overflow: visible !important; + } +} diff --git a/docs/css/bootstrap-responsive.min.css b/docs/css/bootstrap-responsive.min.css new file mode 100644 index 000000000000..8eb78d8e67c3 --- /dev/null +++ b/docs/css/bootstrap-responsive.min.css @@ -0,0 +1,9 @@ +/*! + * Bootstrap Responsive v2.1.1 + * + * Copyright 2012 Twitter, Inc + * Licensed under the Apache License v2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Designed and built with all the love in the world @twitter by @mdo and @fat. + */.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.hidden{display:none;visibility:hidden}.visible-phone{display:none!important}.visible-tablet{display:none!important}.hidden-desktop{display:none!important}.visible-desktop{display:inherit!important}@media(min-width:768px) and (max-width:979px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-tablet{display:inherit!important}.hidden-tablet{display:none!important}}@media(max-width:767px){.hidden-desktop{display:inherit!important}.visible-desktop{display:none!important}.visible-phone{display:inherit!important}.hidden-phone{display:none!important}}@media(min-width:1200px){.row{margin-left:-30px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:30px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:1170px}.span12{width:1170px}.span11{width:1070px}.span10{width:970px}.span9{width:870px}.span8{width:770px}.span7{width:670px}.span6{width:570px}.span5{width:470px}.span4{width:370px}.span3{width:270px}.span2{width:170px}.span1{width:70px}.offset12{margin-left:1230px}.offset11{margin-left:1130px}.offset10{margin-left:1030px}.offset9{margin-left:930px}.offset8{margin-left:830px}.offset7{margin-left:730px}.offset6{margin-left:630px}.offset5{margin-left:530px}.offset4{margin-left:430px}.offset3{margin-left:330px}.offset2{margin-left:230px}.offset1{margin-left:130px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.564102564102564%;*margin-left:2.5109110747408616%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.45299145299145%;*width:91.39979996362975%}.row-fluid .span10{width:82.90598290598291%;*width:82.8527914166212%}.row-fluid .span9{width:74.35897435897436%;*width:74.30578286961266%}.row-fluid .span8{width:65.81196581196582%;*width:65.75877432260411%}.row-fluid .span7{width:57.26495726495726%;*width:57.21176577559556%}.row-fluid .span6{width:48.717948717948715%;*width:48.664757228587014%}.row-fluid .span5{width:40.17094017094017%;*width:40.11774868157847%}.row-fluid .span4{width:31.623931623931625%;*width:31.570740134569924%}.row-fluid .span3{width:23.076923076923077%;*width:23.023731587561375%}.row-fluid .span2{width:14.52991452991453%;*width:14.476723040552828%}.row-fluid .span1{width:5.982905982905983%;*width:5.929714493544281%}.row-fluid .offset12{margin-left:105.12820512820512%;*margin-left:105.02182214948171%}.row-fluid .offset12:first-child{margin-left:102.56410256410257%;*margin-left:102.45771958537915%}.row-fluid .offset11{margin-left:96.58119658119658%;*margin-left:96.47481360247316%}.row-fluid .offset11:first-child{margin-left:94.01709401709402%;*margin-left:93.91071103837061%}.row-fluid .offset10{margin-left:88.03418803418803%;*margin-left:87.92780505546462%}.row-fluid .offset10:first-child{margin-left:85.47008547008548%;*margin-left:85.36370249136206%}.row-fluid .offset9{margin-left:79.48717948717949%;*margin-left:79.38079650845607%}.row-fluid .offset9:first-child{margin-left:76.92307692307693%;*margin-left:76.81669394435352%}.row-fluid .offset8{margin-left:70.94017094017094%;*margin-left:70.83378796144753%}.row-fluid .offset8:first-child{margin-left:68.37606837606839%;*margin-left:68.26968539734497%}.row-fluid .offset7{margin-left:62.393162393162385%;*margin-left:62.28677941443899%}.row-fluid .offset7:first-child{margin-left:59.82905982905982%;*margin-left:59.72267685033642%}.row-fluid .offset6{margin-left:53.84615384615384%;*margin-left:53.739770867430444%}.row-fluid .offset6:first-child{margin-left:51.28205128205128%;*margin-left:51.175668303327875%}.row-fluid .offset5{margin-left:45.299145299145295%;*margin-left:45.1927623204219%}.row-fluid .offset5:first-child{margin-left:42.73504273504273%;*margin-left:42.62865975631933%}.row-fluid .offset4{margin-left:36.75213675213675%;*margin-left:36.645753773413354%}.row-fluid .offset4:first-child{margin-left:34.18803418803419%;*margin-left:34.081651209310785%}.row-fluid .offset3{margin-left:28.205128205128204%;*margin-left:28.0987452264048%}.row-fluid .offset3:first-child{margin-left:25.641025641025642%;*margin-left:25.53464266230224%}.row-fluid .offset2{margin-left:19.65811965811966%;*margin-left:19.551736679396257%}.row-fluid .offset2:first-child{margin-left:17.094017094017094%;*margin-left:16.98763411529369%}.row-fluid .offset1{margin-left:11.11111111111111%;*margin-left:11.004728132387708%}.row-fluid .offset1:first-child{margin-left:8.547008547008547%;*margin-left:8.440625568285142%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:30px}input.span12,textarea.span12,.uneditable-input.span12{width:1156px}input.span11,textarea.span11,.uneditable-input.span11{width:1056px}input.span10,textarea.span10,.uneditable-input.span10{width:956px}input.span9,textarea.span9,.uneditable-input.span9{width:856px}input.span8,textarea.span8,.uneditable-input.span8{width:756px}input.span7,textarea.span7,.uneditable-input.span7{width:656px}input.span6,textarea.span6,.uneditable-input.span6{width:556px}input.span5,textarea.span5,.uneditable-input.span5{width:456px}input.span4,textarea.span4,.uneditable-input.span4{width:356px}input.span3,textarea.span3,.uneditable-input.span3{width:256px}input.span2,textarea.span2,.uneditable-input.span2{width:156px}input.span1,textarea.span1,.uneditable-input.span1{width:56px}.thumbnails{margin-left:-30px}.thumbnails>li{margin-left:30px}.row-fluid .thumbnails{margin-left:0}}@media(min-width:768px) and (max-width:979px){.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:724px}.span12{width:724px}.span11{width:662px}.span10{width:600px}.span9{width:538px}.span8{width:476px}.span7{width:414px}.span6{width:352px}.span5{width:290px}.span4{width:228px}.span3{width:166px}.span2{width:104px}.span1{width:42px}.offset12{margin-left:764px}.offset11{margin-left:702px}.offset10{margin-left:640px}.offset9{margin-left:578px}.offset8{margin-left:516px}.offset7{margin-left:454px}.offset6{margin-left:392px}.offset5{margin-left:330px}.offset4{margin-left:268px}.offset3{margin-left:206px}.offset2{margin-left:144px}.offset1{margin-left:82px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.7624309392265194%;*margin-left:2.709239449864817%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.43646408839778%;*width:91.38327259903608%}.row-fluid .span10{width:82.87292817679558%;*width:82.81973668743387%}.row-fluid .span9{width:74.30939226519337%;*width:74.25620077583166%}.row-fluid .span8{width:65.74585635359117%;*width:65.69266486422946%}.row-fluid .span7{width:57.18232044198895%;*width:57.12912895262725%}.row-fluid .span6{width:48.61878453038674%;*width:48.56559304102504%}.row-fluid .span5{width:40.05524861878453%;*width:40.00205712942283%}.row-fluid .span4{width:31.491712707182323%;*width:31.43852121782062%}.row-fluid .span3{width:22.92817679558011%;*width:22.87498530621841%}.row-fluid .span2{width:14.3646408839779%;*width:14.311449394616199%}.row-fluid .span1{width:5.801104972375691%;*width:5.747913483013988%}.row-fluid .offset12{margin-left:105.52486187845304%;*margin-left:105.41847889972962%}.row-fluid .offset12:first-child{margin-left:102.76243093922652%;*margin-left:102.6560479605031%}.row-fluid .offset11{margin-left:96.96132596685082%;*margin-left:96.8549429881274%}.row-fluid .offset11:first-child{margin-left:94.1988950276243%;*margin-left:94.09251204890089%}.row-fluid .offset10{margin-left:88.39779005524862%;*margin-left:88.2914070765252%}.row-fluid .offset10:first-child{margin-left:85.6353591160221%;*margin-left:85.52897613729868%}.row-fluid .offset9{margin-left:79.8342541436464%;*margin-left:79.72787116492299%}.row-fluid .offset9:first-child{margin-left:77.07182320441989%;*margin-left:76.96544022569647%}.row-fluid .offset8{margin-left:71.2707182320442%;*margin-left:71.16433525332079%}.row-fluid .offset8:first-child{margin-left:68.50828729281768%;*margin-left:68.40190431409427%}.row-fluid .offset7{margin-left:62.70718232044199%;*margin-left:62.600799341718584%}.row-fluid .offset7:first-child{margin-left:59.94475138121547%;*margin-left:59.838368402492065%}.row-fluid .offset6{margin-left:54.14364640883978%;*margin-left:54.037263430116376%}.row-fluid .offset6:first-child{margin-left:51.38121546961326%;*margin-left:51.27483249088986%}.row-fluid .offset5{margin-left:45.58011049723757%;*margin-left:45.47372751851417%}.row-fluid .offset5:first-child{margin-left:42.81767955801105%;*margin-left:42.71129657928765%}.row-fluid .offset4{margin-left:37.01657458563536%;*margin-left:36.91019160691196%}.row-fluid .offset4:first-child{margin-left:34.25414364640884%;*margin-left:34.14776066768544%}.row-fluid .offset3{margin-left:28.45303867403315%;*margin-left:28.346655695309746%}.row-fluid .offset3:first-child{margin-left:25.69060773480663%;*margin-left:25.584224756083227%}.row-fluid .offset2{margin-left:19.88950276243094%;*margin-left:19.783119783707537%}.row-fluid .offset2:first-child{margin-left:17.12707182320442%;*margin-left:17.02068884448102%}.row-fluid .offset1{margin-left:11.32596685082873%;*margin-left:11.219583872105325%}.row-fluid .offset1:first-child{margin-left:8.56353591160221%;*margin-left:8.457152932878806%}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:710px}input.span11,textarea.span11,.uneditable-input.span11{width:648px}input.span10,textarea.span10,.uneditable-input.span10{width:586px}input.span9,textarea.span9,.uneditable-input.span9{width:524px}input.span8,textarea.span8,.uneditable-input.span8{width:462px}input.span7,textarea.span7,.uneditable-input.span7{width:400px}input.span6,textarea.span6,.uneditable-input.span6{width:338px}input.span5,textarea.span5,.uneditable-input.span5{width:276px}input.span4,textarea.span4,.uneditable-input.span4{width:214px}input.span3,textarea.span3,.uneditable-input.span3{width:152px}input.span2,textarea.span2,.uneditable-input.span2{width:90px}input.span1,textarea.span1,.uneditable-input.span1{width:28px}}@media(max-width:767px){body{padding-right:20px;padding-left:20px}.navbar-fixed-top,.navbar-fixed-bottom,.navbar-static-top{margin-right:-20px;margin-left:-20px}.container-fluid{padding:0}.dl-horizontal dt{float:none;width:auto;clear:none;text-align:left}.dl-horizontal dd{margin-left:0}.container{width:auto}.row-fluid{width:100%}.row,.thumbnails{margin-left:0}.thumbnails>li{float:none;margin-left:0}[class*="span"],.row-fluid [class*="span"]{display:block;float:none;width:100%;margin-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.span12,.row-fluid .span12{width:100%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-large,.input-xlarge,.input-xxlarge,input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.input-prepend input,.input-append input,.input-prepend input[class*="span"],.input-append input[class*="span"]{display:inline-block;width:auto}.controls-row [class*="span"]+[class*="span"]{margin-left:0}.modal{position:fixed;top:20px;right:20px;left:20px;width:auto;margin:0}.modal.fade.in{top:auto}}@media(max-width:480px){.nav-collapse{-webkit-transform:translate3d(0,0,0)}.page-header h1 small{display:block;line-height:20px}input[type="checkbox"],input[type="radio"]{border:1px solid #ccc}.form-horizontal .control-label{float:none;width:auto;padding-top:0;text-align:left}.form-horizontal .controls{margin-left:0}.form-horizontal .control-list{padding-top:0}.form-horizontal .form-actions{padding-right:10px;padding-left:10px}.modal{top:10px;right:10px;left:10px}.modal-header .close{padding:10px;margin:-10px}.carousel-caption{position:static}}@media(max-width:979px){body{padding-top:0}.navbar-fixed-top,.navbar-fixed-bottom{position:static}.navbar-fixed-top{margin-bottom:20px}.navbar-fixed-bottom{margin-top:20px}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding:5px}.navbar .container{width:auto;padding:0}.navbar .brand{padding-right:10px;padding-left:10px;margin:0 0 0 -5px}.nav-collapse{clear:both}.nav-collapse .nav{float:none;margin:0 0 10px}.nav-collapse .nav>li{float:none}.nav-collapse .nav>li>a{margin-bottom:2px}.nav-collapse .nav>.divider-vertical{display:none}.nav-collapse .nav .nav-header{color:#777;text-shadow:none}.nav-collapse .nav>li>a,.nav-collapse .dropdown-menu a{padding:9px 15px;font-weight:bold;color:#777;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.nav-collapse .btn{padding:4px 10px 4px;font-weight:normal;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.nav-collapse .dropdown-menu li+li a{margin-bottom:2px}.nav-collapse .nav>li>a:hover,.nav-collapse .dropdown-menu a:hover{background-color:#f2f2f2}.navbar-inverse .nav-collapse .nav>li>a:hover,.navbar-inverse .nav-collapse .dropdown-menu a:hover{background-color:#111}.nav-collapse.in .btn-group{padding:0;margin-top:5px}.nav-collapse .dropdown-menu{position:static;top:auto;left:auto;display:block;float:none;max-width:none;padding:0;margin:0 15px;background-color:transparent;border:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.nav-collapse .dropdown-menu:before,.nav-collapse .dropdown-menu:after{display:none}.nav-collapse .dropdown-menu .divider{display:none}.nav-collapse .nav>li>.dropdown-menu:before,.nav-collapse .nav>li>.dropdown-menu:after{display:none}.nav-collapse .navbar-form,.nav-collapse .navbar-search{float:none;padding:10px 15px;margin:10px 0;border-top:1px solid #f2f2f2;border-bottom:1px solid #f2f2f2;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.1)}.navbar-inverse .nav-collapse .navbar-form,.navbar-inverse .nav-collapse .navbar-search{border-top-color:#111;border-bottom-color:#111}.navbar .nav-collapse .nav.pull-right{float:none;margin-left:0}.nav-collapse,.nav-collapse.collapse{height:0;overflow:hidden}.navbar .btn-navbar{display:block}.navbar-static .navbar-inner{padding-right:10px;padding-left:10px}}@media(min-width:980px){.nav-collapse.collapse{height:auto!important;overflow:visible!important}} diff --git a/docs/css/bootstrap.css b/docs/css/bootstrap.css new file mode 100644 index 000000000000..8dca0b71366f --- /dev/null +++ b/docs/css/bootstrap.css @@ -0,0 +1,5774 @@ +/*! + * Bootstrap v2.1.1 + * + * Copyright 2012 Twitter, Inc + * Licensed under the Apache License v2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Designed and built with all the love in the world @twitter by @mdo and @fat. + */ + +article, +aside, +details, +figcaption, +figure, +footer, +header, +hgroup, +nav, +section { + display: block; +} + +audio, +canvas, +video { + display: inline-block; + *display: inline; + *zoom: 1; +} + +audio:not([controls]) { + display: none; +} + +html { + font-size: 100%; + -webkit-text-size-adjust: 100%; + -ms-text-size-adjust: 100%; +} + +a:focus { + outline: thin dotted #333; + outline: 5px auto -webkit-focus-ring-color; + outline-offset: -2px; +} + +a:hover, +a:active { + outline: 0; +} + +sub, +sup { + position: relative; + font-size: 75%; + line-height: 0; + vertical-align: baseline; +} + +sup { + top: -0.5em; +} + +sub { + bottom: -0.25em; +} + +img { + width: auto\9; + height: auto; + max-width: 100%; + vertical-align: middle; + border: 0; + -ms-interpolation-mode: bicubic; +} + +#map_canvas img { + max-width: none; +} + +button, +input, +select, +textarea { + margin: 0; + font-size: 100%; + vertical-align: middle; +} + +button, +input { + *overflow: visible; + line-height: normal; +} + +button::-moz-focus-inner, +input::-moz-focus-inner { + padding: 0; + border: 0; +} + +button, +input[type="button"], +input[type="reset"], +input[type="submit"] { + cursor: pointer; + -webkit-appearance: button; +} + +input[type="search"] { + -webkit-box-sizing: content-box; + -moz-box-sizing: content-box; + box-sizing: content-box; + -webkit-appearance: textfield; +} + +input[type="search"]::-webkit-search-decoration, +input[type="search"]::-webkit-search-cancel-button { + -webkit-appearance: none; +} + +textarea { + overflow: auto; + vertical-align: top; +} + +.clearfix { + *zoom: 1; +} + +.clearfix:before, +.clearfix:after { + display: table; + line-height: 0; + content: ""; +} + +.clearfix:after { + clear: both; +} + +.hide-text { + font: 0/0 a; + color: transparent; + text-shadow: none; + background-color: transparent; + border: 0; +} + +.input-block-level { + display: block; + width: 100%; + min-height: 30px; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} + +body { + margin: 0; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 14px; + line-height: 20px; + color: #333333; + background-color: #ffffff; +} + +a { + color: #0088cc; + text-decoration: none; +} + +a:hover { + color: #005580; + text-decoration: underline; +} + +.img-rounded { + -webkit-border-radius: 6px; + -moz-border-radius: 6px; + border-radius: 6px; +} + +.img-polaroid { + padding: 4px; + background-color: #fff; + border: 1px solid #ccc; + border: 1px solid rgba(0, 0, 0, 0.2); + -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); +} + +.img-circle { + -webkit-border-radius: 500px; + -moz-border-radius: 500px; + border-radius: 500px; +} + +.row { + margin-left: -20px; + *zoom: 1; +} + +.row:before, +.row:after { + display: table; + line-height: 0; + content: ""; +} + +.row:after { + clear: both; +} + +[class*="span"] { + float: left; + min-height: 1px; + margin-left: 20px; +} + +.container, +.navbar-static-top .container, +.navbar-fixed-top .container, +.navbar-fixed-bottom .container { + width: 940px; +} + +.span12 { + width: 940px; +} + +.span11 { + width: 860px; +} + +.span10 { + width: 780px; +} + +.span9 { + width: 700px; +} + +.span8 { + width: 620px; +} + +.span7 { + width: 540px; +} + +.span6 { + width: 460px; +} + +.span5 { + width: 380px; +} + +.span4 { + width: 300px; +} + +.span3 { + width: 220px; +} + +.span2 { + width: 140px; +} + +.span1 { + width: 60px; +} + +.offset12 { + margin-left: 980px; +} + +.offset11 { + margin-left: 900px; +} + +.offset10 { + margin-left: 820px; +} + +.offset9 { + margin-left: 740px; +} + +.offset8 { + margin-left: 660px; +} + +.offset7 { + margin-left: 580px; +} + +.offset6 { + margin-left: 500px; +} + +.offset5 { + margin-left: 420px; +} + +.offset4 { + margin-left: 340px; +} + +.offset3 { + margin-left: 260px; +} + +.offset2 { + margin-left: 180px; +} + +.offset1 { + margin-left: 100px; +} + +.row-fluid { + width: 100%; + *zoom: 1; +} + +.row-fluid:before, +.row-fluid:after { + display: table; + line-height: 0; + content: ""; +} + +.row-fluid:after { + clear: both; +} + +.row-fluid [class*="span"] { + display: block; + float: left; + width: 100%; + min-height: 30px; + margin-left: 2.127659574468085%; + *margin-left: 2.074468085106383%; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} + +.row-fluid [class*="span"]:first-child { + margin-left: 0; +} + +.row-fluid .span12 { + width: 100%; + *width: 99.94680851063829%; +} + +.row-fluid .span11 { + width: 91.48936170212765%; + *width: 91.43617021276594%; +} + +.row-fluid .span10 { + width: 82.97872340425532%; + *width: 82.92553191489361%; +} + +.row-fluid .span9 { + width: 74.46808510638297%; + *width: 74.41489361702126%; +} + +.row-fluid .span8 { + width: 65.95744680851064%; + *width: 65.90425531914893%; +} + +.row-fluid .span7 { + width: 57.44680851063829%; + *width: 57.39361702127659%; +} + +.row-fluid .span6 { + width: 48.93617021276595%; + *width: 48.88297872340425%; +} + +.row-fluid .span5 { + width: 40.42553191489362%; + *width: 40.37234042553192%; +} + +.row-fluid .span4 { + width: 31.914893617021278%; + *width: 31.861702127659576%; +} + +.row-fluid .span3 { + width: 23.404255319148934%; + *width: 23.351063829787233%; +} + +.row-fluid .span2 { + width: 14.893617021276595%; + *width: 14.840425531914894%; +} + +.row-fluid .span1 { + width: 6.382978723404255%; + *width: 6.329787234042553%; +} + +.row-fluid .offset12 { + margin-left: 104.25531914893617%; + *margin-left: 104.14893617021275%; +} + +.row-fluid .offset12:first-child { + margin-left: 102.12765957446808%; + *margin-left: 102.02127659574467%; +} + +.row-fluid .offset11 { + margin-left: 95.74468085106382%; + *margin-left: 95.6382978723404%; +} + +.row-fluid .offset11:first-child { + margin-left: 93.61702127659574%; + *margin-left: 93.51063829787232%; +} + +.row-fluid .offset10 { + margin-left: 87.23404255319149%; + *margin-left: 87.12765957446807%; +} + +.row-fluid .offset10:first-child { + margin-left: 85.1063829787234%; + *margin-left: 84.99999999999999%; +} + +.row-fluid .offset9 { + margin-left: 78.72340425531914%; + *margin-left: 78.61702127659572%; +} + +.row-fluid .offset9:first-child { + margin-left: 76.59574468085106%; + *margin-left: 76.48936170212764%; +} + +.row-fluid .offset8 { + margin-left: 70.2127659574468%; + *margin-left: 70.10638297872339%; +} + +.row-fluid .offset8:first-child { + margin-left: 68.08510638297872%; + *margin-left: 67.9787234042553%; +} + +.row-fluid .offset7 { + margin-left: 61.70212765957446%; + *margin-left: 61.59574468085106%; +} + +.row-fluid .offset7:first-child { + margin-left: 59.574468085106375%; + *margin-left: 59.46808510638297%; +} + +.row-fluid .offset6 { + margin-left: 53.191489361702125%; + *margin-left: 53.085106382978715%; +} + +.row-fluid .offset6:first-child { + margin-left: 51.063829787234035%; + *margin-left: 50.95744680851063%; +} + +.row-fluid .offset5 { + margin-left: 44.68085106382979%; + *margin-left: 44.57446808510638%; +} + +.row-fluid .offset5:first-child { + margin-left: 42.5531914893617%; + *margin-left: 42.4468085106383%; +} + +.row-fluid .offset4 { + margin-left: 36.170212765957444%; + *margin-left: 36.06382978723405%; +} + +.row-fluid .offset4:first-child { + margin-left: 34.04255319148936%; + *margin-left: 33.93617021276596%; +} + +.row-fluid .offset3 { + margin-left: 27.659574468085104%; + *margin-left: 27.5531914893617%; +} + +.row-fluid .offset3:first-child { + margin-left: 25.53191489361702%; + *margin-left: 25.425531914893618%; +} + +.row-fluid .offset2 { + margin-left: 19.148936170212764%; + *margin-left: 19.04255319148936%; +} + +.row-fluid .offset2:first-child { + margin-left: 17.02127659574468%; + *margin-left: 16.914893617021278%; +} + +.row-fluid .offset1 { + margin-left: 10.638297872340425%; + *margin-left: 10.53191489361702%; +} + +.row-fluid .offset1:first-child { + margin-left: 8.51063829787234%; + *margin-left: 8.404255319148938%; +} + +[class*="span"].hide, +.row-fluid [class*="span"].hide { + display: none; +} + +[class*="span"].pull-right, +.row-fluid [class*="span"].pull-right { + float: right; +} + +.container { + margin-right: auto; + margin-left: auto; + *zoom: 1; +} + +.container:before, +.container:after { + display: table; + line-height: 0; + content: ""; +} + +.container:after { + clear: both; +} + +.container-fluid { + padding-right: 20px; + padding-left: 20px; + *zoom: 1; +} + +.container-fluid:before, +.container-fluid:after { + display: table; + line-height: 0; + content: ""; +} + +.container-fluid:after { + clear: both; +} + +p { + margin: 0 0 10px; +} + +.lead { + margin-bottom: 20px; + font-size: 21px; + font-weight: 200; + line-height: 30px; +} + +small { + font-size: 85%; +} + +strong { + font-weight: bold; +} + +em { + font-style: italic; +} + +cite { + font-style: normal; +} + +.muted { + color: #999999; +} + +.text-warning { + color: #c09853; +} + +.text-error { + color: #b94a48; +} + +.text-info { + color: #3a87ad; +} + +.text-success { + color: #468847; +} + +h1, +h2, +h3, +h4, +h5, +h6 { + margin: 10px 0; + font-family: inherit; + font-weight: bold; + line-height: 1; + color: inherit; + text-rendering: optimizelegibility; +} + +h1 small, +h2 small, +h3 small, +h4 small, +h5 small, +h6 small { + font-weight: normal; + line-height: 1; + color: #999999; +} + +h1 { + font-size: 36px; + line-height: 40px; +} + +h2 { + font-size: 30px; + line-height: 40px; +} + +h3 { + font-size: 24px; + line-height: 40px; +} + +h4 { + font-size: 18px; + line-height: 20px; +} + +h5 { + font-size: 14px; + line-height: 20px; +} + +h6 { + font-size: 12px; + line-height: 20px; +} + +h1 small { + font-size: 24px; +} + +h2 small { + font-size: 18px; +} + +h3 small { + font-size: 14px; +} + +h4 small { + font-size: 14px; +} + +.page-header { + padding-bottom: 9px; + margin: 20px 0 30px; + border-bottom: 1px solid #eeeeee; +} + +ul, +ol { + padding: 0; + margin: 0 0 10px 25px; +} + +ul ul, +ul ol, +ol ol, +ol ul { + margin-bottom: 0; +} + +li { + line-height: 20px; +} + +ul.unstyled, +ol.unstyled { + margin-left: 0; + list-style: none; +} + +dl { + margin-bottom: 20px; +} + +dt, +dd { + line-height: 20px; +} + +dt { + font-weight: bold; +} + +dd { + margin-left: 10px; +} + +.dl-horizontal { + *zoom: 1; +} + +.dl-horizontal:before, +.dl-horizontal:after { + display: table; + line-height: 0; + content: ""; +} + +.dl-horizontal:after { + clear: both; +} + +.dl-horizontal dt { + float: left; + width: 160px; + overflow: hidden; + clear: left; + text-align: right; + text-overflow: ellipsis; + white-space: nowrap; +} + +.dl-horizontal dd { + margin-left: 180px; +} + +hr { + margin: 20px 0; + border: 0; + border-top: 1px solid #eeeeee; + border-bottom: 1px solid #ffffff; +} + +abbr[title] { + cursor: help; + border-bottom: 1px dotted #999999; +} + +abbr.initialism { + font-size: 90%; + text-transform: uppercase; +} + +blockquote { + padding: 0 0 0 15px; + margin: 0 0 20px; + border-left: 5px solid #eeeeee; +} + +blockquote p { + margin-bottom: 0; + font-size: 16px; + font-weight: 300; + line-height: 25px; +} + +blockquote small { + display: block; + line-height: 20px; + color: #999999; +} + +blockquote small:before { + content: '\2014 \00A0'; +} + +blockquote.pull-right { + float: right; + padding-right: 15px; + padding-left: 0; + border-right: 5px solid #eeeeee; + border-left: 0; +} + +blockquote.pull-right p, +blockquote.pull-right small { + text-align: right; +} + +blockquote.pull-right small:before { + content: ''; +} + +blockquote.pull-right small:after { + content: '\00A0 \2014'; +} + +q:before, +q:after, +blockquote:before, +blockquote:after { + content: ""; +} + +address { + display: block; + margin-bottom: 20px; + font-style: normal; + line-height: 20px; +} + +code, +pre { + padding: 0 3px 2px; + font-family: Monaco, Menlo, Consolas, "Courier New", monospace; + font-size: 12px; + color: #333333; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; +} + +code { + padding: 2px 4px; + color: #d14; + background-color: #f7f7f9; + border: 1px solid #e1e1e8; +} + +pre { + display: block; + padding: 9.5px; + margin: 0 0 10px; + font-size: 13px; + line-height: 20px; + word-break: break-all; + word-wrap: break-word; + white-space: pre; + white-space: pre-wrap; + background-color: #f5f5f5; + border: 1px solid #ccc; + border: 1px solid rgba(0, 0, 0, 0.15); + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} + +pre.prettyprint { + margin-bottom: 20px; +} + +pre code { + padding: 0; + color: inherit; + background-color: transparent; + border: 0; +} + +.pre-scrollable { + max-height: 340px; + overflow-y: scroll; +} + +form { + margin: 0 0 20px; +} + +fieldset { + padding: 0; + margin: 0; + border: 0; +} + +legend { + display: block; + width: 100%; + padding: 0; + margin-bottom: 20px; + font-size: 21px; + line-height: 40px; + color: #333333; + border: 0; + border-bottom: 1px solid #e5e5e5; +} + +legend small { + font-size: 15px; + color: #999999; +} + +label, +input, +button, +select, +textarea { + font-size: 14px; + font-weight: normal; + line-height: 20px; +} + +input, +button, +select, +textarea { + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; +} + +label { + display: block; + margin-bottom: 5px; +} + +select, +textarea, +input[type="text"], +input[type="password"], +input[type="datetime"], +input[type="datetime-local"], +input[type="date"], +input[type="month"], +input[type="time"], +input[type="week"], +input[type="number"], +input[type="email"], +input[type="url"], +input[type="search"], +input[type="tel"], +input[type="color"], +.uneditable-input { + display: inline-block; + height: 20px; + padding: 4px 6px; + margin-bottom: 9px; + font-size: 14px; + line-height: 20px; + color: #555555; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; +} + +input, +textarea, +.uneditable-input { + width: 206px; +} + +textarea { + height: auto; +} + +textarea, +input[type="text"], +input[type="password"], +input[type="datetime"], +input[type="datetime-local"], +input[type="date"], +input[type="month"], +input[type="time"], +input[type="week"], +input[type="number"], +input[type="email"], +input[type="url"], +input[type="search"], +input[type="tel"], +input[type="color"], +.uneditable-input { + background-color: #ffffff; + border: 1px solid #cccccc; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -webkit-transition: border linear 0.2s, box-shadow linear 0.2s; + -moz-transition: border linear 0.2s, box-shadow linear 0.2s; + -o-transition: border linear 0.2s, box-shadow linear 0.2s; + transition: border linear 0.2s, box-shadow linear 0.2s; +} + +textarea:focus, +input[type="text"]:focus, +input[type="password"]:focus, +input[type="datetime"]:focus, +input[type="datetime-local"]:focus, +input[type="date"]:focus, +input[type="month"]:focus, +input[type="time"]:focus, +input[type="week"]:focus, +input[type="number"]:focus, +input[type="email"]:focus, +input[type="url"]:focus, +input[type="search"]:focus, +input[type="tel"]:focus, +input[type="color"]:focus, +.uneditable-input:focus { + border-color: rgba(82, 168, 236, 0.8); + outline: 0; + outline: thin dotted \9; + /* IE6-9 */ + + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); +} + +input[type="radio"], +input[type="checkbox"] { + margin: 4px 0 0; + margin-top: 1px \9; + *margin-top: 0; + line-height: normal; + cursor: pointer; +} + +input[type="file"], +input[type="image"], +input[type="submit"], +input[type="reset"], +input[type="button"], +input[type="radio"], +input[type="checkbox"] { + width: auto; +} + +select, +input[type="file"] { + height: 30px; + /* In IE7, the height of the select element cannot be changed by height, only font-size */ + + *margin-top: 4px; + /* For IE7, add top margin to align select with labels */ + + line-height: 30px; +} + +select { + width: 220px; + background-color: #ffffff; + border: 1px solid #cccccc; +} + +select[multiple], +select[size] { + height: auto; +} + +select:focus, +input[type="file"]:focus, +input[type="radio"]:focus, +input[type="checkbox"]:focus { + outline: thin dotted #333; + outline: 5px auto -webkit-focus-ring-color; + outline-offset: -2px; +} + +.uneditable-input, +.uneditable-textarea { + color: #999999; + cursor: not-allowed; + background-color: #fcfcfc; + border-color: #cccccc; + -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025); + -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025); + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.025); +} + +.uneditable-input { + overflow: hidden; + white-space: nowrap; +} + +.uneditable-textarea { + width: auto; + height: auto; +} + +input:-moz-placeholder, +textarea:-moz-placeholder { + color: #999999; +} + +input:-ms-input-placeholder, +textarea:-ms-input-placeholder { + color: #999999; +} + +input::-webkit-input-placeholder, +textarea::-webkit-input-placeholder { + color: #999999; +} + +.radio, +.checkbox { + min-height: 18px; + padding-left: 18px; +} + +.radio input[type="radio"], +.checkbox input[type="checkbox"] { + float: left; + margin-left: -18px; +} + +.controls > .radio:first-child, +.controls > .checkbox:first-child { + padding-top: 5px; +} + +.radio.inline, +.checkbox.inline { + display: inline-block; + padding-top: 5px; + margin-bottom: 0; + vertical-align: middle; +} + +.radio.inline + .radio.inline, +.checkbox.inline + .checkbox.inline { + margin-left: 10px; +} + +.input-mini { + width: 60px; +} + +.input-small { + width: 90px; +} + +.input-medium { + width: 150px; +} + +.input-large { + width: 210px; +} + +.input-xlarge { + width: 270px; +} + +.input-xxlarge { + width: 530px; +} + +input[class*="span"], +select[class*="span"], +textarea[class*="span"], +.uneditable-input[class*="span"], +.row-fluid input[class*="span"], +.row-fluid select[class*="span"], +.row-fluid textarea[class*="span"], +.row-fluid .uneditable-input[class*="span"] { + float: none; + margin-left: 0; +} + +.input-append input[class*="span"], +.input-append .uneditable-input[class*="span"], +.input-prepend input[class*="span"], +.input-prepend .uneditable-input[class*="span"], +.row-fluid input[class*="span"], +.row-fluid select[class*="span"], +.row-fluid textarea[class*="span"], +.row-fluid .uneditable-input[class*="span"], +.row-fluid .input-prepend [class*="span"], +.row-fluid .input-append [class*="span"] { + display: inline-block; +} + +input, +textarea, +.uneditable-input { + margin-left: 0; +} + +.controls-row [class*="span"] + [class*="span"] { + margin-left: 20px; +} + +input.span12, +textarea.span12, +.uneditable-input.span12 { + width: 926px; +} + +input.span11, +textarea.span11, +.uneditable-input.span11 { + width: 846px; +} + +input.span10, +textarea.span10, +.uneditable-input.span10 { + width: 766px; +} + +input.span9, +textarea.span9, +.uneditable-input.span9 { + width: 686px; +} + +input.span8, +textarea.span8, +.uneditable-input.span8 { + width: 606px; +} + +input.span7, +textarea.span7, +.uneditable-input.span7 { + width: 526px; +} + +input.span6, +textarea.span6, +.uneditable-input.span6 { + width: 446px; +} + +input.span5, +textarea.span5, +.uneditable-input.span5 { + width: 366px; +} + +input.span4, +textarea.span4, +.uneditable-input.span4 { + width: 286px; +} + +input.span3, +textarea.span3, +.uneditable-input.span3 { + width: 206px; +} + +input.span2, +textarea.span2, +.uneditable-input.span2 { + width: 126px; +} + +input.span1, +textarea.span1, +.uneditable-input.span1 { + width: 46px; +} + +.controls-row { + *zoom: 1; +} + +.controls-row:before, +.controls-row:after { + display: table; + line-height: 0; + content: ""; +} + +.controls-row:after { + clear: both; +} + +.controls-row [class*="span"] { + float: left; +} + +input[disabled], +select[disabled], +textarea[disabled], +input[readonly], +select[readonly], +textarea[readonly] { + cursor: not-allowed; + background-color: #eeeeee; +} + +input[type="radio"][disabled], +input[type="checkbox"][disabled], +input[type="radio"][readonly], +input[type="checkbox"][readonly] { + background-color: transparent; +} + +.control-group.warning > label, +.control-group.warning .help-block, +.control-group.warning .help-inline { + color: #c09853; +} + +.control-group.warning .checkbox, +.control-group.warning .radio, +.control-group.warning input, +.control-group.warning select, +.control-group.warning textarea { + color: #c09853; +} + +.control-group.warning input, +.control-group.warning select, +.control-group.warning textarea { + border-color: #c09853; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); +} + +.control-group.warning input:focus, +.control-group.warning select:focus, +.control-group.warning textarea:focus { + border-color: #a47e3c; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e; + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e; + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #dbc59e; +} + +.control-group.warning .input-prepend .add-on, +.control-group.warning .input-append .add-on { + color: #c09853; + background-color: #fcf8e3; + border-color: #c09853; +} + +.control-group.error > label, +.control-group.error .help-block, +.control-group.error .help-inline { + color: #b94a48; +} + +.control-group.error .checkbox, +.control-group.error .radio, +.control-group.error input, +.control-group.error select, +.control-group.error textarea { + color: #b94a48; +} + +.control-group.error input, +.control-group.error select, +.control-group.error textarea { + border-color: #b94a48; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); +} + +.control-group.error input:focus, +.control-group.error select:focus, +.control-group.error textarea:focus { + border-color: #953b39; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392; + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392; + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #d59392; +} + +.control-group.error .input-prepend .add-on, +.control-group.error .input-append .add-on { + color: #b94a48; + background-color: #f2dede; + border-color: #b94a48; +} + +.control-group.success > label, +.control-group.success .help-block, +.control-group.success .help-inline { + color: #468847; +} + +.control-group.success .checkbox, +.control-group.success .radio, +.control-group.success input, +.control-group.success select, +.control-group.success textarea { + color: #468847; +} + +.control-group.success input, +.control-group.success select, +.control-group.success textarea { + border-color: #468847; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); +} + +.control-group.success input:focus, +.control-group.success select:focus, +.control-group.success textarea:focus { + border-color: #356635; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b; + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b; + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7aba7b; +} + +.control-group.success .input-prepend .add-on, +.control-group.success .input-append .add-on { + color: #468847; + background-color: #dff0d8; + border-color: #468847; +} + +.control-group.info > label, +.control-group.info .help-block, +.control-group.info .help-inline { + color: #3a87ad; +} + +.control-group.info .checkbox, +.control-group.info .radio, +.control-group.info input, +.control-group.info select, +.control-group.info textarea { + color: #3a87ad; +} + +.control-group.info input, +.control-group.info select, +.control-group.info textarea { + border-color: #3a87ad; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); +} + +.control-group.info input:focus, +.control-group.info select:focus, +.control-group.info textarea:focus { + border-color: #2d6987; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3; + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3; + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 6px #7ab5d3; +} + +.control-group.info .input-prepend .add-on, +.control-group.info .input-append .add-on { + color: #3a87ad; + background-color: #d9edf7; + border-color: #3a87ad; +} + +input:focus:required:invalid, +textarea:focus:required:invalid, +select:focus:required:invalid { + color: #b94a48; + border-color: #ee5f5b; +} + +input:focus:required:invalid:focus, +textarea:focus:required:invalid:focus, +select:focus:required:invalid:focus { + border-color: #e9322d; + -webkit-box-shadow: 0 0 6px #f8b9b7; + -moz-box-shadow: 0 0 6px #f8b9b7; + box-shadow: 0 0 6px #f8b9b7; +} + +.form-actions { + padding: 19px 20px 20px; + margin-top: 20px; + margin-bottom: 20px; + background-color: #f5f5f5; + border-top: 1px solid #e5e5e5; + *zoom: 1; +} + +.form-actions:before, +.form-actions:after { + display: table; + line-height: 0; + content: ""; +} + +.form-actions:after { + clear: both; +} + +.help-block, +.help-inline { + color: #595959; +} + +.help-block { + display: block; + margin-bottom: 10px; +} + +.help-inline { + display: inline-block; + *display: inline; + padding-left: 5px; + vertical-align: middle; + *zoom: 1; +} + +.input-append, +.input-prepend { + margin-bottom: 5px; + font-size: 0; + white-space: nowrap; +} + +.input-append input, +.input-prepend input, +.input-append select, +.input-prepend select, +.input-append .uneditable-input, +.input-prepend .uneditable-input { + position: relative; + margin-bottom: 0; + *margin-left: 0; + font-size: 14px; + vertical-align: top; + -webkit-border-radius: 0 3px 3px 0; + -moz-border-radius: 0 3px 3px 0; + border-radius: 0 3px 3px 0; +} + +.input-append input:focus, +.input-prepend input:focus, +.input-append select:focus, +.input-prepend select:focus, +.input-append .uneditable-input:focus, +.input-prepend .uneditable-input:focus { + z-index: 2; +} + +.input-append .add-on, +.input-prepend .add-on { + display: inline-block; + width: auto; + height: 20px; + min-width: 16px; + padding: 4px 5px; + font-size: 14px; + font-weight: normal; + line-height: 20px; + text-align: center; + text-shadow: 0 1px 0 #ffffff; + background-color: #eeeeee; + border: 1px solid #ccc; +} + +.input-append .add-on, +.input-prepend .add-on, +.input-append .btn, +.input-prepend .btn { + vertical-align: top; + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.input-append .active, +.input-prepend .active { + background-color: #a9dba9; + border-color: #46a546; +} + +.input-prepend .add-on, +.input-prepend .btn { + margin-right: -1px; +} + +.input-prepend .add-on:first-child, +.input-prepend .btn:first-child { + -webkit-border-radius: 3px 0 0 3px; + -moz-border-radius: 3px 0 0 3px; + border-radius: 3px 0 0 3px; +} + +.input-append input, +.input-append select, +.input-append .uneditable-input { + -webkit-border-radius: 3px 0 0 3px; + -moz-border-radius: 3px 0 0 3px; + border-radius: 3px 0 0 3px; +} + +.input-append .add-on, +.input-append .btn { + margin-left: -1px; +} + +.input-append .add-on:last-child, +.input-append .btn:last-child { + -webkit-border-radius: 0 3px 3px 0; + -moz-border-radius: 0 3px 3px 0; + border-radius: 0 3px 3px 0; +} + +.input-prepend.input-append input, +.input-prepend.input-append select, +.input-prepend.input-append .uneditable-input { + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.input-prepend.input-append .add-on:first-child, +.input-prepend.input-append .btn:first-child { + margin-right: -1px; + -webkit-border-radius: 3px 0 0 3px; + -moz-border-radius: 3px 0 0 3px; + border-radius: 3px 0 0 3px; +} + +.input-prepend.input-append .add-on:last-child, +.input-prepend.input-append .btn:last-child { + margin-left: -1px; + -webkit-border-radius: 0 3px 3px 0; + -moz-border-radius: 0 3px 3px 0; + border-radius: 0 3px 3px 0; +} + +input.search-query { + padding-right: 14px; + padding-right: 4px \9; + padding-left: 14px; + padding-left: 4px \9; + /* IE7-8 doesn't have border-radius, so don't indent the padding */ + + margin-bottom: 0; + -webkit-border-radius: 15px; + -moz-border-radius: 15px; + border-radius: 15px; +} + +/* Allow for input prepend/append in search forms */ + +.form-search .input-append .search-query, +.form-search .input-prepend .search-query { + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.form-search .input-append .search-query { + -webkit-border-radius: 14px 0 0 14px; + -moz-border-radius: 14px 0 0 14px; + border-radius: 14px 0 0 14px; +} + +.form-search .input-append .btn { + -webkit-border-radius: 0 14px 14px 0; + -moz-border-radius: 0 14px 14px 0; + border-radius: 0 14px 14px 0; +} + +.form-search .input-prepend .search-query { + -webkit-border-radius: 0 14px 14px 0; + -moz-border-radius: 0 14px 14px 0; + border-radius: 0 14px 14px 0; +} + +.form-search .input-prepend .btn { + -webkit-border-radius: 14px 0 0 14px; + -moz-border-radius: 14px 0 0 14px; + border-radius: 14px 0 0 14px; +} + +.form-search input, +.form-inline input, +.form-horizontal input, +.form-search textarea, +.form-inline textarea, +.form-horizontal textarea, +.form-search select, +.form-inline select, +.form-horizontal select, +.form-search .help-inline, +.form-inline .help-inline, +.form-horizontal .help-inline, +.form-search .uneditable-input, +.form-inline .uneditable-input, +.form-horizontal .uneditable-input, +.form-search .input-prepend, +.form-inline .input-prepend, +.form-horizontal .input-prepend, +.form-search .input-append, +.form-inline .input-append, +.form-horizontal .input-append { + display: inline-block; + *display: inline; + margin-bottom: 0; + vertical-align: middle; + *zoom: 1; +} + +.form-search .hide, +.form-inline .hide, +.form-horizontal .hide { + display: none; +} + +.form-search label, +.form-inline label, +.form-search .btn-group, +.form-inline .btn-group { + display: inline-block; +} + +.form-search .input-append, +.form-inline .input-append, +.form-search .input-prepend, +.form-inline .input-prepend { + margin-bottom: 0; +} + +.form-search .radio, +.form-search .checkbox, +.form-inline .radio, +.form-inline .checkbox { + padding-left: 0; + margin-bottom: 0; + vertical-align: middle; +} + +.form-search .radio input[type="radio"], +.form-search .checkbox input[type="checkbox"], +.form-inline .radio input[type="radio"], +.form-inline .checkbox input[type="checkbox"] { + float: left; + margin-right: 3px; + margin-left: 0; +} + +.control-group { + margin-bottom: 10px; +} + +legend + .control-group { + margin-top: 20px; + -webkit-margin-top-collapse: separate; +} + +.form-horizontal .control-group { + margin-bottom: 20px; + *zoom: 1; +} + +.form-horizontal .control-group:before, +.form-horizontal .control-group:after { + display: table; + line-height: 0; + content: ""; +} + +.form-horizontal .control-group:after { + clear: both; +} + +.form-horizontal .control-label { + float: left; + width: 160px; + padding-top: 5px; + text-align: right; +} + +.form-horizontal .controls { + *display: inline-block; + *padding-left: 20px; + margin-left: 180px; + *margin-left: 0; +} + +.form-horizontal .controls:first-child { + *padding-left: 180px; +} + +.form-horizontal .help-block { + margin-bottom: 0; +} + +.form-horizontal input + .help-block, +.form-horizontal select + .help-block, +.form-horizontal textarea + .help-block { + margin-top: 10px; +} + +.form-horizontal .form-actions { + padding-left: 180px; +} + +table { + max-width: 100%; + background-color: transparent; + border-collapse: collapse; + border-spacing: 0; +} + +.table { + width: 100%; + margin-bottom: 20px; +} + +.table th, +.table td { + padding: 8px; + line-height: 20px; + text-align: left; + vertical-align: top; + border-top: 1px solid #dddddd; +} + +.table th { + font-weight: bold; +} + +.table thead th { + vertical-align: bottom; +} + +.table caption + thead tr:first-child th, +.table caption + thead tr:first-child td, +.table colgroup + thead tr:first-child th, +.table colgroup + thead tr:first-child td, +.table thead:first-child tr:first-child th, +.table thead:first-child tr:first-child td { + border-top: 0; +} + +.table tbody + tbody { + border-top: 2px solid #dddddd; +} + +.table-condensed th, +.table-condensed td { + padding: 4px 5px; +} + +.table-bordered { + border: 1px solid #dddddd; + border-collapse: separate; + *border-collapse: collapse; + border-left: 0; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} + +.table-bordered th, +.table-bordered td { + border-left: 1px solid #dddddd; +} + +.table-bordered caption + thead tr:first-child th, +.table-bordered caption + tbody tr:first-child th, +.table-bordered caption + tbody tr:first-child td, +.table-bordered colgroup + thead tr:first-child th, +.table-bordered colgroup + tbody tr:first-child th, +.table-bordered colgroup + tbody tr:first-child td, +.table-bordered thead:first-child tr:first-child th, +.table-bordered tbody:first-child tr:first-child th, +.table-bordered tbody:first-child tr:first-child td { + border-top: 0; +} + +.table-bordered thead:first-child tr:first-child th:first-child, +.table-bordered tbody:first-child tr:first-child td:first-child { + -webkit-border-top-left-radius: 4px; + border-top-left-radius: 4px; + -moz-border-radius-topleft: 4px; +} + +.table-bordered thead:first-child tr:first-child th:last-child, +.table-bordered tbody:first-child tr:first-child td:last-child { + -webkit-border-top-right-radius: 4px; + border-top-right-radius: 4px; + -moz-border-radius-topright: 4px; +} + +.table-bordered thead:last-child tr:last-child th:first-child, +.table-bordered tbody:last-child tr:last-child td:first-child, +.table-bordered tfoot:last-child tr:last-child td:first-child { + -webkit-border-radius: 0 0 0 4px; + -moz-border-radius: 0 0 0 4px; + border-radius: 0 0 0 4px; + -webkit-border-bottom-left-radius: 4px; + border-bottom-left-radius: 4px; + -moz-border-radius-bottomleft: 4px; +} + +.table-bordered thead:last-child tr:last-child th:last-child, +.table-bordered tbody:last-child tr:last-child td:last-child, +.table-bordered tfoot:last-child tr:last-child td:last-child { + -webkit-border-bottom-right-radius: 4px; + border-bottom-right-radius: 4px; + -moz-border-radius-bottomright: 4px; +} + +.table-bordered caption + thead tr:first-child th:first-child, +.table-bordered caption + tbody tr:first-child td:first-child, +.table-bordered colgroup + thead tr:first-child th:first-child, +.table-bordered colgroup + tbody tr:first-child td:first-child { + -webkit-border-top-left-radius: 4px; + border-top-left-radius: 4px; + -moz-border-radius-topleft: 4px; +} + +.table-bordered caption + thead tr:first-child th:last-child, +.table-bordered caption + tbody tr:first-child td:last-child, +.table-bordered colgroup + thead tr:first-child th:last-child, +.table-bordered colgroup + tbody tr:first-child td:last-child { + -webkit-border-top-right-radius: 4px; + border-top-right-radius: 4px; + -moz-border-radius-topleft: 4px; +} + +.table-striped tbody tr:nth-child(odd) td, +.table-striped tbody tr:nth-child(odd) th { + background-color: #f9f9f9; +} + +.table-hover tbody tr:hover td, +.table-hover tbody tr:hover th { + background-color: #f5f5f5; +} + +table [class*=span], +.row-fluid table [class*=span] { + display: table-cell; + float: none; + margin-left: 0; +} + +.table .span1 { + float: none; + width: 44px; + margin-left: 0; +} + +.table .span2 { + float: none; + width: 124px; + margin-left: 0; +} + +.table .span3 { + float: none; + width: 204px; + margin-left: 0; +} + +.table .span4 { + float: none; + width: 284px; + margin-left: 0; +} + +.table .span5 { + float: none; + width: 364px; + margin-left: 0; +} + +.table .span6 { + float: none; + width: 444px; + margin-left: 0; +} + +.table .span7 { + float: none; + width: 524px; + margin-left: 0; +} + +.table .span8 { + float: none; + width: 604px; + margin-left: 0; +} + +.table .span9 { + float: none; + width: 684px; + margin-left: 0; +} + +.table .span10 { + float: none; + width: 764px; + margin-left: 0; +} + +.table .span11 { + float: none; + width: 844px; + margin-left: 0; +} + +.table .span12 { + float: none; + width: 924px; + margin-left: 0; +} + +.table .span13 { + float: none; + width: 1004px; + margin-left: 0; +} + +.table .span14 { + float: none; + width: 1084px; + margin-left: 0; +} + +.table .span15 { + float: none; + width: 1164px; + margin-left: 0; +} + +.table .span16 { + float: none; + width: 1244px; + margin-left: 0; +} + +.table .span17 { + float: none; + width: 1324px; + margin-left: 0; +} + +.table .span18 { + float: none; + width: 1404px; + margin-left: 0; +} + +.table .span19 { + float: none; + width: 1484px; + margin-left: 0; +} + +.table .span20 { + float: none; + width: 1564px; + margin-left: 0; +} + +.table .span21 { + float: none; + width: 1644px; + margin-left: 0; +} + +.table .span22 { + float: none; + width: 1724px; + margin-left: 0; +} + +.table .span23 { + float: none; + width: 1804px; + margin-left: 0; +} + +.table .span24 { + float: none; + width: 1884px; + margin-left: 0; +} + +.table tbody tr.success td { + background-color: #dff0d8; +} + +.table tbody tr.error td { + background-color: #f2dede; +} + +.table tbody tr.warning td { + background-color: #fcf8e3; +} + +.table tbody tr.info td { + background-color: #d9edf7; +} + +.table-hover tbody tr.success:hover td { + background-color: #d0e9c6; +} + +.table-hover tbody tr.error:hover td { + background-color: #ebcccc; +} + +.table-hover tbody tr.warning:hover td { + background-color: #faf2cc; +} + +.table-hover tbody tr.info:hover td { + background-color: #c4e3f3; +} + +[class^="icon-"], +[class*=" icon-"] { + display: inline-block; + width: 14px; + height: 14px; + margin-top: 1px; + *margin-right: .3em; + line-height: 14px; + vertical-align: text-top; + background-image: url("../img/glyphicons-halflings.png"); + background-position: 14px 14px; + background-repeat: no-repeat; +} + +/* White icons with optional class, or on hover/active states of certain elements */ + +.icon-white, +.nav-tabs > .active > a > [class^="icon-"], +.nav-tabs > .active > a > [class*=" icon-"], +.nav-pills > .active > a > [class^="icon-"], +.nav-pills > .active > a > [class*=" icon-"], +.nav-list > .active > a > [class^="icon-"], +.nav-list > .active > a > [class*=" icon-"], +.navbar-inverse .nav > .active > a > [class^="icon-"], +.navbar-inverse .nav > .active > a > [class*=" icon-"], +.dropdown-menu > li > a:hover > [class^="icon-"], +.dropdown-menu > li > a:hover > [class*=" icon-"], +.dropdown-menu > .active > a > [class^="icon-"], +.dropdown-menu > .active > a > [class*=" icon-"] { + background-image: url("../img/glyphicons-halflings-white.png"); +} + +.icon-glass { + background-position: 0 0; +} + +.icon-music { + background-position: -24px 0; +} + +.icon-search { + background-position: -48px 0; +} + +.icon-envelope { + background-position: -72px 0; +} + +.icon-heart { + background-position: -96px 0; +} + +.icon-star { + background-position: -120px 0; +} + +.icon-star-empty { + background-position: -144px 0; +} + +.icon-user { + background-position: -168px 0; +} + +.icon-film { + background-position: -192px 0; +} + +.icon-th-large { + background-position: -216px 0; +} + +.icon-th { + background-position: -240px 0; +} + +.icon-th-list { + background-position: -264px 0; +} + +.icon-ok { + background-position: -288px 0; +} + +.icon-remove { + background-position: -312px 0; +} + +.icon-zoom-in { + background-position: -336px 0; +} + +.icon-zoom-out { + background-position: -360px 0; +} + +.icon-off { + background-position: -384px 0; +} + +.icon-signal { + background-position: -408px 0; +} + +.icon-cog { + background-position: -432px 0; +} + +.icon-trash { + background-position: -456px 0; +} + +.icon-home { + background-position: 0 -24px; +} + +.icon-file { + background-position: -24px -24px; +} + +.icon-time { + background-position: -48px -24px; +} + +.icon-road { + background-position: -72px -24px; +} + +.icon-download-alt { + background-position: -96px -24px; +} + +.icon-download { + background-position: -120px -24px; +} + +.icon-upload { + background-position: -144px -24px; +} + +.icon-inbox { + background-position: -168px -24px; +} + +.icon-play-circle { + background-position: -192px -24px; +} + +.icon-repeat { + background-position: -216px -24px; +} + +.icon-refresh { + background-position: -240px -24px; +} + +.icon-list-alt { + background-position: -264px -24px; +} + +.icon-lock { + background-position: -287px -24px; +} + +.icon-flag { + background-position: -312px -24px; +} + +.icon-headphones { + background-position: -336px -24px; +} + +.icon-volume-off { + background-position: -360px -24px; +} + +.icon-volume-down { + background-position: -384px -24px; +} + +.icon-volume-up { + background-position: -408px -24px; +} + +.icon-qrcode { + background-position: -432px -24px; +} + +.icon-barcode { + background-position: -456px -24px; +} + +.icon-tag { + background-position: 0 -48px; +} + +.icon-tags { + background-position: -25px -48px; +} + +.icon-book { + background-position: -48px -48px; +} + +.icon-bookmark { + background-position: -72px -48px; +} + +.icon-print { + background-position: -96px -48px; +} + +.icon-camera { + background-position: -120px -48px; +} + +.icon-font { + background-position: -144px -48px; +} + +.icon-bold { + background-position: -167px -48px; +} + +.icon-italic { + background-position: -192px -48px; +} + +.icon-text-height { + background-position: -216px -48px; +} + +.icon-text-width { + background-position: -240px -48px; +} + +.icon-align-left { + background-position: -264px -48px; +} + +.icon-align-center { + background-position: -288px -48px; +} + +.icon-align-right { + background-position: -312px -48px; +} + +.icon-align-justify { + background-position: -336px -48px; +} + +.icon-list { + background-position: -360px -48px; +} + +.icon-indent-left { + background-position: -384px -48px; +} + +.icon-indent-right { + background-position: -408px -48px; +} + +.icon-facetime-video { + background-position: -432px -48px; +} + +.icon-picture { + background-position: -456px -48px; +} + +.icon-pencil { + background-position: 0 -72px; +} + +.icon-map-marker { + background-position: -24px -72px; +} + +.icon-adjust { + background-position: -48px -72px; +} + +.icon-tint { + background-position: -72px -72px; +} + +.icon-edit { + background-position: -96px -72px; +} + +.icon-share { + background-position: -120px -72px; +} + +.icon-check { + background-position: -144px -72px; +} + +.icon-move { + background-position: -168px -72px; +} + +.icon-step-backward { + background-position: -192px -72px; +} + +.icon-fast-backward { + background-position: -216px -72px; +} + +.icon-backward { + background-position: -240px -72px; +} + +.icon-play { + background-position: -264px -72px; +} + +.icon-pause { + background-position: -288px -72px; +} + +.icon-stop { + background-position: -312px -72px; +} + +.icon-forward { + background-position: -336px -72px; +} + +.icon-fast-forward { + background-position: -360px -72px; +} + +.icon-step-forward { + background-position: -384px -72px; +} + +.icon-eject { + background-position: -408px -72px; +} + +.icon-chevron-left { + background-position: -432px -72px; +} + +.icon-chevron-right { + background-position: -456px -72px; +} + +.icon-plus-sign { + background-position: 0 -96px; +} + +.icon-minus-sign { + background-position: -24px -96px; +} + +.icon-remove-sign { + background-position: -48px -96px; +} + +.icon-ok-sign { + background-position: -72px -96px; +} + +.icon-question-sign { + background-position: -96px -96px; +} + +.icon-info-sign { + background-position: -120px -96px; +} + +.icon-screenshot { + background-position: -144px -96px; +} + +.icon-remove-circle { + background-position: -168px -96px; +} + +.icon-ok-circle { + background-position: -192px -96px; +} + +.icon-ban-circle { + background-position: -216px -96px; +} + +.icon-arrow-left { + background-position: -240px -96px; +} + +.icon-arrow-right { + background-position: -264px -96px; +} + +.icon-arrow-up { + background-position: -289px -96px; +} + +.icon-arrow-down { + background-position: -312px -96px; +} + +.icon-share-alt { + background-position: -336px -96px; +} + +.icon-resize-full { + background-position: -360px -96px; +} + +.icon-resize-small { + background-position: -384px -96px; +} + +.icon-plus { + background-position: -408px -96px; +} + +.icon-minus { + background-position: -433px -96px; +} + +.icon-asterisk { + background-position: -456px -96px; +} + +.icon-exclamation-sign { + background-position: 0 -120px; +} + +.icon-gift { + background-position: -24px -120px; +} + +.icon-leaf { + background-position: -48px -120px; +} + +.icon-fire { + background-position: -72px -120px; +} + +.icon-eye-open { + background-position: -96px -120px; +} + +.icon-eye-close { + background-position: -120px -120px; +} + +.icon-warning-sign { + background-position: -144px -120px; +} + +.icon-plane { + background-position: -168px -120px; +} + +.icon-calendar { + background-position: -192px -120px; +} + +.icon-random { + width: 16px; + background-position: -216px -120px; +} + +.icon-comment { + background-position: -240px -120px; +} + +.icon-magnet { + background-position: -264px -120px; +} + +.icon-chevron-up { + background-position: -288px -120px; +} + +.icon-chevron-down { + background-position: -313px -119px; +} + +.icon-retweet { + background-position: -336px -120px; +} + +.icon-shopping-cart { + background-position: -360px -120px; +} + +.icon-folder-close { + background-position: -384px -120px; +} + +.icon-folder-open { + width: 16px; + background-position: -408px -120px; +} + +.icon-resize-vertical { + background-position: -432px -119px; +} + +.icon-resize-horizontal { + background-position: -456px -118px; +} + +.icon-hdd { + background-position: 0 -144px; +} + +.icon-bullhorn { + background-position: -24px -144px; +} + +.icon-bell { + background-position: -48px -144px; +} + +.icon-certificate { + background-position: -72px -144px; +} + +.icon-thumbs-up { + background-position: -96px -144px; +} + +.icon-thumbs-down { + background-position: -120px -144px; +} + +.icon-hand-right { + background-position: -144px -144px; +} + +.icon-hand-left { + background-position: -168px -144px; +} + +.icon-hand-up { + background-position: -192px -144px; +} + +.icon-hand-down { + background-position: -216px -144px; +} + +.icon-circle-arrow-right { + background-position: -240px -144px; +} + +.icon-circle-arrow-left { + background-position: -264px -144px; +} + +.icon-circle-arrow-up { + background-position: -288px -144px; +} + +.icon-circle-arrow-down { + background-position: -312px -144px; +} + +.icon-globe { + background-position: -336px -144px; +} + +.icon-wrench { + background-position: -360px -144px; +} + +.icon-tasks { + background-position: -384px -144px; +} + +.icon-filter { + background-position: -408px -144px; +} + +.icon-briefcase { + background-position: -432px -144px; +} + +.icon-fullscreen { + background-position: -456px -144px; +} + +.dropup, +.dropdown { + position: relative; +} + +.dropdown-toggle { + *margin-bottom: -3px; +} + +.dropdown-toggle:active, +.open .dropdown-toggle { + outline: 0; +} + +.caret { + display: inline-block; + width: 0; + height: 0; + vertical-align: top; + border-top: 4px solid #000000; + border-right: 4px solid transparent; + border-left: 4px solid transparent; + content: ""; +} + +.dropdown .caret { + margin-top: 8px; + margin-left: 2px; +} + +.dropdown-menu { + position: absolute; + top: 100%; + left: 0; + z-index: 1000; + display: none; + float: left; + min-width: 160px; + padding: 5px 0; + margin: 2px 0 0; + list-style: none; + background-color: #ffffff; + border: 1px solid #ccc; + border: 1px solid rgba(0, 0, 0, 0.2); + *border-right-width: 2px; + *border-bottom-width: 2px; + -webkit-border-radius: 6px; + -moz-border-radius: 6px; + border-radius: 6px; + -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); + -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); + box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); + -webkit-background-clip: padding-box; + -moz-background-clip: padding; + background-clip: padding-box; +} + +.dropdown-menu.pull-right { + right: 0; + left: auto; +} + +.dropdown-menu .divider { + *width: 100%; + height: 1px; + margin: 9px 1px; + *margin: -5px 0 5px; + overflow: hidden; + background-color: #e5e5e5; + border-bottom: 1px solid #ffffff; +} + +.dropdown-menu a { + display: block; + padding: 3px 20px; + clear: both; + font-weight: normal; + line-height: 20px; + color: #333333; + white-space: nowrap; +} + +.dropdown-menu li > a:hover, +.dropdown-menu li > a:focus, +.dropdown-submenu:hover > a { + color: #ffffff; + text-decoration: none; + background-color: #0088cc; + background-color: #0081c2; + background-image: -moz-linear-gradient(top, #0088cc, #0077b3); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3)); + background-image: -webkit-linear-gradient(top, #0088cc, #0077b3); + background-image: -o-linear-gradient(top, #0088cc, #0077b3); + background-image: linear-gradient(to bottom, #0088cc, #0077b3); + background-repeat: repeat-x; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0); +} + +.dropdown-menu .active > a, +.dropdown-menu .active > a:hover { + color: #ffffff; + text-decoration: none; + background-color: #0088cc; + background-color: #0081c2; + background-image: linear-gradient(to bottom, #0088cc, #0077b3); + background-image: -moz-linear-gradient(top, #0088cc, #0077b3); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0077b3)); + background-image: -webkit-linear-gradient(top, #0088cc, #0077b3); + background-image: -o-linear-gradient(top, #0088cc, #0077b3); + background-repeat: repeat-x; + outline: 0; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0077b3', GradientType=0); +} + +.dropdown-menu .disabled > a, +.dropdown-menu .disabled > a:hover { + color: #999999; +} + +.dropdown-menu .disabled > a:hover { + text-decoration: none; + cursor: default; + background-color: transparent; +} + +.open { + *z-index: 1000; +} + +.open > .dropdown-menu { + display: block; +} + +.pull-right > .dropdown-menu { + right: 0; + left: auto; +} + +.dropup .caret, +.navbar-fixed-bottom .dropdown .caret { + border-top: 0; + border-bottom: 4px solid #000000; + content: ""; +} + +.dropup .dropdown-menu, +.navbar-fixed-bottom .dropdown .dropdown-menu { + top: auto; + bottom: 100%; + margin-bottom: 1px; +} + +.dropdown-submenu { + position: relative; +} + +.dropdown-submenu > .dropdown-menu { + top: 0; + left: 100%; + margin-top: -6px; + margin-left: -1px; + -webkit-border-radius: 0 6px 6px 6px; + -moz-border-radius: 0 6px 6px 6px; + border-radius: 0 6px 6px 6px; +} + +.dropdown-submenu:hover > .dropdown-menu { + display: block; +} + +.dropdown-submenu > a:after { + display: block; + float: right; + width: 0; + height: 0; + margin-top: 5px; + margin-right: -10px; + border-color: transparent; + border-left-color: #cccccc; + border-style: solid; + border-width: 5px 0 5px 5px; + content: " "; +} + +.dropdown-submenu:hover > a:after { + border-left-color: #ffffff; +} + +.dropdown .dropdown-menu .nav-header { + padding-right: 20px; + padding-left: 20px; +} + +.typeahead { + margin-top: 2px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} + +.well { + min-height: 20px; + padding: 19px; + margin-bottom: 20px; + background-color: #f5f5f5; + border: 1px solid #e3e3e3; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05); + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05); + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.05); +} + +.well blockquote { + border-color: #ddd; + border-color: rgba(0, 0, 0, 0.15); +} + +.well-large { + padding: 24px; + -webkit-border-radius: 6px; + -moz-border-radius: 6px; + border-radius: 6px; +} + +.well-small { + padding: 9px; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; +} + +.fade { + opacity: 0; + -webkit-transition: opacity 0.15s linear; + -moz-transition: opacity 0.15s linear; + -o-transition: opacity 0.15s linear; + transition: opacity 0.15s linear; +} + +.fade.in { + opacity: 1; +} + +.collapse { + position: relative; + height: 0; + overflow: hidden; + -webkit-transition: height 0.35s ease; + -moz-transition: height 0.35s ease; + -o-transition: height 0.35s ease; + transition: height 0.35s ease; +} + +.collapse.in { + height: auto; +} + +.close { + float: right; + font-size: 20px; + font-weight: bold; + line-height: 20px; + color: #000000; + text-shadow: 0 1px 0 #ffffff; + opacity: 0.2; + filter: alpha(opacity=20); +} + +.close:hover { + color: #000000; + text-decoration: none; + cursor: pointer; + opacity: 0.4; + filter: alpha(opacity=40); +} + +button.close { + padding: 0; + cursor: pointer; + background: transparent; + border: 0; + -webkit-appearance: none; +} + +.btn { + display: inline-block; + *display: inline; + padding: 4px 14px; + margin-bottom: 0; + *margin-left: .3em; + font-size: 14px; + line-height: 20px; + *line-height: 20px; + color: #333333; + text-align: center; + text-shadow: 0 1px 1px rgba(255, 255, 255, 0.75); + vertical-align: middle; + cursor: pointer; + background-color: #f5f5f5; + *background-color: #e6e6e6; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#e6e6e6)); + background-image: -webkit-linear-gradient(top, #ffffff, #e6e6e6); + background-image: -o-linear-gradient(top, #ffffff, #e6e6e6); + background-image: linear-gradient(to bottom, #ffffff, #e6e6e6); + background-image: -moz-linear-gradient(top, #ffffff, #e6e6e6); + background-repeat: repeat-x; + border: 1px solid #bbbbbb; + *border: 0; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + border-color: #e6e6e6 #e6e6e6 #bfbfbf; + border-bottom-color: #a2a2a2; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe6e6e6', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); + *zoom: 1; + -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05); + -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05); +} + +.btn:hover, +.btn:active, +.btn.active, +.btn.disabled, +.btn[disabled] { + color: #333333; + background-color: #e6e6e6; + *background-color: #d9d9d9; +} + +.btn:active, +.btn.active { + background-color: #cccccc \9; +} + +.btn:first-child { + *margin-left: 0; +} + +.btn:hover { + color: #333333; + text-decoration: none; + background-color: #e6e6e6; + *background-color: #d9d9d9; + /* Buttons in IE7 don't get borders, so darken on hover */ + + background-position: 0 -15px; + -webkit-transition: background-position 0.1s linear; + -moz-transition: background-position 0.1s linear; + -o-transition: background-position 0.1s linear; + transition: background-position 0.1s linear; +} + +.btn:focus { + outline: thin dotted #333; + outline: 5px auto -webkit-focus-ring-color; + outline-offset: -2px; +} + +.btn.active, +.btn:active { + background-color: #e6e6e6; + background-color: #d9d9d9 \9; + background-image: none; + outline: 0; + -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05); + -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05); + box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05); +} + +.btn.disabled, +.btn[disabled] { + cursor: default; + background-color: #e6e6e6; + background-image: none; + opacity: 0.65; + filter: alpha(opacity=65); + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; +} + +.btn-large { + padding: 9px 14px; + font-size: 16px; + line-height: normal; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-radius: 5px; +} + +.btn-large [class^="icon-"] { + margin-top: 2px; +} + +.btn-small { + padding: 3px 9px; + font-size: 12px; + line-height: 18px; +} + +.btn-small [class^="icon-"] { + margin-top: 0; +} + +.btn-mini { + padding: 2px 6px; + font-size: 11px; + line-height: 17px; +} + +.btn-block { + display: block; + width: 100%; + padding-right: 0; + padding-left: 0; + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; +} + +.btn-block + .btn-block { + margin-top: 5px; +} + +input[type="submit"].btn-block, +input[type="reset"].btn-block, +input[type="button"].btn-block { + width: 100%; +} + +.btn-primary.active, +.btn-warning.active, +.btn-danger.active, +.btn-success.active, +.btn-info.active, +.btn-inverse.active { + color: rgba(255, 255, 255, 0.75); +} + +.btn { + border-color: #c5c5c5; + border-color: rgba(0, 0, 0, 0.15) rgba(0, 0, 0, 0.15) rgba(0, 0, 0, 0.25); +} + +.btn-primary { + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #006dcc; + *background-color: #0044cc; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#0088cc), to(#0044cc)); + background-image: -webkit-linear-gradient(top, #0088cc, #0044cc); + background-image: -o-linear-gradient(top, #0088cc, #0044cc); + background-image: linear-gradient(to bottom, #0088cc, #0044cc); + background-image: -moz-linear-gradient(top, #0088cc, #0044cc); + background-repeat: repeat-x; + border-color: #0044cc #0044cc #002a80; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc', endColorstr='#ff0044cc', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); +} + +.btn-primary:hover, +.btn-primary:active, +.btn-primary.active, +.btn-primary.disabled, +.btn-primary[disabled] { + color: #ffffff; + background-color: #0044cc; + *background-color: #003bb3; +} + +.btn-primary:active, +.btn-primary.active { + background-color: #003399 \9; +} + +.btn-warning { + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #faa732; + *background-color: #f89406; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406)); + background-image: -webkit-linear-gradient(top, #fbb450, #f89406); + background-image: -o-linear-gradient(top, #fbb450, #f89406); + background-image: linear-gradient(to bottom, #fbb450, #f89406); + background-image: -moz-linear-gradient(top, #fbb450, #f89406); + background-repeat: repeat-x; + border-color: #f89406 #f89406 #ad6704; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); +} + +.btn-warning:hover, +.btn-warning:active, +.btn-warning.active, +.btn-warning.disabled, +.btn-warning[disabled] { + color: #ffffff; + background-color: #f89406; + *background-color: #df8505; +} + +.btn-warning:active, +.btn-warning.active { + background-color: #c67605 \9; +} + +.btn-danger { + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #da4f49; + *background-color: #bd362f; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#bd362f)); + background-image: -webkit-linear-gradient(top, #ee5f5b, #bd362f); + background-image: -o-linear-gradient(top, #ee5f5b, #bd362f); + background-image: linear-gradient(to bottom, #ee5f5b, #bd362f); + background-image: -moz-linear-gradient(top, #ee5f5b, #bd362f); + background-repeat: repeat-x; + border-color: #bd362f #bd362f #802420; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffbd362f', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); +} + +.btn-danger:hover, +.btn-danger:active, +.btn-danger.active, +.btn-danger.disabled, +.btn-danger[disabled] { + color: #ffffff; + background-color: #bd362f; + *background-color: #a9302a; +} + +.btn-danger:active, +.btn-danger.active { + background-color: #942a25 \9; +} + +.btn-success { + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #5bb75b; + *background-color: #51a351; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#51a351)); + background-image: -webkit-linear-gradient(top, #62c462, #51a351); + background-image: -o-linear-gradient(top, #62c462, #51a351); + background-image: linear-gradient(to bottom, #62c462, #51a351); + background-image: -moz-linear-gradient(top, #62c462, #51a351); + background-repeat: repeat-x; + border-color: #51a351 #51a351 #387038; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff51a351', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); +} + +.btn-success:hover, +.btn-success:active, +.btn-success.active, +.btn-success.disabled, +.btn-success[disabled] { + color: #ffffff; + background-color: #51a351; + *background-color: #499249; +} + +.btn-success:active, +.btn-success.active { + background-color: #408140 \9; +} + +.btn-info { + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #49afcd; + *background-color: #2f96b4; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#2f96b4)); + background-image: -webkit-linear-gradient(top, #5bc0de, #2f96b4); + background-image: -o-linear-gradient(top, #5bc0de, #2f96b4); + background-image: linear-gradient(to bottom, #5bc0de, #2f96b4); + background-image: -moz-linear-gradient(top, #5bc0de, #2f96b4); + background-repeat: repeat-x; + border-color: #2f96b4 #2f96b4 #1f6377; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2f96b4', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); +} + +.btn-info:hover, +.btn-info:active, +.btn-info.active, +.btn-info.disabled, +.btn-info[disabled] { + color: #ffffff; + background-color: #2f96b4; + *background-color: #2a85a0; +} + +.btn-info:active, +.btn-info.active { + background-color: #24748c \9; +} + +.btn-inverse { + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #363636; + *background-color: #222222; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#444444), to(#222222)); + background-image: -webkit-linear-gradient(top, #444444, #222222); + background-image: -o-linear-gradient(top, #444444, #222222); + background-image: linear-gradient(to bottom, #444444, #222222); + background-image: -moz-linear-gradient(top, #444444, #222222); + background-repeat: repeat-x; + border-color: #222222 #222222 #000000; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff444444', endColorstr='#ff222222', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); +} + +.btn-inverse:hover, +.btn-inverse:active, +.btn-inverse.active, +.btn-inverse.disabled, +.btn-inverse[disabled] { + color: #ffffff; + background-color: #222222; + *background-color: #151515; +} + +.btn-inverse:active, +.btn-inverse.active { + background-color: #080808 \9; +} + +button.btn, +input[type="submit"].btn { + *padding-top: 3px; + *padding-bottom: 3px; +} + +button.btn::-moz-focus-inner, +input[type="submit"].btn::-moz-focus-inner { + padding: 0; + border: 0; +} + +button.btn.btn-large, +input[type="submit"].btn.btn-large { + *padding-top: 7px; + *padding-bottom: 7px; +} + +button.btn.btn-small, +input[type="submit"].btn.btn-small { + *padding-top: 3px; + *padding-bottom: 3px; +} + +button.btn.btn-mini, +input[type="submit"].btn.btn-mini { + *padding-top: 1px; + *padding-bottom: 1px; +} + +.btn-link, +.btn-link:active, +.btn-link[disabled] { + background-color: transparent; + background-image: none; + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; +} + +.btn-link { + color: #0088cc; + cursor: pointer; + border-color: transparent; + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.btn-link:hover { + color: #005580; + text-decoration: underline; + background-color: transparent; +} + +.btn-link[disabled]:hover { + color: #333333; + text-decoration: none; +} + +.btn-group { + position: relative; + *margin-left: .3em; + font-size: 0; + white-space: nowrap; + vertical-align: middle; +} + +.btn-group:first-child { + *margin-left: 0; +} + +.btn-group + .btn-group { + margin-left: 5px; +} + +.btn-toolbar { + margin-top: 10px; + margin-bottom: 10px; + font-size: 0; +} + +.btn-toolbar .btn-group { + display: inline-block; + *display: inline; + /* IE7 inline-block hack */ + + *zoom: 1; +} + +.btn-toolbar .btn + .btn, +.btn-toolbar .btn-group + .btn, +.btn-toolbar .btn + .btn-group { + margin-left: 5px; +} + +.btn-group > .btn { + position: relative; + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.btn-group > .btn + .btn { + margin-left: -1px; +} + +.btn-group > .btn, +.btn-group > .dropdown-menu { + font-size: 14px; +} + +.btn-group > .btn-mini { + font-size: 11px; +} + +.btn-group > .btn-small { + font-size: 12px; +} + +.btn-group > .btn-large { + font-size: 16px; +} + +.btn-group > .btn:first-child { + margin-left: 0; + -webkit-border-bottom-left-radius: 4px; + border-bottom-left-radius: 4px; + -webkit-border-top-left-radius: 4px; + border-top-left-radius: 4px; + -moz-border-radius-bottomleft: 4px; + -moz-border-radius-topleft: 4px; +} + +.btn-group > .btn:last-child, +.btn-group > .dropdown-toggle { + -webkit-border-top-right-radius: 4px; + border-top-right-radius: 4px; + -webkit-border-bottom-right-radius: 4px; + border-bottom-right-radius: 4px; + -moz-border-radius-topright: 4px; + -moz-border-radius-bottomright: 4px; +} + +.btn-group > .btn.large:first-child { + margin-left: 0; + -webkit-border-bottom-left-radius: 6px; + border-bottom-left-radius: 6px; + -webkit-border-top-left-radius: 6px; + border-top-left-radius: 6px; + -moz-border-radius-bottomleft: 6px; + -moz-border-radius-topleft: 6px; +} + +.btn-group > .btn.large:last-child, +.btn-group > .large.dropdown-toggle { + -webkit-border-top-right-radius: 6px; + border-top-right-radius: 6px; + -webkit-border-bottom-right-radius: 6px; + border-bottom-right-radius: 6px; + -moz-border-radius-topright: 6px; + -moz-border-radius-bottomright: 6px; +} + +.btn-group > .btn:hover, +.btn-group > .btn:focus, +.btn-group > .btn:active, +.btn-group > .btn.active { + z-index: 2; +} + +.btn-group .dropdown-toggle:active, +.btn-group.open .dropdown-toggle { + outline: 0; +} + +.btn-group > .btn + .dropdown-toggle { + *padding-top: 5px; + padding-right: 8px; + *padding-bottom: 5px; + padding-left: 8px; + -webkit-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05); + -moz-box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05); + box-shadow: inset 1px 0 0 rgba(255, 255, 255, 0.125), inset 0 1px 0 rgba(255, 255, 255, 0.2), 0 1px 2px rgba(0, 0, 0, 0.05); +} + +.btn-group > .btn-mini + .dropdown-toggle { + *padding-top: 2px; + padding-right: 5px; + *padding-bottom: 2px; + padding-left: 5px; +} + +.btn-group > .btn-small + .dropdown-toggle { + *padding-top: 5px; + *padding-bottom: 4px; +} + +.btn-group > .btn-large + .dropdown-toggle { + *padding-top: 7px; + padding-right: 12px; + *padding-bottom: 7px; + padding-left: 12px; +} + +.btn-group.open .dropdown-toggle { + background-image: none; + -webkit-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05); + -moz-box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05); + box-shadow: inset 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.05); +} + +.btn-group.open .btn.dropdown-toggle { + background-color: #e6e6e6; +} + +.btn-group.open .btn-primary.dropdown-toggle { + background-color: #0044cc; +} + +.btn-group.open .btn-warning.dropdown-toggle { + background-color: #f89406; +} + +.btn-group.open .btn-danger.dropdown-toggle { + background-color: #bd362f; +} + +.btn-group.open .btn-success.dropdown-toggle { + background-color: #51a351; +} + +.btn-group.open .btn-info.dropdown-toggle { + background-color: #2f96b4; +} + +.btn-group.open .btn-inverse.dropdown-toggle { + background-color: #222222; +} + +.btn .caret { + margin-top: 8px; + margin-left: 0; +} + +.btn-mini .caret, +.btn-small .caret, +.btn-large .caret { + margin-top: 6px; +} + +.btn-large .caret { + border-top-width: 5px; + border-right-width: 5px; + border-left-width: 5px; +} + +.dropup .btn-large .caret { + border-top: 0; + border-bottom: 5px solid #000000; +} + +.btn-primary .caret, +.btn-warning .caret, +.btn-danger .caret, +.btn-info .caret, +.btn-success .caret, +.btn-inverse .caret { + border-top-color: #ffffff; + border-bottom-color: #ffffff; +} + +.btn-group-vertical { + display: inline-block; + *display: inline; + /* IE7 inline-block hack */ + + *zoom: 1; +} + +.btn-group-vertical .btn { + display: block; + float: none; + width: 100%; + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.btn-group-vertical .btn + .btn { + margin-top: -1px; + margin-left: 0; +} + +.btn-group-vertical .btn:first-child { + -webkit-border-radius: 4px 4px 0 0; + -moz-border-radius: 4px 4px 0 0; + border-radius: 4px 4px 0 0; +} + +.btn-group-vertical .btn:last-child { + -webkit-border-radius: 0 0 4px 4px; + -moz-border-radius: 0 0 4px 4px; + border-radius: 0 0 4px 4px; +} + +.btn-group-vertical .btn-large:first-child { + -webkit-border-radius: 6px 6px 0 0; + -moz-border-radius: 6px 6px 0 0; + border-radius: 6px 6px 0 0; +} + +.btn-group-vertical .btn-large:last-child { + -webkit-border-radius: 0 0 6px 6px; + -moz-border-radius: 0 0 6px 6px; + border-radius: 0 0 6px 6px; +} + +.alert { + padding: 8px 35px 8px 14px; + margin-bottom: 20px; + color: #c09853; + text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); + background-color: #fcf8e3; + border: 1px solid #fbeed5; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} + +.alert h4 { + margin: 0; +} + +.alert .close { + position: relative; + top: -2px; + right: -21px; + line-height: 20px; +} + +.alert-success { + color: #468847; + background-color: #dff0d8; + border-color: #d6e9c6; +} + +.alert-danger, +.alert-error { + color: #b94a48; + background-color: #f2dede; + border-color: #eed3d7; +} + +.alert-info { + color: #3a87ad; + background-color: #d9edf7; + border-color: #bce8f1; +} + +.alert-block { + padding-top: 14px; + padding-bottom: 14px; +} + +.alert-block > p, +.alert-block > ul { + margin-bottom: 0; +} + +.alert-block p + p { + margin-top: 5px; +} + +.nav { + margin-bottom: 20px; + margin-left: 0; + list-style: none; +} + +.nav > li > a { + display: block; +} + +.nav > li > a:hover { + text-decoration: none; + background-color: #eeeeee; +} + +.nav > .pull-right { + float: right; +} + +.nav-header { + display: block; + padding: 3px 15px; + font-size: 11px; + font-weight: bold; + line-height: 20px; + color: #999999; + text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); + text-transform: uppercase; +} + +.nav li + .nav-header { + margin-top: 9px; +} + +.nav-list { + padding-right: 15px; + padding-left: 15px; + margin-bottom: 0; +} + +.nav-list > li > a, +.nav-list .nav-header { + margin-right: -15px; + margin-left: -15px; + text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); +} + +.nav-list > li > a { + padding: 3px 15px; +} + +.nav-list > .active > a, +.nav-list > .active > a:hover { + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2); + background-color: #0088cc; +} + +.nav-list [class^="icon-"] { + margin-right: 2px; +} + +.nav-list .divider { + *width: 100%; + height: 1px; + margin: 9px 1px; + *margin: -5px 0 5px; + overflow: hidden; + background-color: #e5e5e5; + border-bottom: 1px solid #ffffff; +} + +.nav-tabs, +.nav-pills { + *zoom: 1; +} + +.nav-tabs:before, +.nav-pills:before, +.nav-tabs:after, +.nav-pills:after { + display: table; + line-height: 0; + content: ""; +} + +.nav-tabs:after, +.nav-pills:after { + clear: both; +} + +.nav-tabs > li, +.nav-pills > li { + float: left; +} + +.nav-tabs > li > a, +.nav-pills > li > a { + padding-right: 12px; + padding-left: 12px; + margin-right: 2px; + line-height: 14px; +} + +.nav-tabs { + border-bottom: 1px solid #ddd; +} + +.nav-tabs > li { + margin-bottom: -1px; +} + +.nav-tabs > li > a { + padding-top: 8px; + padding-bottom: 8px; + line-height: 20px; + border: 1px solid transparent; + -webkit-border-radius: 4px 4px 0 0; + -moz-border-radius: 4px 4px 0 0; + border-radius: 4px 4px 0 0; +} + +.nav-tabs > li > a:hover { + border-color: #eeeeee #eeeeee #dddddd; +} + +.nav-tabs > .active > a, +.nav-tabs > .active > a:hover { + color: #555555; + cursor: default; + background-color: #ffffff; + border: 1px solid #ddd; + border-bottom-color: transparent; +} + +.nav-pills > li > a { + padding-top: 8px; + padding-bottom: 8px; + margin-top: 2px; + margin-bottom: 2px; + -webkit-border-radius: 5px; + -moz-border-radius: 5px; + border-radius: 5px; +} + +.nav-pills > .active > a, +.nav-pills > .active > a:hover { + color: #ffffff; + background-color: #0088cc; +} + +.nav-stacked > li { + float: none; +} + +.nav-stacked > li > a { + margin-right: 0; +} + +.nav-tabs.nav-stacked { + border-bottom: 0; +} + +.nav-tabs.nav-stacked > li > a { + border: 1px solid #ddd; + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.nav-tabs.nav-stacked > li:first-child > a { + -webkit-border-top-right-radius: 4px; + border-top-right-radius: 4px; + -webkit-border-top-left-radius: 4px; + border-top-left-radius: 4px; + -moz-border-radius-topright: 4px; + -moz-border-radius-topleft: 4px; +} + +.nav-tabs.nav-stacked > li:last-child > a { + -webkit-border-bottom-right-radius: 4px; + border-bottom-right-radius: 4px; + -webkit-border-bottom-left-radius: 4px; + border-bottom-left-radius: 4px; + -moz-border-radius-bottomright: 4px; + -moz-border-radius-bottomleft: 4px; +} + +.nav-tabs.nav-stacked > li > a:hover { + z-index: 2; + border-color: #ddd; +} + +.nav-pills.nav-stacked > li > a { + margin-bottom: 3px; +} + +.nav-pills.nav-stacked > li:last-child > a { + margin-bottom: 1px; +} + +.nav-tabs .dropdown-menu { + -webkit-border-radius: 0 0 6px 6px; + -moz-border-radius: 0 0 6px 6px; + border-radius: 0 0 6px 6px; +} + +.nav-pills .dropdown-menu { + -webkit-border-radius: 6px; + -moz-border-radius: 6px; + border-radius: 6px; +} + +.nav .dropdown-toggle .caret { + margin-top: 6px; + border-top-color: #0088cc; + border-bottom-color: #0088cc; +} + +.nav .dropdown-toggle:hover .caret { + border-top-color: #005580; + border-bottom-color: #005580; +} + +/* move down carets for tabs */ + +.nav-tabs .dropdown-toggle .caret { + margin-top: 8px; +} + +.nav .active .dropdown-toggle .caret { + border-top-color: #fff; + border-bottom-color: #fff; +} + +.nav-tabs .active .dropdown-toggle .caret { + border-top-color: #555555; + border-bottom-color: #555555; +} + +.nav > .dropdown.active > a:hover { + cursor: pointer; +} + +.nav-tabs .open .dropdown-toggle, +.nav-pills .open .dropdown-toggle, +.nav > li.dropdown.open.active > a:hover { + color: #ffffff; + background-color: #999999; + border-color: #999999; +} + +.nav li.dropdown.open .caret, +.nav li.dropdown.open.active .caret, +.nav li.dropdown.open a:hover .caret { + border-top-color: #ffffff; + border-bottom-color: #ffffff; + opacity: 1; + filter: alpha(opacity=100); +} + +.tabs-stacked .open > a:hover { + border-color: #999999; +} + +.tabbable { + *zoom: 1; +} + +.tabbable:before, +.tabbable:after { + display: table; + line-height: 0; + content: ""; +} + +.tabbable:after { + clear: both; +} + +.tab-content { + overflow: auto; +} + +.tabs-below > .nav-tabs, +.tabs-right > .nav-tabs, +.tabs-left > .nav-tabs { + border-bottom: 0; +} + +.tab-content > .tab-pane, +.pill-content > .pill-pane { + display: none; +} + +.tab-content > .active, +.pill-content > .active { + display: block; +} + +.tabs-below > .nav-tabs { + border-top: 1px solid #ddd; +} + +.tabs-below > .nav-tabs > li { + margin-top: -1px; + margin-bottom: 0; +} + +.tabs-below > .nav-tabs > li > a { + -webkit-border-radius: 0 0 4px 4px; + -moz-border-radius: 0 0 4px 4px; + border-radius: 0 0 4px 4px; +} + +.tabs-below > .nav-tabs > li > a:hover { + border-top-color: #ddd; + border-bottom-color: transparent; +} + +.tabs-below > .nav-tabs > .active > a, +.tabs-below > .nav-tabs > .active > a:hover { + border-color: transparent #ddd #ddd #ddd; +} + +.tabs-left > .nav-tabs > li, +.tabs-right > .nav-tabs > li { + float: none; +} + +.tabs-left > .nav-tabs > li > a, +.tabs-right > .nav-tabs > li > a { + min-width: 74px; + margin-right: 0; + margin-bottom: 3px; +} + +.tabs-left > .nav-tabs { + float: left; + margin-right: 19px; + border-right: 1px solid #ddd; +} + +.tabs-left > .nav-tabs > li > a { + margin-right: -1px; + -webkit-border-radius: 4px 0 0 4px; + -moz-border-radius: 4px 0 0 4px; + border-radius: 4px 0 0 4px; +} + +.tabs-left > .nav-tabs > li > a:hover { + border-color: #eeeeee #dddddd #eeeeee #eeeeee; +} + +.tabs-left > .nav-tabs .active > a, +.tabs-left > .nav-tabs .active > a:hover { + border-color: #ddd transparent #ddd #ddd; + *border-right-color: #ffffff; +} + +.tabs-right > .nav-tabs { + float: right; + margin-left: 19px; + border-left: 1px solid #ddd; +} + +.tabs-right > .nav-tabs > li > a { + margin-left: -1px; + -webkit-border-radius: 0 4px 4px 0; + -moz-border-radius: 0 4px 4px 0; + border-radius: 0 4px 4px 0; +} + +.tabs-right > .nav-tabs > li > a:hover { + border-color: #eeeeee #eeeeee #eeeeee #dddddd; +} + +.tabs-right > .nav-tabs .active > a, +.tabs-right > .nav-tabs .active > a:hover { + border-color: #ddd #ddd #ddd transparent; + *border-left-color: #ffffff; +} + +.nav > .disabled > a { + color: #999999; +} + +.nav > .disabled > a:hover { + text-decoration: none; + cursor: default; + background-color: transparent; +} + +.navbar { + *position: relative; + *z-index: 2; + margin-bottom: 20px; + overflow: visible; + color: #777777; +} + +.navbar-inner { + min-height: 40px; + padding-right: 20px; + padding-left: 20px; + background-color: #fafafa; + background-image: -moz-linear-gradient(top, #ffffff, #f2f2f2); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ffffff), to(#f2f2f2)); + background-image: -webkit-linear-gradient(top, #ffffff, #f2f2f2); + background-image: -o-linear-gradient(top, #ffffff, #f2f2f2); + background-image: linear-gradient(to bottom, #ffffff, #f2f2f2); + background-repeat: repeat-x; + border: 1px solid #d4d4d4; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff2f2f2', GradientType=0); + *zoom: 1; + -webkit-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065); + -moz-box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065); + box-shadow: 0 1px 4px rgba(0, 0, 0, 0.065); +} + +.navbar-inner:before, +.navbar-inner:after { + display: table; + line-height: 0; + content: ""; +} + +.navbar-inner:after { + clear: both; +} + +.navbar .container { + width: auto; +} + +.nav-collapse.collapse { + height: auto; +} + +.navbar .brand { + display: block; + float: left; + padding: 10px 20px 10px; + margin-left: -20px; + font-size: 20px; + font-weight: 200; + color: #777777; + text-shadow: 0 1px 0 #ffffff; +} + +.navbar .brand:hover { + text-decoration: none; +} + +.navbar-text { + margin-bottom: 0; + line-height: 40px; +} + +.navbar-link { + color: #777777; +} + +.navbar-link:hover { + color: #333333; +} + +.navbar .divider-vertical { + height: 40px; + margin: 0 9px; + border-right: 1px solid #ffffff; + border-left: 1px solid #f2f2f2; +} + +.navbar .btn, +.navbar .btn-group { + margin-top: 5px; +} + +.navbar .btn-group .btn, +.navbar .input-prepend .btn, +.navbar .input-append .btn { + margin-top: 0; +} + +.navbar-form { + margin-bottom: 0; + *zoom: 1; +} + +.navbar-form:before, +.navbar-form:after { + display: table; + line-height: 0; + content: ""; +} + +.navbar-form:after { + clear: both; +} + +.navbar-form input, +.navbar-form select, +.navbar-form .radio, +.navbar-form .checkbox { + margin-top: 5px; +} + +.navbar-form input, +.navbar-form select, +.navbar-form .btn { + display: inline-block; + margin-bottom: 0; +} + +.navbar-form input[type="image"], +.navbar-form input[type="checkbox"], +.navbar-form input[type="radio"] { + margin-top: 3px; +} + +.navbar-form .input-append, +.navbar-form .input-prepend { + margin-top: 6px; + white-space: nowrap; +} + +.navbar-form .input-append input, +.navbar-form .input-prepend input { + margin-top: 0; +} + +.navbar-search { + position: relative; + float: left; + margin-top: 5px; + margin-bottom: 0; +} + +.navbar-search .search-query { + padding: 4px 14px; + margin-bottom: 0; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-size: 13px; + font-weight: normal; + line-height: 1; + -webkit-border-radius: 15px; + -moz-border-radius: 15px; + border-radius: 15px; +} + +.navbar-static-top { + position: static; + width: 100%; + margin-bottom: 0; +} + +.navbar-static-top .navbar-inner { + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.navbar-fixed-top, +.navbar-fixed-bottom { + position: fixed; + right: 0; + left: 0; + z-index: 1030; + margin-bottom: 0; +} + +.navbar-fixed-top .navbar-inner, +.navbar-static-top .navbar-inner { + border-width: 0 0 1px; +} + +.navbar-fixed-bottom .navbar-inner { + border-width: 1px 0 0; +} + +.navbar-fixed-top .navbar-inner, +.navbar-fixed-bottom .navbar-inner { + padding-right: 0; + padding-left: 0; + -webkit-border-radius: 0; + -moz-border-radius: 0; + border-radius: 0; +} + +.navbar-static-top .container, +.navbar-fixed-top .container, +.navbar-fixed-bottom .container { + width: 940px; +} + +.navbar-fixed-top { + top: 0; +} + +.navbar-fixed-top .navbar-inner, +.navbar-static-top .navbar-inner { + -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1); + -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1); + box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.1), 0 1px 10px rgba(0, 0, 0, 0.1); +} + +.navbar-fixed-bottom { + bottom: 0; +} + +.navbar-fixed-bottom .navbar-inner { + -webkit-box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1); + -moz-box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1); + box-shadow: inset 0 1px 0 rgba(0, 0, 0, 0.1), 0 -1px 10px rgba(0, 0, 0, 0.1); +} + +.navbar .nav { + position: relative; + left: 0; + display: block; + float: left; + margin: 0 10px 0 0; +} + +.navbar .nav.pull-right { + float: right; + margin-right: 0; +} + +.navbar .nav > li { + float: left; +} + +.navbar .nav > li > a { + float: none; + padding: 10px 15px 10px; + color: #777777; + text-decoration: none; + text-shadow: 0 1px 0 #ffffff; +} + +.navbar .nav .dropdown-toggle .caret { + margin-top: 8px; +} + +.navbar .nav > li > a:focus, +.navbar .nav > li > a:hover { + color: #333333; + text-decoration: none; + background-color: transparent; +} + +.navbar .nav > .active > a, +.navbar .nav > .active > a:hover, +.navbar .nav > .active > a:focus { + color: #555555; + text-decoration: none; + background-color: #e5e5e5; + -webkit-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125); + -moz-box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125); + box-shadow: inset 0 3px 8px rgba(0, 0, 0, 0.125); +} + +.navbar .btn-navbar { + display: none; + float: right; + padding: 7px 10px; + margin-right: 5px; + margin-left: 5px; + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #ededed; + *background-color: #e5e5e5; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f2f2f2), to(#e5e5e5)); + background-image: -webkit-linear-gradient(top, #f2f2f2, #e5e5e5); + background-image: -o-linear-gradient(top, #f2f2f2, #e5e5e5); + background-image: linear-gradient(to bottom, #f2f2f2, #e5e5e5); + background-image: -moz-linear-gradient(top, #f2f2f2, #e5e5e5); + background-repeat: repeat-x; + border-color: #e5e5e5 #e5e5e5 #bfbfbf; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fff2f2f2', endColorstr='#ffe5e5e5', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); + -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075); + -moz-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075); + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.1), 0 1px 0 rgba(255, 255, 255, 0.075); +} + +.navbar .btn-navbar:hover, +.navbar .btn-navbar:active, +.navbar .btn-navbar.active, +.navbar .btn-navbar.disabled, +.navbar .btn-navbar[disabled] { + color: #ffffff; + background-color: #e5e5e5; + *background-color: #d9d9d9; +} + +.navbar .btn-navbar:active, +.navbar .btn-navbar.active { + background-color: #cccccc \9; +} + +.navbar .btn-navbar .icon-bar { + display: block; + width: 18px; + height: 2px; + background-color: #f5f5f5; + -webkit-border-radius: 1px; + -moz-border-radius: 1px; + border-radius: 1px; + -webkit-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25); + -moz-box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25); + box-shadow: 0 1px 0 rgba(0, 0, 0, 0.25); +} + +.btn-navbar .icon-bar + .icon-bar { + margin-top: 3px; +} + +.navbar .nav > li > .dropdown-menu:before { + position: absolute; + top: -7px; + left: 9px; + display: inline-block; + border-right: 7px solid transparent; + border-bottom: 7px solid #ccc; + border-left: 7px solid transparent; + border-bottom-color: rgba(0, 0, 0, 0.2); + content: ''; +} + +.navbar .nav > li > .dropdown-menu:after { + position: absolute; + top: -6px; + left: 10px; + display: inline-block; + border-right: 6px solid transparent; + border-bottom: 6px solid #ffffff; + border-left: 6px solid transparent; + content: ''; +} + +.navbar-fixed-bottom .nav > li > .dropdown-menu:before { + top: auto; + bottom: -7px; + border-top: 7px solid #ccc; + border-bottom: 0; + border-top-color: rgba(0, 0, 0, 0.2); +} + +.navbar-fixed-bottom .nav > li > .dropdown-menu:after { + top: auto; + bottom: -6px; + border-top: 6px solid #ffffff; + border-bottom: 0; +} + +.navbar .nav li.dropdown.open > .dropdown-toggle, +.navbar .nav li.dropdown.active > .dropdown-toggle, +.navbar .nav li.dropdown.open.active > .dropdown-toggle { + color: #555555; + background-color: #e5e5e5; +} + +.navbar .nav li.dropdown > .dropdown-toggle .caret { + border-top-color: #777777; + border-bottom-color: #777777; +} + +.navbar .nav li.dropdown.open > .dropdown-toggle .caret, +.navbar .nav li.dropdown.active > .dropdown-toggle .caret, +.navbar .nav li.dropdown.open.active > .dropdown-toggle .caret { + border-top-color: #555555; + border-bottom-color: #555555; +} + +.navbar .pull-right > li > .dropdown-menu, +.navbar .nav > li > .dropdown-menu.pull-right { + right: 0; + left: auto; +} + +.navbar .pull-right > li > .dropdown-menu:before, +.navbar .nav > li > .dropdown-menu.pull-right:before { + right: 12px; + left: auto; +} + +.navbar .pull-right > li > .dropdown-menu:after, +.navbar .nav > li > .dropdown-menu.pull-right:after { + right: 13px; + left: auto; +} + +.navbar .pull-right > li > .dropdown-menu .dropdown-menu, +.navbar .nav > li > .dropdown-menu.pull-right .dropdown-menu { + right: 100%; + left: auto; + margin-right: -1px; + margin-left: 0; + -webkit-border-radius: 6px 0 6px 6px; + -moz-border-radius: 6px 0 6px 6px; + border-radius: 6px 0 6px 6px; +} + +.navbar-inverse { + color: #999999; +} + +.navbar-inverse .navbar-inner { + background-color: #1b1b1b; + background-image: -moz-linear-gradient(top, #222222, #111111); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#222222), to(#111111)); + background-image: -webkit-linear-gradient(top, #222222, #111111); + background-image: -o-linear-gradient(top, #222222, #111111); + background-image: linear-gradient(to bottom, #222222, #111111); + background-repeat: repeat-x; + border-color: #252525; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff222222', endColorstr='#ff111111', GradientType=0); +} + +.navbar-inverse .brand, +.navbar-inverse .nav > li > a { + color: #999999; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); +} + +.navbar-inverse .brand:hover, +.navbar-inverse .nav > li > a:hover { + color: #ffffff; +} + +.navbar-inverse .nav > li > a:focus, +.navbar-inverse .nav > li > a:hover { + color: #ffffff; + background-color: transparent; +} + +.navbar-inverse .nav .active > a, +.navbar-inverse .nav .active > a:hover, +.navbar-inverse .nav .active > a:focus { + color: #ffffff; + background-color: #111111; +} + +.navbar-inverse .navbar-link { + color: #999999; +} + +.navbar-inverse .navbar-link:hover { + color: #ffffff; +} + +.navbar-inverse .divider-vertical { + border-right-color: #222222; + border-left-color: #111111; +} + +.navbar-inverse .nav li.dropdown.open > .dropdown-toggle, +.navbar-inverse .nav li.dropdown.active > .dropdown-toggle, +.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle { + color: #ffffff; + background-color: #111111; +} + +.navbar-inverse .nav li.dropdown > .dropdown-toggle .caret { + border-top-color: #999999; + border-bottom-color: #999999; +} + +.navbar-inverse .nav li.dropdown.open > .dropdown-toggle .caret, +.navbar-inverse .nav li.dropdown.active > .dropdown-toggle .caret, +.navbar-inverse .nav li.dropdown.open.active > .dropdown-toggle .caret { + border-top-color: #ffffff; + border-bottom-color: #ffffff; +} + +.navbar-inverse .navbar-search .search-query { + color: #ffffff; + background-color: #515151; + border-color: #111111; + -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15); + -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15); + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1), 0 1px 0 rgba(255, 255, 255, 0.15); + -webkit-transition: none; + -moz-transition: none; + -o-transition: none; + transition: none; +} + +.navbar-inverse .navbar-search .search-query:-moz-placeholder { + color: #cccccc; +} + +.navbar-inverse .navbar-search .search-query:-ms-input-placeholder { + color: #cccccc; +} + +.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder { + color: #cccccc; +} + +.navbar-inverse .navbar-search .search-query:focus, +.navbar-inverse .navbar-search .search-query.focused { + padding: 5px 15px; + color: #333333; + text-shadow: 0 1px 0 #ffffff; + background-color: #ffffff; + border: 0; + outline: 0; + -webkit-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15); + -moz-box-shadow: 0 0 3px rgba(0, 0, 0, 0.15); + box-shadow: 0 0 3px rgba(0, 0, 0, 0.15); +} + +.navbar-inverse .btn-navbar { + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #0e0e0e; + *background-color: #040404; + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#151515), to(#040404)); + background-image: -webkit-linear-gradient(top, #151515, #040404); + background-image: -o-linear-gradient(top, #151515, #040404); + background-image: linear-gradient(to bottom, #151515, #040404); + background-image: -moz-linear-gradient(top, #151515, #040404); + background-repeat: repeat-x; + border-color: #040404 #040404 #000000; + border-color: rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.1) rgba(0, 0, 0, 0.25); + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff151515', endColorstr='#ff040404', GradientType=0); + filter: progid:dximagetransform.microsoft.gradient(enabled=false); +} + +.navbar-inverse .btn-navbar:hover, +.navbar-inverse .btn-navbar:active, +.navbar-inverse .btn-navbar.active, +.navbar-inverse .btn-navbar.disabled, +.navbar-inverse .btn-navbar[disabled] { + color: #ffffff; + background-color: #040404; + *background-color: #000000; +} + +.navbar-inverse .btn-navbar:active, +.navbar-inverse .btn-navbar.active { + background-color: #000000 \9; +} + +.breadcrumb { + padding: 8px 15px; + margin: 0 0 20px; + list-style: none; + background-color: #f5f5f5; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} + +.breadcrumb li { + display: inline-block; + *display: inline; + text-shadow: 0 1px 0 #ffffff; + *zoom: 1; +} + +.breadcrumb .divider { + padding: 0 5px; + color: #ccc; +} + +.breadcrumb .active { + color: #999999; +} + +.pagination { + height: 40px; + margin: 20px 0; +} + +.pagination ul { + display: inline-block; + *display: inline; + margin-bottom: 0; + margin-left: 0; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; + *zoom: 1; + -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); + -moz-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); +} + +.pagination ul > li { + display: inline; +} + +.pagination ul > li > a, +.pagination ul > li > span { + float: left; + padding: 0 14px; + line-height: 38px; + text-decoration: none; + background-color: #ffffff; + border: 1px solid #dddddd; + border-left-width: 0; +} + +.pagination ul > li > a:hover, +.pagination ul > .active > a, +.pagination ul > .active > span { + background-color: #f5f5f5; +} + +.pagination ul > .active > a, +.pagination ul > .active > span { + color: #999999; + cursor: default; +} + +.pagination ul > .disabled > span, +.pagination ul > .disabled > a, +.pagination ul > .disabled > a:hover { + color: #999999; + cursor: default; + background-color: transparent; +} + +.pagination ul > li:first-child > a, +.pagination ul > li:first-child > span { + border-left-width: 1px; + -webkit-border-radius: 3px 0 0 3px; + -moz-border-radius: 3px 0 0 3px; + border-radius: 3px 0 0 3px; +} + +.pagination ul > li:last-child > a, +.pagination ul > li:last-child > span { + -webkit-border-radius: 0 3px 3px 0; + -moz-border-radius: 0 3px 3px 0; + border-radius: 0 3px 3px 0; +} + +.pagination-centered { + text-align: center; +} + +.pagination-right { + text-align: right; +} + +.pager { + margin: 20px 0; + text-align: center; + list-style: none; + *zoom: 1; +} + +.pager:before, +.pager:after { + display: table; + line-height: 0; + content: ""; +} + +.pager:after { + clear: both; +} + +.pager li { + display: inline; +} + +.pager a, +.pager span { + display: inline-block; + padding: 5px 14px; + background-color: #fff; + border: 1px solid #ddd; + -webkit-border-radius: 15px; + -moz-border-radius: 15px; + border-radius: 15px; +} + +.pager a:hover { + text-decoration: none; + background-color: #f5f5f5; +} + +.pager .next a, +.pager .next span { + float: right; +} + +.pager .previous a { + float: left; +} + +.pager .disabled a, +.pager .disabled a:hover, +.pager .disabled span { + color: #999999; + cursor: default; + background-color: #fff; +} + +.modal-open .modal .dropdown-menu { + z-index: 2050; +} + +.modal-open .modal .dropdown.open { + *z-index: 2050; +} + +.modal-open .modal .popover { + z-index: 2060; +} + +.modal-open .modal .tooltip { + z-index: 2080; +} + +.modal-backdrop { + position: fixed; + top: 0; + right: 0; + bottom: 0; + left: 0; + z-index: 1040; + background-color: #000000; +} + +.modal-backdrop.fade { + opacity: 0; +} + +.modal-backdrop, +.modal-backdrop.fade.in { + opacity: 0.8; + filter: alpha(opacity=80); +} + +.modal { + position: fixed; + top: 50%; + left: 50%; + z-index: 1050; + width: 560px; + margin: -250px 0 0 -280px; + overflow: auto; + background-color: #ffffff; + border: 1px solid #999; + border: 1px solid rgba(0, 0, 0, 0.3); + *border: 1px solid #999; + -webkit-border-radius: 6px; + -moz-border-radius: 6px; + border-radius: 6px; + -webkit-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3); + -moz-box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3); + box-shadow: 0 3px 7px rgba(0, 0, 0, 0.3); + -webkit-background-clip: padding-box; + -moz-background-clip: padding-box; + background-clip: padding-box; +} + +.modal.fade { + top: -25%; + -webkit-transition: opacity 0.3s linear, top 0.3s ease-out; + -moz-transition: opacity 0.3s linear, top 0.3s ease-out; + -o-transition: opacity 0.3s linear, top 0.3s ease-out; + transition: opacity 0.3s linear, top 0.3s ease-out; +} + +.modal.fade.in { + top: 50%; +} + +.modal-header { + padding: 9px 15px; + border-bottom: 1px solid #eee; +} + +.modal-header .close { + margin-top: 2px; +} + +.modal-header h3 { + margin: 0; + line-height: 30px; +} + +.modal-body { + max-height: 400px; + padding: 15px; + overflow-y: auto; +} + +.modal-form { + margin-bottom: 0; +} + +.modal-footer { + padding: 14px 15px 15px; + margin-bottom: 0; + text-align: right; + background-color: #f5f5f5; + border-top: 1px solid #ddd; + -webkit-border-radius: 0 0 6px 6px; + -moz-border-radius: 0 0 6px 6px; + border-radius: 0 0 6px 6px; + *zoom: 1; + -webkit-box-shadow: inset 0 1px 0 #ffffff; + -moz-box-shadow: inset 0 1px 0 #ffffff; + box-shadow: inset 0 1px 0 #ffffff; +} + +.modal-footer:before, +.modal-footer:after { + display: table; + line-height: 0; + content: ""; +} + +.modal-footer:after { + clear: both; +} + +.modal-footer .btn + .btn { + margin-bottom: 0; + margin-left: 5px; +} + +.modal-footer .btn-group .btn + .btn { + margin-left: -1px; +} + +.tooltip { + position: absolute; + z-index: 1030; + display: block; + padding: 5px; + font-size: 11px; + opacity: 0; + filter: alpha(opacity=0); + visibility: visible; +} + +.tooltip.in { + opacity: 0.8; + filter: alpha(opacity=80); +} + +.tooltip.top { + margin-top: -3px; +} + +.tooltip.right { + margin-left: 3px; +} + +.tooltip.bottom { + margin-top: 3px; +} + +.tooltip.left { + margin-left: -3px; +} + +.tooltip-inner { + max-width: 200px; + padding: 3px 8px; + color: #ffffff; + text-align: center; + text-decoration: none; + background-color: #000000; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} + +.tooltip-arrow { + position: absolute; + width: 0; + height: 0; + border-color: transparent; + border-style: solid; +} + +.tooltip.top .tooltip-arrow { + bottom: 0; + left: 50%; + margin-left: -5px; + border-top-color: #000000; + border-width: 5px 5px 0; +} + +.tooltip.right .tooltip-arrow { + top: 50%; + left: 0; + margin-top: -5px; + border-right-color: #000000; + border-width: 5px 5px 5px 0; +} + +.tooltip.left .tooltip-arrow { + top: 50%; + right: 0; + margin-top: -5px; + border-left-color: #000000; + border-width: 5px 0 5px 5px; +} + +.tooltip.bottom .tooltip-arrow { + top: 0; + left: 50%; + margin-left: -5px; + border-bottom-color: #000000; + border-width: 0 5px 5px; +} + +.popover { + position: absolute; + top: 0; + left: 0; + z-index: 1010; + display: none; + width: 236px; + padding: 1px; + background-color: #ffffff; + border: 1px solid #ccc; + border: 1px solid rgba(0, 0, 0, 0.2); + -webkit-border-radius: 6px; + -moz-border-radius: 6px; + border-radius: 6px; + -webkit-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); + -moz-box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); + box-shadow: 0 5px 10px rgba(0, 0, 0, 0.2); + -webkit-background-clip: padding-box; + -moz-background-clip: padding; + background-clip: padding-box; +} + +.popover.top { + margin-bottom: 10px; +} + +.popover.right { + margin-left: 10px; +} + +.popover.bottom { + margin-top: 10px; +} + +.popover.left { + margin-right: 10px; +} + +.popover-title { + padding: 8px 14px; + margin: 0; + font-size: 14px; + font-weight: normal; + line-height: 18px; + background-color: #f7f7f7; + border-bottom: 1px solid #ebebeb; + -webkit-border-radius: 5px 5px 0 0; + -moz-border-radius: 5px 5px 0 0; + border-radius: 5px 5px 0 0; +} + +.popover-content { + padding: 9px 14px; +} + +.popover-content p, +.popover-content ul, +.popover-content ol { + margin-bottom: 0; +} + +.popover .arrow, +.popover .arrow:after { + position: absolute; + display: inline-block; + width: 0; + height: 0; + border-color: transparent; + border-style: solid; +} + +.popover .arrow:after { + z-index: -1; + content: ""; +} + +.popover.top .arrow { + bottom: -10px; + left: 50%; + margin-left: -10px; + border-top-color: #ffffff; + border-width: 10px 10px 0; +} + +.popover.top .arrow:after { + bottom: -1px; + left: -11px; + border-top-color: rgba(0, 0, 0, 0.25); + border-width: 11px 11px 0; +} + +.popover.right .arrow { + top: 50%; + left: -10px; + margin-top: -10px; + border-right-color: #ffffff; + border-width: 10px 10px 10px 0; +} + +.popover.right .arrow:after { + bottom: -11px; + left: -1px; + border-right-color: rgba(0, 0, 0, 0.25); + border-width: 11px 11px 11px 0; +} + +.popover.bottom .arrow { + top: -10px; + left: 50%; + margin-left: -10px; + border-bottom-color: #ffffff; + border-width: 0 10px 10px; +} + +.popover.bottom .arrow:after { + top: -1px; + left: -11px; + border-bottom-color: rgba(0, 0, 0, 0.25); + border-width: 0 11px 11px; +} + +.popover.left .arrow { + top: 50%; + right: -10px; + margin-top: -10px; + border-left-color: #ffffff; + border-width: 10px 0 10px 10px; +} + +.popover.left .arrow:after { + right: -1px; + bottom: -11px; + border-left-color: rgba(0, 0, 0, 0.25); + border-width: 11px 0 11px 11px; +} + +.thumbnails { + margin-left: -20px; + list-style: none; + *zoom: 1; +} + +.thumbnails:before, +.thumbnails:after { + display: table; + line-height: 0; + content: ""; +} + +.thumbnails:after { + clear: both; +} + +.row-fluid .thumbnails { + margin-left: 0; +} + +.thumbnails > li { + float: left; + margin-bottom: 20px; + margin-left: 20px; +} + +.thumbnail { + display: block; + padding: 4px; + line-height: 20px; + border: 1px solid #ddd; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + -webkit-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055); + -moz-box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.055); + -webkit-transition: all 0.2s ease-in-out; + -moz-transition: all 0.2s ease-in-out; + -o-transition: all 0.2s ease-in-out; + transition: all 0.2s ease-in-out; +} + +a.thumbnail:hover { + border-color: #0088cc; + -webkit-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25); + -moz-box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25); + box-shadow: 0 1px 4px rgba(0, 105, 214, 0.25); +} + +.thumbnail > img { + display: block; + max-width: 100%; + margin-right: auto; + margin-left: auto; +} + +.thumbnail .caption { + padding: 9px; + color: #555555; +} + +.label, +.badge { + font-size: 11.844px; + font-weight: bold; + line-height: 14px; + color: #ffffff; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + white-space: nowrap; + vertical-align: baseline; + background-color: #999999; +} + +.label { + padding: 1px 4px 2px; + -webkit-border-radius: 3px; + -moz-border-radius: 3px; + border-radius: 3px; +} + +.badge { + padding: 1px 9px 2px; + -webkit-border-radius: 9px; + -moz-border-radius: 9px; + border-radius: 9px; +} + +a.label:hover, +a.badge:hover { + color: #ffffff; + text-decoration: none; + cursor: pointer; +} + +.label-important, +.badge-important { + background-color: #b94a48; +} + +.label-important[href], +.badge-important[href] { + background-color: #953b39; +} + +.label-warning, +.badge-warning { + background-color: #f89406; +} + +.label-warning[href], +.badge-warning[href] { + background-color: #c67605; +} + +.label-success, +.badge-success { + background-color: #468847; +} + +.label-success[href], +.badge-success[href] { + background-color: #356635; +} + +.label-info, +.badge-info { + background-color: #3a87ad; +} + +.label-info[href], +.badge-info[href] { + background-color: #2d6987; +} + +.label-inverse, +.badge-inverse { + background-color: #333333; +} + +.label-inverse[href], +.badge-inverse[href] { + background-color: #1a1a1a; +} + +.btn .label, +.btn .badge { + position: relative; + top: -1px; +} + +.btn-mini .label, +.btn-mini .badge { + top: 0; +} + +@-webkit-keyframes progress-bar-stripes { + from { + background-position: 40px 0; + } + to { + background-position: 0 0; + } +} + +@-moz-keyframes progress-bar-stripes { + from { + background-position: 40px 0; + } + to { + background-position: 0 0; + } +} + +@-ms-keyframes progress-bar-stripes { + from { + background-position: 40px 0; + } + to { + background-position: 0 0; + } +} + +@-o-keyframes progress-bar-stripes { + from { + background-position: 0 0; + } + to { + background-position: 40px 0; + } +} + +@keyframes progress-bar-stripes { + from { + background-position: 40px 0; + } + to { + background-position: 0 0; + } +} + +.progress { + height: 20px; + margin-bottom: 20px; + overflow: hidden; + background-color: #f7f7f7; + background-image: -moz-linear-gradient(top, #f5f5f5, #f9f9f9); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#f5f5f5), to(#f9f9f9)); + background-image: -webkit-linear-gradient(top, #f5f5f5, #f9f9f9); + background-image: -o-linear-gradient(top, #f5f5f5, #f9f9f9); + background-image: linear-gradient(to bottom, #f5f5f5, #f9f9f9); + background-repeat: repeat-x; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#fff9f9f9', GradientType=0); + -webkit-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1); + -moz-box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1); + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.1); +} + +.progress .bar { + float: left; + width: 0; + height: 100%; + font-size: 12px; + color: #ffffff; + text-align: center; + text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25); + background-color: #0e90d2; + background-image: -moz-linear-gradient(top, #149bdf, #0480be); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#149bdf), to(#0480be)); + background-image: -webkit-linear-gradient(top, #149bdf, #0480be); + background-image: -o-linear-gradient(top, #149bdf, #0480be); + background-image: linear-gradient(to bottom, #149bdf, #0480be); + background-repeat: repeat-x; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff149bdf', endColorstr='#ff0480be', GradientType=0); + -webkit-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15); + -moz-box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15); + box-shadow: inset 0 -1px 0 rgba(0, 0, 0, 0.15); + -webkit-box-sizing: border-box; + -moz-box-sizing: border-box; + box-sizing: border-box; + -webkit-transition: width 0.6s ease; + -moz-transition: width 0.6s ease; + -o-transition: width 0.6s ease; + transition: width 0.6s ease; +} + +.progress .bar + .bar { + -webkit-box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15); + -moz-box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15); + box-shadow: inset 1px 0 0 rgba(0, 0, 0, 0.15), inset 0 -1px 0 rgba(0, 0, 0, 0.15); +} + +.progress-striped .bar { + background-color: #149bdf; + background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + -webkit-background-size: 40px 40px; + -moz-background-size: 40px 40px; + -o-background-size: 40px 40px; + background-size: 40px 40px; +} + +.progress.active .bar { + -webkit-animation: progress-bar-stripes 2s linear infinite; + -moz-animation: progress-bar-stripes 2s linear infinite; + -ms-animation: progress-bar-stripes 2s linear infinite; + -o-animation: progress-bar-stripes 2s linear infinite; + animation: progress-bar-stripes 2s linear infinite; +} + +.progress-danger .bar, +.progress .bar-danger { + background-color: #dd514c; + background-image: -moz-linear-gradient(top, #ee5f5b, #c43c35); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#ee5f5b), to(#c43c35)); + background-image: -webkit-linear-gradient(top, #ee5f5b, #c43c35); + background-image: -o-linear-gradient(top, #ee5f5b, #c43c35); + background-image: linear-gradient(to bottom, #ee5f5b, #c43c35); + background-repeat: repeat-x; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b', endColorstr='#ffc43c35', GradientType=0); +} + +.progress-danger.progress-striped .bar, +.progress-striped .bar-danger { + background-color: #ee5f5b; + background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); +} + +.progress-success .bar, +.progress .bar-success { + background-color: #5eb95e; + background-image: -moz-linear-gradient(top, #62c462, #57a957); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#62c462), to(#57a957)); + background-image: -webkit-linear-gradient(top, #62c462, #57a957); + background-image: -o-linear-gradient(top, #62c462, #57a957); + background-image: linear-gradient(to bottom, #62c462, #57a957); + background-repeat: repeat-x; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462', endColorstr='#ff57a957', GradientType=0); +} + +.progress-success.progress-striped .bar, +.progress-striped .bar-success { + background-color: #62c462; + background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); +} + +.progress-info .bar, +.progress .bar-info { + background-color: #4bb1cf; + background-image: -moz-linear-gradient(top, #5bc0de, #339bb9); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#5bc0de), to(#339bb9)); + background-image: -webkit-linear-gradient(top, #5bc0de, #339bb9); + background-image: -o-linear-gradient(top, #5bc0de, #339bb9); + background-image: linear-gradient(to bottom, #5bc0de, #339bb9); + background-repeat: repeat-x; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff339bb9', GradientType=0); +} + +.progress-info.progress-striped .bar, +.progress-striped .bar-info { + background-color: #5bc0de; + background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); +} + +.progress-warning .bar, +.progress .bar-warning { + background-color: #faa732; + background-image: -moz-linear-gradient(top, #fbb450, #f89406); + background-image: -webkit-gradient(linear, 0 0, 0 100%, from(#fbb450), to(#f89406)); + background-image: -webkit-linear-gradient(top, #fbb450, #f89406); + background-image: -o-linear-gradient(top, #fbb450, #f89406); + background-image: linear-gradient(to bottom, #fbb450, #f89406); + background-repeat: repeat-x; + filter: progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450', endColorstr='#fff89406', GradientType=0); +} + +.progress-warning.progress-striped .bar, +.progress-striped .bar-warning { + background-color: #fbb450; + background-image: -webkit-gradient(linear, 0 100%, 100% 0, color-stop(0.25, rgba(255, 255, 255, 0.15)), color-stop(0.25, transparent), color-stop(0.5, transparent), color-stop(0.5, rgba(255, 255, 255, 0.15)), color-stop(0.75, rgba(255, 255, 255, 0.15)), color-stop(0.75, transparent), to(transparent)); + background-image: -webkit-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -moz-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: -o-linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); + background-image: linear-gradient(45deg, rgba(255, 255, 255, 0.15) 25%, transparent 25%, transparent 50%, rgba(255, 255, 255, 0.15) 50%, rgba(255, 255, 255, 0.15) 75%, transparent 75%, transparent); +} + +.accordion { + margin-bottom: 20px; +} + +.accordion-group { + margin-bottom: 2px; + border: 1px solid #e5e5e5; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; +} + +.accordion-heading { + border-bottom: 0; +} + +.accordion-heading .accordion-toggle { + display: block; + padding: 8px 15px; +} + +.accordion-toggle { + cursor: pointer; +} + +.accordion-inner { + padding: 9px 15px; + border-top: 1px solid #e5e5e5; +} + +.carousel { + position: relative; + margin-bottom: 20px; + line-height: 1; +} + +.carousel-inner { + position: relative; + width: 100%; + overflow: hidden; +} + +.carousel .item { + position: relative; + display: none; + -webkit-transition: 0.6s ease-in-out left; + -moz-transition: 0.6s ease-in-out left; + -o-transition: 0.6s ease-in-out left; + transition: 0.6s ease-in-out left; +} + +.carousel .item > img { + display: block; + line-height: 1; +} + +.carousel .active, +.carousel .next, +.carousel .prev { + display: block; +} + +.carousel .active { + left: 0; +} + +.carousel .next, +.carousel .prev { + position: absolute; + top: 0; + width: 100%; +} + +.carousel .next { + left: 100%; +} + +.carousel .prev { + left: -100%; +} + +.carousel .next.left, +.carousel .prev.right { + left: 0; +} + +.carousel .active.left { + left: -100%; +} + +.carousel .active.right { + left: 100%; +} + +.carousel-control { + position: absolute; + top: 40%; + left: 15px; + width: 40px; + height: 40px; + margin-top: -20px; + font-size: 60px; + font-weight: 100; + line-height: 30px; + color: #ffffff; + text-align: center; + background: #222222; + border: 3px solid #ffffff; + -webkit-border-radius: 23px; + -moz-border-radius: 23px; + border-radius: 23px; + opacity: 0.5; + filter: alpha(opacity=50); +} + +.carousel-control.right { + right: 15px; + left: auto; +} + +.carousel-control:hover { + color: #ffffff; + text-decoration: none; + opacity: 0.9; + filter: alpha(opacity=90); +} + +.carousel-caption { + position: absolute; + right: 0; + bottom: 0; + left: 0; + padding: 15px; + background: #333333; + background: rgba(0, 0, 0, 0.75); +} + +.carousel-caption h4, +.carousel-caption p { + line-height: 20px; + color: #ffffff; +} + +.carousel-caption h4 { + margin: 0 0 5px; +} + +.carousel-caption p { + margin-bottom: 0; +} + +.hero-unit { + padding: 60px; + margin-bottom: 30px; + background-color: #eeeeee; + -webkit-border-radius: 6px; + -moz-border-radius: 6px; + border-radius: 6px; +} + +.hero-unit h1 { + margin-bottom: 0; + font-size: 60px; + line-height: 1; + letter-spacing: -1px; + color: inherit; +} + +.hero-unit p { + font-size: 18px; + font-weight: 200; + line-height: 30px; + color: inherit; +} + +.pull-right { + float: right; +} + +.pull-left { + float: left; +} + +.hide { + display: none; +} + +.show { + display: block; +} + +.invisible { + visibility: hidden; +} + +.affix { + position: fixed; +} diff --git a/docs/css/bootstrap.min.css b/docs/css/bootstrap.min.css new file mode 100644 index 000000000000..ef013cba233e --- /dev/null +++ b/docs/css/bootstrap.min.css @@ -0,0 +1,9 @@ +/*! + * Bootstrap v2.1.1 + * + * Copyright 2012 Twitter, Inc + * Licensed under the Apache License v2.0 + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Designed and built with all the love in the world @twitter by @mdo and @fat. + */article,aside,details,figcaption,figure,footer,header,hgroup,nav,section{display:block}audio,canvas,video{display:inline-block;*display:inline;*zoom:1}audio:not([controls]){display:none}html{font-size:100%;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%}a:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}a:hover,a:active{outline:0}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{width:auto\9;height:auto;max-width:100%;vertical-align:middle;border:0;-ms-interpolation-mode:bicubic}#map_canvas img{max-width:none}button,input,select,textarea{margin:0;font-size:100%;vertical-align:middle}button,input{*overflow:visible;line-height:normal}button::-moz-focus-inner,input::-moz-focus-inner{padding:0;border:0}button,input[type="button"],input[type="reset"],input[type="submit"]{cursor:pointer;-webkit-appearance:button}input[type="search"]{-webkit-box-sizing:content-box;-moz-box-sizing:content-box;box-sizing:content-box;-webkit-appearance:textfield}input[type="search"]::-webkit-search-decoration,input[type="search"]::-webkit-search-cancel-button{-webkit-appearance:none}textarea{overflow:auto;vertical-align:top}.clearfix{*zoom:1}.clearfix:before,.clearfix:after{display:table;line-height:0;content:""}.clearfix:after{clear:both}.hide-text{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.input-block-level{display:block;width:100%;min-height:30px;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}body{margin:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;line-height:20px;color:#333;background-color:#fff}a{color:#08c;text-decoration:none}a:hover{color:#005580;text-decoration:underline}.img-rounded{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.img-polaroid{padding:4px;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.1);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.1);box-shadow:0 1px 3px rgba(0,0,0,0.1)}.img-circle{-webkit-border-radius:500px;-moz-border-radius:500px;border-radius:500px}.row{margin-left:-20px;*zoom:1}.row:before,.row:after{display:table;line-height:0;content:""}.row:after{clear:both}[class*="span"]{float:left;min-height:1px;margin-left:20px}.container,.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.span12{width:940px}.span11{width:860px}.span10{width:780px}.span9{width:700px}.span8{width:620px}.span7{width:540px}.span6{width:460px}.span5{width:380px}.span4{width:300px}.span3{width:220px}.span2{width:140px}.span1{width:60px}.offset12{margin-left:980px}.offset11{margin-left:900px}.offset10{margin-left:820px}.offset9{margin-left:740px}.offset8{margin-left:660px}.offset7{margin-left:580px}.offset6{margin-left:500px}.offset5{margin-left:420px}.offset4{margin-left:340px}.offset3{margin-left:260px}.offset2{margin-left:180px}.offset1{margin-left:100px}.row-fluid{width:100%;*zoom:1}.row-fluid:before,.row-fluid:after{display:table;line-height:0;content:""}.row-fluid:after{clear:both}.row-fluid [class*="span"]{display:block;float:left;width:100%;min-height:30px;margin-left:2.127659574468085%;*margin-left:2.074468085106383%;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.row-fluid [class*="span"]:first-child{margin-left:0}.row-fluid .span12{width:100%;*width:99.94680851063829%}.row-fluid .span11{width:91.48936170212765%;*width:91.43617021276594%}.row-fluid .span10{width:82.97872340425532%;*width:82.92553191489361%}.row-fluid .span9{width:74.46808510638297%;*width:74.41489361702126%}.row-fluid .span8{width:65.95744680851064%;*width:65.90425531914893%}.row-fluid .span7{width:57.44680851063829%;*width:57.39361702127659%}.row-fluid .span6{width:48.93617021276595%;*width:48.88297872340425%}.row-fluid .span5{width:40.42553191489362%;*width:40.37234042553192%}.row-fluid .span4{width:31.914893617021278%;*width:31.861702127659576%}.row-fluid .span3{width:23.404255319148934%;*width:23.351063829787233%}.row-fluid .span2{width:14.893617021276595%;*width:14.840425531914894%}.row-fluid .span1{width:6.382978723404255%;*width:6.329787234042553%}.row-fluid .offset12{margin-left:104.25531914893617%;*margin-left:104.14893617021275%}.row-fluid .offset12:first-child{margin-left:102.12765957446808%;*margin-left:102.02127659574467%}.row-fluid .offset11{margin-left:95.74468085106382%;*margin-left:95.6382978723404%}.row-fluid .offset11:first-child{margin-left:93.61702127659574%;*margin-left:93.51063829787232%}.row-fluid .offset10{margin-left:87.23404255319149%;*margin-left:87.12765957446807%}.row-fluid .offset10:first-child{margin-left:85.1063829787234%;*margin-left:84.99999999999999%}.row-fluid .offset9{margin-left:78.72340425531914%;*margin-left:78.61702127659572%}.row-fluid .offset9:first-child{margin-left:76.59574468085106%;*margin-left:76.48936170212764%}.row-fluid .offset8{margin-left:70.2127659574468%;*margin-left:70.10638297872339%}.row-fluid .offset8:first-child{margin-left:68.08510638297872%;*margin-left:67.9787234042553%}.row-fluid .offset7{margin-left:61.70212765957446%;*margin-left:61.59574468085106%}.row-fluid .offset7:first-child{margin-left:59.574468085106375%;*margin-left:59.46808510638297%}.row-fluid .offset6{margin-left:53.191489361702125%;*margin-left:53.085106382978715%}.row-fluid .offset6:first-child{margin-left:51.063829787234035%;*margin-left:50.95744680851063%}.row-fluid .offset5{margin-left:44.68085106382979%;*margin-left:44.57446808510638%}.row-fluid .offset5:first-child{margin-left:42.5531914893617%;*margin-left:42.4468085106383%}.row-fluid .offset4{margin-left:36.170212765957444%;*margin-left:36.06382978723405%}.row-fluid .offset4:first-child{margin-left:34.04255319148936%;*margin-left:33.93617021276596%}.row-fluid .offset3{margin-left:27.659574468085104%;*margin-left:27.5531914893617%}.row-fluid .offset3:first-child{margin-left:25.53191489361702%;*margin-left:25.425531914893618%}.row-fluid .offset2{margin-left:19.148936170212764%;*margin-left:19.04255319148936%}.row-fluid .offset2:first-child{margin-left:17.02127659574468%;*margin-left:16.914893617021278%}.row-fluid .offset1{margin-left:10.638297872340425%;*margin-left:10.53191489361702%}.row-fluid .offset1:first-child{margin-left:8.51063829787234%;*margin-left:8.404255319148938%}[class*="span"].hide,.row-fluid [class*="span"].hide{display:none}[class*="span"].pull-right,.row-fluid [class*="span"].pull-right{float:right}.container{margin-right:auto;margin-left:auto;*zoom:1}.container:before,.container:after{display:table;line-height:0;content:""}.container:after{clear:both}.container-fluid{padding-right:20px;padding-left:20px;*zoom:1}.container-fluid:before,.container-fluid:after{display:table;line-height:0;content:""}.container-fluid:after{clear:both}p{margin:0 0 10px}.lead{margin-bottom:20px;font-size:21px;font-weight:200;line-height:30px}small{font-size:85%}strong{font-weight:bold}em{font-style:italic}cite{font-style:normal}.muted{color:#999}.text-warning{color:#c09853}.text-error{color:#b94a48}.text-info{color:#3a87ad}.text-success{color:#468847}h1,h2,h3,h4,h5,h6{margin:10px 0;font-family:inherit;font-weight:bold;line-height:1;color:inherit;text-rendering:optimizelegibility}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small{font-weight:normal;line-height:1;color:#999}h1{font-size:36px;line-height:40px}h2{font-size:30px;line-height:40px}h3{font-size:24px;line-height:40px}h4{font-size:18px;line-height:20px}h5{font-size:14px;line-height:20px}h6{font-size:12px;line-height:20px}h1 small{font-size:24px}h2 small{font-size:18px}h3 small{font-size:14px}h4 small{font-size:14px}.page-header{padding-bottom:9px;margin:20px 0 30px;border-bottom:1px solid #eee}ul,ol{padding:0;margin:0 0 10px 25px}ul ul,ul ol,ol ol,ol ul{margin-bottom:0}li{line-height:20px}ul.unstyled,ol.unstyled{margin-left:0;list-style:none}dl{margin-bottom:20px}dt,dd{line-height:20px}dt{font-weight:bold}dd{margin-left:10px}.dl-horizontal{*zoom:1}.dl-horizontal:before,.dl-horizontal:after{display:table;line-height:0;content:""}.dl-horizontal:after{clear:both}.dl-horizontal dt{float:left;width:160px;overflow:hidden;clear:left;text-align:right;text-overflow:ellipsis;white-space:nowrap}.dl-horizontal dd{margin-left:180px}hr{margin:20px 0;border:0;border-top:1px solid #eee;border-bottom:1px solid #fff}abbr[title]{cursor:help;border-bottom:1px dotted #999}abbr.initialism{font-size:90%;text-transform:uppercase}blockquote{padding:0 0 0 15px;margin:0 0 20px;border-left:5px solid #eee}blockquote p{margin-bottom:0;font-size:16px;font-weight:300;line-height:25px}blockquote small{display:block;line-height:20px;color:#999}blockquote small:before{content:'\2014 \00A0'}blockquote.pull-right{float:right;padding-right:15px;padding-left:0;border-right:5px solid #eee;border-left:0}blockquote.pull-right p,blockquote.pull-right small{text-align:right}blockquote.pull-right small:before{content:''}blockquote.pull-right small:after{content:'\00A0 \2014'}q:before,q:after,blockquote:before,blockquote:after{content:""}address{display:block;margin-bottom:20px;font-style:normal;line-height:20px}code,pre{padding:0 3px 2px;font-family:Monaco,Menlo,Consolas,"Courier New",monospace;font-size:12px;color:#333;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}code{padding:2px 4px;color:#d14;background-color:#f7f7f9;border:1px solid #e1e1e8}pre{display:block;padding:9.5px;margin:0 0 10px;font-size:13px;line-height:20px;word-break:break-all;word-wrap:break-word;white-space:pre;white-space:pre-wrap;background-color:#f5f5f5;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.15);-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}pre.prettyprint{margin-bottom:20px}pre code{padding:0;color:inherit;background-color:transparent;border:0}.pre-scrollable{max-height:340px;overflow-y:scroll}form{margin:0 0 20px}fieldset{padding:0;margin:0;border:0}legend{display:block;width:100%;padding:0;margin-bottom:20px;font-size:21px;line-height:40px;color:#333;border:0;border-bottom:1px solid #e5e5e5}legend small{font-size:15px;color:#999}label,input,button,select,textarea{font-size:14px;font-weight:normal;line-height:20px}input,button,select,textarea{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}label{display:block;margin-bottom:5px}select,textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{display:inline-block;height:20px;padding:4px 6px;margin-bottom:9px;font-size:14px;line-height:20px;color:#555;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}input,textarea,.uneditable-input{width:206px}textarea{height:auto}textarea,input[type="text"],input[type="password"],input[type="datetime"],input[type="datetime-local"],input[type="date"],input[type="month"],input[type="time"],input[type="week"],input[type="number"],input[type="email"],input[type="url"],input[type="search"],input[type="tel"],input[type="color"],.uneditable-input{background-color:#fff;border:1px solid #ccc;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-webkit-transition:border linear .2s,box-shadow linear .2s;-moz-transition:border linear .2s,box-shadow linear .2s;-o-transition:border linear .2s,box-shadow linear .2s;transition:border linear .2s,box-shadow linear .2s}textarea:focus,input[type="text"]:focus,input[type="password"]:focus,input[type="datetime"]:focus,input[type="datetime-local"]:focus,input[type="date"]:focus,input[type="month"]:focus,input[type="time"]:focus,input[type="week"]:focus,input[type="number"]:focus,input[type="email"]:focus,input[type="url"]:focus,input[type="search"]:focus,input[type="tel"]:focus,input[type="color"]:focus,.uneditable-input:focus{border-color:rgba(82,168,236,0.8);outline:0;outline:thin dotted \9;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 8px rgba(82,168,236,0.6)}input[type="radio"],input[type="checkbox"]{margin:4px 0 0;margin-top:1px \9;*margin-top:0;line-height:normal;cursor:pointer}input[type="file"],input[type="image"],input[type="submit"],input[type="reset"],input[type="button"],input[type="radio"],input[type="checkbox"]{width:auto}select,input[type="file"]{height:30px;*margin-top:4px;line-height:30px}select{width:220px;background-color:#fff;border:1px solid #ccc}select[multiple],select[size]{height:auto}select:focus,input[type="file"]:focus,input[type="radio"]:focus,input[type="checkbox"]:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.uneditable-input,.uneditable-textarea{color:#999;cursor:not-allowed;background-color:#fcfcfc;border-color:#ccc;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.025);box-shadow:inset 0 1px 2px rgba(0,0,0,0.025)}.uneditable-input{overflow:hidden;white-space:nowrap}.uneditable-textarea{width:auto;height:auto}input:-moz-placeholder,textarea:-moz-placeholder{color:#999}input:-ms-input-placeholder,textarea:-ms-input-placeholder{color:#999}input::-webkit-input-placeholder,textarea::-webkit-input-placeholder{color:#999}.radio,.checkbox{min-height:18px;padding-left:18px}.radio input[type="radio"],.checkbox input[type="checkbox"]{float:left;margin-left:-18px}.controls>.radio:first-child,.controls>.checkbox:first-child{padding-top:5px}.radio.inline,.checkbox.inline{display:inline-block;padding-top:5px;margin-bottom:0;vertical-align:middle}.radio.inline+.radio.inline,.checkbox.inline+.checkbox.inline{margin-left:10px}.input-mini{width:60px}.input-small{width:90px}.input-medium{width:150px}.input-large{width:210px}.input-xlarge{width:270px}.input-xxlarge{width:530px}input[class*="span"],select[class*="span"],textarea[class*="span"],.uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"]{float:none;margin-left:0}.input-append input[class*="span"],.input-append .uneditable-input[class*="span"],.input-prepend input[class*="span"],.input-prepend .uneditable-input[class*="span"],.row-fluid input[class*="span"],.row-fluid select[class*="span"],.row-fluid textarea[class*="span"],.row-fluid .uneditable-input[class*="span"],.row-fluid .input-prepend [class*="span"],.row-fluid .input-append [class*="span"]{display:inline-block}input,textarea,.uneditable-input{margin-left:0}.controls-row [class*="span"]+[class*="span"]{margin-left:20px}input.span12,textarea.span12,.uneditable-input.span12{width:926px}input.span11,textarea.span11,.uneditable-input.span11{width:846px}input.span10,textarea.span10,.uneditable-input.span10{width:766px}input.span9,textarea.span9,.uneditable-input.span9{width:686px}input.span8,textarea.span8,.uneditable-input.span8{width:606px}input.span7,textarea.span7,.uneditable-input.span7{width:526px}input.span6,textarea.span6,.uneditable-input.span6{width:446px}input.span5,textarea.span5,.uneditable-input.span5{width:366px}input.span4,textarea.span4,.uneditable-input.span4{width:286px}input.span3,textarea.span3,.uneditable-input.span3{width:206px}input.span2,textarea.span2,.uneditable-input.span2{width:126px}input.span1,textarea.span1,.uneditable-input.span1{width:46px}.controls-row{*zoom:1}.controls-row:before,.controls-row:after{display:table;line-height:0;content:""}.controls-row:after{clear:both}.controls-row [class*="span"]{float:left}input[disabled],select[disabled],textarea[disabled],input[readonly],select[readonly],textarea[readonly]{cursor:not-allowed;background-color:#eee}input[type="radio"][disabled],input[type="checkbox"][disabled],input[type="radio"][readonly],input[type="checkbox"][readonly]{background-color:transparent}.control-group.warning>label,.control-group.warning .help-block,.control-group.warning .help-inline{color:#c09853}.control-group.warning .checkbox,.control-group.warning .radio,.control-group.warning input,.control-group.warning select,.control-group.warning textarea{color:#c09853}.control-group.warning input,.control-group.warning select,.control-group.warning textarea{border-color:#c09853;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.warning input:focus,.control-group.warning select:focus,.control-group.warning textarea:focus{border-color:#a47e3c;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #dbc59e}.control-group.warning .input-prepend .add-on,.control-group.warning .input-append .add-on{color:#c09853;background-color:#fcf8e3;border-color:#c09853}.control-group.error>label,.control-group.error .help-block,.control-group.error .help-inline{color:#b94a48}.control-group.error .checkbox,.control-group.error .radio,.control-group.error input,.control-group.error select,.control-group.error textarea{color:#b94a48}.control-group.error input,.control-group.error select,.control-group.error textarea{border-color:#b94a48;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.error input:focus,.control-group.error select:focus,.control-group.error textarea:focus{border-color:#953b39;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #d59392}.control-group.error .input-prepend .add-on,.control-group.error .input-append .add-on{color:#b94a48;background-color:#f2dede;border-color:#b94a48}.control-group.success>label,.control-group.success .help-block,.control-group.success .help-inline{color:#468847}.control-group.success .checkbox,.control-group.success .radio,.control-group.success input,.control-group.success select,.control-group.success textarea{color:#468847}.control-group.success input,.control-group.success select,.control-group.success textarea{border-color:#468847;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.success input:focus,.control-group.success select:focus,.control-group.success textarea:focus{border-color:#356635;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7aba7b}.control-group.success .input-prepend .add-on,.control-group.success .input-append .add-on{color:#468847;background-color:#dff0d8;border-color:#468847}.control-group.info>label,.control-group.info .help-block,.control-group.info .help-inline{color:#3a87ad}.control-group.info .checkbox,.control-group.info .radio,.control-group.info input,.control-group.info select,.control-group.info textarea{color:#3a87ad}.control-group.info input,.control-group.info select,.control-group.info textarea{border-color:#3a87ad;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075);box-shadow:inset 0 1px 1px rgba(0,0,0,0.075)}.control-group.info input:focus,.control-group.info select:focus,.control-group.info textarea:focus{border-color:#2d6987;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7ab5d3;-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7ab5d3;box-shadow:inset 0 1px 1px rgba(0,0,0,0.075),0 0 6px #7ab5d3}.control-group.info .input-prepend .add-on,.control-group.info .input-append .add-on{color:#3a87ad;background-color:#d9edf7;border-color:#3a87ad}input:focus:required:invalid,textarea:focus:required:invalid,select:focus:required:invalid{color:#b94a48;border-color:#ee5f5b}input:focus:required:invalid:focus,textarea:focus:required:invalid:focus,select:focus:required:invalid:focus{border-color:#e9322d;-webkit-box-shadow:0 0 6px #f8b9b7;-moz-box-shadow:0 0 6px #f8b9b7;box-shadow:0 0 6px #f8b9b7}.form-actions{padding:19px 20px 20px;margin-top:20px;margin-bottom:20px;background-color:#f5f5f5;border-top:1px solid #e5e5e5;*zoom:1}.form-actions:before,.form-actions:after{display:table;line-height:0;content:""}.form-actions:after{clear:both}.help-block,.help-inline{color:#595959}.help-block{display:block;margin-bottom:10px}.help-inline{display:inline-block;*display:inline;padding-left:5px;vertical-align:middle;*zoom:1}.input-append,.input-prepend{margin-bottom:5px;font-size:0;white-space:nowrap}.input-append input,.input-prepend input,.input-append select,.input-prepend select,.input-append .uneditable-input,.input-prepend .uneditable-input{position:relative;margin-bottom:0;*margin-left:0;font-size:14px;vertical-align:top;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-append input:focus,.input-prepend input:focus,.input-append select:focus,.input-prepend select:focus,.input-append .uneditable-input:focus,.input-prepend .uneditable-input:focus{z-index:2}.input-append .add-on,.input-prepend .add-on{display:inline-block;width:auto;height:20px;min-width:16px;padding:4px 5px;font-size:14px;font-weight:normal;line-height:20px;text-align:center;text-shadow:0 1px 0 #fff;background-color:#eee;border:1px solid #ccc}.input-append .add-on,.input-prepend .add-on,.input-append .btn,.input-prepend .btn{vertical-align:top;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-append .active,.input-prepend .active{background-color:#a9dba9;border-color:#46a546}.input-prepend .add-on,.input-prepend .btn{margin-right:-1px}.input-prepend .add-on:first-child,.input-prepend .btn:first-child{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append input,.input-append select,.input-append .uneditable-input{-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-append .add-on,.input-append .btn{margin-left:-1px}.input-append .add-on:last-child,.input-append .btn:last-child{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.input-prepend.input-append input,.input-prepend.input-append select,.input-prepend.input-append .uneditable-input{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.input-prepend.input-append .add-on:first-child,.input-prepend.input-append .btn:first-child{margin-right:-1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.input-prepend.input-append .add-on:last-child,.input-prepend.input-append .btn:last-child{margin-left:-1px;-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}input.search-query{padding-right:14px;padding-right:4px \9;padding-left:14px;padding-left:4px \9;margin-bottom:0;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.form-search .input-append .search-query,.form-search .input-prepend .search-query{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.form-search .input-append .search-query{-webkit-border-radius:14px 0 0 14px;-moz-border-radius:14px 0 0 14px;border-radius:14px 0 0 14px}.form-search .input-append .btn{-webkit-border-radius:0 14px 14px 0;-moz-border-radius:0 14px 14px 0;border-radius:0 14px 14px 0}.form-search .input-prepend .search-query{-webkit-border-radius:0 14px 14px 0;-moz-border-radius:0 14px 14px 0;border-radius:0 14px 14px 0}.form-search .input-prepend .btn{-webkit-border-radius:14px 0 0 14px;-moz-border-radius:14px 0 0 14px;border-radius:14px 0 0 14px}.form-search input,.form-inline input,.form-horizontal input,.form-search textarea,.form-inline textarea,.form-horizontal textarea,.form-search select,.form-inline select,.form-horizontal select,.form-search .help-inline,.form-inline .help-inline,.form-horizontal .help-inline,.form-search .uneditable-input,.form-inline .uneditable-input,.form-horizontal .uneditable-input,.form-search .input-prepend,.form-inline .input-prepend,.form-horizontal .input-prepend,.form-search .input-append,.form-inline .input-append,.form-horizontal .input-append{display:inline-block;*display:inline;margin-bottom:0;vertical-align:middle;*zoom:1}.form-search .hide,.form-inline .hide,.form-horizontal .hide{display:none}.form-search label,.form-inline label,.form-search .btn-group,.form-inline .btn-group{display:inline-block}.form-search .input-append,.form-inline .input-append,.form-search .input-prepend,.form-inline .input-prepend{margin-bottom:0}.form-search .radio,.form-search .checkbox,.form-inline .radio,.form-inline .checkbox{padding-left:0;margin-bottom:0;vertical-align:middle}.form-search .radio input[type="radio"],.form-search .checkbox input[type="checkbox"],.form-inline .radio input[type="radio"],.form-inline .checkbox input[type="checkbox"]{float:left;margin-right:3px;margin-left:0}.control-group{margin-bottom:10px}legend+.control-group{margin-top:20px;-webkit-margin-top-collapse:separate}.form-horizontal .control-group{margin-bottom:20px;*zoom:1}.form-horizontal .control-group:before,.form-horizontal .control-group:after{display:table;line-height:0;content:""}.form-horizontal .control-group:after{clear:both}.form-horizontal .control-label{float:left;width:160px;padding-top:5px;text-align:right}.form-horizontal .controls{*display:inline-block;*padding-left:20px;margin-left:180px;*margin-left:0}.form-horizontal .controls:first-child{*padding-left:180px}.form-horizontal .help-block{margin-bottom:0}.form-horizontal input+.help-block,.form-horizontal select+.help-block,.form-horizontal textarea+.help-block{margin-top:10px}.form-horizontal .form-actions{padding-left:180px}table{max-width:100%;background-color:transparent;border-collapse:collapse;border-spacing:0}.table{width:100%;margin-bottom:20px}.table th,.table td{padding:8px;line-height:20px;text-align:left;vertical-align:top;border-top:1px solid #ddd}.table th{font-weight:bold}.table thead th{vertical-align:bottom}.table caption+thead tr:first-child th,.table caption+thead tr:first-child td,.table colgroup+thead tr:first-child th,.table colgroup+thead tr:first-child td,.table thead:first-child tr:first-child th,.table thead:first-child tr:first-child td{border-top:0}.table tbody+tbody{border-top:2px solid #ddd}.table-condensed th,.table-condensed td{padding:4px 5px}.table-bordered{border:1px solid #ddd;border-collapse:separate;*border-collapse:collapse;border-left:0;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.table-bordered th,.table-bordered td{border-left:1px solid #ddd}.table-bordered caption+thead tr:first-child th,.table-bordered caption+tbody tr:first-child th,.table-bordered caption+tbody tr:first-child td,.table-bordered colgroup+thead tr:first-child th,.table-bordered colgroup+tbody tr:first-child th,.table-bordered colgroup+tbody tr:first-child td,.table-bordered thead:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child th,.table-bordered tbody:first-child tr:first-child td{border-top:0}.table-bordered thead:first-child tr:first-child th:first-child,.table-bordered tbody:first-child tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered thead:first-child tr:first-child th:last-child,.table-bordered tbody:first-child tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-radius-topright:4px}.table-bordered thead:last-child tr:last-child th:first-child,.table-bordered tbody:last-child tr:last-child td:first-child,.table-bordered tfoot:last-child tr:last-child td:first-child{-webkit-border-radius:0 0 0 4px;-moz-border-radius:0 0 0 4px;border-radius:0 0 0 4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomleft:4px}.table-bordered thead:last-child tr:last-child th:last-child,.table-bordered tbody:last-child tr:last-child td:last-child,.table-bordered tfoot:last-child tr:last-child td:last-child{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-bottomright:4px}.table-bordered caption+thead tr:first-child th:first-child,.table-bordered caption+tbody tr:first-child td:first-child,.table-bordered colgroup+thead tr:first-child th:first-child,.table-bordered colgroup+tbody tr:first-child td:first-child{-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topleft:4px}.table-bordered caption+thead tr:first-child th:last-child,.table-bordered caption+tbody tr:first-child td:last-child,.table-bordered colgroup+thead tr:first-child th:last-child,.table-bordered colgroup+tbody tr:first-child td:last-child{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-moz-border-radius-topleft:4px}.table-striped tbody tr:nth-child(odd) td,.table-striped tbody tr:nth-child(odd) th{background-color:#f9f9f9}.table-hover tbody tr:hover td,.table-hover tbody tr:hover th{background-color:#f5f5f5}table [class*=span],.row-fluid table [class*=span]{display:table-cell;float:none;margin-left:0}.table .span1{float:none;width:44px;margin-left:0}.table .span2{float:none;width:124px;margin-left:0}.table .span3{float:none;width:204px;margin-left:0}.table .span4{float:none;width:284px;margin-left:0}.table .span5{float:none;width:364px;margin-left:0}.table .span6{float:none;width:444px;margin-left:0}.table .span7{float:none;width:524px;margin-left:0}.table .span8{float:none;width:604px;margin-left:0}.table .span9{float:none;width:684px;margin-left:0}.table .span10{float:none;width:764px;margin-left:0}.table .span11{float:none;width:844px;margin-left:0}.table .span12{float:none;width:924px;margin-left:0}.table .span13{float:none;width:1004px;margin-left:0}.table .span14{float:none;width:1084px;margin-left:0}.table .span15{float:none;width:1164px;margin-left:0}.table .span16{float:none;width:1244px;margin-left:0}.table .span17{float:none;width:1324px;margin-left:0}.table .span18{float:none;width:1404px;margin-left:0}.table .span19{float:none;width:1484px;margin-left:0}.table .span20{float:none;width:1564px;margin-left:0}.table .span21{float:none;width:1644px;margin-left:0}.table .span22{float:none;width:1724px;margin-left:0}.table .span23{float:none;width:1804px;margin-left:0}.table .span24{float:none;width:1884px;margin-left:0}.table tbody tr.success td{background-color:#dff0d8}.table tbody tr.error td{background-color:#f2dede}.table tbody tr.warning td{background-color:#fcf8e3}.table tbody tr.info td{background-color:#d9edf7}.table-hover tbody tr.success:hover td{background-color:#d0e9c6}.table-hover tbody tr.error:hover td{background-color:#ebcccc}.table-hover tbody tr.warning:hover td{background-color:#faf2cc}.table-hover tbody tr.info:hover td{background-color:#c4e3f3}[class^="icon-"],[class*=" icon-"]{display:inline-block;width:14px;height:14px;margin-top:1px;*margin-right:.3em;line-height:14px;vertical-align:text-top;background-image:url("../img/glyphicons-halflings.png");background-position:14px 14px;background-repeat:no-repeat}.icon-white,.nav-tabs>.active>a>[class^="icon-"],.nav-tabs>.active>a>[class*=" icon-"],.nav-pills>.active>a>[class^="icon-"],.nav-pills>.active>a>[class*=" icon-"],.nav-list>.active>a>[class^="icon-"],.nav-list>.active>a>[class*=" icon-"],.navbar-inverse .nav>.active>a>[class^="icon-"],.navbar-inverse .nav>.active>a>[class*=" icon-"],.dropdown-menu>li>a:hover>[class^="icon-"],.dropdown-menu>li>a:hover>[class*=" icon-"],.dropdown-menu>.active>a>[class^="icon-"],.dropdown-menu>.active>a>[class*=" icon-"]{background-image:url("../img/glyphicons-halflings-white.png")}.icon-glass{background-position:0 0}.icon-music{background-position:-24px 0}.icon-search{background-position:-48px 0}.icon-envelope{background-position:-72px 0}.icon-heart{background-position:-96px 0}.icon-star{background-position:-120px 0}.icon-star-empty{background-position:-144px 0}.icon-user{background-position:-168px 0}.icon-film{background-position:-192px 0}.icon-th-large{background-position:-216px 0}.icon-th{background-position:-240px 0}.icon-th-list{background-position:-264px 0}.icon-ok{background-position:-288px 0}.icon-remove{background-position:-312px 0}.icon-zoom-in{background-position:-336px 0}.icon-zoom-out{background-position:-360px 0}.icon-off{background-position:-384px 0}.icon-signal{background-position:-408px 0}.icon-cog{background-position:-432px 0}.icon-trash{background-position:-456px 0}.icon-home{background-position:0 -24px}.icon-file{background-position:-24px -24px}.icon-time{background-position:-48px -24px}.icon-road{background-position:-72px -24px}.icon-download-alt{background-position:-96px -24px}.icon-download{background-position:-120px -24px}.icon-upload{background-position:-144px -24px}.icon-inbox{background-position:-168px -24px}.icon-play-circle{background-position:-192px -24px}.icon-repeat{background-position:-216px -24px}.icon-refresh{background-position:-240px -24px}.icon-list-alt{background-position:-264px -24px}.icon-lock{background-position:-287px -24px}.icon-flag{background-position:-312px -24px}.icon-headphones{background-position:-336px -24px}.icon-volume-off{background-position:-360px -24px}.icon-volume-down{background-position:-384px -24px}.icon-volume-up{background-position:-408px -24px}.icon-qrcode{background-position:-432px -24px}.icon-barcode{background-position:-456px -24px}.icon-tag{background-position:0 -48px}.icon-tags{background-position:-25px -48px}.icon-book{background-position:-48px -48px}.icon-bookmark{background-position:-72px -48px}.icon-print{background-position:-96px -48px}.icon-camera{background-position:-120px -48px}.icon-font{background-position:-144px -48px}.icon-bold{background-position:-167px -48px}.icon-italic{background-position:-192px -48px}.icon-text-height{background-position:-216px -48px}.icon-text-width{background-position:-240px -48px}.icon-align-left{background-position:-264px -48px}.icon-align-center{background-position:-288px -48px}.icon-align-right{background-position:-312px -48px}.icon-align-justify{background-position:-336px -48px}.icon-list{background-position:-360px -48px}.icon-indent-left{background-position:-384px -48px}.icon-indent-right{background-position:-408px -48px}.icon-facetime-video{background-position:-432px -48px}.icon-picture{background-position:-456px -48px}.icon-pencil{background-position:0 -72px}.icon-map-marker{background-position:-24px -72px}.icon-adjust{background-position:-48px -72px}.icon-tint{background-position:-72px -72px}.icon-edit{background-position:-96px -72px}.icon-share{background-position:-120px -72px}.icon-check{background-position:-144px -72px}.icon-move{background-position:-168px -72px}.icon-step-backward{background-position:-192px -72px}.icon-fast-backward{background-position:-216px -72px}.icon-backward{background-position:-240px -72px}.icon-play{background-position:-264px -72px}.icon-pause{background-position:-288px -72px}.icon-stop{background-position:-312px -72px}.icon-forward{background-position:-336px -72px}.icon-fast-forward{background-position:-360px -72px}.icon-step-forward{background-position:-384px -72px}.icon-eject{background-position:-408px -72px}.icon-chevron-left{background-position:-432px -72px}.icon-chevron-right{background-position:-456px -72px}.icon-plus-sign{background-position:0 -96px}.icon-minus-sign{background-position:-24px -96px}.icon-remove-sign{background-position:-48px -96px}.icon-ok-sign{background-position:-72px -96px}.icon-question-sign{background-position:-96px -96px}.icon-info-sign{background-position:-120px -96px}.icon-screenshot{background-position:-144px -96px}.icon-remove-circle{background-position:-168px -96px}.icon-ok-circle{background-position:-192px -96px}.icon-ban-circle{background-position:-216px -96px}.icon-arrow-left{background-position:-240px -96px}.icon-arrow-right{background-position:-264px -96px}.icon-arrow-up{background-position:-289px -96px}.icon-arrow-down{background-position:-312px -96px}.icon-share-alt{background-position:-336px -96px}.icon-resize-full{background-position:-360px -96px}.icon-resize-small{background-position:-384px -96px}.icon-plus{background-position:-408px -96px}.icon-minus{background-position:-433px -96px}.icon-asterisk{background-position:-456px -96px}.icon-exclamation-sign{background-position:0 -120px}.icon-gift{background-position:-24px -120px}.icon-leaf{background-position:-48px -120px}.icon-fire{background-position:-72px -120px}.icon-eye-open{background-position:-96px -120px}.icon-eye-close{background-position:-120px -120px}.icon-warning-sign{background-position:-144px -120px}.icon-plane{background-position:-168px -120px}.icon-calendar{background-position:-192px -120px}.icon-random{width:16px;background-position:-216px -120px}.icon-comment{background-position:-240px -120px}.icon-magnet{background-position:-264px -120px}.icon-chevron-up{background-position:-288px -120px}.icon-chevron-down{background-position:-313px -119px}.icon-retweet{background-position:-336px -120px}.icon-shopping-cart{background-position:-360px -120px}.icon-folder-close{background-position:-384px -120px}.icon-folder-open{width:16px;background-position:-408px -120px}.icon-resize-vertical{background-position:-432px -119px}.icon-resize-horizontal{background-position:-456px -118px}.icon-hdd{background-position:0 -144px}.icon-bullhorn{background-position:-24px -144px}.icon-bell{background-position:-48px -144px}.icon-certificate{background-position:-72px -144px}.icon-thumbs-up{background-position:-96px -144px}.icon-thumbs-down{background-position:-120px -144px}.icon-hand-right{background-position:-144px -144px}.icon-hand-left{background-position:-168px -144px}.icon-hand-up{background-position:-192px -144px}.icon-hand-down{background-position:-216px -144px}.icon-circle-arrow-right{background-position:-240px -144px}.icon-circle-arrow-left{background-position:-264px -144px}.icon-circle-arrow-up{background-position:-288px -144px}.icon-circle-arrow-down{background-position:-312px -144px}.icon-globe{background-position:-336px -144px}.icon-wrench{background-position:-360px -144px}.icon-tasks{background-position:-384px -144px}.icon-filter{background-position:-408px -144px}.icon-briefcase{background-position:-432px -144px}.icon-fullscreen{background-position:-456px -144px}.dropup,.dropdown{position:relative}.dropdown-toggle{*margin-bottom:-3px}.dropdown-toggle:active,.open .dropdown-toggle{outline:0}.caret{display:inline-block;width:0;height:0;vertical-align:top;border-top:4px solid #000;border-right:4px solid transparent;border-left:4px solid transparent;content:""}.dropdown .caret{margin-top:8px;margin-left:2px}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:160px;padding:5px 0;margin:2px 0 0;list-style:none;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);*border-right-width:2px;*border-bottom-width:2px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.dropdown-menu.pull-right{right:0;left:auto}.dropdown-menu .divider{*width:100%;height:1px;margin:9px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.dropdown-menu a{display:block;padding:3px 20px;clear:both;font-weight:normal;line-height:20px;color:#333;white-space:nowrap}.dropdown-menu li>a:hover,.dropdown-menu li>a:focus,.dropdown-submenu:hover>a{color:#fff;text-decoration:none;background-color:#08c;background-color:#0081c2;background-image:-moz-linear-gradient(top,#08c,#0077b3);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#0077b3));background-image:-webkit-linear-gradient(top,#08c,#0077b3);background-image:-o-linear-gradient(top,#08c,#0077b3);background-image:linear-gradient(to bottom,#08c,#0077b3);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0077b3',GradientType=0)}.dropdown-menu .active>a,.dropdown-menu .active>a:hover{color:#fff;text-decoration:none;background-color:#08c;background-color:#0081c2;background-image:linear-gradient(to bottom,#08c,#0077b3);background-image:-moz-linear-gradient(top,#08c,#0077b3);background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#0077b3));background-image:-webkit-linear-gradient(top,#08c,#0077b3);background-image:-o-linear-gradient(top,#08c,#0077b3);background-repeat:repeat-x;outline:0;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0077b3',GradientType=0)}.dropdown-menu .disabled>a,.dropdown-menu .disabled>a:hover{color:#999}.dropdown-menu .disabled>a:hover{text-decoration:none;cursor:default;background-color:transparent}.open{*z-index:1000}.open>.dropdown-menu{display:block}.pull-right>.dropdown-menu{right:0;left:auto}.dropup .caret,.navbar-fixed-bottom .dropdown .caret{border-top:0;border-bottom:4px solid #000;content:""}.dropup .dropdown-menu,.navbar-fixed-bottom .dropdown .dropdown-menu{top:auto;bottom:100%;margin-bottom:1px}.dropdown-submenu{position:relative}.dropdown-submenu>.dropdown-menu{top:0;left:100%;margin-top:-6px;margin-left:-1px;-webkit-border-radius:0 6px 6px 6px;-moz-border-radius:0 6px 6px 6px;border-radius:0 6px 6px 6px}.dropdown-submenu:hover>.dropdown-menu{display:block}.dropdown-submenu>a:after{display:block;float:right;width:0;height:0;margin-top:5px;margin-right:-10px;border-color:transparent;border-left-color:#ccc;border-style:solid;border-width:5px 0 5px 5px;content:" "}.dropdown-submenu:hover>a:after{border-left-color:#fff}.dropdown .dropdown-menu .nav-header{padding-right:20px;padding-left:20px}.typeahead{margin-top:2px;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.well{min-height:20px;padding:19px;margin-bottom:20px;background-color:#f5f5f5;border:1px solid #e3e3e3;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 1px rgba(0,0,0,0.05);box-shadow:inset 0 1px 1px rgba(0,0,0,0.05)}.well blockquote{border-color:#ddd;border-color:rgba(0,0,0,0.15)}.well-large{padding:24px;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.well-small{padding:9px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.fade{opacity:0;-webkit-transition:opacity .15s linear;-moz-transition:opacity .15s linear;-o-transition:opacity .15s linear;transition:opacity .15s linear}.fade.in{opacity:1}.collapse{position:relative;height:0;overflow:hidden;-webkit-transition:height .35s ease;-moz-transition:height .35s ease;-o-transition:height .35s ease;transition:height .35s ease}.collapse.in{height:auto}.close{float:right;font-size:20px;font-weight:bold;line-height:20px;color:#000;text-shadow:0 1px 0 #fff;opacity:.2;filter:alpha(opacity=20)}.close:hover{color:#000;text-decoration:none;cursor:pointer;opacity:.4;filter:alpha(opacity=40)}button.close{padding:0;cursor:pointer;background:transparent;border:0;-webkit-appearance:none}.btn{display:inline-block;*display:inline;padding:4px 14px;margin-bottom:0;*margin-left:.3em;font-size:14px;line-height:20px;*line-height:20px;color:#333;text-align:center;text-shadow:0 1px 1px rgba(255,255,255,0.75);vertical-align:middle;cursor:pointer;background-color:#f5f5f5;*background-color:#e6e6e6;background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#e6e6e6));background-image:-webkit-linear-gradient(top,#fff,#e6e6e6);background-image:-o-linear-gradient(top,#fff,#e6e6e6);background-image:linear-gradient(to bottom,#fff,#e6e6e6);background-image:-moz-linear-gradient(top,#fff,#e6e6e6);background-repeat:repeat-x;border:1px solid #bbb;*border:0;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);border-color:#e6e6e6 #e6e6e6 #bfbfbf;border-bottom-color:#a2a2a2;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff',endColorstr='#ffe6e6e6',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);*zoom:1;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn:hover,.btn:active,.btn.active,.btn.disabled,.btn[disabled]{color:#333;background-color:#e6e6e6;*background-color:#d9d9d9}.btn:active,.btn.active{background-color:#ccc \9}.btn:first-child{*margin-left:0}.btn:hover{color:#333;text-decoration:none;background-color:#e6e6e6;*background-color:#d9d9d9;background-position:0 -15px;-webkit-transition:background-position .1s linear;-moz-transition:background-position .1s linear;-o-transition:background-position .1s linear;transition:background-position .1s linear}.btn:focus{outline:thin dotted #333;outline:5px auto -webkit-focus-ring-color;outline-offset:-2px}.btn.active,.btn:active{background-color:#e6e6e6;background-color:#d9d9d9 \9;background-image:none;outline:0;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn.disabled,.btn[disabled]{cursor:default;background-color:#e6e6e6;background-image:none;opacity:.65;filter:alpha(opacity=65);-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-large{padding:9px 14px;font-size:16px;line-height:normal;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.btn-large [class^="icon-"]{margin-top:2px}.btn-small{padding:3px 9px;font-size:12px;line-height:18px}.btn-small [class^="icon-"]{margin-top:0}.btn-mini{padding:2px 6px;font-size:11px;line-height:17px}.btn-block{display:block;width:100%;padding-right:0;padding-left:0;-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box}.btn-block+.btn-block{margin-top:5px}input[type="submit"].btn-block,input[type="reset"].btn-block,input[type="button"].btn-block{width:100%}.btn-primary.active,.btn-warning.active,.btn-danger.active,.btn-success.active,.btn-info.active,.btn-inverse.active{color:rgba(255,255,255,0.75)}.btn{border-color:#c5c5c5;border-color:rgba(0,0,0,0.15) rgba(0,0,0,0.15) rgba(0,0,0,0.25)}.btn-primary{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#006dcc;*background-color:#04c;background-image:-webkit-gradient(linear,0 0,0 100%,from(#08c),to(#04c));background-image:-webkit-linear-gradient(top,#08c,#04c);background-image:-o-linear-gradient(top,#08c,#04c);background-image:linear-gradient(to bottom,#08c,#04c);background-image:-moz-linear-gradient(top,#08c,#04c);background-repeat:repeat-x;border-color:#04c #04c #002a80;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff0088cc',endColorstr='#ff0044cc',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-primary:hover,.btn-primary:active,.btn-primary.active,.btn-primary.disabled,.btn-primary[disabled]{color:#fff;background-color:#04c;*background-color:#003bb3}.btn-primary:active,.btn-primary.active{background-color:#039 \9}.btn-warning{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#faa732;*background-color:#f89406;background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(to bottom,#fbb450,#f89406);background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-repeat:repeat-x;border-color:#f89406 #f89406 #ad6704;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450',endColorstr='#fff89406',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-warning:hover,.btn-warning:active,.btn-warning.active,.btn-warning.disabled,.btn-warning[disabled]{color:#fff;background-color:#f89406;*background-color:#df8505}.btn-warning:active,.btn-warning.active{background-color:#c67605 \9}.btn-danger{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#da4f49;*background-color:#bd362f;background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#bd362f));background-image:-webkit-linear-gradient(top,#ee5f5b,#bd362f);background-image:-o-linear-gradient(top,#ee5f5b,#bd362f);background-image:linear-gradient(to bottom,#ee5f5b,#bd362f);background-image:-moz-linear-gradient(top,#ee5f5b,#bd362f);background-repeat:repeat-x;border-color:#bd362f #bd362f #802420;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b',endColorstr='#ffbd362f',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-danger:hover,.btn-danger:active,.btn-danger.active,.btn-danger.disabled,.btn-danger[disabled]{color:#fff;background-color:#bd362f;*background-color:#a9302a}.btn-danger:active,.btn-danger.active{background-color:#942a25 \9}.btn-success{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#5bb75b;*background-color:#51a351;background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#51a351));background-image:-webkit-linear-gradient(top,#62c462,#51a351);background-image:-o-linear-gradient(top,#62c462,#51a351);background-image:linear-gradient(to bottom,#62c462,#51a351);background-image:-moz-linear-gradient(top,#62c462,#51a351);background-repeat:repeat-x;border-color:#51a351 #51a351 #387038;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462',endColorstr='#ff51a351',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-success:hover,.btn-success:active,.btn-success.active,.btn-success.disabled,.btn-success[disabled]{color:#fff;background-color:#51a351;*background-color:#499249}.btn-success:active,.btn-success.active{background-color:#408140 \9}.btn-info{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#49afcd;*background-color:#2f96b4;background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#2f96b4));background-image:-webkit-linear-gradient(top,#5bc0de,#2f96b4);background-image:-o-linear-gradient(top,#5bc0de,#2f96b4);background-image:linear-gradient(to bottom,#5bc0de,#2f96b4);background-image:-moz-linear-gradient(top,#5bc0de,#2f96b4);background-repeat:repeat-x;border-color:#2f96b4 #2f96b4 #1f6377;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de',endColorstr='#ff2f96b4',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-info:hover,.btn-info:active,.btn-info.active,.btn-info.disabled,.btn-info[disabled]{color:#fff;background-color:#2f96b4;*background-color:#2a85a0}.btn-info:active,.btn-info.active{background-color:#24748c \9}.btn-inverse{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#363636;*background-color:#222;background-image:-webkit-gradient(linear,0 0,0 100%,from(#444),to(#222));background-image:-webkit-linear-gradient(top,#444,#222);background-image:-o-linear-gradient(top,#444,#222);background-image:linear-gradient(to bottom,#444,#222);background-image:-moz-linear-gradient(top,#444,#222);background-repeat:repeat-x;border-color:#222 #222 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff444444',endColorstr='#ff222222',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.btn-inverse:hover,.btn-inverse:active,.btn-inverse.active,.btn-inverse.disabled,.btn-inverse[disabled]{color:#fff;background-color:#222;*background-color:#151515}.btn-inverse:active,.btn-inverse.active{background-color:#080808 \9}button.btn,input[type="submit"].btn{*padding-top:3px;*padding-bottom:3px}button.btn::-moz-focus-inner,input[type="submit"].btn::-moz-focus-inner{padding:0;border:0}button.btn.btn-large,input[type="submit"].btn.btn-large{*padding-top:7px;*padding-bottom:7px}button.btn.btn-small,input[type="submit"].btn.btn-small{*padding-top:3px;*padding-bottom:3px}button.btn.btn-mini,input[type="submit"].btn.btn-mini{*padding-top:1px;*padding-bottom:1px}.btn-link,.btn-link:active,.btn-link[disabled]{background-color:transparent;background-image:none;-webkit-box-shadow:none;-moz-box-shadow:none;box-shadow:none}.btn-link{color:#08c;cursor:pointer;border-color:transparent;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-link:hover{color:#005580;text-decoration:underline;background-color:transparent}.btn-link[disabled]:hover{color:#333;text-decoration:none}.btn-group{position:relative;*margin-left:.3em;font-size:0;white-space:nowrap;vertical-align:middle}.btn-group:first-child{*margin-left:0}.btn-group+.btn-group{margin-left:5px}.btn-toolbar{margin-top:10px;margin-bottom:10px;font-size:0}.btn-toolbar .btn-group{display:inline-block;*display:inline;*zoom:1}.btn-toolbar .btn+.btn,.btn-toolbar .btn-group+.btn,.btn-toolbar .btn+.btn-group{margin-left:5px}.btn-group>.btn{position:relative;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group>.btn+.btn{margin-left:-1px}.btn-group>.btn,.btn-group>.dropdown-menu{font-size:14px}.btn-group>.btn-mini{font-size:11px}.btn-group>.btn-small{font-size:12px}.btn-group>.btn-large{font-size:16px}.btn-group>.btn:first-child{margin-left:0;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-bottomleft:4px;-moz-border-radius-topleft:4px}.btn-group>.btn:last-child,.btn-group>.dropdown-toggle{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-bottomright:4px}.btn-group>.btn.large:first-child{margin-left:0;-webkit-border-bottom-left-radius:6px;border-bottom-left-radius:6px;-webkit-border-top-left-radius:6px;border-top-left-radius:6px;-moz-border-radius-bottomleft:6px;-moz-border-radius-topleft:6px}.btn-group>.btn.large:last-child,.btn-group>.large.dropdown-toggle{-webkit-border-top-right-radius:6px;border-top-right-radius:6px;-webkit-border-bottom-right-radius:6px;border-bottom-right-radius:6px;-moz-border-radius-topright:6px;-moz-border-radius-bottomright:6px}.btn-group>.btn:hover,.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active{z-index:2}.btn-group .dropdown-toggle:active,.btn-group.open .dropdown-toggle{outline:0}.btn-group>.btn+.dropdown-toggle{*padding-top:5px;padding-right:8px;*padding-bottom:5px;padding-left:8px;-webkit-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 1px 0 0 rgba(255,255,255,0.125),inset 0 1px 0 rgba(255,255,255,0.2),0 1px 2px rgba(0,0,0,0.05)}.btn-group>.btn-mini+.dropdown-toggle{*padding-top:2px;padding-right:5px;*padding-bottom:2px;padding-left:5px}.btn-group>.btn-small+.dropdown-toggle{*padding-top:5px;*padding-bottom:4px}.btn-group>.btn-large+.dropdown-toggle{*padding-top:7px;padding-right:12px;*padding-bottom:7px;padding-left:12px}.btn-group.open .dropdown-toggle{background-image:none;-webkit-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05);box-shadow:inset 0 2px 4px rgba(0,0,0,0.15),0 1px 2px rgba(0,0,0,0.05)}.btn-group.open .btn.dropdown-toggle{background-color:#e6e6e6}.btn-group.open .btn-primary.dropdown-toggle{background-color:#04c}.btn-group.open .btn-warning.dropdown-toggle{background-color:#f89406}.btn-group.open .btn-danger.dropdown-toggle{background-color:#bd362f}.btn-group.open .btn-success.dropdown-toggle{background-color:#51a351}.btn-group.open .btn-info.dropdown-toggle{background-color:#2f96b4}.btn-group.open .btn-inverse.dropdown-toggle{background-color:#222}.btn .caret{margin-top:8px;margin-left:0}.btn-mini .caret,.btn-small .caret,.btn-large .caret{margin-top:6px}.btn-large .caret{border-top-width:5px;border-right-width:5px;border-left-width:5px}.dropup .btn-large .caret{border-top:0;border-bottom:5px solid #000}.btn-primary .caret,.btn-warning .caret,.btn-danger .caret,.btn-info .caret,.btn-success .caret,.btn-inverse .caret{border-top-color:#fff;border-bottom-color:#fff}.btn-group-vertical{display:inline-block;*display:inline;*zoom:1}.btn-group-vertical .btn{display:block;float:none;width:100%;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.btn-group-vertical .btn+.btn{margin-top:-1px;margin-left:0}.btn-group-vertical .btn:first-child{-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.btn-group-vertical .btn:last-child{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.btn-group-vertical .btn-large:first-child{-webkit-border-radius:6px 6px 0 0;-moz-border-radius:6px 6px 0 0;border-radius:6px 6px 0 0}.btn-group-vertical .btn-large:last-child{-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px}.alert{padding:8px 35px 8px 14px;margin-bottom:20px;color:#c09853;text-shadow:0 1px 0 rgba(255,255,255,0.5);background-color:#fcf8e3;border:1px solid #fbeed5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.alert h4{margin:0}.alert .close{position:relative;top:-2px;right:-21px;line-height:20px}.alert-success{color:#468847;background-color:#dff0d8;border-color:#d6e9c6}.alert-danger,.alert-error{color:#b94a48;background-color:#f2dede;border-color:#eed3d7}.alert-info{color:#3a87ad;background-color:#d9edf7;border-color:#bce8f1}.alert-block{padding-top:14px;padding-bottom:14px}.alert-block>p,.alert-block>ul{margin-bottom:0}.alert-block p+p{margin-top:5px}.nav{margin-bottom:20px;margin-left:0;list-style:none}.nav>li>a{display:block}.nav>li>a:hover{text-decoration:none;background-color:#eee}.nav>.pull-right{float:right}.nav-header{display:block;padding:3px 15px;font-size:11px;font-weight:bold;line-height:20px;color:#999;text-shadow:0 1px 0 rgba(255,255,255,0.5);text-transform:uppercase}.nav li+.nav-header{margin-top:9px}.nav-list{padding-right:15px;padding-left:15px;margin-bottom:0}.nav-list>li>a,.nav-list .nav-header{margin-right:-15px;margin-left:-15px;text-shadow:0 1px 0 rgba(255,255,255,0.5)}.nav-list>li>a{padding:3px 15px}.nav-list>.active>a,.nav-list>.active>a:hover{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.2);background-color:#08c}.nav-list [class^="icon-"]{margin-right:2px}.nav-list .divider{*width:100%;height:1px;margin:9px 1px;*margin:-5px 0 5px;overflow:hidden;background-color:#e5e5e5;border-bottom:1px solid #fff}.nav-tabs,.nav-pills{*zoom:1}.nav-tabs:before,.nav-pills:before,.nav-tabs:after,.nav-pills:after{display:table;line-height:0;content:""}.nav-tabs:after,.nav-pills:after{clear:both}.nav-tabs>li,.nav-pills>li{float:left}.nav-tabs>li>a,.nav-pills>li>a{padding-right:12px;padding-left:12px;margin-right:2px;line-height:14px}.nav-tabs{border-bottom:1px solid #ddd}.nav-tabs>li{margin-bottom:-1px}.nav-tabs>li>a{padding-top:8px;padding-bottom:8px;line-height:20px;border:1px solid transparent;-webkit-border-radius:4px 4px 0 0;-moz-border-radius:4px 4px 0 0;border-radius:4px 4px 0 0}.nav-tabs>li>a:hover{border-color:#eee #eee #ddd}.nav-tabs>.active>a,.nav-tabs>.active>a:hover{color:#555;cursor:default;background-color:#fff;border:1px solid #ddd;border-bottom-color:transparent}.nav-pills>li>a{padding-top:8px;padding-bottom:8px;margin-top:2px;margin-bottom:2px;-webkit-border-radius:5px;-moz-border-radius:5px;border-radius:5px}.nav-pills>.active>a,.nav-pills>.active>a:hover{color:#fff;background-color:#08c}.nav-stacked>li{float:none}.nav-stacked>li>a{margin-right:0}.nav-tabs.nav-stacked{border-bottom:0}.nav-tabs.nav-stacked>li>a{border:1px solid #ddd;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.nav-tabs.nav-stacked>li:first-child>a{-webkit-border-top-right-radius:4px;border-top-right-radius:4px;-webkit-border-top-left-radius:4px;border-top-left-radius:4px;-moz-border-radius-topright:4px;-moz-border-radius-topleft:4px}.nav-tabs.nav-stacked>li:last-child>a{-webkit-border-bottom-right-radius:4px;border-bottom-right-radius:4px;-webkit-border-bottom-left-radius:4px;border-bottom-left-radius:4px;-moz-border-radius-bottomright:4px;-moz-border-radius-bottomleft:4px}.nav-tabs.nav-stacked>li>a:hover{z-index:2;border-color:#ddd}.nav-pills.nav-stacked>li>a{margin-bottom:3px}.nav-pills.nav-stacked>li:last-child>a{margin-bottom:1px}.nav-tabs .dropdown-menu{-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px}.nav-pills .dropdown-menu{-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.nav .dropdown-toggle .caret{margin-top:6px;border-top-color:#08c;border-bottom-color:#08c}.nav .dropdown-toggle:hover .caret{border-top-color:#005580;border-bottom-color:#005580}.nav-tabs .dropdown-toggle .caret{margin-top:8px}.nav .active .dropdown-toggle .caret{border-top-color:#fff;border-bottom-color:#fff}.nav-tabs .active .dropdown-toggle .caret{border-top-color:#555;border-bottom-color:#555}.nav>.dropdown.active>a:hover{cursor:pointer}.nav-tabs .open .dropdown-toggle,.nav-pills .open .dropdown-toggle,.nav>li.dropdown.open.active>a:hover{color:#fff;background-color:#999;border-color:#999}.nav li.dropdown.open .caret,.nav li.dropdown.open.active .caret,.nav li.dropdown.open a:hover .caret{border-top-color:#fff;border-bottom-color:#fff;opacity:1;filter:alpha(opacity=100)}.tabs-stacked .open>a:hover{border-color:#999}.tabbable{*zoom:1}.tabbable:before,.tabbable:after{display:table;line-height:0;content:""}.tabbable:after{clear:both}.tab-content{overflow:auto}.tabs-below>.nav-tabs,.tabs-right>.nav-tabs,.tabs-left>.nav-tabs{border-bottom:0}.tab-content>.tab-pane,.pill-content>.pill-pane{display:none}.tab-content>.active,.pill-content>.active{display:block}.tabs-below>.nav-tabs{border-top:1px solid #ddd}.tabs-below>.nav-tabs>li{margin-top:-1px;margin-bottom:0}.tabs-below>.nav-tabs>li>a{-webkit-border-radius:0 0 4px 4px;-moz-border-radius:0 0 4px 4px;border-radius:0 0 4px 4px}.tabs-below>.nav-tabs>li>a:hover{border-top-color:#ddd;border-bottom-color:transparent}.tabs-below>.nav-tabs>.active>a,.tabs-below>.nav-tabs>.active>a:hover{border-color:transparent #ddd #ddd #ddd}.tabs-left>.nav-tabs>li,.tabs-right>.nav-tabs>li{float:none}.tabs-left>.nav-tabs>li>a,.tabs-right>.nav-tabs>li>a{min-width:74px;margin-right:0;margin-bottom:3px}.tabs-left>.nav-tabs{float:left;margin-right:19px;border-right:1px solid #ddd}.tabs-left>.nav-tabs>li>a{margin-right:-1px;-webkit-border-radius:4px 0 0 4px;-moz-border-radius:4px 0 0 4px;border-radius:4px 0 0 4px}.tabs-left>.nav-tabs>li>a:hover{border-color:#eee #ddd #eee #eee}.tabs-left>.nav-tabs .active>a,.tabs-left>.nav-tabs .active>a:hover{border-color:#ddd transparent #ddd #ddd;*border-right-color:#fff}.tabs-right>.nav-tabs{float:right;margin-left:19px;border-left:1px solid #ddd}.tabs-right>.nav-tabs>li>a{margin-left:-1px;-webkit-border-radius:0 4px 4px 0;-moz-border-radius:0 4px 4px 0;border-radius:0 4px 4px 0}.tabs-right>.nav-tabs>li>a:hover{border-color:#eee #eee #eee #ddd}.tabs-right>.nav-tabs .active>a,.tabs-right>.nav-tabs .active>a:hover{border-color:#ddd #ddd #ddd transparent;*border-left-color:#fff}.nav>.disabled>a{color:#999}.nav>.disabled>a:hover{text-decoration:none;cursor:default;background-color:transparent}.navbar{*position:relative;*z-index:2;margin-bottom:20px;overflow:visible;color:#777}.navbar-inner{min-height:40px;padding-right:20px;padding-left:20px;background-color:#fafafa;background-image:-moz-linear-gradient(top,#fff,#f2f2f2);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fff),to(#f2f2f2));background-image:-webkit-linear-gradient(top,#fff,#f2f2f2);background-image:-o-linear-gradient(top,#fff,#f2f2f2);background-image:linear-gradient(to bottom,#fff,#f2f2f2);background-repeat:repeat-x;border:1px solid #d4d4d4;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffffffff',endColorstr='#fff2f2f2',GradientType=0);*zoom:1;-webkit-box-shadow:0 1px 4px rgba(0,0,0,0.065);-moz-box-shadow:0 1px 4px rgba(0,0,0,0.065);box-shadow:0 1px 4px rgba(0,0,0,0.065)}.navbar-inner:before,.navbar-inner:after{display:table;line-height:0;content:""}.navbar-inner:after{clear:both}.navbar .container{width:auto}.nav-collapse.collapse{height:auto}.navbar .brand{display:block;float:left;padding:10px 20px 10px;margin-left:-20px;font-size:20px;font-weight:200;color:#777;text-shadow:0 1px 0 #fff}.navbar .brand:hover{text-decoration:none}.navbar-text{margin-bottom:0;line-height:40px}.navbar-link{color:#777}.navbar-link:hover{color:#333}.navbar .divider-vertical{height:40px;margin:0 9px;border-right:1px solid #fff;border-left:1px solid #f2f2f2}.navbar .btn,.navbar .btn-group{margin-top:5px}.navbar .btn-group .btn,.navbar .input-prepend .btn,.navbar .input-append .btn{margin-top:0}.navbar-form{margin-bottom:0;*zoom:1}.navbar-form:before,.navbar-form:after{display:table;line-height:0;content:""}.navbar-form:after{clear:both}.navbar-form input,.navbar-form select,.navbar-form .radio,.navbar-form .checkbox{margin-top:5px}.navbar-form input,.navbar-form select,.navbar-form .btn{display:inline-block;margin-bottom:0}.navbar-form input[type="image"],.navbar-form input[type="checkbox"],.navbar-form input[type="radio"]{margin-top:3px}.navbar-form .input-append,.navbar-form .input-prepend{margin-top:6px;white-space:nowrap}.navbar-form .input-append input,.navbar-form .input-prepend input{margin-top:0}.navbar-search{position:relative;float:left;margin-top:5px;margin-bottom:0}.navbar-search .search-query{padding:4px 14px;margin-bottom:0;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:13px;font-weight:normal;line-height:1;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.navbar-static-top{position:static;width:100%;margin-bottom:0}.navbar-static-top .navbar-inner{-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-fixed-top,.navbar-fixed-bottom{position:fixed;right:0;left:0;z-index:1030;margin-bottom:0}.navbar-fixed-top .navbar-inner,.navbar-static-top .navbar-inner{border-width:0 0 1px}.navbar-fixed-bottom .navbar-inner{border-width:1px 0 0}.navbar-fixed-top .navbar-inner,.navbar-fixed-bottom .navbar-inner{padding-right:0;padding-left:0;-webkit-border-radius:0;-moz-border-radius:0;border-radius:0}.navbar-static-top .container,.navbar-fixed-top .container,.navbar-fixed-bottom .container{width:940px}.navbar-fixed-top{top:0}.navbar-fixed-top .navbar-inner,.navbar-static-top .navbar-inner{-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.1),0 1px 10px rgba(0,0,0,0.1)}.navbar-fixed-bottom{bottom:0}.navbar-fixed-bottom .navbar-inner{-webkit-box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1);box-shadow:inset 0 1px 0 rgba(0,0,0,0.1),0 -1px 10px rgba(0,0,0,0.1)}.navbar .nav{position:relative;left:0;display:block;float:left;margin:0 10px 0 0}.navbar .nav.pull-right{float:right;margin-right:0}.navbar .nav>li{float:left}.navbar .nav>li>a{float:none;padding:10px 15px 10px;color:#777;text-decoration:none;text-shadow:0 1px 0 #fff}.navbar .nav .dropdown-toggle .caret{margin-top:8px}.navbar .nav>li>a:focus,.navbar .nav>li>a:hover{color:#333;text-decoration:none;background-color:transparent}.navbar .nav>.active>a,.navbar .nav>.active>a:hover,.navbar .nav>.active>a:focus{color:#555;text-decoration:none;background-color:#e5e5e5;-webkit-box-shadow:inset 0 3px 8px rgba(0,0,0,0.125);-moz-box-shadow:inset 0 3px 8px rgba(0,0,0,0.125);box-shadow:inset 0 3px 8px rgba(0,0,0,0.125)}.navbar .btn-navbar{display:none;float:right;padding:7px 10px;margin-right:5px;margin-left:5px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#ededed;*background-color:#e5e5e5;background-image:-webkit-gradient(linear,0 0,0 100%,from(#f2f2f2),to(#e5e5e5));background-image:-webkit-linear-gradient(top,#f2f2f2,#e5e5e5);background-image:-o-linear-gradient(top,#f2f2f2,#e5e5e5);background-image:linear-gradient(to bottom,#f2f2f2,#e5e5e5);background-image:-moz-linear-gradient(top,#f2f2f2,#e5e5e5);background-repeat:repeat-x;border-color:#e5e5e5 #e5e5e5 #bfbfbf;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fff2f2f2',endColorstr='#ffe5e5e5',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);-moz-box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075);box-shadow:inset 0 1px 0 rgba(255,255,255,0.1),0 1px 0 rgba(255,255,255,0.075)}.navbar .btn-navbar:hover,.navbar .btn-navbar:active,.navbar .btn-navbar.active,.navbar .btn-navbar.disabled,.navbar .btn-navbar[disabled]{color:#fff;background-color:#e5e5e5;*background-color:#d9d9d9}.navbar .btn-navbar:active,.navbar .btn-navbar.active{background-color:#ccc \9}.navbar .btn-navbar .icon-bar{display:block;width:18px;height:2px;background-color:#f5f5f5;-webkit-border-radius:1px;-moz-border-radius:1px;border-radius:1px;-webkit-box-shadow:0 1px 0 rgba(0,0,0,0.25);-moz-box-shadow:0 1px 0 rgba(0,0,0,0.25);box-shadow:0 1px 0 rgba(0,0,0,0.25)}.btn-navbar .icon-bar+.icon-bar{margin-top:3px}.navbar .nav>li>.dropdown-menu:before{position:absolute;top:-7px;left:9px;display:inline-block;border-right:7px solid transparent;border-bottom:7px solid #ccc;border-left:7px solid transparent;border-bottom-color:rgba(0,0,0,0.2);content:''}.navbar .nav>li>.dropdown-menu:after{position:absolute;top:-6px;left:10px;display:inline-block;border-right:6px solid transparent;border-bottom:6px solid #fff;border-left:6px solid transparent;content:''}.navbar-fixed-bottom .nav>li>.dropdown-menu:before{top:auto;bottom:-7px;border-top:7px solid #ccc;border-bottom:0;border-top-color:rgba(0,0,0,0.2)}.navbar-fixed-bottom .nav>li>.dropdown-menu:after{top:auto;bottom:-6px;border-top:6px solid #fff;border-bottom:0}.navbar .nav li.dropdown.open>.dropdown-toggle,.navbar .nav li.dropdown.active>.dropdown-toggle,.navbar .nav li.dropdown.open.active>.dropdown-toggle{color:#555;background-color:#e5e5e5}.navbar .nav li.dropdown>.dropdown-toggle .caret{border-top-color:#777;border-bottom-color:#777}.navbar .nav li.dropdown.open>.dropdown-toggle .caret,.navbar .nav li.dropdown.active>.dropdown-toggle .caret,.navbar .nav li.dropdown.open.active>.dropdown-toggle .caret{border-top-color:#555;border-bottom-color:#555}.navbar .pull-right>li>.dropdown-menu,.navbar .nav>li>.dropdown-menu.pull-right{right:0;left:auto}.navbar .pull-right>li>.dropdown-menu:before,.navbar .nav>li>.dropdown-menu.pull-right:before{right:12px;left:auto}.navbar .pull-right>li>.dropdown-menu:after,.navbar .nav>li>.dropdown-menu.pull-right:after{right:13px;left:auto}.navbar .pull-right>li>.dropdown-menu .dropdown-menu,.navbar .nav>li>.dropdown-menu.pull-right .dropdown-menu{right:100%;left:auto;margin-right:-1px;margin-left:0;-webkit-border-radius:6px 0 6px 6px;-moz-border-radius:6px 0 6px 6px;border-radius:6px 0 6px 6px}.navbar-inverse{color:#999}.navbar-inverse .navbar-inner{background-color:#1b1b1b;background-image:-moz-linear-gradient(top,#222,#111);background-image:-webkit-gradient(linear,0 0,0 100%,from(#222),to(#111));background-image:-webkit-linear-gradient(top,#222,#111);background-image:-o-linear-gradient(top,#222,#111);background-image:linear-gradient(to bottom,#222,#111);background-repeat:repeat-x;border-color:#252525;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff222222',endColorstr='#ff111111',GradientType=0)}.navbar-inverse .brand,.navbar-inverse .nav>li>a{color:#999;text-shadow:0 -1px 0 rgba(0,0,0,0.25)}.navbar-inverse .brand:hover,.navbar-inverse .nav>li>a:hover{color:#fff}.navbar-inverse .nav>li>a:focus,.navbar-inverse .nav>li>a:hover{color:#fff;background-color:transparent}.navbar-inverse .nav .active>a,.navbar-inverse .nav .active>a:hover,.navbar-inverse .nav .active>a:focus{color:#fff;background-color:#111}.navbar-inverse .navbar-link{color:#999}.navbar-inverse .navbar-link:hover{color:#fff}.navbar-inverse .divider-vertical{border-right-color:#222;border-left-color:#111}.navbar-inverse .nav li.dropdown.open>.dropdown-toggle,.navbar-inverse .nav li.dropdown.active>.dropdown-toggle,.navbar-inverse .nav li.dropdown.open.active>.dropdown-toggle{color:#fff;background-color:#111}.navbar-inverse .nav li.dropdown>.dropdown-toggle .caret{border-top-color:#999;border-bottom-color:#999}.navbar-inverse .nav li.dropdown.open>.dropdown-toggle .caret,.navbar-inverse .nav li.dropdown.active>.dropdown-toggle .caret,.navbar-inverse .nav li.dropdown.open.active>.dropdown-toggle .caret{border-top-color:#fff;border-bottom-color:#fff}.navbar-inverse .navbar-search .search-query{color:#fff;background-color:#515151;border-color:#111;-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1),0 1px 0 rgba(255,255,255,0.15);-webkit-transition:none;-moz-transition:none;-o-transition:none;transition:none}.navbar-inverse .navbar-search .search-query:-moz-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query:-ms-input-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query::-webkit-input-placeholder{color:#ccc}.navbar-inverse .navbar-search .search-query:focus,.navbar-inverse .navbar-search .search-query.focused{padding:5px 15px;color:#333;text-shadow:0 1px 0 #fff;background-color:#fff;border:0;outline:0;-webkit-box-shadow:0 0 3px rgba(0,0,0,0.15);-moz-box-shadow:0 0 3px rgba(0,0,0,0.15);box-shadow:0 0 3px rgba(0,0,0,0.15)}.navbar-inverse .btn-navbar{color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e0e0e;*background-color:#040404;background-image:-webkit-gradient(linear,0 0,0 100%,from(#151515),to(#040404));background-image:-webkit-linear-gradient(top,#151515,#040404);background-image:-o-linear-gradient(top,#151515,#040404);background-image:linear-gradient(to bottom,#151515,#040404);background-image:-moz-linear-gradient(top,#151515,#040404);background-repeat:repeat-x;border-color:#040404 #040404 #000;border-color:rgba(0,0,0,0.1) rgba(0,0,0,0.1) rgba(0,0,0,0.25);filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff151515',endColorstr='#ff040404',GradientType=0);filter:progid:dximagetransform.microsoft.gradient(enabled=false)}.navbar-inverse .btn-navbar:hover,.navbar-inverse .btn-navbar:active,.navbar-inverse .btn-navbar.active,.navbar-inverse .btn-navbar.disabled,.navbar-inverse .btn-navbar[disabled]{color:#fff;background-color:#040404;*background-color:#000}.navbar-inverse .btn-navbar:active,.navbar-inverse .btn-navbar.active{background-color:#000 \9}.breadcrumb{padding:8px 15px;margin:0 0 20px;list-style:none;background-color:#f5f5f5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.breadcrumb li{display:inline-block;*display:inline;text-shadow:0 1px 0 #fff;*zoom:1}.breadcrumb .divider{padding:0 5px;color:#ccc}.breadcrumb .active{color:#999}.pagination{height:40px;margin:20px 0}.pagination ul{display:inline-block;*display:inline;margin-bottom:0;margin-left:0;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px;*zoom:1;-webkit-box-shadow:0 1px 2px rgba(0,0,0,0.05);-moz-box-shadow:0 1px 2px rgba(0,0,0,0.05);box-shadow:0 1px 2px rgba(0,0,0,0.05)}.pagination ul>li{display:inline}.pagination ul>li>a,.pagination ul>li>span{float:left;padding:0 14px;line-height:38px;text-decoration:none;background-color:#fff;border:1px solid #ddd;border-left-width:0}.pagination ul>li>a:hover,.pagination ul>.active>a,.pagination ul>.active>span{background-color:#f5f5f5}.pagination ul>.active>a,.pagination ul>.active>span{color:#999;cursor:default}.pagination ul>.disabled>span,.pagination ul>.disabled>a,.pagination ul>.disabled>a:hover{color:#999;cursor:default;background-color:transparent}.pagination ul>li:first-child>a,.pagination ul>li:first-child>span{border-left-width:1px;-webkit-border-radius:3px 0 0 3px;-moz-border-radius:3px 0 0 3px;border-radius:3px 0 0 3px}.pagination ul>li:last-child>a,.pagination ul>li:last-child>span{-webkit-border-radius:0 3px 3px 0;-moz-border-radius:0 3px 3px 0;border-radius:0 3px 3px 0}.pagination-centered{text-align:center}.pagination-right{text-align:right}.pager{margin:20px 0;text-align:center;list-style:none;*zoom:1}.pager:before,.pager:after{display:table;line-height:0;content:""}.pager:after{clear:both}.pager li{display:inline}.pager a,.pager span{display:inline-block;padding:5px 14px;background-color:#fff;border:1px solid #ddd;-webkit-border-radius:15px;-moz-border-radius:15px;border-radius:15px}.pager a:hover{text-decoration:none;background-color:#f5f5f5}.pager .next a,.pager .next span{float:right}.pager .previous a{float:left}.pager .disabled a,.pager .disabled a:hover,.pager .disabled span{color:#999;cursor:default;background-color:#fff}.modal-open .modal .dropdown-menu{z-index:2050}.modal-open .modal .dropdown.open{*z-index:2050}.modal-open .modal .popover{z-index:2060}.modal-open .modal .tooltip{z-index:2080}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop,.modal-backdrop.fade.in{opacity:.8;filter:alpha(opacity=80)}.modal{position:fixed;top:50%;left:50%;z-index:1050;width:560px;margin:-250px 0 0 -280px;overflow:auto;background-color:#fff;border:1px solid #999;border:1px solid rgba(0,0,0,0.3);*border:1px solid #999;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 3px 7px rgba(0,0,0,0.3);-moz-box-shadow:0 3px 7px rgba(0,0,0,0.3);box-shadow:0 3px 7px rgba(0,0,0,0.3);-webkit-background-clip:padding-box;-moz-background-clip:padding-box;background-clip:padding-box}.modal.fade{top:-25%;-webkit-transition:opacity .3s linear,top .3s ease-out;-moz-transition:opacity .3s linear,top .3s ease-out;-o-transition:opacity .3s linear,top .3s ease-out;transition:opacity .3s linear,top .3s ease-out}.modal.fade.in{top:50%}.modal-header{padding:9px 15px;border-bottom:1px solid #eee}.modal-header .close{margin-top:2px}.modal-header h3{margin:0;line-height:30px}.modal-body{max-height:400px;padding:15px;overflow-y:auto}.modal-form{margin-bottom:0}.modal-footer{padding:14px 15px 15px;margin-bottom:0;text-align:right;background-color:#f5f5f5;border-top:1px solid #ddd;-webkit-border-radius:0 0 6px 6px;-moz-border-radius:0 0 6px 6px;border-radius:0 0 6px 6px;*zoom:1;-webkit-box-shadow:inset 0 1px 0 #fff;-moz-box-shadow:inset 0 1px 0 #fff;box-shadow:inset 0 1px 0 #fff}.modal-footer:before,.modal-footer:after{display:table;line-height:0;content:""}.modal-footer:after{clear:both}.modal-footer .btn+.btn{margin-bottom:0;margin-left:5px}.modal-footer .btn-group .btn+.btn{margin-left:-1px}.tooltip{position:absolute;z-index:1030;display:block;padding:5px;font-size:11px;opacity:0;filter:alpha(opacity=0);visibility:visible}.tooltip.in{opacity:.8;filter:alpha(opacity=80)}.tooltip.top{margin-top:-3px}.tooltip.right{margin-left:3px}.tooltip.bottom{margin-top:3px}.tooltip.left{margin-left:-3px}.tooltip-inner{max-width:200px;padding:3px 8px;color:#fff;text-align:center;text-decoration:none;background-color:#000;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.tooltip-arrow{position:absolute;width:0;height:0;border-color:transparent;border-style:solid}.tooltip.top .tooltip-arrow{bottom:0;left:50%;margin-left:-5px;border-top-color:#000;border-width:5px 5px 0}.tooltip.right .tooltip-arrow{top:50%;left:0;margin-top:-5px;border-right-color:#000;border-width:5px 5px 5px 0}.tooltip.left .tooltip-arrow{top:50%;right:0;margin-top:-5px;border-left-color:#000;border-width:5px 0 5px 5px}.tooltip.bottom .tooltip-arrow{top:0;left:50%;margin-left:-5px;border-bottom-color:#000;border-width:0 5px 5px}.popover{position:absolute;top:0;left:0;z-index:1010;display:none;width:236px;padding:1px;background-color:#fff;border:1px solid #ccc;border:1px solid rgba(0,0,0,0.2);-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px;-webkit-box-shadow:0 5px 10px rgba(0,0,0,0.2);-moz-box-shadow:0 5px 10px rgba(0,0,0,0.2);box-shadow:0 5px 10px rgba(0,0,0,0.2);-webkit-background-clip:padding-box;-moz-background-clip:padding;background-clip:padding-box}.popover.top{margin-bottom:10px}.popover.right{margin-left:10px}.popover.bottom{margin-top:10px}.popover.left{margin-right:10px}.popover-title{padding:8px 14px;margin:0;font-size:14px;font-weight:normal;line-height:18px;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;-webkit-border-radius:5px 5px 0 0;-moz-border-radius:5px 5px 0 0;border-radius:5px 5px 0 0}.popover-content{padding:9px 14px}.popover-content p,.popover-content ul,.popover-content ol{margin-bottom:0}.popover .arrow,.popover .arrow:after{position:absolute;display:inline-block;width:0;height:0;border-color:transparent;border-style:solid}.popover .arrow:after{z-index:-1;content:""}.popover.top .arrow{bottom:-10px;left:50%;margin-left:-10px;border-top-color:#fff;border-width:10px 10px 0}.popover.top .arrow:after{bottom:-1px;left:-11px;border-top-color:rgba(0,0,0,0.25);border-width:11px 11px 0}.popover.right .arrow{top:50%;left:-10px;margin-top:-10px;border-right-color:#fff;border-width:10px 10px 10px 0}.popover.right .arrow:after{bottom:-11px;left:-1px;border-right-color:rgba(0,0,0,0.25);border-width:11px 11px 11px 0}.popover.bottom .arrow{top:-10px;left:50%;margin-left:-10px;border-bottom-color:#fff;border-width:0 10px 10px}.popover.bottom .arrow:after{top:-1px;left:-11px;border-bottom-color:rgba(0,0,0,0.25);border-width:0 11px 11px}.popover.left .arrow{top:50%;right:-10px;margin-top:-10px;border-left-color:#fff;border-width:10px 0 10px 10px}.popover.left .arrow:after{right:-1px;bottom:-11px;border-left-color:rgba(0,0,0,0.25);border-width:11px 0 11px 11px}.thumbnails{margin-left:-20px;list-style:none;*zoom:1}.thumbnails:before,.thumbnails:after{display:table;line-height:0;content:""}.thumbnails:after{clear:both}.row-fluid .thumbnails{margin-left:0}.thumbnails>li{float:left;margin-bottom:20px;margin-left:20px}.thumbnail{display:block;padding:4px;line-height:20px;border:1px solid #ddd;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;-webkit-box-shadow:0 1px 3px rgba(0,0,0,0.055);-moz-box-shadow:0 1px 3px rgba(0,0,0,0.055);box-shadow:0 1px 3px rgba(0,0,0,0.055);-webkit-transition:all .2s ease-in-out;-moz-transition:all .2s ease-in-out;-o-transition:all .2s ease-in-out;transition:all .2s ease-in-out}a.thumbnail:hover{border-color:#08c;-webkit-box-shadow:0 1px 4px rgba(0,105,214,0.25);-moz-box-shadow:0 1px 4px rgba(0,105,214,0.25);box-shadow:0 1px 4px rgba(0,105,214,0.25)}.thumbnail>img{display:block;max-width:100%;margin-right:auto;margin-left:auto}.thumbnail .caption{padding:9px;color:#555}.label,.badge{font-size:11.844px;font-weight:bold;line-height:14px;color:#fff;text-shadow:0 -1px 0 rgba(0,0,0,0.25);white-space:nowrap;vertical-align:baseline;background-color:#999}.label{padding:1px 4px 2px;-webkit-border-radius:3px;-moz-border-radius:3px;border-radius:3px}.badge{padding:1px 9px 2px;-webkit-border-radius:9px;-moz-border-radius:9px;border-radius:9px}a.label:hover,a.badge:hover{color:#fff;text-decoration:none;cursor:pointer}.label-important,.badge-important{background-color:#b94a48}.label-important[href],.badge-important[href]{background-color:#953b39}.label-warning,.badge-warning{background-color:#f89406}.label-warning[href],.badge-warning[href]{background-color:#c67605}.label-success,.badge-success{background-color:#468847}.label-success[href],.badge-success[href]{background-color:#356635}.label-info,.badge-info{background-color:#3a87ad}.label-info[href],.badge-info[href]{background-color:#2d6987}.label-inverse,.badge-inverse{background-color:#333}.label-inverse[href],.badge-inverse[href]{background-color:#1a1a1a}.btn .label,.btn .badge{position:relative;top:-1px}.btn-mini .label,.btn-mini .badge{top:0}@-webkit-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-moz-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-ms-keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}@-o-keyframes progress-bar-stripes{from{background-position:0 0}to{background-position:40px 0}}@keyframes progress-bar-stripes{from{background-position:40px 0}to{background-position:0 0}}.progress{height:20px;margin-bottom:20px;overflow:hidden;background-color:#f7f7f7;background-image:-moz-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#f5f5f5),to(#f9f9f9));background-image:-webkit-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:-o-linear-gradient(top,#f5f5f5,#f9f9f9);background-image:linear-gradient(to bottom,#f5f5f5,#f9f9f9);background-repeat:repeat-x;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fff5f5f5',endColorstr='#fff9f9f9',GradientType=0);-webkit-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);-moz-box-shadow:inset 0 1px 2px rgba(0,0,0,0.1);box-shadow:inset 0 1px 2px rgba(0,0,0,0.1)}.progress .bar{float:left;width:0;height:100%;font-size:12px;color:#fff;text-align:center;text-shadow:0 -1px 0 rgba(0,0,0,0.25);background-color:#0e90d2;background-image:-moz-linear-gradient(top,#149bdf,#0480be);background-image:-webkit-gradient(linear,0 0,0 100%,from(#149bdf),to(#0480be));background-image:-webkit-linear-gradient(top,#149bdf,#0480be);background-image:-o-linear-gradient(top,#149bdf,#0480be);background-image:linear-gradient(to bottom,#149bdf,#0480be);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff149bdf',endColorstr='#ff0480be',GradientType=0);-webkit-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 0 -1px 0 rgba(0,0,0,0.15);-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-transition:width .6s ease;-moz-transition:width .6s ease;-o-transition:width .6s ease;transition:width .6s ease}.progress .bar+.bar{-webkit-box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15);-moz-box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15);box-shadow:inset 1px 0 0 rgba(0,0,0,0.15),inset 0 -1px 0 rgba(0,0,0,0.15)}.progress-striped .bar{background-color:#149bdf;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);-webkit-background-size:40px 40px;-moz-background-size:40px 40px;-o-background-size:40px 40px;background-size:40px 40px}.progress.active .bar{-webkit-animation:progress-bar-stripes 2s linear infinite;-moz-animation:progress-bar-stripes 2s linear infinite;-ms-animation:progress-bar-stripes 2s linear infinite;-o-animation:progress-bar-stripes 2s linear infinite;animation:progress-bar-stripes 2s linear infinite}.progress-danger .bar,.progress .bar-danger{background-color:#dd514c;background-image:-moz-linear-gradient(top,#ee5f5b,#c43c35);background-image:-webkit-gradient(linear,0 0,0 100%,from(#ee5f5b),to(#c43c35));background-image:-webkit-linear-gradient(top,#ee5f5b,#c43c35);background-image:-o-linear-gradient(top,#ee5f5b,#c43c35);background-image:linear-gradient(to bottom,#ee5f5b,#c43c35);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ffee5f5b',endColorstr='#ffc43c35',GradientType=0)}.progress-danger.progress-striped .bar,.progress-striped .bar-danger{background-color:#ee5f5b;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-success .bar,.progress .bar-success{background-color:#5eb95e;background-image:-moz-linear-gradient(top,#62c462,#57a957);background-image:-webkit-gradient(linear,0 0,0 100%,from(#62c462),to(#57a957));background-image:-webkit-linear-gradient(top,#62c462,#57a957);background-image:-o-linear-gradient(top,#62c462,#57a957);background-image:linear-gradient(to bottom,#62c462,#57a957);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff62c462',endColorstr='#ff57a957',GradientType=0)}.progress-success.progress-striped .bar,.progress-striped .bar-success{background-color:#62c462;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-info .bar,.progress .bar-info{background-color:#4bb1cf;background-image:-moz-linear-gradient(top,#5bc0de,#339bb9);background-image:-webkit-gradient(linear,0 0,0 100%,from(#5bc0de),to(#339bb9));background-image:-webkit-linear-gradient(top,#5bc0de,#339bb9);background-image:-o-linear-gradient(top,#5bc0de,#339bb9);background-image:linear-gradient(to bottom,#5bc0de,#339bb9);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#ff5bc0de',endColorstr='#ff339bb9',GradientType=0)}.progress-info.progress-striped .bar,.progress-striped .bar-info{background-color:#5bc0de;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.progress-warning .bar,.progress .bar-warning{background-color:#faa732;background-image:-moz-linear-gradient(top,#fbb450,#f89406);background-image:-webkit-gradient(linear,0 0,0 100%,from(#fbb450),to(#f89406));background-image:-webkit-linear-gradient(top,#fbb450,#f89406);background-image:-o-linear-gradient(top,#fbb450,#f89406);background-image:linear-gradient(to bottom,#fbb450,#f89406);background-repeat:repeat-x;filter:progid:dximagetransform.microsoft.gradient(startColorstr='#fffbb450',endColorstr='#fff89406',GradientType=0)}.progress-warning.progress-striped .bar,.progress-striped .bar-warning{background-color:#fbb450;background-image:-webkit-gradient(linear,0 100%,100% 0,color-stop(0.25,rgba(255,255,255,0.15)),color-stop(0.25,transparent),color-stop(0.5,transparent),color-stop(0.5,rgba(255,255,255,0.15)),color-stop(0.75,rgba(255,255,255,0.15)),color-stop(0.75,transparent),to(transparent));background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-moz-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,0.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,0.15) 50%,rgba(255,255,255,0.15) 75%,transparent 75%,transparent)}.accordion{margin-bottom:20px}.accordion-group{margin-bottom:2px;border:1px solid #e5e5e5;-webkit-border-radius:4px;-moz-border-radius:4px;border-radius:4px}.accordion-heading{border-bottom:0}.accordion-heading .accordion-toggle{display:block;padding:8px 15px}.accordion-toggle{cursor:pointer}.accordion-inner{padding:9px 15px;border-top:1px solid #e5e5e5}.carousel{position:relative;margin-bottom:20px;line-height:1}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel .item{position:relative;display:none;-webkit-transition:.6s ease-in-out left;-moz-transition:.6s ease-in-out left;-o-transition:.6s ease-in-out left;transition:.6s ease-in-out left}.carousel .item>img{display:block;line-height:1}.carousel .active,.carousel .next,.carousel .prev{display:block}.carousel .active{left:0}.carousel .next,.carousel .prev{position:absolute;top:0;width:100%}.carousel .next{left:100%}.carousel .prev{left:-100%}.carousel .next.left,.carousel .prev.right{left:0}.carousel .active.left{left:-100%}.carousel .active.right{left:100%}.carousel-control{position:absolute;top:40%;left:15px;width:40px;height:40px;margin-top:-20px;font-size:60px;font-weight:100;line-height:30px;color:#fff;text-align:center;background:#222;border:3px solid #fff;-webkit-border-radius:23px;-moz-border-radius:23px;border-radius:23px;opacity:.5;filter:alpha(opacity=50)}.carousel-control.right{right:15px;left:auto}.carousel-control:hover{color:#fff;text-decoration:none;opacity:.9;filter:alpha(opacity=90)}.carousel-caption{position:absolute;right:0;bottom:0;left:0;padding:15px;background:#333;background:rgba(0,0,0,0.75)}.carousel-caption h4,.carousel-caption p{line-height:20px;color:#fff}.carousel-caption h4{margin:0 0 5px}.carousel-caption p{margin-bottom:0}.hero-unit{padding:60px;margin-bottom:30px;background-color:#eee;-webkit-border-radius:6px;-moz-border-radius:6px;border-radius:6px}.hero-unit h1{margin-bottom:0;font-size:60px;line-height:1;letter-spacing:-1px;color:inherit}.hero-unit p{font-size:18px;font-weight:200;line-height:30px;color:inherit}.pull-right{float:right}.pull-left{float:left}.hide{display:none}.show{display:block}.invisible{visibility:hidden}.affix{position:fixed} diff --git a/docs/css/custom.css b/docs/css/custom.css new file mode 100644 index 000000000000..ce15235e889a --- /dev/null +++ b/docs/css/custom.css @@ -0,0 +1,592 @@ +@font-face { + font-family: 'Conv_framd'; + src: url('../fonts/framd.eot'); + src: url('../fonts/framd.eot?#iefix') format('embedded-opentype'), + url('../fonts/framd.woff') format('woff'), + url('../fonts/framd.ttf') format('truetype'), + url('../fonts/framd.svg#heroregular') format('svg'); + font-weight: normal; + font-style: normal; +} + + +html, body { + position:relative; + height:100%; + min-height:100%; + height:100%; + color:#252525; + font:400 18px/26px 'Open Sans', Arial, Helvetica, sans-serif; + padding:0; + margin:0; + word-wrap:break-word; +} +a { + color:#6ab6dd; + position:relative; +} +a:hover { + text-decoration:underline; + color:#2c79a1; +} +.wrapper { + min-height:100%; +} +header { + margin:0 0 100px; +} +header .span12 { + padding:0 0 7px 0; +} +.logo.custom { + display:inline-block; + margin:0; + vertical-align:25px; +} +.logo.custom a { + background:url(../img/logo.png) no-repeat; + width: 110px; + height: 49px; + display:block; + text-indent:-9999px; +} +.custom.navbar { + margin:10px 0; +} +.custom.navbar .nav li { + padding:0 !important; +} +.custom.navbar .nav li a, .navbar .brand, .custom.navbar .nav li.pull-right span { + font:300 14px/20px 'Open Sans', Arial, Helvetica, sans-serif; + padding:10px 10px; +} +.navbar .brand.active { + color:#fff; +} +.custom.navbar .nav li.pull-right span { + padding-right:0; + color:#e76d4c !important; + display:block; +} +.custom.navbar .nav li a.doc-link { + padding:5px 10px 0; +} +.custom.navbar .nav li a.doc-link span { + display:inline-block; + background:url(../img/icon-git.png) no-repeat; + width: 28px; + height: 28px; + vertical-align:-7px; + margin-left:5px; +} +.custom.navbar .nav li a.doc-link:hover span, .custom.navbar .nav li.active a.doc-link span { + background-position:0 -28px; +} +.custom.navbar .nav { + float:none; +} +.navbar .nav > li.pull-right { + float:right; + padding:10px 0; +} +h1 { + font:300 48px/48px 'Open Sans', Arial, Helvetica, sans-serif; + margin:0 0 45px; +} +h1.index { + text-align:center; +} +h1 span { + display:block; + font:400 14px/28px 'Open Sans', Arial, Helvetica, sans-serif; +} +h2 { + font:30px/30px 'Conv_framd', Arial, Helvetica, sans-serif; + margin:0 0 20px; + color:#0f1e35; +} +h3 { + font:300 30px/36px 'Open Sans', Arial, Helvetica, sans-serif; + margin:0 0 33px; + text-align:center; +} +.btn { + display:block; + margin:0 auto 65px; + background:#6ab6dd; + border:none; + box-shadow:inset -3px -3px 3px #5592b1; + height:53px; + width:205px; + font:30px/53px 'Conv_framd', Arial, Helvetica, sans-serif; + color:#252424; + text-shadow:none; + padding:0; + z-index:100; + position:relative; +} +.btn a { + color:#252424; +} +.btn:hover { + background:#83c6e9; +} +.index-content { + margin:0 auto 60px; + text-align:center; +} +.third { + margin:0 auto 35px; +} +.third-item { + text-align:left; +} +.third-item:first-child { + margin:0; +} +.third-item a { + display:block; + font-family:'Open Sans', Arial, Helvetica, sans-serif; + font-weight:700; + font-size:30px; + margin:0 auto 20px; + color:#252424; + text-align:center; +} +.container.custom { + padding:0; + margin:0 auto; +} +.container.custom.main-cont { + padding-bottom:230px; +} +.text-part { + padding-top:70px; +} +.row-fluid.index-page { + padding-top:100px; +} +.index-page .content { + padding:15px 0 0; +} +.index-page h3 { + text-align:left; +} +.index-page .sidebar { + padding:65px 0 30px; +} +.container.custom p { + margin:0 0 17px; +} +.homepage .index-page .content h2 { + margin:0 0 20px; +} +.container.custom .unstyled { + margin:0; + color:#353535; +} +.container.custom .unstyled li { + margin:0 0 17px; +} +.container.custom .unstyled li span { + font-family:'Open Sans', Arial, Helvetica, sans-serif; + font-weight:700; + display:block; +} +.container.custom .unstyled li a { + display:inline; +} +.homepage h4 { + font:24px/24px 'Conv_framd', Arial, Helvetica, sans-serif; + margin:0 0 15px; +} +.container.custom .sidebar .unstyled { + margin:0 0 100px; +} +.container.custom .sidebar .unstyled li { + margin:0 0 12px; + border-bottom:1px solid #adadad; + padding: 5px 7px; +} +.grey-box { + background:#e5e4e3; + border-radius:3px; + -moz-border-radius:3px; + -webkit-border-radius:3px; + position:relative; + padding:20px 10px 130px; + color:#000; +} +footer { + text-align:center; + font-size:14px; + color:#000; + margin:-135px 0 0; +} +footer .container.custom { + border-top:1px solid #e1e1e1; + padding:20px 0 25px; +} +footer .span9 { + text-align:left; +} +footer .container.custom ul.unstyled { + display:inline-block; + margin:0 120px 30px 30px; + text-align:left; + vertical-align:top; +} +footer .container.custom ul.unstyled li { + font:300 14px/26px 'Open Sans', Arial, Helvetica, sans-serif; + margin:0; +} +footer .container.custom .unstyled li a { + color:#000; + font-weight:300; +} +footer .container.custom .unstyled li:first-child a { + font-weight:400; +} +footer ul li a:hover { + text-decoration:underline; + color:#fff; +} +footer .logo-block { + text-align:right; +} +footer .container.custom p { + display:inline-block; + margin:28px 0 0 10px; + text-align:left; +} +.contact-item { + margin:0 0 30px 30px; + text-align:left; + font-weight:300; +} +.contact-item a { + color:#000; +} +footer .contact-item span { + font-weight:400; + display:block; +} +footer .contact-item:first-child span { + text-transform:uppercase; +} +footer .span4 { + text-align:left; +} +footer .span5 { + padding-top: 75px; +} +.soc { + text-align:left; + margin:5px 0 0 0; +} +.soc a { + display:inline-block; + width:35px; + height:34px; + background:url(../img/icons-soc.png) no-repeat; +} +.soc a.github { + background-position:0 -34px; +} +.soc a.meet { + background-position:0 -68px; +} +.soc a.rss { + background-position:0 -102px; +} + +.text-item { + margin:0 0 75px; +} +.container.custom p.note { + text-align:center; + padding:30px 0 0; +} +.text-item strong { + font-weight:normal; + font-family:'Open Sans', Arial, Helvetica, sans-serif; + font-weight:700; +} +h2.date { + font-family:'Open Sans', Arial, Helvetica, sans-serif; + font-weight:400; +} +.blog h2.date { + margin:0 0 25px; +} +h2.date span { + display:block; + margin:0 0 5px; + padding:0 0 15px; + font-size:20px; + border-bottom:1px solid #ccc; +} +.blog h2.date a { + font-weight:700; +} +.blog.inner h2.date span:first-child { + display:block; + font-size:30px; + font-weight:700; + padding:0; + border:none; +} +.blog.inner h3 { + text-align:left; + font-size:25px; + font-weight:700; + margin:0 0 15px; +} +.blog.inner ul li { + margin-left: 50px; + line-height:26px; +} +.recent h3 { + font-size: 25px; + font-weight: 700; + margin: 0 0 15px; + text-align: left; +} +.recent ul li.active a { + color:#252525; +} +.border { + width:130px; + margin: 45px auto; + border-top:1px solid #dfdfdf; + border-top:1px solid #81807f; +} +.text-img { + display:block; + margin:0 auto 17px; +} +.indent p, .indent ul { + padding:0 0 0 50px; +} +.span3 { + margin-left:0; +} +.nav.nav-list.bs-docs-sidenav { + border:1px solid #e5e5e5; + border-radius:5px; + box-shadow:0 0 3px #f9f9f9; + padding:0; + width:auto; +} +.nav.nav-list.bs-docs-sidenav li { + border-top:1px solid #e5e5e5; +} +.nav.nav-list.bs-docs-sidenav li:first-child { + border:none; +} +.nav.nav-list.bs-docs-sidenav li:first-child a { + border-radius:5px 5px 0 0; + -moz-border-radius:5px 5px 0 0; + -webkit-border-radius:5px 5px 0 0; +} +.nav.nav-list.bs-docs-sidenav li:last-child, .nav.nav-list.bs-docs-sidenav li:last-child a { + border-radius:0 0 5px 5px; + -moz-border-radius:0 0 5px 5px; + -webkit-border-radius:0 0 5px 5px; +} +.nav.nav-list.bs-docs-sidenav li a { + padding:10px; + margin:0; + font-weight:400; + font-size:14px; + line-height:18px; +} +.icon-chevron-right { + float: right; + margin-right: -6px; + margin-top: 2px; + opacity: 0.25; +} +.indent ul li { + line-height:26px; +} +.span8 h3 { + text-align:left; + margin:0 0 50px; +} +.span8 h3 a { + font-weight:800; +} +.span8 h4 { + font:700 18px/26px 'Open Sans', Arial, Helvetica, sans-serif; + margin:0 0 20px; +} +.span8 p span { + font-weight:700; +} +header.index-head { + background:#f9f9f9; + margin:0 0 30px; +} +header.index-head .span12 { + margin-bottom:80px; +} +.index-content h2 { + text-align:center; +} +.third-item img { + display:block; + margin:0 auto 70px; +} +.container.custom .third-item p { + margin:0 0 0 20px; +} +.row-fluid { + margin:0; + padding:0; +} +.nav-list [class^="icon-"] { + margin-right:-2px; +} +@media (min-width: 1200px) { + .custom.navbar .nav li a, .navbar .brand, .custom.navbar .nav li.pull-right span { + padding: 10px 20px; + font-size:16px; + } + .nav.nav-list.bs-docs-sidenav { + width:258px; + } + .container.custom .recent ul.unstyled { + margin-right:100px; + } +} +@media (max-width: 980px) { + .container.custom { + width:95%; + } + .bs-docs-sidenav.affix { + position: static; + top: 0; + width: 100%; + } + .nav.nav-list.bs-docs-sidenav { + width:100%; + margin-bottom:20px; + } +} +@media only screen +and (min-device-width : 710px) +and (max-device-width : 770px) { + .container.custom { + width:700px; + position:relative; + } + .custom.navbar .nav li { + font-size:22px; + padding:0 10px; + } + .nav.nav-list.bs-docs-sidenav.affix { + position:fixed; + top:175px; + width:218px; + } +} +@media only screen and (min-device-width : 770px) +and (max-device-width : 810px) { + .container.custom { + width:700px; + } + .custom.navbar .nav li { + font-size:15px; + padding:0 15px; + } + .custom.navbar .nav { + margin-left:30px; + } +} +@media only screen +and (min-device-width : 320px) +and (max-device-width : 480px) { + .container.custom { + width:100%; + margin:0; + } + footer .logo-block { + text-align: left; + padding-left:30px; + } + .offset1, .row-fluid .offset1:first-child { + margin:0; + } + .indent p, .indent ul { + padding:0; + } + .indent, .blog, .recent h3, .recent ul, .text-part { + padding:0 20px; + } + .index-head h3 { + font-size:20px; + } + .index-content { + padding:0 20px; + } + .content h2, .content h3, .content ul.unstyled, .sidebar h3 { + padding:0 20px; + } + h1 { + padding:0 20px; + } +} +@media (max-width: 320px) { + .container.custom { + width:100%; + } + footer .logo-block { + text-align: left; + padding-left:30px; + } + footer .container.custom p { + margin-top:10px; + } + .offset1, .row-fluid .offset1:first-child { + margin:0; + } + .indent p, .indent ul { + padding:0; + } + .indent, .blog, .recent h3, .recent ul, .text-part { + padding:0 20px; + } + .index-head h3 { + font-size:25px; + line-height:30px; + padding:0 20px; + } + .index-content { + padding:0 20px; + } + .content h2, .content h3, .content ul.unstyled, .sidebar h3 { + padding:0 20px; + } + h1 { + padding:0 20px; + } +} +.container.custom .faq-page p { + margin-bottom:10px; +} +.index-head h3 { + margin-bottom:50px; +} +h1.center { + text-align:center; +} +.btn.btn-navbar { + height:auto; + width:auto; + margin:10px 0 0; +} +.navbar-inner { + z-index:100000; + position:relative; +} diff --git a/docs/css/default.html b/docs/css/default.html new file mode 100644 index 000000000000..12106274e0f6 --- /dev/null +++ b/docs/css/default.html @@ -0,0 +1,147 @@ + + + + + Druid | {{page.title}} + + + + + + + + + + + +
+ +
+
+
+ +
+
+ {% if page.id == 'home' %} +

Druid is open-source infrastructure for real²time exploratory analytics on large datasets.

+ + {% endif %} +
+ +
+ + {{ content }} + +
+
+ + + + + + diff --git a/docs/css/docs.html b/docs/css/docs.html new file mode 100644 index 000000000000..cb38f8c336e4 --- /dev/null +++ b/docs/css/docs.html @@ -0,0 +1,8 @@ +--- +layout: default +--- +
+ + {{ content }} + +
diff --git a/docs/css/main.css b/docs/css/main.css new file mode 100755 index 000000000000..50a818048adc --- /dev/null +++ b/docs/css/main.css @@ -0,0 +1,160 @@ +/*****************************************************************************/ +/* +/* Common +/* +/*****************************************************************************/ + +/* Global Reset */ +* { + margin: 0; + padding: 0; +} + +html, body { height: 100%; } + +body { + background-color: #FFF; + font: 13.34px Helvetica, Arial, sans-serif; + font-size: small; + text-align: center; +} + +h1, h2, h3, h4, h5, h6 { + font-size: 100%; } + +h1 { margin-bottom: 1em; } +p { margin: 1em 0; } + +a { color: #00a; } +a:hover { color: #000; } +a:visited { color: #a0a; } + +/*****************************************************************************/ +/* +/* Home +/* +/*****************************************************************************/ +ul.posts { + list-style-type: none; + margin-bottom: 2em; +} + +ul.posts li { + line-height: 1.75em; +} + +ul.posts span { + color: #aaa; + font-family: Monaco, "Courier New", monospace; + font-size: 80%; +} + +/*****************************************************************************/ +/* +/* Site +/* +/*****************************************************************************/ + +.site { + font-size: 115%; + text-align: justify; + width: 42em; + margin: 3em auto 2em; + line-height: 1.5em; +} + +.site .header a { + font-weight: bold; + text-decoration: none; +} + +.site .header h1.title { + display: inline-block; + margin-bottom: 2em; +} + +.site .header h1.title a { + color: #a00; +} + +.site .header h1.title a:hover { + color: #000; +} + +.site .header a.extra { + color: #aaa; + margin-left: 1em; +} + +.site .header a.extra:hover { + color: #000; +} + +.site .meta { + color: #aaa; +} + +.site .footer { + font-size: 80%; + color: #666; + border-top: 4px solid #eee; + margin-top: 2em; + overflow: hidden; +} + +.site .footer .contact { + float: left; + margin-right: 3em; +} + +.site .footer .contact a { + color: #8085C1; +} + +.site .footer .rss { + margin-top: 1.1em; + margin-right: -.2em; + float: right; +} + +.site .footer .rss img { + border: 0; +} + +/*****************************************************************************/ +/* +/* Posts +/* +/*****************************************************************************/ + +/* standard */ +.post pre { + border: 1px solid #ddd; + background-color: #eef; + padding: 0 .4em; +} + +.post ul, .post ol { + margin-left: 1.35em; +} + +.post code { + border: 1px solid #ddd; + background-color: #eef; + padding: 0 .2em; +} + +.post pre code { + border: none; +} + +/* terminal */ +.post pre.terminal { + border: 1px solid #000; + background-color: #333; + color: #FFF; +} + +.post pre.terminal code { + background-color: #333; +} diff --git a/docs/css/page.html b/docs/css/page.html new file mode 100644 index 000000000000..b5ebfb17ace0 --- /dev/null +++ b/docs/css/page.html @@ -0,0 +1,11 @@ +--- +layout: default +--- + +
+
+ + {{ content }} + +
+
diff --git a/docs/css/pie.htc b/docs/css/pie.htc new file mode 100644 index 000000000000..ca3b54700cc7 --- /dev/null +++ b/docs/css/pie.htc @@ -0,0 +1,96 @@ + + + + + + + + + diff --git a/docs/css/post.html b/docs/css/post.html new file mode 100644 index 000000000000..affb233fd23a --- /dev/null +++ b/docs/css/post.html @@ -0,0 +1,44 @@ +--- +layout: default +sectionid: blog +--- + +
+
+

Recent posts

+ +
+ +
+
+

+ {{ page.title }} + {{ page.date | date: "%B %e, %Y" }} · {{ page.author | upcase }} +

+ + {% if page.image %}{{ page.title }}{% endif %} + + {{ content }} + +
+ + + comments powered by Disqus + +
+
+
diff --git a/docs/css/syntax.css b/docs/css/syntax.css new file mode 100644 index 000000000000..1e651cf79db8 --- /dev/null +++ b/docs/css/syntax.css @@ -0,0 +1,60 @@ +.highlight { background: #ffffff; } +.highlight .c { color: #999988; font-style: italic } /* Comment */ +.highlight .err { color: #a61717; background-color: #e3d2d2 } /* Error */ +.highlight .k { font-weight: bold } /* Keyword */ +.highlight .o { font-weight: bold } /* Operator */ +.highlight .cm { color: #999988; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #999999; font-weight: bold } /* Comment.Preproc */ +.highlight .c1 { color: #999988; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #999999; font-weight: bold; font-style: italic } /* Comment.Special */ +.highlight .gd { color: #000000; background-color: #ffdddd } /* Generic.Deleted */ +.highlight .gd .x { color: #000000; background-color: #ffaaaa } /* Generic.Deleted.Specific */ +.highlight .ge { font-style: italic } /* Generic.Emph */ +.highlight .gr { color: #aa0000 } /* Generic.Error */ +.highlight .gh { color: #999999 } /* Generic.Heading */ +.highlight .gi { color: #000000; background-color: #ddffdd } /* Generic.Inserted */ +.highlight .gi .x { color: #000000; background-color: #aaffaa } /* Generic.Inserted.Specific */ +.highlight .go { color: #888888 } /* Generic.Output */ +.highlight .gp { color: #555555 } /* Generic.Prompt */ +.highlight .gs { font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #aaaaaa } /* Generic.Subheading */ +.highlight .gt { color: #aa0000 } /* Generic.Traceback */ +.highlight .kc { font-weight: bold } /* Keyword.Constant */ +.highlight .kd { font-weight: bold } /* Keyword.Declaration */ +.highlight .kp { font-weight: bold } /* Keyword.Pseudo */ +.highlight .kr { font-weight: bold } /* Keyword.Reserved */ +.highlight .kt { color: #445588; font-weight: bold } /* Keyword.Type */ +.highlight .m { color: #009999 } /* Literal.Number */ +.highlight .s { color: #d14 } /* Literal.String */ +.highlight .na { color: #008080 } /* Name.Attribute */ +.highlight .nb { color: #0086B3 } /* Name.Builtin */ +.highlight .nc { color: #445588; font-weight: bold } /* Name.Class */ +.highlight .no { color: #008080 } /* Name.Constant */ +.highlight .ni { color: #800080 } /* Name.Entity */ +.highlight .ne { color: #990000; font-weight: bold } /* Name.Exception */ +.highlight .nf { color: #990000; font-weight: bold } /* Name.Function */ +.highlight .nn { color: #555555 } /* Name.Namespace */ +.highlight .nt { color: #000080 } /* Name.Tag */ +.highlight .nv { color: #008080 } /* Name.Variable */ +.highlight .ow { font-weight: bold } /* Operator.Word */ +.highlight .w { color: #bbbbbb } /* Text.Whitespace */ +.highlight .mf { color: #009999 } /* Literal.Number.Float */ +.highlight .mh { color: #009999 } /* Literal.Number.Hex */ +.highlight .mi { color: #009999 } /* Literal.Number.Integer */ +.highlight .mo { color: #009999 } /* Literal.Number.Oct */ +.highlight .sb { color: #d14 } /* Literal.String.Backtick */ +.highlight .sc { color: #d14 } /* Literal.String.Char */ +.highlight .sd { color: #d14 } /* Literal.String.Doc */ +.highlight .s2 { color: #d14 } /* Literal.String.Double */ +.highlight .se { color: #d14 } /* Literal.String.Escape */ +.highlight .sh { color: #d14 } /* Literal.String.Heredoc */ +.highlight .si { color: #d14 } /* Literal.String.Interpol */ +.highlight .sx { color: #d14 } /* Literal.String.Other */ +.highlight .sr { color: #009926 } /* Literal.String.Regex */ +.highlight .s1 { color: #d14 } /* Literal.String.Single */ +.highlight .ss { color: #990073 } /* Literal.String.Symbol */ +.highlight .bp { color: #999999 } /* Name.Builtin.Pseudo */ +.highlight .vc { color: #008080 } /* Name.Variable.Class */ +.highlight .vg { color: #008080 } /* Name.Variable.Global */ +.highlight .vi { color: #008080 } /* Name.Variable.Instance */ +.highlight .il { color: #009999 } /* Literal.Number.Integer.Long */ \ No newline at end of file diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 000000000000..c7268192374a --- /dev/null +++ b/docs/index.html @@ -0,0 +1,13 @@ +--- +layout: default +title: Your New Jekyll Site +--- + +
+

Blog Posts

+
    + {% for post in site.posts %} +
  • {{ post.date | date_to_string }} » {{ post.title }}
  • + {% endfor %} +
+
\ No newline at end of file diff --git a/examples/bin/ec2/run.sh b/examples/bin/ec2/run.sh index 933de0a2b422..df7438ccf5bc 100644 --- a/examples/bin/ec2/run.sh +++ b/examples/bin/ec2/run.sh @@ -7,16 +7,16 @@ cd druid-services-* 2>&1 > /dev/null mkdir logs 2>&1 > /dev/null # Now start a realtime node -nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=config/realtime/realtime.spec -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/realtime com.metamx.druid.realtime.RealtimeMain 2>&1 > logs/realtime.log & +nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Ddruid.realtime.specFile=config/realtime/realtime.spec -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/realtime io.druid.cli.Main server realtime 2>&1 > logs/realtime.log & # And a master node -nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/master com.metamx.druid.http.MasterMain 2>&1 > logs/master.log & +nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/master io.druid.cli.Main server coordinator 2>&1 > logs/master.log & # And a compute node -nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/compute com.metamx.druid.http.ComputeMain 2>&1 > logs/compute.log & +nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/compute io.druid.cli.Main server historical 2>&1 > logs/compute.log & # And a broker node -nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/broker com.metamx.druid.http.BrokerMain 2>&1 > logs/broker.log & +nohup java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/druid-services-0.5.5-SNAPSHOT-selfcontained.jar:config/broker io.druid.cli.Main server broker 2>&1 > logs/broker.log & echo "Hit CTRL-C to continue..." exit 0 diff --git a/examples/bin/run_example_server.sh b/examples/bin/run_example_server.sh index 38e372e3e3cc..dc0483b932a2 100755 --- a/examples/bin/run_example_server.sh +++ b/examples/bin/run_example_server.sh @@ -63,4 +63,4 @@ DRUID_CP=${DRUID_CP}:${SCRIPT_DIR}/config/realtime echo "Running command:" -(set -x; java ${JAVA_ARGS} -classpath ${DRUID_CP} druid.examples.RealtimeStandaloneMain) +(set -x; java ${JAVA_ARGS} -classpath ${DRUID_CP} io.druid.cli.Main druid server realtimeStandalone) diff --git a/examples/pom.xml b/examples/pom.xml index 83de18965230..2a8310e92ba2 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -1,4 +1,23 @@ + + 4.0.0 com.metamx.druid @@ -9,7 +28,7 @@ com.metamx druid - 0.5.59-SNAPSHOT + 0.6.0-SNAPSHOT @@ -29,99 +48,6 @@ ${project.parent.version} - - com.metamx - emitter - - - com.metamx - http-client - - - com.metamx - java-util - - - com.metamx - server-metrics - - - - com.davekoelle - alphanum - - - commons-codec - commons-codec - - - org.skife.config - config-magic - - - com.google.guava - guava - - - com.google.inject - guice - - - com.google.inject.extensions - guice-servlet - - - com.ibm.icu - icu4j - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.jaxrs - jackson-jaxrs-json-provider - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.dataformat - jackson-dataformat-smile - - - org.jdbi - jdbi - - - com.sun.jersey - jersey-server - - - com.sun.jersey - jersey-core - - - com.sun.jersey.contribs - jersey-guice - - - org.mortbay.jetty - jetty - - - joda-time - joda-time - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - org.twitter4j twitter4j-core @@ -137,18 +63,6 @@ twitter4j-stream 3.0.3 - - - - junit - junit - test - - - org.easymock - easymock - test - diff --git a/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java b/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java deleted file mode 100644 index ebdd59d46c1b..000000000000 --- a/examples/src/main/java/druid/examples/RealtimeStandaloneMain.java +++ /dev/null @@ -1,161 +0,0 @@ -package druid.examples; - -import com.fasterxml.jackson.databind.jsontype.NamedType; -import com.google.common.collect.ImmutableList; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.client.ServerView; -import com.metamx.druid.coordination.DataSegmentAnnouncer; -import com.metamx.druid.loading.DataSegmentPusher; -import com.metamx.druid.log.LogLevelAdjuster; -import com.metamx.druid.realtime.RealtimeNode; -import com.metamx.druid.realtime.SegmentPublisher; -import druid.examples.flights.FlightsFirehoseFactory; -import druid.examples.rand.RandomFirehoseFactory; -import druid.examples.twitter.TwitterSpritzerFirehoseFactory; -import druid.examples.web.WebFirehoseFactory; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.Executor; - -/** - * Standalone Demo Realtime process. - */ -public class RealtimeStandaloneMain -{ - private static final Logger log = new Logger(RealtimeStandaloneMain.class); - - public static void main(String[] args) throws Exception - { - LogLevelAdjuster.register(); - - final Lifecycle lifecycle = new Lifecycle(); - - RealtimeNode rn = RealtimeNode.builder().build(); - lifecycle.addManagedInstance(rn); - - // register the Firehoses - rn.registerJacksonSubtype( - new NamedType(TwitterSpritzerFirehoseFactory.class, "twitzer"), - new NamedType(FlightsFirehoseFactory.class, "flights"), - new NamedType(RandomFirehoseFactory.class, "rand"), - new NamedType(WebFirehoseFactory.class, "webstream") - - ); - - // Create dummy objects for the various interfaces that interact with the DB, ZK and deep storage - rn.setSegmentPublisher(new NoopSegmentPublisher()); - rn.setAnnouncer(new NoopDataSegmentAnnouncer()); - rn.setDataSegmentPusher(new NoopDataSegmentPusher()); - rn.setServerView(new NoopServerView()); - rn.setInventoryView(new NoopInventoryView()); - - Runtime.getRuntime().addShutdownHook( - new Thread( - new Runnable() - { - @Override - public void run() - { - log.info("Running shutdown hook"); - lifecycle.stop(); - } - } - ) - ); - - try { - lifecycle.start(); - } - catch (Throwable t) { - log.info(t, "Throwable caught at startup, committing seppuku"); - t.printStackTrace(); - System.exit(2); - } - - lifecycle.join(); - } - - private static class NoopServerView implements ServerView - { - @Override - public void registerServerCallback( - Executor exec, ServerCallback callback - ) - { - - } - - @Override - public void registerSegmentCallback( - Executor exec, SegmentCallback callback - ) - { - - } - } - - private static class NoopInventoryView implements InventoryView - { - @Override - public DruidServer getInventoryValue(String string) - { - return null; - } - - @Override - public Iterable getInventory() - { - return ImmutableList.of(); - } - } - - private static class NoopDataSegmentPusher implements DataSegmentPusher - { - @Override - public DataSegment push(File file, DataSegment segment) throws IOException - { - return segment; - } - } - - private static class NoopSegmentPublisher implements SegmentPublisher - { - @Override - public void publishSegment(DataSegment segment) throws IOException - { - // do nothing - } - } - - private static class NoopDataSegmentAnnouncer implements DataSegmentAnnouncer - { - @Override - public void announceSegment(DataSegment segment) throws IOException - { - // do nothing - } - - @Override - public void unannounceSegment(DataSegment segment) throws IOException - { - // do nothing - } - - @Override - public void announceSegments(Iterable segments) throws IOException - { - // do nothing - } - - @Override - public void unannounceSegments(Iterable segments) throws IOException - { - // do nothing - } - } -} \ No newline at end of file diff --git a/examples/src/main/java/druid/examples/flights/FlightsConverter.java b/examples/src/main/java/druid/examples/flights/FlightsConverter.java index 53ea6638e603..2edb012b62b8 100644 --- a/examples/src/main/java/druid/examples/flights/FlightsConverter.java +++ b/examples/src/main/java/druid/examples/flights/FlightsConverter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -22,7 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.Closeables; import com.metamx.common.parsers.CSVParser; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.jackson.DefaultObjectMapper; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Period; diff --git a/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java b/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java index 66b8de4450c9..42f62440736f 100644 --- a/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/flights/FlightsFirehoseFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -25,10 +25,10 @@ import com.google.common.base.Throwables; import com.google.common.collect.Iterators; import com.google.common.io.Closeables; -import com.metamx.druid.indexer.data.StringInputRowParser; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.realtime.firehose.Firehose; -import com.metamx.druid.realtime.firehose.FirehoseFactory; +import io.druid.data.input.Firehose; +import io.druid.data.input.FirehoseFactory; +import io.druid.data.input.InputRow; +import io.druid.data.input.StringInputRowParser; import java.io.BufferedReader; import java.io.File; diff --git a/examples/src/main/java/druid/examples/guice/RealtimeExampleModule.java b/examples/src/main/java/druid/examples/guice/RealtimeExampleModule.java new file mode 100644 index 000000000000..816a8d367da0 --- /dev/null +++ b/examples/src/main/java/druid/examples/guice/RealtimeExampleModule.java @@ -0,0 +1,163 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package druid.examples.guice; + +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.common.collect.ImmutableList; +import com.google.inject.Binder; +import com.google.inject.TypeLiteral; +import com.metamx.common.guava.LazySequence; +import com.metamx.common.logger.Logger; +import druid.examples.flights.FlightsFirehoseFactory; +import druid.examples.rand.RandomFirehoseFactory; +import druid.examples.twitter.TwitterSpritzerFirehoseFactory; +import druid.examples.web.WebFirehoseFactory; +import io.druid.client.DruidServer; +import io.druid.client.InventoryView; +import io.druid.client.ServerView; +import io.druid.guice.FireDepartmentsProvider; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.guice.ManageLifecycle; +import io.druid.guice.NoopSegmentPublisherProvider; +import io.druid.guice.RealtimeManagerConfig; +import io.druid.initialization.DruidModule; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.realtime.FireDepartment; +import io.druid.segment.realtime.RealtimeManager; +import io.druid.segment.realtime.SegmentPublisher; +import io.druid.server.coordination.DataSegmentAnnouncer; +import io.druid.timeline.DataSegment; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.Executor; + +/** + */ +public class RealtimeExampleModule implements DruidModule +{ + private static final Logger log = new Logger(RealtimeExampleModule.class); + + @Override + public void configure(Binder binder) + { + binder.bind(SegmentPublisher.class).toProvider(NoopSegmentPublisherProvider.class); + binder.bind(DataSegmentPusher.class).to(NoopDataSegmentPusher.class); + binder.bind(DataSegmentAnnouncer.class).to(NoopDataSegmentAnnouncer.class); + binder.bind(InventoryView.class).to(NoopInventoryView.class); + binder.bind(ServerView.class).to(NoopServerView.class); + + JsonConfigProvider.bind(binder, "druid.realtime", RealtimeManagerConfig.class); + binder.bind( + new TypeLiteral>() + { + } + ).toProvider(FireDepartmentsProvider.class).in(LazySingleton.class); + binder.bind(RealtimeManager.class).in(ManageLifecycle.class); + } + + @Override + public List getJacksonModules() + { + return Arrays.asList( + new SimpleModule("RealtimeExampleModule") + .registerSubtypes( + new NamedType(TwitterSpritzerFirehoseFactory.class, "twitzer"), + new NamedType(FlightsFirehoseFactory.class, "flights"), + new NamedType(RandomFirehoseFactory.class, "rand"), + new NamedType(WebFirehoseFactory.class, "webstream") + ) + ); + } + + private static class NoopServerView implements ServerView + { + @Override + public void registerServerCallback( + Executor exec, ServerCallback callback + ) + { + // do nothing + } + + @Override + public void registerSegmentCallback( + Executor exec, SegmentCallback callback + ) + { + // do nothing + } + } + + private static class NoopInventoryView implements InventoryView + { + @Override + public DruidServer getInventoryValue(String string) + { + return null; + } + + @Override + public Iterable getInventory() + { + return ImmutableList.of(); + } + } + + private static class NoopDataSegmentPusher implements DataSegmentPusher + { + @Override + public DataSegment push(File file, DataSegment segment) throws IOException + { + return segment; + } + } + + private static class NoopDataSegmentAnnouncer implements DataSegmentAnnouncer + { + @Override + public void announceSegment(DataSegment segment) throws IOException + { + // do nothing + } + + @Override + public void unannounceSegment(DataSegment segment) throws IOException + { + // do nothing + } + + @Override + public void announceSegments(Iterable segments) throws IOException + { + // do nothing + } + + @Override + public void unannounceSegments(Iterable segments) throws IOException + { + // do nothing + } + } +} diff --git a/examples/src/main/java/druid/examples/rabbitmq/RabbitMQProducerMain.java b/examples/src/main/java/druid/examples/rabbitmq/RabbitMQProducerMain.java index a070bfff929d..0e9bd2be4ddd 100644 --- a/examples/src/main/java/druid/examples/rabbitmq/RabbitMQProducerMain.java +++ b/examples/src/main/java/druid/examples/rabbitmq/RabbitMQProducerMain.java @@ -3,10 +3,21 @@ import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.ConnectionFactory; -import org.apache.commons.cli.*; +import org.apache.commons.cli.BasicParser; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import java.text.SimpleDateFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Random; /** * diff --git a/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java b/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java index 71a0c1fbb925..6c02600650c6 100644 --- a/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/rand/RandomFirehoseFactory.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.rand; import com.fasterxml.jackson.annotation.JsonCreator; @@ -5,10 +24,10 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.collect.Maps; import com.metamx.common.logger.Logger; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.realtime.firehose.Firehose; -import com.metamx.druid.realtime.firehose.FirehoseFactory; +import io.druid.data.input.Firehose; +import io.druid.data.input.FirehoseFactory; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; import java.io.IOException; import java.util.LinkedList; diff --git a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java index e87b5e44de2b..dcb5915b8a0d 100644 --- a/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/twitter/TwitterSpritzerFirehoseFactory.java @@ -1,3 +1,22 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + package druid.examples.twitter; import com.fasterxml.jackson.annotation.JsonCreator; @@ -5,19 +24,19 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.collect.Lists; import com.metamx.common.logger.Logger; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.realtime.firehose.Firehose; -import com.metamx.druid.realtime.firehose.FirehoseFactory; +import io.druid.data.input.Firehose; +import io.druid.data.input.FirehoseFactory; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; import twitter4j.ConnectionLifeCycleListener; import twitter4j.HashtagEntity; +import twitter4j.StallWarning; import twitter4j.Status; import twitter4j.StatusDeletionNotice; import twitter4j.StatusListener; import twitter4j.TwitterStream; import twitter4j.TwitterStreamFactory; import twitter4j.User; -import twitter4j.StallWarning; import java.io.IOException; import java.util.Arrays; diff --git a/examples/src/main/java/druid/examples/web/InputSupplierUpdateStream.java b/examples/src/main/java/druid/examples/web/InputSupplierUpdateStream.java index 40d7cf71616d..4546d8cb2a9e 100644 --- a/examples/src/main/java/druid/examples/web/InputSupplierUpdateStream.java +++ b/examples/src/main/java/druid/examples/web/InputSupplierUpdateStream.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -24,8 +24,8 @@ import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.InputSupplier; -import com.metamx.druid.jackson.DefaultObjectMapper; import com.metamx.emitter.EmittingLogger; +import io.druid.jackson.DefaultObjectMapper; import java.io.BufferedReader; import java.io.IOException; diff --git a/examples/src/main/java/druid/examples/web/WebFirehoseFactory.java b/examples/src/main/java/druid/examples/web/WebFirehoseFactory.java index 885089a20014..2c7ccfa09a5d 100644 --- a/examples/src/main/java/druid/examples/web/WebFirehoseFactory.java +++ b/examples/src/main/java/druid/examples/web/WebFirehoseFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -24,12 +24,12 @@ import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Throwables; import com.metamx.common.parsers.TimestampParser; -import com.metamx.druid.guava.Runnables; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.input.MapBasedInputRow; -import com.metamx.druid.realtime.firehose.Firehose; -import com.metamx.druid.realtime.firehose.FirehoseFactory; import com.metamx.emitter.EmittingLogger; +import io.druid.common.guava.Runnables; +import io.druid.data.input.Firehose; +import io.druid.data.input.FirehoseFactory; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; import org.joda.time.DateTime; import java.io.IOException; diff --git a/examples/src/test/java/druid/examples/web/WebFirehoseFactoryTest.java b/examples/src/test/java/druid/examples/web/WebFirehoseFactoryTest.java index 755bcfd197ac..bb7f6b25a242 100644 --- a/examples/src/test/java/druid/examples/web/WebFirehoseFactoryTest.java +++ b/examples/src/test/java/druid/examples/web/WebFirehoseFactoryTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -21,8 +21,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.realtime.firehose.Firehose; +import io.druid.data.input.Firehose; +import io.druid.data.input.InputRow; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Before; diff --git a/indexing-common/src/main/java/com/metamx/druid/index/column/ValueType.java b/indexing-common/src/main/java/com/metamx/druid/index/column/ValueType.java deleted file mode 100644 index 245ff682bd7f..000000000000 --- a/indexing-common/src/main/java/com/metamx/druid/index/column/ValueType.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.metamx.druid.index.column; - -/** -*/ -public enum ValueType -{ - FLOAT, - LONG, - STRING, - COMPLEX -} diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/Index.java b/indexing-common/src/main/java/com/metamx/druid/index/v1/Index.java deleted file mode 100644 index fcec6135b866..000000000000 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/Index.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.index.v1; - -import com.metamx.collections.spatial.ImmutableRTree; -import com.metamx.common.logger.Logger; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; -import org.joda.time.Interval; - -import java.util.HashMap; -import java.util.Map; - -/** - * In-memory representation of a segment - */ -public class Index -{ - private static final Logger log = new Logger(Index.class); - private static final ImmutableConciseSet emptySet = new ImmutableConciseSet(); - - final Map dimToPositionMap = new HashMap(); - final Map metricToPositionMap = new HashMap(); - - final String[] dimensions; - final String[] metrics; - final Interval dataInterval; - final long[] timeOffsets; - final Map metricVals; - final Map> dimIdLookup; - final Map reverseDimLookup; - final Map indexes; - final Map spatialIndexes; - final Map dimensionValues; - - /* - * If we name the various occurrences of String and int then the types are more informative: - * - * dimToPositionMap : Dim -> DimColumn - * dimensions : DimColumn -> Dim - * metricToPositionMap : Met -> MetColumn - * metrics : MetColumn -> Met - * - * dataInterval : Interval - * - * timeOffsets : Milli[] - * metricVals : float[] (size() == timeOffsets.size() * metrics.size()) - * - * dimIdLookup : Dim -> Value -> ValueID - * reverseDimLookup : Dim -> ValueID -> Value - * indexes : Dim -> ValueID -> Row[] - * - * dimensionValues : Dim -> - * getRowValues : Row -> ExpansionID - * getDimensionExpansions : ExpansionID -> ValueID[] - * getDimValues : Row -> ValueID[] - */ - - public Index( - String[] dimensions, - String[] metrics, - Interval dataInterval, - long[] timeOffsets, - Map metricVals, - Map> dimIdLookup, - Map reverseDimLookup, - Map indexes, - Map spatialIndexes, - Map dimensionValues - ) - { - this.dimensions = dimensions; - this.metrics = metrics; - this.dataInterval = dataInterval; - this.timeOffsets = timeOffsets; - this.metricVals = metricVals; - this.dimIdLookup = dimIdLookup; - this.reverseDimLookup = reverseDimLookup; - this.indexes = indexes; - this.spatialIndexes = spatialIndexes; - this.dimensionValues = dimensionValues; - - for (int i = 0; i < dimensions.length; i++) { - dimToPositionMap.put(dimensions[i].toLowerCase(), i); - } - for (int i = 0; i < metrics.length; i++) { - metricToPositionMap.put(metrics[i].toLowerCase(), i); - } - } - - public ImmutableConciseSet getInvertedIndex(String dimension, String value) - { - final Map lookup = dimIdLookup.get(dimension); - if (lookup == null) { - return emptySet; - } - - final Integer integer = lookup.get(value); - if (integer == null) { - return emptySet; - } - - try { - return indexes.get(dimension)[integer]; - } - catch (NullPointerException e) { - log.warn( - e, - "NPE on dimension[%s], value[%s], with index over interval[%s]", - dimension, - value, - dataInterval - ); - return emptySet; - } - } - - public ImmutableConciseSet getInvertedIndex(String dimension, int valueIndex) - { - try { - return indexes.get(dimension)[valueIndex]; - } - catch (NullPointerException e) { - log.warn( - e, - "NPE on dimension[%s], valueIndex[%d], with index over interval[%s]", - dimension, - valueIndex, - dataInterval - ); - return emptySet; - } - } - - public ImmutableRTree getSpatialIndex(String dimension) - { - try { - return spatialIndexes.get(dimension); - } - catch (NullPointerException e) { - log.warn( - e, - "NPE on dimension[%s] over interval[%s]", - dimension, - dataInterval - ); - return new ImmutableRTree(); - } - } -} diff --git a/indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java b/indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java deleted file mode 100644 index d792d80de01e..000000000000 --- a/indexing-common/src/main/java/com/metamx/druid/index/v1/MMappedIndex.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.index.v1; - -import com.google.common.base.Function; -import com.google.common.collect.Iterables; -import com.google.common.collect.Maps; -import com.google.common.primitives.Ints; -import com.metamx.collections.spatial.ImmutableRTree; -import com.metamx.common.io.smoosh.SmooshedFileMapper; -import com.metamx.common.logger.Logger; -import com.metamx.druid.kv.ConciseCompressedIndexedInts; -import com.metamx.druid.kv.GenericIndexed; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.kv.IndexedList; -import com.metamx.druid.kv.IndexedLongs; -import com.metamx.druid.kv.VSizeIndexed; -import com.metamx.druid.kv.VSizeIndexedInts; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; -import org.joda.time.Interval; - -import java.io.IOException; -import java.nio.ByteOrder; -import java.nio.LongBuffer; -import java.util.Arrays; -import java.util.Map; - -/** - */ -public class MMappedIndex -{ - private static final Logger log = new Logger(MMappedIndex.class); - private static final ImmutableConciseSet emptySet = new ImmutableConciseSet(); - - final GenericIndexed availableDimensions; - final GenericIndexed availableMetrics; - final Interval dataInterval; - final CompressedLongsIndexedSupplier timestamps; - final Map metrics; - final Map> dimValueLookups; - final Map dimColumns; - final Map> invertedIndexes; - final Map spatialIndexes; - final SmooshedFileMapper fileMapper; - - private final Map metricIndexes = Maps.newHashMap(); - - public MMappedIndex( - GenericIndexed availableDimensions, - GenericIndexed availableMetrics, - Interval dataInterval, - CompressedLongsIndexedSupplier timestamps, - Map metrics, - Map> dimValueLookups, - Map dimColumns, - Map> invertedIndexes, - Map spatialIndexes, - SmooshedFileMapper fileMapper - ) - { - this.availableDimensions = availableDimensions; - this.availableMetrics = availableMetrics; - this.dataInterval = dataInterval; - this.timestamps = timestamps; - this.metrics = metrics; - this.dimValueLookups = dimValueLookups; - this.dimColumns = dimColumns; - this.invertedIndexes = invertedIndexes; - this.spatialIndexes = spatialIndexes; - this.fileMapper = fileMapper; - - for (int i = 0; i < availableMetrics.size(); i++) { - metricIndexes.put(availableMetrics.get(i), i); - } - } - - public CompressedLongsIndexedSupplier getTimestamps() - { - return timestamps; - } - - public GenericIndexed getAvailableDimensions() - { - return availableDimensions; - } - - public GenericIndexed getAvailableMetrics() - { - return availableMetrics; - } - - public Map getMetrics() - { - return metrics; - } - - public Integer getMetricIndex(String metricName) - { - return metricIndexes.get(metricName); - } - - public Interval getDataInterval() - { - return dataInterval; - } - - public IndexedLongs getReadOnlyTimestamps() - { - return timestamps.get(); - } - - public MetricHolder getMetricHolder(String metric) - { - final MetricHolder retVal = metrics.get(metric); - - if (retVal == null) { - return null; - } - - return retVal; - } - - public GenericIndexed getDimValueLookup(String dimension) - { - return dimValueLookups.get(dimension); - } - - public VSizeIndexed getDimColumn(String dimension) - { - return dimColumns.get(dimension); - } - - public Map> getInvertedIndexes() - { - return invertedIndexes; - } - - public Map getSpatialIndexes() - { - return spatialIndexes; - } - - public ImmutableConciseSet getInvertedIndex(String dimension, String value) - { - final GenericIndexed lookup = dimValueLookups.get(dimension); - if (lookup == null) { - return emptySet; - } - - int indexOf = lookup.indexOf(value); - if (indexOf < 0) { - return emptySet; - } - - ImmutableConciseSet retVal = invertedIndexes.get(dimension).get(indexOf); - return (retVal == null) ? emptySet : retVal; - } - - public SmooshedFileMapper getFileMapper() - { - return fileMapper; - } - - public void close() throws IOException - { - if (fileMapper != null) { - fileMapper.close(); - } - } - - public static MMappedIndex fromIndex(Index index) - { - log.info("Converting timestamps"); - CompressedLongsIndexedSupplier timestamps = CompressedLongsIndexedSupplier.fromLongBuffer( - LongBuffer.wrap(index.timeOffsets), ByteOrder.nativeOrder() - ); - - log.info("Converting dimValueLookups"); - Map> dimValueLookups = Maps.newHashMap(); - for (Map.Entry entry : index.reverseDimLookup.entrySet()) { - String[] theValues = Arrays.copyOf(entry.getValue(), entry.getValue().length); - Arrays.sort(theValues); - - dimValueLookups.put(entry.getKey(), GenericIndexed.fromArray(theValues, GenericIndexed.stringStrategy)); - } - - Map dimColumns = Maps.newHashMap(); - Map> invertedIndexes = Maps.newLinkedHashMap(); - Map spatialIndexes = Maps.newLinkedHashMap(); - - for (String dimension : Arrays.asList(index.dimensions)) { - final String[] dimVals = index.reverseDimLookup.get(dimension); - final DimensionColumn dimColumn = index.dimensionValues.get(dimension); - - log.info( - "Converting dim[%s] with valueCardinality[%,d] and expansionCardinality[%,d]", - dimension, - dimColumn.getDimensionExpansions().length, - dimVals.length - ); - - final Indexed lookup = dimValueLookups.get(dimension); - final int[] expansionConversion = new int[dimVals.length]; - final int[] reverseExpansionConversion = new int[dimVals.length]; - for (int i = 0; i < expansionConversion.length; i++) { - expansionConversion[i] = lookup.indexOf(dimVals[i]); - reverseExpansionConversion[expansionConversion[i]] = i; - } - - int[][] originalDimExpansions = dimColumn.getDimensionExpansions(); - final int[][] dimensionExpansions = new int[originalDimExpansions.length][]; - for (int i = 0; i < originalDimExpansions.length; i++) { - int[] originalDimExpansion = originalDimExpansions[i]; - int[] mappedValues = new int[originalDimExpansion.length]; - - for (int j = 0; j < originalDimExpansion.length; j++) { - mappedValues[j] = expansionConversion[originalDimExpansion[j]]; - } - Arrays.sort(mappedValues); - - dimensionExpansions[i] = mappedValues; - } - - final int[] originalRows = dimColumn.getDimensionRowValues(); - final ImmutableConciseSet[] origInvertedIndexes = index.indexes.get(dimension); - - dimColumns.put( - dimension, - VSizeIndexed.fromIterable( - Iterables.transform( - Ints.asList(originalRows), - new Function() - { - @Override - public VSizeIndexedInts apply(Integer input) - { - return VSizeIndexedInts.fromArray(dimensionExpansions[input], dimVals.length - 1); - } - } - ) - ) - ); - - invertedIndexes.put( - dimension, - GenericIndexed.fromIterable( - Iterables.transform( - new IndexedList(lookup), - new Function() - { - @Override - public ImmutableConciseSet apply(String input) - { - return origInvertedIndexes[reverseExpansionConversion[lookup.indexOf(input)]]; - } - } - ), - ConciseCompressedIndexedInts.objectStrategy - ) - ); - - spatialIndexes.put(dimension, index.getSpatialIndex(dimension)); - } - - log.info("Making MMappedIndex"); - return new MMappedIndex( - GenericIndexed.fromArray(index.dimensions, GenericIndexed.stringStrategy), - GenericIndexed.fromArray(index.metrics, GenericIndexed.stringStrategy), - index.dataInterval, - timestamps, - index.metricVals, - dimValueLookups, - dimColumns, - invertedIndexes, - spatialIndexes, - null - ); - } -} diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/ByteBufferInputRowParser.java b/indexing-common/src/main/java/com/metamx/druid/indexer/data/ByteBufferInputRowParser.java deleted file mode 100644 index bc3721bf7198..000000000000 --- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/ByteBufferInputRowParser.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.metamx.druid.indexer.data; - -import java.nio.ByteBuffer; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = StringInputRowParser.class) -@JsonSubTypes(value = { - @JsonSubTypes.Type(name = "protobuf", value = ProtoBufInputRowParser.class), - @JsonSubTypes.Type(name = "string", value = StringInputRowParser.class) -}) -public interface ByteBufferInputRowParser extends InputRowParser { -} diff --git a/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java b/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java deleted file mode 100644 index 231cbd441024..000000000000 --- a/indexing-common/src/main/java/com/metamx/druid/indexer/data/InputRowParser.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.metamx.druid.indexer.data; - -import com.metamx.common.exception.FormattedException; -import com.metamx.druid.input.InputRow; - -public interface InputRowParser -{ - public InputRow parse(T input) throws FormattedException; - public void addDimensionExclusion(String dimension); -} diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferSerializer.java b/indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferSerializer.java deleted file mode 100644 index 93f494519419..000000000000 --- a/indexing-common/src/main/java/com/metamx/druid/kv/ByteBufferSerializer.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.metamx.druid.kv; - -import com.google.common.primitives.Ints; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; - -/** - */ -public class ByteBufferSerializer -{ - public static T read(ByteBuffer buffer, ObjectStrategy strategy) - { - int size = buffer.getInt(); - ByteBuffer bufferToUse = buffer.asReadOnlyBuffer(); - bufferToUse.limit(bufferToUse.position() + size); - buffer.position(bufferToUse.limit()); - - return strategy.fromByteBuffer(bufferToUse, size); - } - - public static void writeToChannel(T obj, ObjectStrategy strategy, WritableByteChannel channel) - throws IOException - { - byte[] toWrite = strategy.toBytes(obj); - channel.write(ByteBuffer.allocate(Ints.BYTES).putInt(0, toWrite.length)); - channel.write(ByteBuffer.wrap(toWrite)); - } -} diff --git a/indexing-common/src/main/java/com/metamx/druid/kv/SingleIndexedInts.java b/indexing-common/src/main/java/com/metamx/druid/kv/SingleIndexedInts.java deleted file mode 100644 index cba9195e0f67..000000000000 --- a/indexing-common/src/main/java/com/metamx/druid/kv/SingleIndexedInts.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.metamx.druid.kv; - -import com.google.common.collect.Iterators; - -import java.util.Iterator; - -/** -*/ -public class SingleIndexedInts implements IndexedInts -{ - private final int value; - - public SingleIndexedInts(int value) { - this.value = value; - } - - @Override - public int size() - { - return 1; - } - - @Override - public int get(int index) - { - return value; - } - - @Override - public Iterator iterator() - { - return Iterators.singletonIterator(value); - } -} diff --git a/indexing-hadoop/pom.xml b/indexing-hadoop/pom.xml index 460e9ab6e42b..569a61db92b7 100644 --- a/indexing-hadoop/pom.xml +++ b/indexing-hadoop/pom.xml @@ -1,7 +1,7 @@ com.metamx emitter - 0.2.3 + 0.2.5 com.metamx @@ -80,7 +85,7 @@ com.metamx server-metrics - 0.0.3 + 0.0.4 @@ -113,11 +118,35 @@ commons-lang 2.6 + + com.amazonaws + aws-java-sdk + 1.3.27 + + + javax.mail + mail + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + com.ning compress-lzf 0.8.4 + + io.airlift + airline + 0.5 + org.skife.config config-magic @@ -167,12 +196,17 @@ com.google.inject guice - 3.0 + 4.0-beta com.google.inject.extensions guice-servlet - 3.0 + 4.0-beta + + + com.google.inject.extensions + guice-multibindings + 4.0-beta com.ibm.icu @@ -214,6 +248,26 @@ jackson-jaxrs-json-provider 2.2.2 + + org.codehaus.jackson + jackson-core-asl + 1.9.11 + + + org.codehaus.jackson + jackson-mapper-asl + 1.9.11 + + + org.hibernate + hibernate-validator + 5.0.1.Final + + + javax.validation + validation-api + 1.1.0.Final + javax.inject javax.inject @@ -232,17 +286,17 @@ com.sun.jersey jersey-core - 1.9.1 + 1.17.1 com.sun.jersey.contribs jersey-guice - 1.9.1 + 1.17.1 com.sun.jersey jersey-server - 1.9.1 + 1.17.1 net.java.dev.jets3t @@ -250,20 +304,19 @@ 0.8.1 - org.mortbay.jetty - jetty - 6.1.26 - - - org.mortbay.jetty - servlet-api - - + org.eclipse.jetty + jetty-server + 8.1.11.v20130520 + + + org.eclipse.jetty + jetty-servlet + 8.1.11.v20130520 - org.mortbay.jetty - jetty-util - 6.1.26 + org.eclipse.jetty + jetty-servlets + 8.1.11.v20130520 joda-time @@ -282,8 +335,8 @@ javax.servlet - servlet-api - 2.5 + javax.servlet-api + 3.0.1 org.slf4j @@ -330,6 +383,16 @@ protobuf-java 2.4.0a + + io.tesla.aether + tesla-aether + 0.0.2 + + + org.eclipse.aether + aether-api + 0.9.0.M2 + @@ -399,8 +462,8 @@ maven-compiler-plugin 2.5.1 - 1.6 - 1.6 + 1.7 + 1.7 @@ -429,7 +492,7 @@ maven-shade-plugin - 1.7.1 + 2.1 maven-site-plugin diff --git a/indexing-common/pom.xml b/processing/pom.xml similarity index 70% rename from indexing-common/pom.xml rename to processing/pom.xml index aa63c90ded09..405c89970782 100644 --- a/indexing-common/pom.xml +++ b/processing/pom.xml @@ -1,7 +1,7 @@ - + 4.0.0 com.metamx.druid druid-server @@ -28,18 +29,13 @@ com.metamx druid - 0.5.59-SNAPSHOT + 0.6.0-SNAPSHOT com.metamx.druid - druid-client - ${project.parent.version} - - - com.metamx.druid - druid-indexing-common + druid-processing ${project.parent.version} @@ -72,6 +68,10 @@ commons-io commons-io + + com.amazonaws + aws-java-sdk + com.ning compress-lzf @@ -100,10 +100,6 @@ com.google.inject guice - - com.google.inject.extensions - guice-servlet - com.fasterxml.jackson.core jackson-core @@ -145,8 +141,8 @@ jets3t - org.mortbay.jetty - jetty + org.eclipse.jetty + jetty-server joda-time @@ -162,7 +158,7 @@ javax.servlet - servlet-api + javax.servlet-api org.slf4j @@ -172,6 +168,15 @@ org.apache.hadoop hadoop-core + + io.tesla.aether + tesla-aether + + + org.eclipse.aether + aether-api + + @@ -199,7 +204,30 @@ astyanax 1.0.1 - + + org.antlr + antlr4-runtime + + + com.google.code.simple-spring-memcached + spymemcached + + + net.jpountz.lz4 + lz4 + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-servlet + + + org.eclipse.jetty + jetty-servlets + @@ -214,21 +242,19 @@ com.metamx.druid - druid-indexing-common + druid-processing ${project.parent.version} test-jar test - com.metamx.druid - druid-client - ${project.parent.version} - test-jar + org.apache.curator + curator-test test - org.apache.curator - curator-test + com.google.caliper + caliper test @@ -247,6 +273,17 @@ + + org.antlr + antlr4-maven-plugin + + + + antlr4 + + + + diff --git a/client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 b/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 similarity index 84% rename from client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 rename to server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 index 9dd297dc8c0c..1cafead058d0 100644 --- a/client/src/main/antlr4/com/metamx/druid/sql/antlr4/DruidSQL.g4 +++ b/server/src/main/antlr4/io/druid/sql/antlr4/DruidSQL.g4 @@ -1,18 +1,50 @@ grammar DruidSQL; @header { -import com.metamx.druid.aggregation.post.*; -import com.metamx.druid.aggregation.*; -import com.metamx.druid.query.filter.*; -import com.metamx.druid.query.dimension.*; -import com.metamx.druid.*; - -import com.google.common.base.*; +import com.google.common.base.Joiner; import com.google.common.collect.Lists; -import org.joda.time.*; - -import java.text.*; -import java.util.*; +import io.druid.granularity.PeriodGranularity; +import io.druid.granularity.QueryGranularity; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.CountAggregatorFactory; +import io.druid.query.aggregation.DoubleSumAggregatorFactory; +import io.druid.query.aggregation.MaxAggregatorFactory; +import io.druid.query.aggregation.MinAggregatorFactory; +import io.druid.query.aggregation.PostAggregator; +import io.druid.query.aggregation.post.ArithmeticPostAggregator; +import io.druid.query.aggregation.post.ConstantPostAggregator; +import io.druid.query.aggregation.post.FieldAccessPostAggregator; +import io.druid.query.dimension.DefaultDimensionSpec; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.filter.AndDimFilter; +import io.druid.query.filter.DimFilter; +import io.druid.query.filter.NotDimFilter; +import io.druid.query.filter.OrDimFilter; +import io.druid.query.filter.RegexDimFilter; +import io.druid.query.filter.SelectorDimFilter; +import org.antlr.v4.runtime.NoViableAltException; +import org.antlr.v4.runtime.Parser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; +import org.antlr.v4.runtime.atn.ATN; +import org.antlr.v4.runtime.atn.ATNSimulator; +import org.antlr.v4.runtime.atn.ParserATNSimulator; +import org.antlr.v4.runtime.atn.PredictionContextCache; +import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.TerminalNode; +import org.joda.time.DateTime; +import org.joda.time.Period; + +import java.text.NumberFormat; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; } @parser::members { diff --git a/server/src/main/java/com/metamx/TsvToJson.java b/server/src/main/java/com/metamx/TsvToJson.java deleted file mode 100644 index d0778bf1476d..000000000000 --- a/server/src/main/java/com/metamx/TsvToJson.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.collect.Maps; -import com.metamx.common.IAE; - -import org.joda.time.DateTime; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.util.Map; - -/** - */ -public class TsvToJson -{ - public static void main(String[] args) throws IOException - { - ObjectMapper mapper = new ObjectMapper(); - - String[] fields = args[0].split(","); - File inFile = new File(args[1]); - File outFile = new File(args[2]); - - FieldHandler[] handlers = new FieldHandler[fields.length]; - for (int i = 0; i < fields.length; i++) { - String field = fields[i]; - String[] fieldParts = field.split(":"); - String fieldName = fieldParts[0]; - if (fieldParts.length < 2 || "string".equalsIgnoreCase(fieldParts[1])) { - handlers[i] = new StringField(fieldName); - } - else if ("number".equalsIgnoreCase(fieldParts[1])) { - handlers[i] = new NumberField(fieldName); - } - else if ("ISO8601".equals(fieldParts[1])) { - handlers[i] = new IsoToNumberField(fieldName); - } - else { - throw new IAE("Unknown type[%s]", fieldParts[1]); - } - } - - BufferedReader in = null; - BufferedWriter out = null; - try { - in = new BufferedReader(new InputStreamReader(new FileInputStream(inFile), Charsets.UTF_8)); - out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outFile), Charsets.UTF_8)); - String line = null; - int count = 0; - long currTime = System.currentTimeMillis(); - long startTime = currTime; - while ((line = in.readLine()) != null) { - if (count % 1000000 == 0) { - long nowTime = System.currentTimeMillis(); - System.out.printf("Processed [%,d] lines in %,d millis. Incremental time %,d millis.%n", count, nowTime - startTime, nowTime - currTime); - currTime = nowTime; - } - ++count; - String[] splits = line.split("\t"); - - if (splits.length == 30) { - continue; - } - - if (splits.length != handlers.length) { - throw new IAE("splits.length[%d] != handlers.length[%d]; line[%s]", splits.length, handlers.length, line); - } - - Map jsonMap = Maps.newLinkedHashMap(); - for (int i = 0; i < handlers.length; ++i) { - jsonMap.put(handlers[i].getFieldName(), handlers[i].process(splits[i])); - } - - final String str = mapper.writeValueAsString(jsonMap); - out.write(str); - out.write("\n"); - } - System.out.printf("Completed %,d lines in %,d millis.%n", count, System.currentTimeMillis() - startTime); - out.flush(); - } finally { - if (out != null) { - out.close(); - } - if (in != null) { - in.close(); - } - } - } - - public static interface FieldHandler - { - public String getFieldName(); - public Object process(String value); - } - - public static class StringField implements FieldHandler - { - private final String fieldName; - - public StringField( - String fieldName - ) - { - this.fieldName = fieldName; - } - - @Override - public String getFieldName() - { - return fieldName; - } - - @Override - public Object process(String value) - { - return value; - } - } - - public static class NumberField implements FieldHandler - { - private final String fieldName; - - public NumberField( - String fieldName - ) - { - this.fieldName = fieldName; - } - - @Override - public String getFieldName() - { - return fieldName; - } - - @Override - public Object process(String value) - { - try { - return Long.parseLong(value); - } catch (NumberFormatException e) { - return Double.parseDouble(value); - } - } - } - - public static class IsoToNumberField implements FieldHandler - { - - private final String fieldName; - - public IsoToNumberField( - String fieldName - ) - { - this.fieldName = fieldName; - } - - @Override - public String getFieldName() - { - return fieldName; - } - - @Override - public Object process(String value) - { - return new DateTime(value).getMillis(); - } - } -} diff --git a/server/src/main/java/com/metamx/druid/BaseServerNode.java b/server/src/main/java/com/metamx/druid/BaseServerNode.java deleted file mode 100644 index 17eee84dc6ee..000000000000 --- a/server/src/main/java/com/metamx/druid/BaseServerNode.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.metamx.common.concurrent.ExecutorServiceConfig; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.collect.StupidPool; -import com.metamx.druid.initialization.ServerInit; -import com.metamx.druid.query.DefaultQueryRunnerFactoryConglomerate; -import com.metamx.druid.query.MetricsEmittingExecutorService; -import com.metamx.druid.query.PrioritizedExecutorService; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryRunnerFactoryConglomerate; - -import com.metamx.emitter.service.ServiceMetricEvent; -import org.skife.config.ConfigurationObjectFactory; - -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ExecutorService; - -/** - */ -public abstract class BaseServerNode extends QueryableNode -{ - private final Map, QueryRunnerFactory> additionalFactories = Maps.newLinkedHashMap(); - private DruidProcessingConfig processingConfig = null; - private QueryRunnerFactoryConglomerate conglomerate = null; - private StupidPool computeScratchPool = null; - private ExecutorService queryExecutorService = null; - - public BaseServerNode( - String nodeType, - Logger log, - Properties props, - Lifecycle lifecycle, - ObjectMapper jsonMapper, - ObjectMapper smileMapper, - ConfigurationObjectFactory configFactory - ) - { - super(nodeType, log, props, lifecycle, jsonMapper, smileMapper, configFactory); - } - - public QueryRunnerFactoryConglomerate getConglomerate() - { - initializeQueryRunnerFactoryConglomerate(); - return conglomerate; - } - - public StupidPool getComputeScratchPool() - { - initializeComputeScratchPool(); - return computeScratchPool; - } - - public DruidProcessingConfig getProcessingConfig() - { - initializeProcessingConfig(); - return processingConfig; - } - - public ExecutorService getQueryExecutorService() - { - initializeQueryExecutorService(); - return queryExecutorService; - } - - @SuppressWarnings("unchecked") - public T setConglomerate(QueryRunnerFactoryConglomerate conglomerate) - { - checkFieldNotSetAndSet("conglomerate", conglomerate); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setComputeScratchPool(StupidPool computeScratchPool) - { - checkFieldNotSetAndSet("computeScratchPool", computeScratchPool); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setProcessingConfig(DruidProcessingConfig processingConfig) - { - checkFieldNotSetAndSet("processingConfig", processingConfig); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T setQueryExecutorService(ExecutorService queryExecutorService) - { - checkFieldNotSetAndSet("queryExecutorService", queryExecutorService); - return (T) this; - } - - @SuppressWarnings("unchecked") - public T registerQueryRunnerFactory(Class queryClazz, QueryRunnerFactory factory) - { - Preconditions.checkState( - conglomerate == null, - "Registering a QueryRunnerFactory only works when a separate conglomerate is not specified." - ); - Preconditions.checkState( - !additionalFactories.containsKey(queryClazz), "Registered factory for class[%s] multiple times", queryClazz - ); - additionalFactories.put(queryClazz, factory); - return (T) this; - } - - private void initializeComputeScratchPool() - { - if (computeScratchPool == null) { - setComputeScratchPool(ServerInit.makeComputeScratchPool(getProcessingConfig())); - } - } - - private void initializeQueryRunnerFactoryConglomerate() - { - if (conglomerate == null) { - final Map, QueryRunnerFactory> factories = ServerInit.initDefaultQueryTypes( - getConfigFactory(), getComputeScratchPool() - ); - - for (Map.Entry, QueryRunnerFactory> entry : additionalFactories.entrySet()) { - factories.put(entry.getKey(), entry.getValue()); - } - - setConglomerate(new DefaultQueryRunnerFactoryConglomerate(factories)); - } - } - - private void initializeProcessingConfig() - { - if (processingConfig == null) { - setProcessingConfig( - getConfigFactory().buildWithReplacements( - DruidProcessingConfig.class, ImmutableMap.of("base_path", "druid.processing") - ) - ); - } - } - - private void initializeQueryExecutorService() - { - if (queryExecutorService == null) { - final PrioritizedExecutorService innerExecutorService = PrioritizedExecutorService.create( - getLifecycle(), - getConfigFactory().buildWithReplacements( - ExecutorServiceConfig.class, ImmutableMap.of("base_path", "druid.processing") - ) - ); - - setQueryExecutorService( - new MetricsEmittingExecutorService( - innerExecutorService, - getEmitter(), - new ServiceMetricEvent.Builder() - ) - ); - } - } -} diff --git a/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java b/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java deleted file mode 100644 index 2aa7e4f59bf5..000000000000 --- a/server/src/main/java/com/metamx/druid/BaseStorageAdapter.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid; - -import com.google.common.collect.Sets; -import com.metamx.common.guava.FunctionalIterable; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.index.v1.ConciseOffset; -import com.metamx.druid.index.v1.processing.IntersectingOffset; -import com.metamx.druid.index.v1.processing.Offset; -import com.metamx.druid.kv.Indexed; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.query.search.SearchQuerySpec; -import it.uniroma3.mat.extendedset.intset.ImmutableConciseSet; - -import java.util.List; -import java.util.TreeSet; - -/** - */ -public abstract class BaseStorageAdapter implements StorageAdapter -{ - public abstract Indexed getAvailableDimensions(); - - public abstract Indexed getDimValueLookup(String dimension); - - public abstract ImmutableConciseSet getInvertedIndex(String dimension, String dimVal); - - public abstract ImmutableConciseSet getInvertedIndex(String dimension, int idx); - - public abstract Offset getFilterOffset(Filter filter); - - @Override - public Iterable searchDimensions(final SearchQuery query, final Filter filter) - { - final List dimensions = query.getDimensions(); - final SearchQuerySpec searchQuerySpec = query.getQuery(); - - final TreeSet retVal = Sets.newTreeSet(query.getSort().getComparator()); - - Iterable dimsToSearch; - if (dimensions == null || dimensions.isEmpty()) { - dimsToSearch = getAvailableDimensions(); - } else { - dimsToSearch = dimensions; - } - - Offset filterOffset = (filter == null) ? null : getFilterOffset(filter); - - for (String dimension : dimsToSearch) { - Iterable dims = getDimValueLookup(dimension); - if (dims != null) { - for (String dimVal : dims) { - dimVal = dimVal == null ? "" : dimVal; - if (searchQuerySpec.accept(dimVal)) { - if (filterOffset != null) { - Offset lhs = new ConciseOffset(getInvertedIndex(dimension, dimVal)); - Offset rhs = filterOffset.clone(); - - if (new IntersectingOffset(lhs, rhs).withinBounds()) { - retVal.add(new SearchHit(dimension, dimVal)); - } - } else { - retVal.add(new SearchHit(dimension, dimVal)); - } - } - } - } - } - - return new FunctionalIterable(retVal).limit(query.getLimit()); - } -} - diff --git a/server/src/main/java/com/metamx/druid/http/ComputeMain.java b/server/src/main/java/com/metamx/druid/http/ComputeMain.java deleted file mode 100644 index 828d1568214e..000000000000 --- a/server/src/main/java/com/metamx/druid/http/ComputeMain.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.log.LogLevelAdjuster; - -/** - */ -public class ComputeMain -{ - private static final Logger log = new Logger(ComputeMain.class); - - public static void main(String[] args) throws Exception - { - LogLevelAdjuster.register(); - - Lifecycle lifecycle = new Lifecycle(); - - lifecycle.addManagedInstance( - ComputeNode.builder().build() - ); - - try { - lifecycle.start(); - } - catch (Throwable t) { - log.info(t, "Throwable caught at startup, committing seppuku"); - System.exit(2); - } - - lifecycle.join(); - } -} diff --git a/server/src/main/java/com/metamx/druid/http/ComputeNode.java b/server/src/main/java/com/metamx/druid/http/ComputeNode.java deleted file mode 100644 index e677db381e32..000000000000 --- a/server/src/main/java/com/metamx/druid/http/ComputeNode.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.smile.SmileFactory; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; -import com.metamx.common.ISE; -import com.metamx.common.concurrent.ExecutorServiceConfig; -import com.metamx.common.config.Config; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.BaseServerNode; -import com.metamx.druid.coordination.ServerManager; -import com.metamx.druid.coordination.ZkCoordinator; -import com.metamx.druid.coordination.ZkCoordinatorConfig; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.initialization.ServerInit; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.loading.SegmentLoader; -import com.metamx.druid.metrics.ServerMonitor; -import com.metamx.druid.query.MetricsEmittingExecutorService; -import com.metamx.druid.query.PrioritizedExecutorService; -import com.metamx.druid.query.QueryRunnerFactoryConglomerate; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.emitter.service.ServiceMetricEvent; -import com.metamx.metrics.Monitor; -import org.jets3t.service.S3ServiceException; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.ServletHolder; -import org.skife.config.ConfigurationObjectFactory; - -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutorService; - -/** - */ -public class ComputeNode extends BaseServerNode -{ - private static final Logger log = new Logger(ComputeNode.class); - - public static Builder builder() - { - return new Builder(); - } - - private SegmentLoader segmentLoader; - - public ComputeNode( - Properties props, - Lifecycle lifecycle, - ObjectMapper jsonMapper, - ObjectMapper smileMapper, - ConfigurationObjectFactory configFactory - ) - { - super("historical", log, props, lifecycle, jsonMapper, smileMapper, configFactory); - } - - public ComputeNode setSegmentLoader(SegmentLoader segmentLoader) - { - Preconditions.checkState(this.segmentLoader == null, "Cannot set segmentLoader once it has already been set."); - this.segmentLoader = segmentLoader; - return this; - } - - public SegmentLoader getSegmentLoader() - { - initializeSegmentLoader(); - return segmentLoader; - } - - protected void doInit() throws Exception - { - final Lifecycle lifecycle = getLifecycle(); - final ServiceEmitter emitter = getEmitter(); - final List monitors = getMonitors(); - final QueryRunnerFactoryConglomerate conglomerate = getConglomerate(); - - final ServerManager serverManager = new ServerManager( - getSegmentLoader(), - conglomerate, - emitter, - getQueryExecutorService() - ); - - final ZkCoordinator coordinator = new ZkCoordinator( - getJsonMapper(), - getConfigFactory().build(ZkCoordinatorConfig.class), - getZkPaths(), - getDruidServerMetadata(), - getAnnouncer(), - getCuratorFramework(), - serverManager - ); - lifecycle.addManagedInstance(coordinator); - - monitors.add(new ServerMonitor(getDruidServerMetadata(), serverManager)); - startMonitoring(monitors); - - final Context root = new Context(getServer(), "/", Context.SESSIONS); - root.addServlet(new ServletHolder(new StatusServlet()), "/status"); - root.addServlet( - new ServletHolder( - new QueryServlet(getJsonMapper(), getSmileMapper(), serverManager, emitter, getRequestLogger()) - ), - "/druid/v2/*" - ); - } - - private void initializeSegmentLoader() - { - if (segmentLoader == null) { - try { - setSegmentLoader( - ServerInit.makeDefaultQueryableLoader(getConfigFactory(), getProps()) - ); - } - catch (S3ServiceException e) { - throw Throwables.propagate(e); - } - } - } - - public static class Builder - { - private ObjectMapper jsonMapper = null; - private ObjectMapper smileMapper = null; - private Lifecycle lifecycle = null; - private Properties props = null; - private ConfigurationObjectFactory configFactory = null; - - public Builder withMappers(ObjectMapper jsonMapper, ObjectMapper smileMapper) - { - this.jsonMapper = jsonMapper; - this.smileMapper = smileMapper; - return this; - } - - public Builder withProps(Properties props) - { - this.props = props; - return this; - } - - public Builder withConfigFactory(ConfigurationObjectFactory configFactory) - { - this.configFactory = configFactory; - return this; - } - - public ComputeNode build() - { - if (jsonMapper == null && smileMapper == null) { - jsonMapper = new DefaultObjectMapper(); - smileMapper = new DefaultObjectMapper(new SmileFactory()); - smileMapper.getJsonFactory().setCodec(smileMapper); - } else if (jsonMapper == null || smileMapper == null) { - throw new ISE( - "Only jsonMapper[%s] or smileMapper[%s] was set, must set neither or both.", - jsonMapper, - smileMapper - ); - } - - if (lifecycle == null) { - lifecycle = new Lifecycle(); - } - - if (props == null) { - props = Initialization.loadProperties(); - } - - if (configFactory == null) { - configFactory = Config.createFactory(props); - } - - return new ComputeNode(props, lifecycle, jsonMapper, smileMapper, configFactory); - } - } -} \ No newline at end of file diff --git a/server/src/main/java/com/metamx/druid/http/MasterMain.java b/server/src/main/java/com/metamx/druid/http/MasterMain.java deleted file mode 100644 index 53382fe81bb8..000000000000 --- a/server/src/main/java/com/metamx/druid/http/MasterMain.java +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Charsets; -import com.google.common.base.Throwables; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.servlet.GuiceFilter; -import com.metamx.common.IAE; -import com.metamx.common.concurrent.ScheduledExecutorFactory; -import com.metamx.common.concurrent.ScheduledExecutors; -import com.metamx.common.config.Config; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.client.BatchServerInventoryView; -import com.metamx.druid.client.ServerInventoryView; -import com.metamx.druid.client.ServerInventoryViewConfig; -import com.metamx.druid.client.SingleServerInventoryView; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.config.ConfigManager; -import com.metamx.druid.config.ConfigManagerConfig; -import com.metamx.druid.config.JacksonConfigManager; -import com.metamx.druid.curator.discovery.ServiceAnnouncer; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.db.DatabaseRuleManagerConfig; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.db.DatabaseSegmentManagerConfig; -import com.metamx.druid.db.DbConnector; -import com.metamx.druid.db.DbConnectorConfig; -import com.metamx.druid.initialization.CuratorConfig; -import com.metamx.druid.initialization.Initialization; -import com.metamx.druid.initialization.ServerConfig; -import com.metamx.druid.initialization.ServiceDiscoveryConfig; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.log.LogLevelAdjuster; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterConfig; -import com.metamx.druid.master.LoadQueueTaskMaster; -import com.metamx.druid.utils.PropUtils; -import com.metamx.emitter.EmittingLogger; -import com.metamx.emitter.core.Emitters; -import com.metamx.emitter.service.ServiceEmitter; -import com.metamx.http.client.HttpClient; -import com.metamx.http.client.HttpClientConfig; -import com.metamx.http.client.HttpClientInit; -import com.metamx.http.client.response.ToStringResponseHandler; -import com.metamx.metrics.JvmMonitor; -import com.metamx.metrics.Monitor; -import com.metamx.metrics.MonitorScheduler; -import com.metamx.metrics.MonitorSchedulerConfig; -import com.metamx.metrics.SysMonitor; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.x.discovery.ServiceDiscovery; -import org.apache.curator.x.discovery.ServiceProvider; -import org.joda.time.Duration; -import org.mortbay.jetty.Server; -import org.mortbay.jetty.servlet.Context; -import org.mortbay.jetty.servlet.DefaultServlet; -import org.mortbay.jetty.servlet.FilterHolder; -import org.mortbay.jetty.servlet.ServletHolder; -import org.mortbay.servlet.GzipFilter; -import org.skife.config.ConfigurationObjectFactory; -import org.skife.jdbi.v2.DBI; - -import java.net.URL; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; - -/** - */ -public class MasterMain -{ - private static final Logger log = new Logger(MasterMain.class); - - public static void main(String[] args) throws Exception - { - LogLevelAdjuster.register(); - - final ObjectMapper jsonMapper = new DefaultObjectMapper(); - final Properties props = Initialization.loadProperties(); - final ConfigurationObjectFactory configFactory = Config.createFactory(props); - final Lifecycle lifecycle = new Lifecycle(); - - final HttpClientConfig.Builder httpClientConfigBuilder = HttpClientConfig.builder().withNumConnections(1); - - final String emitterTimeout = props.getProperty("druid.emitter.timeOut"); - if (emitterTimeout != null) { - httpClientConfigBuilder.withReadTimeout(new Duration(emitterTimeout)); - } - final HttpClient httpClient = HttpClientInit.createClient(httpClientConfigBuilder.build(), lifecycle); - - final ServiceEmitter emitter = new ServiceEmitter( - PropUtils.getProperty(props, "druid.service"), - PropUtils.getProperty(props, "druid.host"), - Emitters.create(props, httpClient, jsonMapper, lifecycle) - ); - EmittingLogger.registerEmitter(emitter); - - final ScheduledExecutorFactory scheduledExecutorFactory = ScheduledExecutors.createFactory(lifecycle); - - final ServiceDiscoveryConfig serviceDiscoveryConfig = configFactory.build(ServiceDiscoveryConfig.class); - CuratorFramework serviceDiscoveryCuratorFramework = Initialization.makeCuratorFramework( - serviceDiscoveryConfig, - lifecycle - ); - final CuratorConfig curatorConfig = configFactory.build(CuratorConfig.class); - CuratorFramework curatorFramework = Initialization.makeCuratorFramework( - curatorConfig, - lifecycle - ); - - final ZkPathsConfig zkPaths = configFactory.build(ZkPathsConfig.class); - - final ExecutorService exec = Executors.newFixedThreadPool( - 1, new ThreadFactoryBuilder().setDaemon(true).setNameFormat("ServerInventoryView-%s").build() - ); - - final ServerInventoryViewConfig serverInventoryViewConfig = configFactory.build(ServerInventoryViewConfig.class); - final String announcerType = serverInventoryViewConfig.getAnnouncerType(); - - final ServerInventoryView serverInventoryView; - if ("legacy".equalsIgnoreCase(announcerType)) { - serverInventoryView = new SingleServerInventoryView( - serverInventoryViewConfig, - zkPaths, - curatorFramework, - exec, - jsonMapper - ); - } else if ("batch".equalsIgnoreCase(announcerType)) { - serverInventoryView = new BatchServerInventoryView( - serverInventoryViewConfig, - zkPaths, - curatorFramework, - exec, - jsonMapper - ); - } else { - throw new IAE("Unknown type %s", announcerType); - } - - lifecycle.addManagedInstance(serverInventoryView); - - final DbConnectorConfig dbConnectorConfig = configFactory.build(DbConnectorConfig.class); - final DatabaseRuleManagerConfig databaseRuleManagerConfig = configFactory.build(DatabaseRuleManagerConfig.class); - final DBI dbi = new DbConnector(dbConnectorConfig).getDBI(); - DbConnector.createSegmentTable(dbi, PropUtils.getProperty(props, "druid.database.segmentTable")); - DbConnector.createRuleTable(dbi, PropUtils.getProperty(props, "druid.database.ruleTable")); - DatabaseRuleManager.createDefaultRule( - dbi, databaseRuleManagerConfig.getRuleTable(), databaseRuleManagerConfig.getDefaultDatasource(), jsonMapper - ); - - final DatabaseSegmentManager databaseSegmentManager = new DatabaseSegmentManager( - jsonMapper, - scheduledExecutorFactory.create(1, "DatabaseSegmentManager-Exec--%d"), - configFactory.build(DatabaseSegmentManagerConfig.class), - dbi - ); - final DatabaseRuleManager databaseRuleManager = new DatabaseRuleManager( - jsonMapper, - scheduledExecutorFactory.create(1, "DatabaseRuleManager-Exec--%d"), - databaseRuleManagerConfig, - dbi - ); - - final ScheduledExecutorService globalScheduledExec = scheduledExecutorFactory.create(1, "Global--%d"); - final List monitors = Lists.newArrayList(); - monitors.add(new JvmMonitor()); - if (Boolean.parseBoolean(props.getProperty("druid.monitoring.monitorSystem", "false"))) { - monitors.add(new SysMonitor()); - } - - final MonitorScheduler healthMonitor = new MonitorScheduler( - configFactory.build(MonitorSchedulerConfig.class), - globalScheduledExec, - emitter, - monitors - ); - lifecycle.addManagedInstance(healthMonitor); - - final DruidMasterConfig druidMasterConfig = configFactory.build(DruidMasterConfig.class); - - final ServiceDiscovery serviceDiscovery = Initialization.makeServiceDiscoveryClient( - serviceDiscoveryCuratorFramework, - serviceDiscoveryConfig, - lifecycle - ); - final ServiceAnnouncer serviceAnnouncer = Initialization.makeServiceAnnouncer( - serviceDiscoveryConfig, serviceDiscovery - ); - Initialization.announceDefaultService(serviceDiscoveryConfig, serviceAnnouncer, lifecycle); - - ServiceProvider serviceProvider = null; - if (druidMasterConfig.getMergerServiceName() != null) { - serviceProvider = Initialization.makeServiceProvider( - druidMasterConfig.getMergerServiceName(), - serviceDiscovery, - lifecycle - ); - } - IndexingServiceClient indexingServiceClient = new IndexingServiceClient(httpClient, jsonMapper, serviceProvider); - - final ConfigManagerConfig configManagerConfig = configFactory.build(ConfigManagerConfig.class); - DbConnector.createConfigTable(dbi, configManagerConfig.getConfigTable()); - JacksonConfigManager configManager = new JacksonConfigManager( - new ConfigManager(dbi, configManagerConfig), jsonMapper - ); - - final LoadQueueTaskMaster taskMaster = new LoadQueueTaskMaster( - curatorFramework, - jsonMapper, - scheduledExecutorFactory.create(1, "Master-PeonExec--%d"), - druidMasterConfig - ); - - final DruidMaster master = new DruidMaster( - druidMasterConfig, - zkPaths, - configManager, - databaseSegmentManager, - serverInventoryView, - databaseRuleManager, - curatorFramework, - emitter, - scheduledExecutorFactory, - indexingServiceClient, - taskMaster - ); - lifecycle.addManagedInstance(master); - - try { - lifecycle.start(); - } - catch (Throwable t) { - log.error(t, "Error when starting up. Failing."); - System.exit(1); - } - - Runtime.getRuntime().addShutdownHook( - new Thread( - new Runnable() - { - @Override - public void run() - { - log.info("Running shutdown hook"); - lifecycle.stop(); - } - } - ) - ); - - final Injector injector = Guice.createInjector( - new MasterServletModule( - serverInventoryView, - databaseSegmentManager, - databaseRuleManager, - master, - jsonMapper, - indexingServiceClient, - configManager - ) - ); - - final Server server = Initialization.makeJettyServer(configFactory.build(ServerConfig.class)); - - final RedirectInfo redirectInfo = new RedirectInfo() - { - @Override - public boolean doLocal() - { - return master.isClusterMaster(); - } - - @Override - public URL getRedirectURL(String queryString, String requestURI) - { - try { - final String currentMaster = master.getCurrentMaster(); - if (currentMaster == null) { - return null; - } - - String location = String.format("http://%s%s", currentMaster, requestURI); - - if (queryString != null) { - location = String.format("%s?%s", location, queryString); - } - - return new URL(location); - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - }; - - final Context staticContext = new Context(server, "/static", Context.SESSIONS); - staticContext.addServlet(new ServletHolder(new RedirectServlet(redirectInfo)), "/*"); - - staticContext.setResourceBase(ComputeMain.class.getClassLoader().getResource("static").toExternalForm()); - - final Context root = new Context(server, "/", Context.SESSIONS); - root.addServlet(new ServletHolder(new StatusServlet()), "/status"); - root.addServlet(new ServletHolder(new DefaultServlet()), "/*"); - root.addEventListener(new GuiceServletConfig(injector)); - root.addFilter(GzipFilter.class, "/*", 0); - root.addFilter( - new FilterHolder( - new RedirectFilter( - new ToStringResponseHandler(Charsets.UTF_8), - redirectInfo - ) - ), "/*", 0 - ); - root.addFilter(GuiceFilter.class, "/info/*", 0); - root.addFilter(GuiceFilter.class, "/master/*", 0); - - server.start(); - server.join(); - } -} diff --git a/server/src/main/java/com/metamx/druid/http/MasterServletModule.java b/server/src/main/java/com/metamx/druid/http/MasterServletModule.java deleted file mode 100644 index 51d57492914a..000000000000 --- a/server/src/main/java/com/metamx/druid/http/MasterServletModule.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; -import com.google.inject.Provides; -import com.google.inject.util.Providers; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.config.JacksonConfigManager; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.master.DruidMaster; -import com.sun.jersey.guice.JerseyServletModule; -import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; - -import javax.inject.Singleton; - -/** - */ -public class MasterServletModule extends JerseyServletModule -{ - private final InventoryView serverInventoryView; - private final DatabaseSegmentManager segmentInventoryManager; - private final DatabaseRuleManager databaseRuleManager; - private final DruidMaster master; - private final ObjectMapper jsonMapper; - private final IndexingServiceClient indexingServiceClient; - private final JacksonConfigManager configManager; - - - public MasterServletModule( - InventoryView serverInventoryView, - DatabaseSegmentManager segmentInventoryManager, - DatabaseRuleManager databaseRuleManager, - DruidMaster master, - ObjectMapper jsonMapper, - IndexingServiceClient indexingServiceClient, - JacksonConfigManager configManager - ) - { - this.serverInventoryView = serverInventoryView; - this.segmentInventoryManager = segmentInventoryManager; - this.databaseRuleManager = databaseRuleManager; - this.master = master; - this.jsonMapper = jsonMapper; - this.indexingServiceClient = indexingServiceClient; - this.configManager = configManager; - } - - @Override - protected void configureServlets() - { - bind(MasterSegmentSettingsResource.class); - bind(InfoResource.class); - bind(MasterResource.class); - bind(InventoryView.class).toInstance(serverInventoryView); - bind(DatabaseSegmentManager.class).toInstance(segmentInventoryManager); - bind(DatabaseRuleManager.class).toInstance(databaseRuleManager); - bind(DruidMaster.class).toInstance(master); - bind(JacksonConfigManager.class).toInstance(configManager); - if (indexingServiceClient == null) { - bind(IndexingServiceClient.class).toProvider(Providers.of(null)); - } - else { - bind(IndexingServiceClient.class).toInstance(indexingServiceClient); - } - - serve("/*").with(GuiceContainer.class); - } - - @Provides - @Singleton - public JacksonJsonProvider getJacksonJsonProvider() - { - final JacksonJsonProvider provider = new JacksonJsonProvider(); - provider.setMapper(jsonMapper); - return provider; - } -} diff --git a/server/src/main/java/com/metamx/druid/http/ServerMain.java b/server/src/main/java/com/metamx/druid/http/ServerMain.java deleted file mode 100644 index 11dcb8795843..000000000000 --- a/server/src/main/java/com/metamx/druid/http/ServerMain.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.http; - -/** - */ -@Deprecated -public class ServerMain -{ - public static void main(String[] args) throws Exception - { - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("!@(*&$#!(*@(@*&$! You are running with ServerMain!!!! PLZ Stop. Use ComputeMain instead."); - System.out.println("K thx bye."); - ComputeMain.main(args); - } -} diff --git a/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java b/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java deleted file mode 100644 index 1e47528d8012..000000000000 --- a/server/src/main/java/com/metamx/druid/index/v1/SegmentIdAttachedStorageAdapter.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.index.v1; - -import com.metamx.druid.Capabilities; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.index.v1.processing.Cursor; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; -import org.joda.time.DateTime; -import org.joda.time.Interval; - -/** - */ -public class SegmentIdAttachedStorageAdapter implements StorageAdapter -{ - private final String segmentId; - private final StorageAdapter delegate; - - public SegmentIdAttachedStorageAdapter( - String segmentId, - StorageAdapter delegate - ) - { - this.segmentId = segmentId; - this.delegate = delegate; - } - - @Override - public String getSegmentIdentifier() - { - return segmentId; - } - - @Override - public Interval getInterval() - { - return delegate.getInterval(); - } - - @Override - public Iterable searchDimensions(SearchQuery query, Filter filter) - { - return delegate.searchDimensions(query, filter); - } - - @Override - public Iterable makeCursors(Filter filter, Interval interval, QueryGranularity gran) - { - return delegate.makeCursors(filter, interval, gran); - } - - @Override - public Capabilities getCapabilities() - { - return delegate.getCapabilities(); - } - - @Override - public DateTime getMaxTime() - { - return delegate.getMaxTime(); - } - - @Override - public DateTime getMinTime() - { - return delegate.getMinTime(); - } - - @Override - public int getDimensionCardinality(String dimension) - { - return delegate.getDimensionCardinality(dimension); - } - - public StorageAdapter getDelegate() - { - return delegate; - } -} diff --git a/server/src/main/java/com/metamx/druid/initialization/ServerInit.java b/server/src/main/java/com/metamx/druid/initialization/ServerInit.java deleted file mode 100644 index 0ac5e67a4def..000000000000 --- a/server/src/main/java/com/metamx/druid/initialization/ServerInit.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.initialization; - -import java.lang.reflect.InvocationTargetException; -import java.nio.ByteBuffer; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.hadoop.conf.Configuration; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.security.AWSCredentials; -import org.skife.config.ConfigurationObjectFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.base.Supplier; -import com.google.common.base.Throwables; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.metamx.common.ISE; -import com.metamx.common.logger.Logger; -import com.metamx.druid.DruidProcessingConfig; -import com.metamx.druid.Query; -import com.metamx.druid.collect.StupidPool; -import com.metamx.druid.loading.DataSegmentPusher; -import com.metamx.druid.loading.DelegatingSegmentLoader; -import com.metamx.druid.loading.HdfsDataSegmentPuller; -import com.metamx.druid.loading.HdfsDataSegmentPusher; -import com.metamx.druid.loading.HdfsDataSegmentPusherConfig; -import com.metamx.druid.loading.LocalDataSegmentPuller; -import com.metamx.druid.loading.LocalDataSegmentPusher; -import com.metamx.druid.loading.LocalDataSegmentPusherConfig; -import com.metamx.druid.loading.MMappedQueryableIndexFactory; -import com.metamx.druid.loading.QueryableIndexFactory; -import com.metamx.druid.loading.S3DataSegmentPuller; -import com.metamx.druid.loading.S3DataSegmentPusher; -import com.metamx.druid.loading.S3DataSegmentPusherConfig; -import com.metamx.druid.loading.SegmentLoader; -import com.metamx.druid.loading.SegmentLoaderConfig; -import com.metamx.druid.loading.SingleSegmentLoader; -import com.metamx.druid.loading.cassandra.CassandraDataSegmentConfig; -import com.metamx.druid.loading.cassandra.CassandraDataSegmentPuller; -import com.metamx.druid.loading.cassandra.CassandraDataSegmentPusher; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.group.GroupByQuery; -import com.metamx.druid.query.group.GroupByQueryEngine; -import com.metamx.druid.query.group.GroupByQueryEngineConfig; -import com.metamx.druid.query.group.GroupByQueryRunnerFactory; -import com.metamx.druid.query.group.GroupByQueryRunnerFactoryConfig; -import com.metamx.druid.query.metadata.SegmentMetadataQuery; -import com.metamx.druid.query.metadata.SegmentMetadataQueryRunnerFactory; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.query.search.SearchQueryRunnerFactory; -import com.metamx.druid.query.timeboundary.TimeBoundaryQuery; -import com.metamx.druid.query.timeboundary.TimeBoundaryQueryRunnerFactory; -import com.metamx.druid.query.timeseries.TimeseriesQuery; -import com.metamx.druid.query.timeseries.TimeseriesQueryRunnerFactory; -import com.metamx.druid.utils.PropUtils; - -/** - */ -public class ServerInit -{ - private static Logger log = new Logger(ServerInit.class); - - public static SegmentLoader makeDefaultQueryableLoader( - final ConfigurationObjectFactory configFactory, - final Properties props - ) throws S3ServiceException - { - SegmentLoaderConfig config = configFactory.build(SegmentLoaderConfig.class); - DelegatingSegmentLoader delegateLoader = new DelegatingSegmentLoader(); - final QueryableIndexFactory factory = new MMappedQueryableIndexFactory(); - - final RestS3Service s3Client = new RestS3Service( - new AWSCredentials( - props.getProperty("com.metamx.aws.accessKey", ""), - props.getProperty("com.metamx.aws.secretKey", "") - ) - ); - final S3DataSegmentPuller segmentGetter = new S3DataSegmentPuller(s3Client); - final SingleSegmentLoader s3segmentLoader = new SingleSegmentLoader(segmentGetter, factory, config); - - delegateLoader.setLoaderTypes( - ImmutableMap.builder() - .put("local", new SingleSegmentLoader(new LocalDataSegmentPuller(), factory, config)) - .put("hdfs", new SingleSegmentLoader(new HdfsDataSegmentPuller(new Configuration()), factory, config)) - .put("s3", s3segmentLoader) - .put("s3_zip", s3segmentLoader) - .put("c*",new SingleSegmentLoader(new CassandraDataSegmentPuller(configFactory.build(CassandraDataSegmentConfig.class)), factory, config)) - .build() - ); - return delegateLoader; - } - - public static StupidPool makeComputeScratchPool(DruidProcessingConfig config) - { - try { - Class vmClass = Class.forName("sun.misc.VM"); - Object maxDirectMemoryObj = vmClass.getMethod("maxDirectMemory").invoke(null); - - if (maxDirectMemoryObj == null || !(maxDirectMemoryObj instanceof Number)) { - log.info("Cannot determine maxDirectMemory from[%s]", maxDirectMemoryObj); - } else { - long maxDirectMemory = ((Number) maxDirectMemoryObj).longValue(); - - final long memoryNeeded = (long) config.intermediateComputeSizeBytes() * (config.getNumThreads() + 1); - if (maxDirectMemory < memoryNeeded) { - throw new ISE( - "Not enough direct memory. Please adjust -XX:MaxDirectMemorySize or druid.computation.buffer.size: " - + "maxDirectMemory[%,d], memoryNeeded[%,d], druid.computation.buffer.size[%,d], numThreads[%,d]", - maxDirectMemory, memoryNeeded, config.intermediateComputeSizeBytes(), config.getNumThreads() - ); - } - } - } - catch (ClassNotFoundException e) { - log.info("No VM class, cannot do memory check."); - } - catch (NoSuchMethodException e) { - log.info("VM.maxDirectMemory doesn't exist, cannot do memory check."); - } - catch (InvocationTargetException e) { - log.warn(e, "static method shouldn't throw this"); - } - catch (IllegalAccessException e) { - log.warn(e, "public method, shouldn't throw this"); - } - - return new ComputeScratchPool(config.intermediateComputeSizeBytes()); - } - - public static Map, QueryRunnerFactory> initDefaultQueryTypes( - ConfigurationObjectFactory configFactory, - StupidPool computationBufferPool - ) - { - Map, QueryRunnerFactory> queryRunners = Maps.newLinkedHashMap(); - queryRunners.put(TimeseriesQuery.class, new TimeseriesQueryRunnerFactory()); - queryRunners.put( - GroupByQuery.class, - new GroupByQueryRunnerFactory( - new GroupByQueryEngine( - configFactory.build(GroupByQueryEngineConfig.class), - computationBufferPool - ), - configFactory.build(GroupByQueryRunnerFactoryConfig.class) - ) - ); - queryRunners.put(SearchQuery.class, new SearchQueryRunnerFactory()); - queryRunners.put(TimeBoundaryQuery.class, new TimeBoundaryQueryRunnerFactory()); - queryRunners.put(SegmentMetadataQuery.class, new SegmentMetadataQueryRunnerFactory()); - return queryRunners; - } - - public static DataSegmentPusher getSegmentPusher( - final Properties props, - final ConfigurationObjectFactory configFactory, - final ObjectMapper jsonMapper - ) - { - if (Boolean.parseBoolean(props.getProperty("druid.pusher.local", "false"))) { - return new LocalDataSegmentPusher(configFactory.build(LocalDataSegmentPusherConfig.class), jsonMapper); - } - else if (Boolean.parseBoolean(props.getProperty("druid.pusher.cassandra", "false"))) { - final CassandraDataSegmentConfig config = configFactory.build(CassandraDataSegmentConfig.class); - - return new CassandraDataSegmentPusher(config, jsonMapper); - } - else if (Boolean.parseBoolean(props.getProperty("druid.pusher.hdfs", "false"))) { - final HdfsDataSegmentPusherConfig config = configFactory.build(HdfsDataSegmentPusherConfig.class); - - return new HdfsDataSegmentPusher(config, new Configuration(), jsonMapper); - } - else { - - final RestS3Service s3Client; - try { - s3Client = new RestS3Service( - new AWSCredentials( - PropUtils.getProperty(props, "com.metamx.aws.accessKey"), - PropUtils.getProperty(props, "com.metamx.aws.secretKey") - ) - ); - } - catch (S3ServiceException e) { - throw Throwables.propagate(e); - } - - return new S3DataSegmentPusher(s3Client, configFactory.build(S3DataSegmentPusherConfig.class), jsonMapper); - } - } - - private static class ComputeScratchPool extends StupidPool - { - private static final Logger log = new Logger(ComputeScratchPool.class); - - public ComputeScratchPool(final int computationBufferSize) - { - super( - new Supplier() - { - final AtomicLong count = new AtomicLong(0); - - @Override - public ByteBuffer get() - { - log.info( - "Allocating new computeScratchPool[%,d] of size[%,d]", count.getAndIncrement(), computationBufferSize - ); - return ByteBuffer.allocateDirect(computationBufferSize); - } - } - ); - } - } -} diff --git a/server/src/main/java/com/metamx/druid/loading/DataSegmentPuller.java b/server/src/main/java/com/metamx/druid/loading/DataSegmentPuller.java deleted file mode 100644 index 306d7f449af6..000000000000 --- a/server/src/main/java/com/metamx/druid/loading/DataSegmentPuller.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.loading; - -import com.metamx.druid.client.DataSegment; - -import java.io.File; - -/** - */ -public interface DataSegmentPuller -{ - /** - * Pull down segment files for the given DataSegment and put them in the given directory. - * - * @param segment The segment to pull down files for - * @param dir The directory to store the files in - * @throws SegmentLoadingException if there are any errors - */ - public void getSegmentFiles(DataSegment segment, File dir) throws SegmentLoadingException; - - /** - * Returns the last modified time of the given segment. - * - * Note, this is not actually used at this point and doesn't need to actually be implemented. It's just still here - * to not break compatibility. - * - * @param segment The segment to check the last modified time for - * @return the last modified time in millis from the epoch - * @throws SegmentLoadingException if there are any errors - */ - @Deprecated - public long getLastModified(DataSegment segment) throws SegmentLoadingException; -} diff --git a/server/src/main/java/com/metamx/druid/loading/DelegatingSegmentLoader.java b/server/src/main/java/com/metamx/druid/loading/DelegatingSegmentLoader.java deleted file mode 100644 index f113051d506b..000000000000 --- a/server/src/main/java/com/metamx/druid/loading/DelegatingSegmentLoader.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.loading; - -import com.metamx.common.MapUtils; -import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.Segment; - -import javax.inject.Inject; -import java.util.Map; - -/** - */ -public class DelegatingSegmentLoader implements SegmentLoader -{ - private static final Logger log = new Logger(DelegatingSegmentLoader.class); - - private volatile Map loaderTypes; - - @Inject - public void setLoaderTypes( - Map loaderTypes - ) - { - this.loaderTypes = loaderTypes; - } - - @Override - public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException - { - return getLoader(segment.getLoadSpec()).isSegmentLoaded(segment); - } - - @Override - public Segment getSegment(DataSegment segment) throws SegmentLoadingException - { - return getLoader(segment.getLoadSpec()).getSegment(segment); - } - - @Override - public void cleanup(DataSegment segment) throws SegmentLoadingException - { - getLoader(segment.getLoadSpec()).cleanup(segment); - } - - private SegmentLoader getLoader(Map loadSpec) throws SegmentLoadingException - { - String type = MapUtils.getString(loadSpec, "type"); - SegmentLoader loader = loaderTypes.get(type); - - if (loader == null) { - throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, loaderTypes.keySet()); - } - return loader; - } -} diff --git a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusherConfig.java b/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusherConfig.java deleted file mode 100644 index 3fbbe2d311f7..000000000000 --- a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusherConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.loading; - -import org.skife.config.Config; -import org.skife.config.Default; - -/** - */ -public abstract class S3DataSegmentPusherConfig -{ - @Config("druid.pusher.s3.bucket") - public abstract String getBucket(); - - @Config("druid.pusher.s3.baseKey") - @Default("") - public abstract String getBaseKey(); - - @Config("druid.pusher.s3.disableAcl") - @Default("false") - public abstract boolean getDisableAcl(); -} diff --git a/server/src/main/java/com/metamx/druid/master/BalancerStrategy.java b/server/src/main/java/com/metamx/druid/master/BalancerStrategy.java deleted file mode 100644 index 84944114b49b..000000000000 --- a/server/src/main/java/com/metamx/druid/master/BalancerStrategy.java +++ /dev/null @@ -1,35 +0,0 @@ -/* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -package com.metamx.druid.master; - -import com.metamx.druid.client.DataSegment; - -import java.util.List; - -public interface BalancerStrategy -{ - public ServerHolder findNewSegmentHomeBalancer(final DataSegment proposalSegment, final List serverHolders); - - public ServerHolder findNewSegmentHomeReplicator(final DataSegment proposalSegment, final List serverHolders); - - public BalancerSegmentHolder pickSegmentToMove(final List serverHolders); - - public void emitStats(String tier, MasterStats stats, List serverHolderList); -} diff --git a/server/src/main/java/com/metamx/druid/master/BalancerStrategyFactory.java b/server/src/main/java/com/metamx/druid/master/BalancerStrategyFactory.java deleted file mode 100644 index c215c1b39a91..000000000000 --- a/server/src/main/java/com/metamx/druid/master/BalancerStrategyFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -/* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; - -import org.joda.time.DateTime; - -public interface BalancerStrategyFactory -{ - public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp); -} diff --git a/server/src/main/java/com/metamx/druid/master/CostBalancerStrategyFactory.java b/server/src/main/java/com/metamx/druid/master/CostBalancerStrategyFactory.java deleted file mode 100644 index 76e315b16251..000000000000 --- a/server/src/main/java/com/metamx/druid/master/CostBalancerStrategyFactory.java +++ /dev/null @@ -1,31 +0,0 @@ -/* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; - -import org.joda.time.DateTime; - -public class CostBalancerStrategyFactory implements BalancerStrategyFactory -{ - - @Override - public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp) - { - return new CostBalancerStrategy(referenceTimestamp); - } -} diff --git a/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategyFactory.java b/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategyFactory.java deleted file mode 100644 index 1994c0a5fea7..000000000000 --- a/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategyFactory.java +++ /dev/null @@ -1,30 +0,0 @@ -/* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; - -import org.joda.time.DateTime; - -public class RandomBalancerStrategyFactory implements BalancerStrategyFactory -{ - @Override - public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp) - { - return new RandomBalancerStrategy(); - } -} diff --git a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactoryConfig.java b/server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactoryConfig.java deleted file mode 100644 index 52e3227c1ff2..000000000000 --- a/server/src/main/java/com/metamx/druid/query/group/GroupByQueryRunnerFactoryConfig.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.metamx.druid.query.group; - -import org.skife.config.Config; - -/** - */ -public abstract class GroupByQueryRunnerFactoryConfig -{ - @Config("druid.query.groupBy.singleThreaded") - public boolean isSingleThreaded() - { - return false; - } -} diff --git a/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java b/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java deleted file mode 100644 index 516742ba59b5..000000000000 --- a/server/src/main/java/com/metamx/druid/query/search/SearchQueryRunnerFactory.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.query.search; - -import com.google.common.collect.Iterators; -import com.metamx.common.ISE; -import com.metamx.common.guava.BaseSequence; -import com.metamx.common.guava.Sequence; -import com.metamx.druid.Query; -import com.metamx.druid.SearchResultBuilder; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.brita.Filters; -import com.metamx.druid.query.ChainedExecutionQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; - -import java.util.Iterator; -import java.util.concurrent.ExecutorService; - -/** - */ -public class SearchQueryRunnerFactory implements QueryRunnerFactory, SearchQuery> -{ - private static final SearchQueryQueryToolChest toolChest = new SearchQueryQueryToolChest(); - - @Override - public QueryRunner> createRunner(final Segment segment) - { - return new SearchQueryRunner(segment); - } - - @Override - public QueryRunner> mergeRunners( - ExecutorService queryExecutor, Iterable>> queryRunners - ) - { - return new ChainedExecutionQueryRunner>( - queryExecutor, toolChest.getOrdering(), queryRunners - ); - } - - @Override - public QueryToolChest, SearchQuery> getToolchest() - { - return toolChest; - } - - private static class SearchQueryRunner implements QueryRunner> - { - private final StorageAdapter adapter; - - public SearchQueryRunner(Segment segment) - { - this.adapter = segment.asStorageAdapter(); - } - - @Override - public Sequence> run(final Query> input) - { - if (!(input instanceof SearchQuery)) { - throw new ISE("Got a [%s] which isn't a %s", input.getClass(), SearchQuery.class); - } - - final SearchQuery query = (SearchQuery) input; - - return new BaseSequence, Iterator>>( - new BaseSequence.IteratorMaker, Iterator>>() - { - @Override - public Iterator> make() - { - return Iterators.singletonIterator( - new SearchResultBuilder( - adapter.getInterval().getStart(), - adapter.searchDimensions( - query, - Filters.convertDimensionFilters(query.getDimensionsFilter()) - ) - ).build() - ); - } - - @Override - public void cleanup(Iterator> toClean) - { - - } - } - ); - } - } -} diff --git a/server/src/main/java/com/metamx/druid/utils/CLI.java b/server/src/main/java/com/metamx/druid/utils/CLI.java deleted file mode 100644 index 3c2b5ae14977..000000000000 --- a/server/src/main/java/com/metamx/druid/utils/CLI.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.utils; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.OptionGroup; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; - -/** - */ -public class CLI -{ - private final Options opts; - - public CLI() - { - this.opts = new Options(); - } - - public CLI addOptionGroup(OptionGroup group) - { - opts.addOptionGroup(group); - return this; - } - - public CLI addOption(String opt, boolean hasArg, String description) - { - opts.addOption(opt, hasArg, description); - return this; - } - - public CLI addOption(String opt, String longOpt, boolean hasArg, String description) - { - opts.addOption(opt, longOpt, hasArg, description); - return this; - } - - public CLI addOption(Option opt) - { - opts.addOption(opt); - return this; - } - - public CLI addRequiredOption(String opt, String longOpt, boolean hasArg, String description) - { - opts.addOption(new RequiredOption(opt, longOpt, hasArg, description)); - return this; - } - - public CommandLine parse(String[] args) - { - if (args.length == 0) { - new HelpFormatter().printHelp("", opts); - return null; - } - - CommandLine cli = null; - try { - cli = new PosixParser().parse(opts, args, false); - } - catch (ParseException e) { - System.out.println(e.getMessage()); - new HelpFormatter().printHelp("", opts); - return null; - } - - if (cli.hasOption("help")) { - new HelpFormatter().printHelp("", opts); - return null; - } - - return cli; - } -} diff --git a/server/src/main/java/com/metamx/druid/utils/DruidSetup.java b/server/src/main/java/com/metamx/druid/utils/DruidSetup.java deleted file mode 100644 index ef5185ac7037..000000000000 --- a/server/src/main/java/com/metamx/druid/utils/DruidSetup.java +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.utils; - -import com.google.common.base.Charsets; -import com.google.common.base.Throwables; -import com.google.common.collect.Sets; -import com.google.common.io.Closeables; -import com.metamx.common.config.Config; -import com.metamx.druid.initialization.ZkPathsConfig; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryOneTime; -import org.skife.config.ConfigurationObjectFactory; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintStream; -import java.util.Properties; -import java.util.Set; - -/** - * Set up the shared Druid ensemble space. - * This affects the Zookeeper which holds common properties, and znode paths for coordination, - * and also performs metadata table creation in the database (MySQL). - * By storing ensemble-wide properties in zookeeper, cluster administration is simplified. - * Each service instance can also have local property overrides in the file runtime.properties - * located in the classpath. - *

- * The design rules are noted here with rationale - *

- *

- *

- * Design Rule Notes:
- * (a) Properties set on the commandline of services take precedence over  runtime.properties which
- *       takes precedence over properties stored in zookeeper.
- *
- *       Rationale:  organizing principle.
- *
- * (a) Services load properties on startup only.
- *
- *       Rationale: stepwise changes are safer and easier to manage.
- *
- * (b) Only DruidSetup creates properties and znode paths (zpaths) on zookeeper and no other tool or service
- *       will make ensemble-wide settings automatically.
- *
- *       Rationale: one place for this logic, under manual control, and avoid accidental
- *       namespace/partition creation.
- *
- * (c) DruidSetup creates reasonable zpaths but supports overrides to enable tactical
- *   version transitions (just in case).  If zpaths are overridden, then they must all be
- *   overridden together since they are not independent.
- *
- *       Rationale:  convention beats configuration most of the time; sometimes configuration is needed
- *       negotiate unusual cases.
- *
- * (d) Properties settings stored on zookeeper are not cumulative; previous properties are removed before
- *   new ones are stored.
- *       Rationale:  Keep the operations at the granularity of a file of properties, avoid
- *       dependence on order of setup operations, enable dumping of current settings.
- * 
- * - * @author pbaclace - */ -public class DruidSetup -{ - private final static String MODIFIED_PROP = "__MODIFIED"; - private final static Set IGNORED_PROPS = Sets.newHashSet(MODIFIED_PROP); - - public static void main(final String[] args) - { - CuratorFramework curator = null; - - try { - if (args.length < 2 || args.length > 3) { - printUsage(); - System.exit(1); - } - String cmd = args[0]; - if ("dump".equals(cmd) && args.length == 3) { - final String zkConnect = args[1]; - curator = connectToZK(zkConnect); - curator.start(); - String zpathBase = args[2]; - dumpFromZk(curator, zkConnect, zpathBase, System.out); - } else if ("put".equals(cmd) && args.length == 3) { - final String zkConnect = args[1]; - curator = connectToZK(zkConnect); - curator.start(); - final String pfile = args[2]; - putToZk(curator, pfile); - } else { - printUsage(); - System.exit(1); - } - } - finally { - Closeables.closeQuietly(curator); - } - } - - /** - * Load properties from local file, validate and tweak. - *

- * This can only be used for setup, not service run time because of some assembly here. - * - * @param pfile path to runtime.properties file to be read. - */ - private static Properties loadProperties(String pfile) - { - InputStream is = null; - try { - is = new FileInputStream(pfile); - } - catch (FileNotFoundException e) { - System.err.println("File not found: " + pfile); - System.err.println("No changes made."); - System.exit(4); - } - - try { - Properties props = new Properties(); - props.load(new InputStreamReader(is, Charsets.UTF_8)); - return props; - } - catch (IOException e) { - throw reportErrorAndExit(pfile, e); - } - finally { - Closeables.closeQuietly(is); - } - } - - /** - * @param curator zookeeper client. - * @param zPathBase znode base path. - * @param zkConnect ZK coordinates in the form host1:port1[,host2:port2[, ...]] - * @param out - */ - private static void dumpFromZk(CuratorFramework curator, String zkConnect, final String zPathBase, PrintStream out) - { - ZkPathsConfig config = new ZkPathsConfig() - { - @Override - public String getZkBasePath() - { - return zPathBase; - } - }; - - try { - if (curator.checkExists().forPath(config.getPropertiesPath()) != null) { - byte[] data = curator.getData().forPath(config.getPropertiesPath()); - Properties currProps = new Properties(); - currProps.load(new InputStreamReader(new ByteArrayInputStream(data), Charsets.UTF_8)); - - if (! currProps.isEmpty()) { - out.printf("# Begin Properties Listing for zpath[%s]%n", config.getPropertiesPath()); - try { - currProps.store(new OutputStreamWriter(out, Charsets.UTF_8), "Druid"); - } - catch (IOException ignored) { - } - out.printf("# End Properties for zkConnect[%s] zpath[%s]%n", zkConnect, config.getPropertiesPath()); - } - else { - out.printf("# Properties at zpath[%s] empty.%n", config.getPropertiesPath()); - } - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - - private static void putToZk(CuratorFramework curator, String pfile) - { - final Properties props = loadProperties(pfile); - ConfigurationObjectFactory configFactory = Config.createFactory(props); - final ZkPathsConfig zkPaths = configFactory.build(ZkPathsConfig.class); - - createZNodes(curator, zkPaths, System.out); - updatePropertiesZK(curator, zkPaths, props, System.out); - } - - /** - * @param curator zookeeper client. - * @param zkPaths znode base path. - * @param props the properties to store. - * @param out the PrintStream for human readable update summary (usually System.out). - */ - private static void updatePropertiesZK(CuratorFramework curator, ZkPathsConfig zkPaths, Properties props, PrintStream out) - { - Properties currProps = new Properties(); - try { - if (curator.checkExists().forPath(zkPaths.getPropertiesPath()) != null) { - final byte[] data = curator.getData().forPath(zkPaths.getPropertiesPath()); - currProps.load(new InputStreamReader(new ByteArrayInputStream(data), Charsets.UTF_8)); - } - boolean propsDiffer = false; - if (currProps.isEmpty()) { - out.println("No properties currently stored in zk"); - propsDiffer = true; - } else { // determine whether anything is different - int countNew = 0; - int countDiffer = 0; - int countRemoved = 0; - int countNoChange = 0; - StringBuilder changes = new StringBuilder(1024); - for (String pname : props.stringPropertyNames()) { - if (IGNORED_PROPS.contains(pname)) { - continue; // ignore meta props, if any - } - final String pvalue = props.getProperty(pname); - final String pvalueCurr = currProps.getProperty(pname); - if (pvalueCurr == null) { - countNew++; - } else { - if (pvalueCurr.equals(pvalue)) { - countNoChange++; - } else { - countDiffer++; - changes.append(String.format("CHANGED[%s]: PREV=%s --- NOW=%s%n", pname, pvalueCurr, pvalue)); - } - } - } - for (String pname : currProps.stringPropertyNames()) { - if (IGNORED_PROPS.contains(pname)) { - continue; // ignore meta props, if any - } - if (props.getProperty(pname) == null) { - countRemoved++; - changes.append(String.format("REMOVED: %s=%s%n", pname, currProps.getProperty(pname))); - } - } - if (countNew + countRemoved + countDiffer > 0) { - out.printf( - "Properties differ: %,d new, %,d changed, %,d removed, %,d unchanged, previously updated %s%n", - countNew, countDiffer, countRemoved, countNoChange, currProps.getProperty(MODIFIED_PROP) - ); - out.println(changes); - propsDiffer = true; - } else { - out.printf("Current properties identical to file given, %,d total properties set.%n", countNoChange); - } - } - if (propsDiffer) { - ByteArrayOutputStream propsBytes = new ByteArrayOutputStream(); - props.store(new OutputStreamWriter(propsBytes, Charsets.UTF_8), "Common Druid properties"); - - if (currProps.isEmpty()) { - curator.setData().forPath(zkPaths.getPropertiesPath(), propsBytes.toByteArray()); - } - else { - curator.create().forPath(zkPaths.getPropertiesPath(), propsBytes.toByteArray()); - } - out.printf("Properties updated, %,d total properties set.%n", props.size()); - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - - /** - * @param curator zookeeper client. - * @param zkPaths znode base path. - * @param out the PrintStream for human readable update summary. - */ - private static void createZNodes(CuratorFramework curator, ZkPathsConfig zkPaths, PrintStream out) - { - createPath(curator, zkPaths.getAnnouncementsPath(), out); - createPath(curator, zkPaths.getMasterPath(), out); - createPath(curator, zkPaths.getLoadQueuePath(), out); - createPath(curator, zkPaths.getServedSegmentsPath(), out); - createPath(curator, zkPaths.getLiveSegmentsPath(), out); - createPath(curator, zkPaths.getPropertiesPath(), out); - } - - private static void createPath(CuratorFramework curator, String thePath, PrintStream out) - { - try { - if (curator.checkExists().forPath(thePath) != null) { - out.printf("Path[%s] exists already%n", thePath); - } else { - out.printf("Creating ZK path[%s]%n", thePath); - curator.create().creatingParentsIfNeeded().forPath(thePath); - } - } - catch (Exception e) { - throw Throwables.propagate(e); - } - } - - private static RuntimeException reportErrorAndExit(String pfile, IOException ioe) - { - System.err.println("Could not read file: " + pfile); - System.err.println(" because of: " + ioe); - System.err.println("No changes made."); - System.exit(4); - - return new RuntimeException(); - } - - private static CuratorFramework connectToZK(String zkConnect) - { - return CuratorFrameworkFactory.builder() - .connectString(zkConnect) - .retryPolicy(new RetryOneTime(5000)) - .build(); - } - - /** - * Print usage to stdout. - */ - private static void printUsage() - { - System.out.println( - "Usage: CMD [args]\n" - + " Where CMD is a particular command:\n" - + " CMD choices:\n" - + " dump zkConnect baseZkPath # dump info from zk at given coordinates\n" - + " put zkConnect propfile # store paths and propfile into zk at given coordinates\n" - + " args:\n" - + " zkConnect: ZK coordinates in the form host1:port1[,host2:port2[, ...]]\n" - + " baseZkPath: like /druid or /mydruid etc. to uniquely identify a Druid ensemble\n" - + " and should be equal to property druid.zk.paths.base\n" - + " propfile: Java properties file with common properties for all services in ensemble\n" - + " Notes:\n" - + " dump command makes no modifications and shows zk properties at baseZkPath.\n" - + " put command can safely be invoked more than once, will not disturb existing queues,\n" - + " and properties are not cumulative.\n" - + " A zookeeper can service more than one Druid ensemble if baseZkPath is distinct.\n" - + " Druid services only load properties during process startup.\n" - + " Properties defined on a service command line take precedence over the runtime.properties\n" - + " file which takes precedence over properties stored in zookeeper.\n" - + "" - ); - } -} diff --git a/server/src/main/java/com/metamx/druid/utils/RequiredOption.java b/server/src/main/java/com/metamx/druid/utils/RequiredOption.java deleted file mode 100644 index 5ec23aac0812..000000000000 --- a/server/src/main/java/com/metamx/druid/utils/RequiredOption.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package com.metamx.druid.utils; - -import org.apache.commons.cli.Option; - -/** - */ -public class RequiredOption extends Option -{ - public RequiredOption(String opt, String description) - throws IllegalArgumentException - { - super(opt, description); - setRequired(true); - } - - public RequiredOption(String opt, boolean hasArg, String description) - throws IllegalArgumentException - { - super(opt, hasArg, description); - setRequired(true); - } - - public RequiredOption(String opt, String longOpt, boolean hasArg, String description) - throws IllegalArgumentException - { - super(opt, longOpt, hasArg, description); - setRequired(true); - } -} diff --git a/client/src/main/java/com/metamx/druid/client/BatchServerInventoryView.java b/server/src/main/java/io/druid/client/BatchServerInventoryView.java similarity index 80% rename from client/src/main/java/com/metamx/druid/client/BatchServerInventoryView.java rename to server/src/main/java/io/druid/client/BatchServerInventoryView.java index 58497e293db6..3a1bf636aa21 100644 --- a/client/src/main/java/com/metamx/druid/client/BatchServerInventoryView.java +++ b/server/src/main/java/io/druid/client/BatchServerInventoryView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,61 +17,46 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.MapMaker; import com.google.common.collect.Sets; +import com.google.inject.Inject; import com.metamx.common.ISE; -import com.metamx.druid.curator.inventory.InventoryManagerConfig; -import com.metamx.druid.initialization.ZkPathsConfig; import com.metamx.emitter.EmittingLogger; +import io.druid.guice.ManageLifecycle; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import java.util.Set; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutorService; /** */ +@ManageLifecycle public class BatchServerInventoryView extends ServerInventoryView> { private static final EmittingLogger log = new EmittingLogger(BatchServerInventoryView.class); final ConcurrentMap> zNodes = new MapMaker().makeMap(); + @Inject public BatchServerInventoryView( - final ServerInventoryViewConfig config, final ZkPathsConfig zkPaths, final CuratorFramework curator, - final ExecutorService exec, final ObjectMapper jsonMapper ) { super( - config, log, - new InventoryManagerConfig() - { - @Override - public String getContainerPath() - { - return zkPaths.getAnnouncementsPath(); - } - - @Override - public String getInventoryPath() - { - return zkPaths.getLiveSegmentsPath(); - } - }, + zkPaths.getAnnouncementsPath(), + zkPaths.getLiveSegmentsPath(), curator, - exec, jsonMapper, - new TypeReference>() - { - } + new TypeReference>(){} ); } diff --git a/server/src/main/java/io/druid/client/BatchServerInventoryViewProvider.java b/server/src/main/java/io/druid/client/BatchServerInventoryViewProvider.java new file mode 100644 index 000000000000..609a47754aa6 --- /dev/null +++ b/server/src/main/java/io/druid/client/BatchServerInventoryViewProvider.java @@ -0,0 +1,50 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.server.initialization.ZkPathsConfig; +import org.apache.curator.framework.CuratorFramework; + +import javax.validation.constraints.NotNull; + +/** + */ +public class BatchServerInventoryViewProvider implements ServerInventoryViewProvider +{ + @JacksonInject + @NotNull + private ZkPathsConfig zkPaths = null; + + @JacksonInject + @NotNull + private CuratorFramework curator = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public ServerInventoryView get() + { + return new BatchServerInventoryView(zkPaths, curator, jsonMapper); + } +} diff --git a/client/src/main/java/com/metamx/druid/client/BrokerServerView.java b/server/src/main/java/io/druid/client/BrokerServerView.java similarity index 92% rename from client/src/main/java/com/metamx/druid/client/BrokerServerView.java rename to server/src/main/java/io/druid/client/BrokerServerView.java index a61b6b0c00f8..a60cfb18fea8 100644 --- a/client/src/main/java/com/metamx/druid/client/BrokerServerView.java +++ b/server/src/main/java/io/druid/client/BrokerServerView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,23 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; import com.google.common.collect.Ordering; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.selector.QueryableDruidServer; -import com.metamx.druid.client.selector.ServerSelector; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChestWarehouse; import com.metamx.http.client.HttpClient; +import io.druid.client.selector.QueryableDruidServer; +import io.druid.client.selector.ServerSelector; +import io.druid.concurrent.Execs; +import io.druid.guice.annotations.Client; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.timeline.DataSegment; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; import java.util.Iterator; import java.util.Map; @@ -54,12 +58,12 @@ public class BrokerServerView implements TimelineServerView private final HttpClient httpClient; private final ServerView baseView; + @Inject public BrokerServerView( QueryToolChestWarehouse warehose, ObjectMapper smileMapper, - HttpClient httpClient, - ServerView baseView, - ExecutorService exec + @Client HttpClient httpClient, + ServerView baseView ) { this.warehose = warehose; @@ -71,6 +75,7 @@ public BrokerServerView( this.selectors = Maps.newHashMap(); this.timelines = Maps.newHashMap(); + ExecutorService exec = Execs.singleThreaded("BrokerServerView-%s"); baseView.registerSegmentCallback( exec, new ServerView.SegmentCallback() diff --git a/client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java b/server/src/main/java/io/druid/client/CachingClusteredClient.java similarity index 93% rename from client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java rename to server/src/main/java/io/druid/client/CachingClusteredClient.java index 72519cbeb8c2..7560494170a5 100644 --- a/client/src/main/java/com/metamx/druid/client/CachingClusteredClient.java +++ b/server/src/main/java/io/druid/client/CachingClusteredClient.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -32,32 +32,31 @@ import com.google.common.collect.Ordering; import com.google.common.collect.Sets; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.metamx.common.ISE; +import com.google.inject.Inject; import com.metamx.common.Pair; import com.metamx.common.guava.BaseSequence; import com.metamx.common.guava.LazySequence; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; -import com.metamx.common.logger.Logger; -import com.metamx.druid.Query; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.client.cache.Cache; -import com.metamx.druid.client.selector.QueryableDruidServer; -import com.metamx.druid.client.selector.ServerSelector; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.query.CacheStrategy; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.Queries; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.query.segment.MultipleSpecificSegmentSpec; -import com.metamx.druid.query.segment.SegmentDescriptor; -import com.metamx.druid.result.BySegmentResultValueClass; -import com.metamx.druid.result.Result; import com.metamx.emitter.EmittingLogger; +import io.druid.client.cache.Cache; +import io.druid.client.selector.QueryableDruidServer; +import io.druid.client.selector.ServerSelector; +import io.druid.query.BySegmentResultValueClass; +import io.druid.query.CacheStrategy; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.Result; +import io.druid.query.SegmentDescriptor; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.spec.MultipleSpecificSegmentSpec; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; import org.joda.time.DateTime; import org.joda.time.Interval; @@ -82,6 +81,7 @@ public class CachingClusteredClient implements QueryRunner private final Cache cache; private final ObjectMapper objectMapper; + @Inject public CachingClusteredClient( QueryToolChestWarehouse warehouse, TimelineServerView serverView, diff --git a/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java b/server/src/main/java/io/druid/client/DirectDruidClient.java similarity index 94% rename from client/src/main/java/com/metamx/druid/client/DirectDruidClient.java rename to server/src/main/java/io/druid/client/DirectDruidClient.java index 92a4e5b94264..80844ccbedaf 100644 --- a/client/src/main/java/com/metamx/druid/client/DirectDruidClient.java +++ b/server/src/main/java/io/druid/client/DirectDruidClient.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; @@ -35,18 +35,18 @@ import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.logger.Logger; -import com.metamx.druid.Query; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.result.BySegmentResultValueClass; -import com.metamx.druid.result.Result; import com.metamx.http.client.HttpClient; import com.metamx.http.client.io.AppendableByteArrayInputStream; import com.metamx.http.client.response.ClientResponse; import com.metamx.http.client.response.InputStreamResponseHandler; +import io.druid.query.BySegmentResultValueClass; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryToolChest; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.aggregation.MetricManipulationFn; import org.jboss.netty.handler.codec.http.HttpChunk; import org.jboss.netty.handler.codec.http.HttpHeaders; import org.jboss.netty.handler.codec.http.HttpResponse; @@ -199,14 +199,16 @@ public void cleanup(JsonParserIterator iterFromMake) if (!isBySegment) { retVal = Sequences.map( retVal, - toolChest.makeMetricManipulatorFn(query, new MetricManipulationFn() + toolChest.makeMetricManipulatorFn( + query, new MetricManipulationFn() { @Override public Object manipulate(AggregatorFactory factory, Object object) { return factory.deserialize(object); } - }) + } + ) ); } diff --git a/client/src/main/java/com/metamx/druid/client/DruidDataSource.java b/server/src/main/java/io/druid/client/DruidDataSource.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/DruidDataSource.java rename to server/src/main/java/io/druid/client/DruidDataSource.java index ebe566052f81..45137eb7da17 100644 --- a/client/src/main/java/com/metamx/druid/client/DruidDataSource.java +++ b/server/src/main/java/io/druid/client/DruidDataSource.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Maps; +import io.druid.timeline.DataSegment; import java.util.Collections; import java.util.Map; diff --git a/client/src/main/java/com/metamx/druid/client/DruidServer.java b/server/src/main/java/io/druid/client/DruidServer.java similarity index 95% rename from client/src/main/java/com/metamx/druid/client/DruidServer.java rename to server/src/main/java/io/druid/client/DruidServer.java index a3acc3c00d81..53acc501513a 100644 --- a/client/src/main/java/com/metamx/druid/client/DruidServer.java +++ b/server/src/main/java/io/druid/client/DruidServer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.ImmutableMap; import com.metamx.common.logger.Logger; -import com.metamx.druid.coordination.DruidServerMetadata; +import io.druid.server.DruidNode; +import io.druid.server.coordination.DruidServerMetadata; +import io.druid.timeline.DataSegment; import java.util.Collections; import java.util.Map; @@ -47,13 +49,14 @@ public class DruidServer implements Comparable private volatile long currSize; public DruidServer( + DruidNode node, DruidServerConfig config, String type ) { this( - config.getServerName(), - config.getHost(), + node.getHost(), + node.getHost(), config.getMaxSize(), type, config.getTier() diff --git a/server/src/main/java/com/metamx/druid/loading/SegmentLoadingException.java b/server/src/main/java/io/druid/client/DruidServerConfig.java similarity index 65% rename from server/src/main/java/com/metamx/druid/loading/SegmentLoadingException.java rename to server/src/main/java/io/druid/client/DruidServerConfig.java index d52fd6e3a822..14cfaee290ac 100644 --- a/server/src/main/java/com/metamx/druid/loading/SegmentLoadingException.java +++ b/server/src/main/java/io/druid/client/DruidServerConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,26 +17,30 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.client; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; /** */ -public class SegmentLoadingException extends Exception +public class DruidServerConfig { - public SegmentLoadingException( - String formatString, - Object... objs - ) + @JsonProperty + @Min(0) + private long maxSize = -1; + + @JsonProperty + private String tier = "_default_tier"; + + public long getMaxSize() { - super(String.format(formatString, objs)); + return maxSize; } - public SegmentLoadingException( - Throwable cause, - String formatString, - Object... objs - ) + public String getTier() { - super(String.format(formatString, objs), cause); + return tier; } } diff --git a/server/src/main/java/com/metamx/druid/loading/DataSegmentKiller.java b/server/src/main/java/io/druid/client/InventoryView.java similarity index 77% rename from server/src/main/java/com/metamx/druid/loading/DataSegmentKiller.java rename to server/src/main/java/io/druid/client/InventoryView.java index 85483eaa5058..d955ea21c422 100644 --- a/server/src/main/java/com/metamx/druid/loading/DataSegmentKiller.java +++ b/server/src/main/java/io/druid/client/InventoryView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; - -import com.metamx.druid.client.DataSegment; +package io.druid.client; /** */ -public interface DataSegmentKiller +public interface InventoryView { - public void kill(DataSegment segments) throws SegmentLoadingException; + public DruidServer getInventoryValue(String string); + public Iterable getInventory(); } diff --git a/client/src/main/java/com/metamx/druid/client/ServerInventoryView.java b/server/src/main/java/io/druid/client/ServerInventoryView.java similarity index 85% rename from client/src/main/java/com/metamx/druid/client/ServerInventoryView.java rename to server/src/main/java/io/druid/client/ServerInventoryView.java index bd5cd70c668f..f9187828ac39 100644 --- a/client/src/main/java/com/metamx/druid/client/ServerInventoryView.java +++ b/server/src/main/java/io/druid/client/ServerInventoryView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; @@ -27,51 +27,59 @@ import com.google.common.collect.MapMaker; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.inventory.CuratorInventoryManager; -import com.metamx.druid.curator.inventory.CuratorInventoryManagerStrategy; -import com.metamx.druid.curator.inventory.InventoryManagerConfig; import com.metamx.emitter.EmittingLogger; +import io.druid.concurrent.Execs; +import io.druid.curator.inventory.CuratorInventoryManager; +import io.druid.curator.inventory.CuratorInventoryManagerStrategy; +import io.druid.curator.inventory.InventoryManagerConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import java.io.IOException; -import java.util.Iterator; import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; /** */ public abstract class ServerInventoryView implements ServerView, InventoryView { - private final ServerInventoryViewConfig config; - private final Logger log; + + private final EmittingLogger log; private final CuratorInventoryManager inventoryManager; private final AtomicBoolean started = new AtomicBoolean(false); private final ConcurrentMap serverCallbacks = new MapMaker().makeMap(); private final ConcurrentMap segmentCallbacks = new MapMaker().makeMap(); - private final Map removedSegments = new MapMaker().makeMap(); - public ServerInventoryView( - final ServerInventoryViewConfig config, - final Logger log, - final InventoryManagerConfig inventoryManagerConfig, + final EmittingLogger log, + final String announcementsPath, + final String inventoryPath, final CuratorFramework curator, - final ExecutorService exec, final ObjectMapper jsonMapper, final TypeReference typeReference ) { - this.config = config; this.log = log; - this.inventoryManager = new CuratorInventoryManager( + this.inventoryManager = new CuratorInventoryManager<>( curator, - inventoryManagerConfig, - exec, + new InventoryManagerConfig() + { + @Override + public String getContainerPath() + { + return announcementsPath; + } + + @Override + public String getInventoryPath() + { + return inventoryPath; + } + }, + Execs.singleThreaded("ServerInventoryView-%s"), new CuratorInventoryManagerStrategy() { @Override @@ -164,26 +172,6 @@ public DruidServer removeInventory(final DruidServer container, String inventory ); } - public int lookupSegmentLifetime(DataSegment segment) - { - Integer lifetime = removedSegments.get(segment.getIdentifier()); - return (lifetime == null) ? 0 : lifetime; - } - - public void decrementRemovedSegmentsLifetime() - { - for (Iterator> mapIter = removedSegments.entrySet().iterator(); mapIter.hasNext(); ) { - Map.Entry segment = mapIter.next(); - int lifetime = segment.getValue() - 1; - - if (lifetime < 0) { - mapIter.remove(); - } else { - segment.setValue(lifetime); - } - } - } - @LifecycleStart public void start() throws Exception { @@ -335,8 +323,6 @@ public CallbackAction apply(SegmentCallback input) } } ); - - removedSegments.put(inventoryKey, config.getRemovedSegmentLifetime()); } protected abstract DruidServer addInnerInventory( diff --git a/server/src/main/java/io/druid/client/ServerInventoryViewProvider.java b/server/src/main/java/io/druid/client/ServerInventoryViewProvider.java new file mode 100644 index 000000000000..fa48fba4661a --- /dev/null +++ b/server/src/main/java/io/druid/client/ServerInventoryViewProvider.java @@ -0,0 +1,35 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +/** + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = SingleServerInventoryProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "legacy", value = SingleServerInventoryProvider.class), + @JsonSubTypes.Type(name = "batch", value = BatchServerInventoryViewProvider.class) +}) +public interface ServerInventoryViewProvider extends Provider +{ +} diff --git a/client/src/main/java/com/metamx/druid/client/ServerView.java b/server/src/main/java/io/druid/client/ServerView.java similarity index 97% rename from client/src/main/java/com/metamx/druid/client/ServerView.java rename to server/src/main/java/io/druid/client/ServerView.java index 61afd6d1222f..cbd740269b84 100644 --- a/client/src/main/java/com/metamx/druid/client/ServerView.java +++ b/server/src/main/java/io/druid/client/ServerView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; + +import io.druid.timeline.DataSegment; import java.util.concurrent.Executor; diff --git a/server/src/main/java/io/druid/client/SingleServerInventoryProvider.java b/server/src/main/java/io/druid/client/SingleServerInventoryProvider.java new file mode 100644 index 000000000000..7ebce938791f --- /dev/null +++ b/server/src/main/java/io/druid/client/SingleServerInventoryProvider.java @@ -0,0 +1,50 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.server.initialization.ZkPathsConfig; +import org.apache.curator.framework.CuratorFramework; + +import javax.validation.constraints.NotNull; + +/** + */ +public class SingleServerInventoryProvider implements ServerInventoryViewProvider +{ + @JacksonInject + @NotNull + private ZkPathsConfig zkPaths = null; + + @JacksonInject + @NotNull + private CuratorFramework curator = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public ServerInventoryView get() + { + return new SingleServerInventoryView(zkPaths, curator, jsonMapper); + } +} diff --git a/client/src/main/java/com/metamx/druid/client/SingleServerInventoryView.java b/server/src/main/java/io/druid/client/SingleServerInventoryView.java similarity index 74% rename from client/src/main/java/com/metamx/druid/client/SingleServerInventoryView.java rename to server/src/main/java/io/druid/client/SingleServerInventoryView.java index 781c4ed0c772..801b78c5d2ac 100644 --- a/client/src/main/java/com/metamx/druid/client/SingleServerInventoryView.java +++ b/server/src/main/java/io/druid/client/SingleServerInventoryView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,50 +17,36 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.druid.curator.inventory.InventoryManagerConfig; -import com.metamx.druid.initialization.ZkPathsConfig; +import com.google.inject.Inject; import com.metamx.emitter.EmittingLogger; +import io.druid.guice.ManageLifecycle; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; -import java.util.concurrent.ExecutorService; - /** */ +@ManageLifecycle public class SingleServerInventoryView extends ServerInventoryView { private static final EmittingLogger log = new EmittingLogger(SingleServerInventoryView.class); + @Inject public SingleServerInventoryView( - final ServerInventoryViewConfig config, final ZkPathsConfig zkPaths, final CuratorFramework curator, - final ExecutorService exec, final ObjectMapper jsonMapper ) { super( - config, log, - new InventoryManagerConfig() - { - @Override - public String getContainerPath() - { - return zkPaths.getAnnouncementsPath(); - } - - @Override - public String getInventoryPath() - { - return zkPaths.getServedSegmentsPath(); - } - }, + zkPaths.getAnnouncementsPath(), + zkPaths.getServedSegmentsPath(), curator, - exec, jsonMapper, new TypeReference(){} ); diff --git a/client/src/main/java/com/metamx/druid/client/TimelineServerView.java b/server/src/main/java/io/druid/client/TimelineServerView.java similarity index 81% rename from client/src/main/java/com/metamx/druid/client/TimelineServerView.java rename to server/src/main/java/io/druid/client/TimelineServerView.java index 33747f18af12..7082c599c751 100644 --- a/client/src/main/java/com/metamx/druid/client/TimelineServerView.java +++ b/server/src/main/java/io/druid/client/TimelineServerView.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.selector.ServerSelector; -import com.metamx.druid.query.QueryRunner; +import io.druid.client.selector.ServerSelector; +import io.druid.query.QueryRunner; +import io.druid.timeline.VersionedIntervalTimeline; /** */ diff --git a/client/src/main/java/com/metamx/druid/client/cache/ByteCountingLRUMap.java b/server/src/main/java/io/druid/client/cache/ByteCountingLRUMap.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/cache/ByteCountingLRUMap.java rename to server/src/main/java/io/druid/client/cache/ByteCountingLRUMap.java index 59a60f060e55..b351dfc68378 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/ByteCountingLRUMap.java +++ b/server/src/main/java/io/druid/client/cache/ByteCountingLRUMap.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.metamx.common.logger.Logger; diff --git a/client/src/main/java/com/metamx/druid/client/cache/Cache.java b/server/src/main/java/io/druid/client/cache/Cache.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/cache/Cache.java rename to server/src/main/java/io/druid/client/cache/Cache.java index 6e9463deb565..6ed875edb0b9 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/Cache.java +++ b/server/src/main/java/io/druid/client/cache/Cache.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; diff --git a/client/src/main/java/com/metamx/druid/client/cache/CacheMonitor.java b/server/src/main/java/io/druid/client/cache/CacheMonitor.java similarity index 95% rename from client/src/main/java/com/metamx/druid/client/cache/CacheMonitor.java rename to server/src/main/java/io/druid/client/cache/CacheMonitor.java index e89c1113e8de..baa4920e60fc 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/CacheMonitor.java +++ b/server/src/main/java/io/druid/client/cache/CacheMonitor.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,8 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; +import com.google.inject.Inject; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; import com.metamx.metrics.AbstractMonitor; @@ -31,6 +32,7 @@ public class CacheMonitor extends AbstractMonitor private volatile CacheStats prevCacheStats = null; + @Inject public CacheMonitor( Cache cache ) diff --git a/server/src/main/java/io/druid/client/cache/CacheProvider.java b/server/src/main/java/io/druid/client/cache/CacheProvider.java new file mode 100644 index 000000000000..87597dba1c79 --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/CacheProvider.java @@ -0,0 +1,33 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = LocalCacheProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "local", value = LocalCacheProvider.class), + @JsonSubTypes.Type(name = "memcached", value = MemcachedCacheProvider.class) +}) +public interface CacheProvider extends Provider +{ +} diff --git a/client/src/main/java/com/metamx/druid/client/cache/CacheSerde.java b/server/src/main/java/io/druid/client/cache/CacheSerde.java similarity index 91% rename from client/src/main/java/com/metamx/druid/client/cache/CacheSerde.java rename to server/src/main/java/io/druid/client/cache/CacheSerde.java index e7e60fcc0964..16dbce6ad6dd 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/CacheSerde.java +++ b/server/src/main/java/io/druid/client/cache/CacheSerde.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; /** */ diff --git a/client/src/main/java/com/metamx/druid/client/cache/CacheStats.java b/server/src/main/java/io/druid/client/cache/CacheStats.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/cache/CacheStats.java rename to server/src/main/java/io/druid/client/cache/CacheStats.java index 33f0a145082d..ea917649154f 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/CacheStats.java +++ b/server/src/main/java/io/druid/client/cache/CacheStats.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; /** */ diff --git a/client/src/main/java/com/metamx/druid/client/cache/LZ4Transcoder.java b/server/src/main/java/io/druid/client/cache/LZ4Transcoder.java similarity index 96% rename from client/src/main/java/com/metamx/druid/client/cache/LZ4Transcoder.java rename to server/src/main/java/io/druid/client/cache/LZ4Transcoder.java index 4728430b4e7b..7427c2b3db51 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/LZ4Transcoder.java +++ b/server/src/main/java/io/druid/client/cache/LZ4Transcoder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2013 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.primitives.Ints; import net.jpountz.lz4.LZ4Compressor; diff --git a/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java b/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java new file mode 100644 index 000000000000..4f78457b5b99 --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/LocalCacheProvider.java @@ -0,0 +1,48 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; + +/** + */ +public class LocalCacheProvider implements CacheProvider +{ + @JsonProperty + @Min(0) + private long sizeInBytes = 0; + + @JsonProperty + @Min(0) + private int initialSize = 500000; + + @JsonProperty + @Min(0) + private int logEvictionCount = 0; + + + @Override + public Cache get() + { + return new MapCache(new ByteCountingLRUMap(initialSize, logEvictionCount, sizeInBytes)); + } +} diff --git a/client/src/main/java/com/metamx/druid/client/cache/MapCache.java b/server/src/main/java/io/druid/client/cache/MapCache.java similarity index 91% rename from client/src/main/java/com/metamx/druid/client/cache/MapCache.java rename to server/src/main/java/io/druid/client/cache/MapCache.java index bf549ec31b97..dda2929bc89d 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/MapCache.java +++ b/server/src/main/java/io/druid/client/cache/MapCache.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; @@ -33,6 +33,11 @@ */ public class MapCache implements Cache { + public static Cache create(long sizeInBytes) + { + return new MapCache(new ByteCountingLRUMap(sizeInBytes)); + } + private final Map baseMap; private final ByteCountingLRUMap byteCountingLRUMap; @@ -44,17 +49,6 @@ public class MapCache implements Cache private final AtomicLong hitCount = new AtomicLong(0); private final AtomicLong missCount = new AtomicLong(0); - public static com.metamx.druid.client.cache.Cache create(final MapCacheConfig config) - { - return new MapCache( - new ByteCountingLRUMap( - config.getInitialSize(), - config.getLogEvictionCount(), - config.getSizeInBytes() - ) - ); - } - MapCache( ByteCountingLRUMap byteCountingLRUMap ) diff --git a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java b/server/src/main/java/io/druid/client/cache/MemcachedCache.java similarity index 98% rename from client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java rename to server/src/main/java/io/druid/client/cache/MemcachedCache.java index fb6fa72ce46e..155c75e3f86c 100644 --- a/client/src/main/java/com/metamx/druid/client/cache/MemcachedCache.java +++ b/server/src/main/java/io/druid/client/cache/MemcachedCache.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -32,7 +32,6 @@ import net.spy.memcached.MemcachedClient; import net.spy.memcached.MemcachedClientIF; import net.spy.memcached.internal.BulkFuture; -import net.spy.memcached.transcoders.SerializingTranscoder; import org.apache.commons.codec.digest.DigestUtils; import javax.annotation.Nullable; diff --git a/server/src/main/java/io/druid/client/cache/MemcachedCacheConfig.java b/server/src/main/java/io/druid/client/cache/MemcachedCacheConfig.java new file mode 100644 index 000000000000..2cc06cf36377 --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/MemcachedCacheConfig.java @@ -0,0 +1,68 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.NotNull; + +public class MemcachedCacheConfig +{ + @JsonProperty + private int expiration = 2592000; // What is this number? + + @JsonProperty + private int timeout = 500; + + @JsonProperty + @NotNull + private String hosts; + + @JsonProperty + private int maxObjectSize = 50 * 1024 * 1024; + + @JsonProperty + private String memcachedPrefix = "druid"; + + public int getExpiration() + { + return expiration; + } + + public int getTimeout() + { + return timeout; + } + + public String getHosts() + { + return hosts; + } + + public int getMaxObjectSize() + { + return maxObjectSize; + } + + public String getMemcachedPrefix() + { + return memcachedPrefix; + } +} diff --git a/server/src/main/java/io/druid/client/cache/MemcachedCacheProvider.java b/server/src/main/java/io/druid/client/cache/MemcachedCacheProvider.java new file mode 100644 index 000000000000..afdab26dd09b --- /dev/null +++ b/server/src/main/java/io/druid/client/cache/MemcachedCacheProvider.java @@ -0,0 +1,29 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; + +public class MemcachedCacheProvider extends MemcachedCacheConfig implements CacheProvider +{ + @Override + public Cache get() + { + return MemcachedCache.create(this); + } +} diff --git a/client/src/main/java/com/metamx/druid/client/indexing/ClientAppendQuery.java b/server/src/main/java/io/druid/client/indexing/ClientAppendQuery.java similarity index 92% rename from client/src/main/java/com/metamx/druid/client/indexing/ClientAppendQuery.java rename to server/src/main/java/io/druid/client/indexing/ClientAppendQuery.java index 6e9a2eb4163e..1f83252a6815 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/ClientAppendQuery.java +++ b/server/src/main/java/io/druid/client/indexing/ClientAppendQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.indexing; +package io.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.util.List; diff --git a/client/src/main/java/com/metamx/druid/client/indexing/ClientConversionQuery.java b/server/src/main/java/io/druid/client/indexing/ClientConversionQuery.java similarity index 51% rename from client/src/main/java/com/metamx/druid/client/indexing/ClientConversionQuery.java rename to server/src/main/java/io/druid/client/indexing/ClientConversionQuery.java index c0b96bb80b41..631d0aca86ff 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/ClientConversionQuery.java +++ b/server/src/main/java/io/druid/client/indexing/ClientConversionQuery.java @@ -1,7 +1,26 @@ -package com.metamx.druid.client.indexing; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; /** diff --git a/client/src/main/java/com/metamx/druid/client/indexing/ClientKillQuery.java b/server/src/main/java/io/druid/client/indexing/ClientKillQuery.java similarity index 94% rename from client/src/main/java/com/metamx/druid/client/indexing/ClientKillQuery.java rename to server/src/main/java/io/druid/client/indexing/ClientKillQuery.java index 3ae8dffb2254..6f280cef7fa0 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/ClientKillQuery.java +++ b/server/src/main/java/io/druid/client/indexing/ClientKillQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.indexing; +package io.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/client/src/main/java/com/metamx/druid/client/indexing/ClientMergeQuery.java b/server/src/main/java/io/druid/client/indexing/ClientMergeQuery.java similarity index 91% rename from client/src/main/java/com/metamx/druid/client/indexing/ClientMergeQuery.java rename to server/src/main/java/io/druid/client/indexing/ClientMergeQuery.java index e363618a107a..3aa6efefe19e 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/ClientMergeQuery.java +++ b/server/src/main/java/io/druid/client/indexing/ClientMergeQuery.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.indexing; +package io.druid.client.indexing; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.client.DataSegment; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.timeline.DataSegment; import java.util.List; diff --git a/server/src/main/java/io/druid/client/indexing/IndexingService.java b/server/src/main/java/io/druid/client/indexing/IndexingService.java new file mode 100644 index 000000000000..e79bab7fbeeb --- /dev/null +++ b/server/src/main/java/io/druid/client/indexing/IndexingService.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.indexing; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + */ +@BindingAnnotation +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface IndexingService +{ +} diff --git a/client/src/main/java/com/metamx/druid/client/indexing/IndexingServiceClient.java b/server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java similarity index 85% rename from client/src/main/java/com/metamx/druid/client/indexing/IndexingServiceClient.java rename to server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java index ea17b6b68387..628ed978c427 100644 --- a/client/src/main/java/com/metamx/druid/client/indexing/IndexingServiceClient.java +++ b/server/src/main/java/io/druid/client/indexing/IndexingServiceClient.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.indexing; +package io.druid.client.indexing; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; +import com.google.inject.Inject; import com.metamx.common.IAE; import com.metamx.common.ISE; -import com.metamx.druid.client.DataSegment; import com.metamx.http.client.HttpClient; import com.metamx.http.client.response.InputStreamResponseHandler; -import org.apache.curator.x.discovery.ServiceInstance; -import org.apache.curator.x.discovery.ServiceProvider; +import io.druid.client.selector.DiscoverySelector; +import io.druid.client.selector.Server; +import io.druid.guice.annotations.Global; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; import java.io.InputStream; @@ -41,12 +43,13 @@ public class IndexingServiceClient private final HttpClient client; private final ObjectMapper jsonMapper; - private final ServiceProvider serviceProvider; + private final DiscoverySelector serviceProvider; + @Inject public IndexingServiceClient( - HttpClient client, + @Global HttpClient client, ObjectMapper jsonMapper, - ServiceProvider serviceProvider + @IndexingService DiscoverySelector serviceProvider ) { this.client = client; @@ -103,12 +106,12 @@ private InputStream runQuery(String endpoint, Object queryObject) private String baseUrl() { try { - final ServiceInstance instance = serviceProvider.getInstance(); + final Server instance = serviceProvider.pick(); if (instance == null) { throw new ISE("Cannot find instance of indexingService"); } - return String.format("http://%s:%s/druid/indexer/v1", instance.getAddress(), instance.getPort()); + return String.format("http://%s:%s/druid/indexer/v1", instance.getHost(), instance.getPort()); } catch (Exception e) { throw Throwables.propagate(e); diff --git a/server/src/main/java/io/druid/client/indexing/IndexingServiceSelector.java b/server/src/main/java/io/druid/client/indexing/IndexingServiceSelector.java new file mode 100644 index 000000000000..7c5eaaf8d724 --- /dev/null +++ b/server/src/main/java/io/druid/client/indexing/IndexingServiceSelector.java @@ -0,0 +1,94 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.indexing; + +import com.google.inject.Inject; +import com.metamx.common.lifecycle.LifecycleStart; +import com.metamx.common.lifecycle.LifecycleStop; +import com.metamx.common.logger.Logger; +import io.druid.client.selector.DiscoverySelector; +import io.druid.client.selector.Server; +import org.apache.curator.x.discovery.ServiceInstance; +import org.apache.curator.x.discovery.ServiceProvider; + +import javax.annotation.Nullable; +import java.io.IOException; + +/** +*/ +public class IndexingServiceSelector implements DiscoverySelector +{ + private static final Logger log = new Logger(IndexingServiceSelector.class); + + private final ServiceProvider serviceProvider; + + @Inject + public IndexingServiceSelector( + @Nullable @IndexingService ServiceProvider serviceProvider + ) { + this.serviceProvider = serviceProvider; + } + + @Override + public Server pick() + { + final ServiceInstance instance; + try { + instance = serviceProvider.getInstance(); + } + catch (Exception e) { + log.info(e, ""); + return null; + } + + return new Server() + { + @Override + public String getHost() + { + return instance.getAddress(); + } + + @Override + public int getPort() + { + return instance.getPort(); + } + + @Override + public String getScheme() + { + return "http"; + } + }; + } + + @LifecycleStart + public void start() throws Exception + { + serviceProvider.start(); + } + + @LifecycleStop + public void stop() throws IOException + { + serviceProvider.close(); + } +} diff --git a/server/src/main/java/io/druid/client/indexing/IndexingServiceSelectorConfig.java b/server/src/main/java/io/druid/client/indexing/IndexingServiceSelectorConfig.java new file mode 100644 index 000000000000..976225f6665f --- /dev/null +++ b/server/src/main/java/io/druid/client/indexing/IndexingServiceSelectorConfig.java @@ -0,0 +1,35 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.indexing; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +public class IndexingServiceSelectorConfig +{ + @JsonProperty + private String serviceName = null; + + public String getServiceName() + { + return serviceName; + } +} diff --git a/server/src/main/java/io/druid/client/selector/DiscoverySelector.java b/server/src/main/java/io/druid/client/selector/DiscoverySelector.java new file mode 100644 index 000000000000..d878d7ed921d --- /dev/null +++ b/server/src/main/java/io/druid/client/selector/DiscoverySelector.java @@ -0,0 +1,27 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.selector; + +/** + */ +public interface DiscoverySelector +{ + public T pick(); +} diff --git a/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java b/server/src/main/java/io/druid/client/selector/QueryableDruidServer.java similarity index 86% rename from client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java rename to server/src/main/java/io/druid/client/selector/QueryableDruidServer.java index 2528facb8d89..8f4f7f48e822 100644 --- a/client/src/main/java/com/metamx/druid/client/selector/QueryableDruidServer.java +++ b/server/src/main/java/io/druid/client/selector/QueryableDruidServer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.selector; +package io.druid.client.selector; -import com.metamx.druid.client.DirectDruidClient; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DirectDruidClient; +import io.druid.client.DruidServer; /** */ diff --git a/server/src/main/java/io/druid/client/selector/Server.java b/server/src/main/java/io/druid/client/selector/Server.java new file mode 100644 index 000000000000..664d621136e5 --- /dev/null +++ b/server/src/main/java/io/druid/client/selector/Server.java @@ -0,0 +1,29 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.selector; + +/** + */ +public interface Server +{ + public String getScheme(); + public String getHost(); + public int getPort(); +} diff --git a/client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java b/server/src/main/java/io/druid/client/selector/ServerSelector.java similarity index 91% rename from client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java rename to server/src/main/java/io/druid/client/selector/ServerSelector.java index 388761aaaa20..d10c561c72d4 100644 --- a/client/src/main/java/com/metamx/druid/client/selector/ServerSelector.java +++ b/server/src/main/java/io/druid/client/selector/ServerSelector.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.selector; +package io.druid.client.selector; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +import io.druid.timeline.DataSegment; import java.util.Collections; import java.util.Comparator; @@ -30,7 +29,7 @@ /** */ -public class ServerSelector +public class ServerSelector implements DiscoverySelector { private static final Comparator comparator = new Comparator() { diff --git a/client/src/main/java/com/metamx/druid/initialization/CuratorConfig.java b/server/src/main/java/io/druid/curator/CuratorConfig.java similarity index 93% rename from client/src/main/java/com/metamx/druid/initialization/CuratorConfig.java rename to server/src/main/java/io/druid/curator/CuratorConfig.java index be46aea41f68..7ac0247b342c 100644 --- a/client/src/main/java/com/metamx/druid/initialization/CuratorConfig.java +++ b/server/src/main/java/io/druid/curator/CuratorConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.initialization; +package io.druid.curator; import org.skife.config.Config; import org.skife.config.Default; diff --git a/server/src/main/java/io/druid/curator/CuratorModule.java b/server/src/main/java/io/druid/curator/CuratorModule.java new file mode 100644 index 000000000000..94fba357dd2b --- /dev/null +++ b/server/src/main/java/io/druid/curator/CuratorModule.java @@ -0,0 +1,79 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.common.logger.Logger; +import io.druid.guice.ConfigProvider; +import io.druid.guice.LazySingleton; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.BoundedExponentialBackoffRetry; + +import java.io.IOException; + +/** + */ +public class CuratorModule implements Module +{ + private static final Logger log = new Logger(CuratorModule.class); + + @Override + public void configure(Binder binder) + { + ConfigProvider.bind(binder, CuratorConfig.class); + } + + @Provides @LazySingleton + public CuratorFramework makeCurator(CuratorConfig config, Lifecycle lifecycle) throws IOException + { + final CuratorFramework framework = + CuratorFrameworkFactory.builder() + .connectString(config.getZkHosts()) + .sessionTimeoutMs(config.getZkSessionTimeoutMs()) + .retryPolicy(new BoundedExponentialBackoffRetry(1000, 45000, 30)) + .compressionProvider(new PotentiallyGzippedCompressionProvider(config.enableCompression())) + .build(); + + lifecycle.addHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + log.info("Starting Curator"); + framework.start(); + } + + @Override + public void stop() + { + log.info("Stopping Curator"); + framework.close(); + } + } + ); + + return framework; + } +} diff --git a/client/src/main/java/com/metamx/druid/curator/PotentiallyGzippedCompressionProvider.java b/server/src/main/java/io/druid/curator/PotentiallyGzippedCompressionProvider.java similarity index 93% rename from client/src/main/java/com/metamx/druid/curator/PotentiallyGzippedCompressionProvider.java rename to server/src/main/java/io/druid/curator/PotentiallyGzippedCompressionProvider.java index abddd9574fcb..c2fda7139be9 100644 --- a/client/src/main/java/com/metamx/druid/curator/PotentiallyGzippedCompressionProvider.java +++ b/server/src/main/java/io/druid/curator/PotentiallyGzippedCompressionProvider.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator; +package io.druid.curator; import org.apache.curator.framework.api.CompressionProvider; import org.apache.curator.framework.imps.GzipCompressionProvider; import java.io.IOException; -import java.util.zip.ZipException; /** */ diff --git a/client/src/main/java/com/metamx/druid/curator/ShutdownNowIgnoringExecutorService.java b/server/src/main/java/io/druid/curator/ShutdownNowIgnoringExecutorService.java similarity index 74% rename from client/src/main/java/com/metamx/druid/curator/ShutdownNowIgnoringExecutorService.java rename to server/src/main/java/io/druid/curator/ShutdownNowIgnoringExecutorService.java index b4a8128f6d9e..7ac77fcc5eab 100644 --- a/client/src/main/java/com/metamx/druid/curator/ShutdownNowIgnoringExecutorService.java +++ b/server/src/main/java/io/druid/curator/ShutdownNowIgnoringExecutorService.java @@ -1,4 +1,23 @@ -package com.metamx.druid.curator; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator; import com.google.common.collect.ImmutableList; diff --git a/client/src/main/java/com/metamx/druid/curator/announcement/Announcer.java b/server/src/main/java/io/druid/curator/announcement/Announcer.java similarity index 97% rename from client/src/main/java/com/metamx/druid/curator/announcement/Announcer.java rename to server/src/main/java/io/druid/curator/announcement/Announcer.java index abb96b76f68a..8b017c41a247 100644 --- a/client/src/main/java/com/metamx/druid/curator/announcement/Announcer.java +++ b/server/src/main/java/io/druid/curator/announcement/Announcer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.announcement; +package io.druid.curator.announcement; import com.google.common.base.Throwables; import com.google.common.collect.Lists; @@ -30,9 +30,9 @@ import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.ShutdownNowIgnoringExecutorService; -import com.metamx.druid.curator.cache.PathChildrenCacheFactory; -import com.metamx.druid.curator.cache.SimplePathChildrenCacheFactory; +import io.druid.curator.ShutdownNowIgnoringExecutorService; +import io.druid.curator.cache.PathChildrenCacheFactory; +import io.druid.curator.cache.SimplePathChildrenCacheFactory; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; diff --git a/client/src/main/java/com/metamx/druid/curator/cache/PathChildrenCacheFactory.java b/server/src/main/java/io/druid/curator/cache/PathChildrenCacheFactory.java similarity index 92% rename from client/src/main/java/com/metamx/druid/curator/cache/PathChildrenCacheFactory.java rename to server/src/main/java/io/druid/curator/cache/PathChildrenCacheFactory.java index 37335d8aa88d..d722ba0ba5dd 100644 --- a/client/src/main/java/com/metamx/druid/curator/cache/PathChildrenCacheFactory.java +++ b/server/src/main/java/io/druid/curator/cache/PathChildrenCacheFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.cache; +package io.druid.curator.cache; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; diff --git a/client/src/main/java/com/metamx/druid/curator/cache/SimplePathChildrenCacheFactory.java b/server/src/main/java/io/druid/curator/cache/SimplePathChildrenCacheFactory.java similarity index 96% rename from client/src/main/java/com/metamx/druid/curator/cache/SimplePathChildrenCacheFactory.java rename to server/src/main/java/io/druid/curator/cache/SimplePathChildrenCacheFactory.java index fb7a3044ee4e..ef0d3c50b9b4 100644 --- a/client/src/main/java/com/metamx/druid/curator/cache/SimplePathChildrenCacheFactory.java +++ b/server/src/main/java/io/druid/curator/cache/SimplePathChildrenCacheFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.cache; +package io.druid.curator.cache; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; diff --git a/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java b/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java new file mode 100644 index 000000000000..633b987a88a2 --- /dev/null +++ b/server/src/main/java/io/druid/curator/discovery/CuratorServiceAnnouncer.java @@ -0,0 +1,122 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator.discovery; + +import com.google.common.base.Throwables; +import com.google.common.collect.Maps; +import com.google.inject.Inject; +import com.metamx.emitter.EmittingLogger; +import io.druid.server.DruidNode; +import org.apache.curator.x.discovery.ServiceDiscovery; +import org.apache.curator.x.discovery.ServiceInstance; + +import java.util.Map; + +/** + * Uses the Curator Service Discovery recipe to announce services. + */ +public class CuratorServiceAnnouncer implements ServiceAnnouncer +{ + private static final EmittingLogger log = new EmittingLogger(CuratorServiceAnnouncer.class); + + private final ServiceDiscovery discovery; + private final Map> instanceMap = Maps.newHashMap(); + private final Object monitor = new Object(); + + @Inject + public CuratorServiceAnnouncer( + ServiceDiscovery discovery + ) + { + this.discovery = discovery; + } + + @Override + public void announce(DruidNode service) + { + final String serviceName = getServiceName(service); + + final ServiceInstance instance; + synchronized (monitor) { + if (instanceMap.containsKey(serviceName)) { + log.warn("Ignoring request to announce service[%s]", service); + return; + } else { + try { + instance = ServiceInstance.builder() + .name(serviceName) + .address(service.getHost()) + .port(service.getPort()) + .build(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + instanceMap.put(serviceName, instance); + } + } + + try { + log.info("Announcing service[%s]", service); + discovery.registerService(instance); + } + catch (Exception e) { + log.warn("Failed to announce service[%s]", service); + synchronized (monitor) { + instanceMap.remove(serviceName); + } + } + } + + @Override + public void unannounce(DruidNode service) + { + final String serviceName = getServiceName(service); + final ServiceInstance instance; + + synchronized (monitor) { + instance = instanceMap.get(serviceName); + if (instance == null) { + log.warn("Ignoring request to unannounce service[%s]", service); + return; + } + } + + log.info("Unannouncing service[%s]", service); + try { + discovery.unregisterService(instance); + } + catch (Exception e) { + log.makeAlert(e, "Failed to unannounce service[%s], zombie znode perhaps in existence.", serviceName) + .addData("service", service) + .emit(); + } + finally { + synchronized (monitor) { + instanceMap.remove(serviceName); + } + } + } + + private String getServiceName(DruidNode service) { + return service.getServiceName().replaceAll("/", ":"); + } +} diff --git a/server/src/main/java/io/druid/curator/discovery/DiscoveryModule.java b/server/src/main/java/io/druid/curator/discovery/DiscoveryModule.java new file mode 100644 index 000000000000..20d52e8e77fe --- /dev/null +++ b/server/src/main/java/io/druid/curator/discovery/DiscoveryModule.java @@ -0,0 +1,233 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator.discovery; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.TypeLiteral; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.metamx.common.lifecycle.Lifecycle; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.server.DruidNode; +import io.druid.server.initialization.CuratorDiscoveryConfig; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.x.discovery.ServiceDiscovery; +import org.apache.curator.x.discovery.ServiceDiscoveryBuilder; + +import javax.annotation.Nullable; +import java.lang.annotation.Annotation; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * The DiscoveryModule allows for the registration of Keys of DruidNode objects, which it intends to be + * automatically announced at the end of the lifecycle start. + * + * In order for this to work a ServiceAnnouncer instance *must* be injected and instantiated first. + * This can often be achieved by registering ServiceAnnouncer.class with the LifecycleModule. + */ +public class DiscoveryModule implements Module +{ + private static final String NAME = "DiscoveryModule:internal"; + + public final List>> nodesToAnnounce = new CopyOnWriteArrayList>>(); + public boolean configured = false; + + /** + * Requests that the un-annotated DruidNode instance be injected and published as part of the lifecycle. + * + * That is, this module will announce the DruidNode instance returned by + * injector.getInstance(Key.get(DruidNode.class)) automatically. + * Announcement will happen in the LAST stage of the Lifecycle + * + * @return this, for chaining. + */ + public DiscoveryModule registerDefault() + { + return registerKey(Key.get(new TypeLiteral>(){})); + } + + /** + * Requests that the annotated DruidNode instance be injected and published as part of the lifecycle. + * + * That is, this module will announce the DruidNode instance returned by + * injector.getInstance(Key.get(DruidNode.class, annotation)) automatically. + * Announcement will happen in the LAST stage of the Lifecycle + * + * @param annotation The annotation instance to use in finding the DruidNode instance, usually a Named annotation + * @return this, for chaining. + */ + public DiscoveryModule register(Annotation annotation) + { + return registerKey(Key.get(new TypeLiteral>(){}, annotation)); + } + + /** + * Requests that the annotated DruidNode instance be injected and published as part of the lifecycle. + * + * That is, this module will announce the DruidNode instance returned by + * injector.getInstance(Key.get(DruidNode.class, annotation)) automatically. + * Announcement will happen in the LAST stage of the Lifecycle + * + * @param annotation The annotation class to use in finding the DruidNode instance + * @return this, for chaining + */ + public DiscoveryModule register(Class annotation) + { + return registerKey(Key.get(new TypeLiteral>(){}, annotation)); + } + + /** + * Requests that the keyed DruidNode instance be injected and published as part of the lifecycle. + * + * That is, this module will announce the DruidNode instance returned by + * injector.getInstance(Key.get(DruidNode.class, annotation)) automatically. + * Announcement will happen in the LAST stage of the Lifecycle + * + * @param key The key to use in finding the DruidNode instance + * @return this, for chaining + */ + public DiscoveryModule registerKey(Key> key) + { + synchronized (nodesToAnnounce) { + Preconditions.checkState(!configured, "Cannot register key[%s] after configuration.", key); + } + nodesToAnnounce.add(key); + return this; + } + + @Override + public void configure(Binder binder) + { + synchronized (nodesToAnnounce) { + configured = true; + JsonConfigProvider.bind(binder, "druid.discovery.curator", CuratorDiscoveryConfig.class); + + binder.bind(CuratorServiceAnnouncer.class).in(LazySingleton.class); + + // We bind this eagerly so that it gets instantiated and registers stuff with Lifecycle as a side-effect + binder.bind(ServiceAnnouncer.class) + .to(Key.get(CuratorServiceAnnouncer.class, Names.named(NAME))) + .asEagerSingleton(); + } + } + + @Provides @LazySingleton @Named(NAME) + public CuratorServiceAnnouncer getServiceAnnouncer( + final CuratorServiceAnnouncer announcer, + final Injector injector, + final Lifecycle lifecycle + ) + { + lifecycle.addHandler( + new Lifecycle.Handler() + { + private volatile List> nodes = null; + + @Override + public void start() throws Exception + { + if (nodes == null) { + nodes = Lists.transform( + nodesToAnnounce, + new Function>, Supplier>() + { + @Nullable + @Override + public Supplier apply( + @Nullable Key> input + ) + { + return injector.getInstance(input); + } + } + ); + } + + for (Supplier node : nodes) { + announcer.announce(node.get()); + } + } + + @Override + public void stop() + { + if (nodes != null) { + for (Supplier node : nodes) { + announcer.unannounce(node.get()); + } + } + } + }, + Lifecycle.Stage.LAST + ); + + return announcer; + } + + @Provides @LazySingleton + public ServiceDiscovery getServiceDiscovery( + CuratorFramework curator, + Supplier config, + Lifecycle lifecycle + ) throws Exception + { + final ServiceDiscovery serviceDiscovery = + ServiceDiscoveryBuilder.builder(Void.class) + .basePath(config.get().getPath()) + .client(curator) + .build(); + + lifecycle.addHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + serviceDiscovery.start(); + } + + @Override + public void stop() + { + try { + serviceDiscovery.close(); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + }, + Lifecycle.Stage.LAST + ); + + return serviceDiscovery; + } +} diff --git a/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java b/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java new file mode 100644 index 000000000000..672b900d5bfb --- /dev/null +++ b/server/src/main/java/io/druid/curator/discovery/NoopServiceAnnouncer.java @@ -0,0 +1,40 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator.discovery; + +import io.druid.server.DruidNode; + +/** + * Does nothing. + */ +public class NoopServiceAnnouncer implements ServiceAnnouncer +{ + @Override + public void announce(DruidNode node) + { + + } + + @Override + public void unannounce(DruidNode node) + { + + } +} diff --git a/server/src/main/java/com/metamx/druid/db/DatabaseSegmentManagerConfig.java b/server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java similarity index 65% rename from server/src/main/java/com/metamx/druid/db/DatabaseSegmentManagerConfig.java rename to server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java index 25e90cd16bdc..46da8f5b0ff6 100644 --- a/server/src/main/java/com/metamx/druid/db/DatabaseSegmentManagerConfig.java +++ b/server/src/main/java/io/druid/curator/discovery/ServiceAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.curator.discovery; -import org.joda.time.Duration; -import org.skife.config.Config; -import org.skife.config.Default; +import io.druid.server.DruidNode; /** + * Announces our ability to serve a particular function. Multiple users may announce the same service, in which + * case they are treated as interchangeable instances of that service. */ -public abstract class DatabaseSegmentManagerConfig +public interface ServiceAnnouncer { - @Config("druid.database.segmentTable") - public abstract String getSegmentTable(); + public void announce(DruidNode node); - @Config("druid.database.poll.duration") - @Default("PT1M") - public abstract Duration getPollDuration(); + public void unannounce(DruidNode node); } diff --git a/client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManager.java b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java similarity index 97% rename from client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManager.java rename to server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java index 7c35ad126187..dcff14024e30 100644 --- a/client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManager.java +++ b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManager.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.inventory; +package io.druid.curator.inventory; import com.google.common.base.Function; import com.google.common.collect.Iterables; @@ -26,9 +26,9 @@ import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.ShutdownNowIgnoringExecutorService; -import com.metamx.druid.curator.cache.PathChildrenCacheFactory; -import com.metamx.druid.curator.cache.SimplePathChildrenCacheFactory; +import io.druid.curator.ShutdownNowIgnoringExecutorService; +import io.druid.curator.cache.PathChildrenCacheFactory; +import io.druid.curator.cache.SimplePathChildrenCacheFactory; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; diff --git a/client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerStrategy.java b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManagerStrategy.java similarity index 94% rename from client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerStrategy.java rename to server/src/main/java/io/druid/curator/inventory/CuratorInventoryManagerStrategy.java index 8cab619e16f8..785e35534dfb 100644 --- a/client/src/main/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerStrategy.java +++ b/server/src/main/java/io/druid/curator/inventory/CuratorInventoryManagerStrategy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.inventory; +package io.druid.curator.inventory; /** */ diff --git a/client/src/main/java/com/metamx/druid/curator/inventory/InventoryManagerConfig.java b/server/src/main/java/io/druid/curator/inventory/InventoryManagerConfig.java similarity index 95% rename from client/src/main/java/com/metamx/druid/curator/inventory/InventoryManagerConfig.java rename to server/src/main/java/io/druid/curator/inventory/InventoryManagerConfig.java index 418c402ff836..78158b713be5 100644 --- a/client/src/main/java/com/metamx/druid/curator/inventory/InventoryManagerConfig.java +++ b/server/src/main/java/io/druid/curator/inventory/InventoryManagerConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.inventory; +package io.druid.curator.inventory; /** */ diff --git a/server/src/main/java/com/metamx/druid/db/DatabaseRuleManager.java b/server/src/main/java/io/druid/db/DatabaseRuleManager.java similarity index 81% rename from server/src/main/java/com/metamx/druid/db/DatabaseRuleManager.java rename to server/src/main/java/io/druid/db/DatabaseRuleManager.java index 44ea802082cb..c6968362caaf 100644 --- a/server/src/main/java/com/metamx/druid/db/DatabaseRuleManager.java +++ b/server/src/main/java/io/druid/db/DatabaseRuleManager.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,28 +17,32 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.db; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.inject.Inject; import com.metamx.common.MapUtils; import com.metamx.common.concurrent.ScheduledExecutors; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.master.rules.PeriodLoadRule; -import com.metamx.druid.master.rules.Rule; - +import io.druid.concurrent.Execs; +import io.druid.guice.ManageLifecycle; +import io.druid.guice.annotations.Json; +import io.druid.server.master.rules.PeriodLoadRule; +import io.druid.server.master.rules.Rule; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Period; -import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.FoldController; import org.skife.jdbi.v2.Folder3; import org.skife.jdbi.v2.Handle; +import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.StatementContext; import org.skife.jdbi.v2.tweak.HandleCallback; @@ -52,12 +56,13 @@ /** */ +@ManageLifecycle public class DatabaseRuleManager { public static void createDefaultRule( - final DBI dbi, + final IDBI dbi, final String ruleTable, - final String defaultDatasource, + final String defaultTier, final ObjectMapper jsonMapper ) { @@ -72,7 +77,7 @@ public Void withHandle(Handle handle) throws Exception String.format( "SELECT id from %s where datasource='%s';", ruleTable, - defaultDatasource + defaultTier ) ); @@ -94,8 +99,8 @@ public Void withHandle(Handle handle) throws Exception ruleTable ) ) - .bind("id", String.format("%s_%s", defaultDatasource, version)) - .bind("dataSource", defaultDatasource) + .bind("id", String.format("%s_%s", defaultTier, version)) + .bind("dataSource", defaultTier) .bind("version", version) .bind("payload", jsonMapper.writeValueAsString(defaultRules)) .execute(); @@ -113,25 +118,28 @@ public Void withHandle(Handle handle) throws Exception private static final Logger log = new Logger(DatabaseRuleManager.class); private final ObjectMapper jsonMapper; - private final ScheduledExecutorService exec; - private final DatabaseRuleManagerConfig config; - private final DBI dbi; + private final Supplier config; + private final Supplier dbTables; + private final IDBI dbi; private final AtomicReference>> rules; + private volatile ScheduledExecutorService exec; + private final Object lock = new Object(); private volatile boolean started = false; + @Inject public DatabaseRuleManager( - ObjectMapper jsonMapper, - ScheduledExecutorService exec, - DatabaseRuleManagerConfig config, - DBI dbi + @Json ObjectMapper jsonMapper, + Supplier config, + Supplier dbTables, + IDBI dbi ) { this.jsonMapper = jsonMapper; - this.exec = exec; this.config = config; + this.dbTables = dbTables; this.dbi = dbi; this.rules = new AtomicReference>>( @@ -147,10 +155,13 @@ public void start() return; } + this.exec = Execs.scheduledSingleThreaded("DatabaseRuleManager-Exec--%d"); + + createDefaultRule(dbi, getRulesTable(), config.get().getDefaultTier(), jsonMapper); ScheduledExecutors.scheduleWithFixedDelay( exec, new Duration(0), - config.getRulesPollDuration(), + config.get().getPollDuration().toStandardDuration(), new Runnable() { @Override @@ -176,6 +187,8 @@ public void stop() rules.set(new ConcurrentHashMap>()); started = false; + exec.shutdownNow(); + exec = null; } } @@ -192,8 +205,11 @@ public Map> withHandle(Handle handle) throws Exception return handle.createQuery( // Return latest version rule by dataSource String.format( - "SELECT %1$s.dataSource, %1$s.payload FROM %1$s INNER JOIN(SELECT dataSource, max(version) as version FROM %1$s GROUP BY dataSource) ds ON %1$s.datasource = ds.datasource and %1$s.version = ds.version", - config.getRuleTable() + "SELECT r.dataSource, r.payload " + + "FROM %1$s r " + + "INNER JOIN(SELECT dataSource, max(version) as version, payload FROM %1$s GROUP BY dataSource) ds " + + "ON r.datasource = ds.datasource and r.version = ds.version", + getRulesTable() ) ).fold( Maps.>newHashMap(), @@ -255,8 +271,8 @@ public List getRulesWithDefault(final String dataSource) if (theRules.get(dataSource) != null) { retVal.addAll(theRules.get(dataSource)); } - if (theRules.get(config.getDefaultDatasource()) != null) { - retVal.addAll(theRules.get(config.getDefaultDatasource())); + if (theRules.get(config.get().getDefaultTier()) != null) { + retVal.addAll(theRules.get(config.get().getDefaultTier())); } return retVal; } @@ -275,7 +291,7 @@ public Void withHandle(Handle handle) throws Exception handle.createStatement( String.format( "INSERT INTO %s (id, dataSource, version, payload) VALUES (:id, :dataSource, :version, :payload)", - config.getRuleTable() + getRulesTable() ) ) .bind("id", String.format("%s_%s", dataSource, version)) @@ -303,4 +319,6 @@ public Void withHandle(Handle handle) throws Exception return true; } + + private String getRulesTable() {return dbTables.get().getRulesTable();} } diff --git a/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java b/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java new file mode 100644 index 000000000000..8b770a57955e --- /dev/null +++ b/server/src/main/java/io/druid/db/DatabaseRuleManagerConfig.java @@ -0,0 +1,44 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.db; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +/** + */ +public class DatabaseRuleManagerConfig +{ + @JsonProperty + private String defaultTier = "_default"; + + @JsonProperty + private Period pollDuration = new Period("PT1M"); + + public String getDefaultTier() + { + return defaultTier; + } + + public Period getPollDuration() + { + return pollDuration; + } +} diff --git a/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java b/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java new file mode 100644 index 000000000000..dfc25bea2836 --- /dev/null +++ b/server/src/main/java/io/druid/db/DatabaseRuleManagerProvider.java @@ -0,0 +1,85 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.db; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.metamx.common.lifecycle.Lifecycle; + +/** + */ +public class DatabaseRuleManagerProvider implements Provider +{ + private final ObjectMapper jsonMapper; + private final Supplier config; + private final Supplier dbTables; + private final DbConnector dbConnector; + private final Lifecycle lifecycle; + + @Inject + public DatabaseRuleManagerProvider( + ObjectMapper jsonMapper, + Supplier config, + Supplier dbTables, + DbConnector dbConnector, + Lifecycle lifecycle + ) + { + this.jsonMapper = jsonMapper; + this.config = config; + this.dbTables = dbTables; + this.dbConnector = dbConnector; + this.lifecycle = lifecycle; + } + + @Override + public DatabaseRuleManager get() + { + try { + lifecycle.addMaybeStartHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + dbConnector.createRulesTable(); + DatabaseRuleManager.createDefaultRule( + dbConnector.getDBI(), dbTables.get().getRulesTable(), config.get().getDefaultTier(), jsonMapper + ); + } + + @Override + public void stop() + { + + } + } + ); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + + return new DatabaseRuleManager(jsonMapper, config, dbTables, dbConnector.getDBI()); + } +} diff --git a/server/src/main/java/com/metamx/druid/db/DatabaseSegmentManager.java b/server/src/main/java/io/druid/db/DatabaseSegmentManager.java similarity index 88% rename from server/src/main/java/com/metamx/druid/db/DatabaseSegmentManager.java rename to server/src/main/java/io/druid/db/DatabaseSegmentManager.java index 2096e0bd70a6..203d082b806d 100644 --- a/server/src/main/java/com/metamx/druid/db/DatabaseSegmentManager.java +++ b/server/src/main/java/io/druid/db/DatabaseSegmentManager.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,30 +17,34 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.db; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Ordering; +import com.google.inject.Inject; import com.metamx.common.MapUtils; import com.metamx.common.concurrent.ScheduledExecutors; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.partition.PartitionChunk; +import io.druid.client.DruidDataSource; +import io.druid.concurrent.Execs; +import io.druid.guice.ManageLifecycle; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.Interval; import org.skife.jdbi.v2.Batch; -import org.skife.jdbi.v2.DBI; import org.skife.jdbi.v2.FoldController; import org.skife.jdbi.v2.Folder3; import org.skife.jdbi.v2.Handle; +import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.StatementContext; import org.skife.jdbi.v2.tweak.HandleCallback; @@ -55,6 +59,7 @@ /** */ +@ManageLifecycle public class DatabaseSegmentManager { private static final Logger log = new Logger(DatabaseSegmentManager.class); @@ -62,23 +67,26 @@ public class DatabaseSegmentManager private final Object lock = new Object(); private final ObjectMapper jsonMapper; - private final ScheduledExecutorService exec; - private final DatabaseSegmentManagerConfig config; + private final Supplier config; + private final Supplier dbTables; private final AtomicReference> dataSources; - private final DBI dbi; + private final IDBI dbi; + + private volatile ScheduledExecutorService exec; private volatile boolean started = false; + @Inject public DatabaseSegmentManager( ObjectMapper jsonMapper, - ScheduledExecutorService exec, - DatabaseSegmentManagerConfig config, - DBI dbi + Supplier config, + Supplier dbTables, + IDBI dbi ) { this.jsonMapper = jsonMapper; - this.exec = exec; this.config = config; + this.dbTables = dbTables; this.dataSources = new AtomicReference>( new ConcurrentHashMap() ); @@ -93,10 +101,13 @@ public void start() return; } + this.exec = Execs.scheduledSingleThreaded("DatabaseSegmentManager-Exec--%d"); + + final Duration delay = config.get().getPollDuration().toStandardDuration(); ScheduledExecutors.scheduleWithFixedDelay( exec, - new Duration(0), - config.getPollDuration(), + delay, + delay, new Runnable() { @Override @@ -121,6 +132,8 @@ public void stop() started = false; dataSources.set(new ConcurrentHashMap()); + exec.shutdownNow(); + exec = null; } } @@ -134,7 +147,7 @@ public boolean enableDatasource(final String ds) public VersionedIntervalTimeline withHandle(Handle handle) throws Exception { return handle.createQuery( - String.format("SELECT payload FROM %s WHERE dataSource = :dataSource", config.getSegmentTable()) + String.format("SELECT payload FROM %s WHERE dataSource = :dataSource", getSegmentsTable()) ) .bind("dataSource", ds) .fold( @@ -200,8 +213,8 @@ public Void withHandle(Handle handle) throws Exception for (DataSegment segment : segments) { batch.add( String.format( - "UPDATE %s SET used=true WHERE id = '%s'", - config.getSegmentTable(), + "UPDATE %s SET used=1 WHERE id = '%s'", + getSegmentsTable(), segment.getIdentifier() ) ); @@ -231,7 +244,7 @@ public boolean enableSegment(final String segmentId) public Void withHandle(Handle handle) throws Exception { handle.createStatement( - String.format("UPDATE %s SET used=true WHERE id = :id", config.getSegmentTable()) + String.format("UPDATE %s SET used=1 WHERE id = :id", getSegmentsTable()) ) .bind("id", segmentId) .execute(); @@ -265,7 +278,7 @@ public boolean removeDatasource(final String ds) public Void withHandle(Handle handle) throws Exception { handle.createStatement( - String.format("UPDATE %s SET used=false WHERE dataSource = :dataSource", config.getSegmentTable()) + String.format("UPDATE %s SET used=0 WHERE dataSource = :dataSource", getSegmentsTable()) ) .bind("dataSource", ds) .execute(); @@ -295,7 +308,7 @@ public boolean removeSegment(String ds, final String segmentID) public Void withHandle(Handle handle) throws Exception { handle.createStatement( - String.format("UPDATE %s SET used=false WHERE id = :segmentID", config.getSegmentTable()) + String.format("UPDATE %s SET used=0 WHERE id = :segmentID", getSegmentsTable()) ).bind("segmentID", segmentID) .execute(); @@ -351,7 +364,7 @@ public Collection getAllDatasourceNames() public List withHandle(Handle handle) throws Exception { return handle.createQuery( - String.format("SELECT DISTINCT(datasource) FROM %s", config.getSegmentTable()) + String.format("SELECT DISTINCT(datasource) FROM %s", getSegmentsTable()) ) .fold( Lists.newArrayList(), @@ -395,7 +408,7 @@ public void poll() public List> withHandle(Handle handle) throws Exception { return handle.createQuery( - String.format("SELECT payload FROM %s WHERE used=true", config.getSegmentTable()) + String.format("SELECT payload FROM %s WHERE used=1", getSegmentsTable()) ).list(); } } @@ -448,4 +461,8 @@ public List> withHandle(Handle handle) throws Exception log.error(e, "Problem polling DB."); } } + + private String getSegmentsTable() { + return dbTables.get().getSegmentsTable(); + } } \ No newline at end of file diff --git a/server/src/main/java/com/metamx/druid/index/v1/Searchable.java b/server/src/main/java/io/druid/db/DatabaseSegmentManagerConfig.java similarity index 70% rename from server/src/main/java/com/metamx/druid/index/v1/Searchable.java rename to server/src/main/java/io/druid/db/DatabaseSegmentManagerConfig.java index a4f6ad70c789..a87d2db59e5a 100644 --- a/server/src/main/java/com/metamx/druid/index/v1/Searchable.java +++ b/server/src/main/java/io/druid/db/DatabaseSegmentManagerConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,20 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.index.v1; +package io.druid.db; -import com.metamx.druid.index.brita.Filter; -import com.metamx.druid.query.search.SearchHit; -import com.metamx.druid.query.search.SearchQuery; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; /** */ -public interface Searchable +public class DatabaseSegmentManagerConfig { - public Iterable searchDimensions(SearchQuery query, Filter filter); + @JsonProperty + private Period pollDuration = new Period("PT1M"); + + public Period getPollDuration() + { + return pollDuration; + } } diff --git a/server/src/main/java/io/druid/db/DatabaseSegmentManagerProvider.java b/server/src/main/java/io/druid/db/DatabaseSegmentManagerProvider.java new file mode 100644 index 000000000000..c7cda31f8d5f --- /dev/null +++ b/server/src/main/java/io/druid/db/DatabaseSegmentManagerProvider.java @@ -0,0 +1,81 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.db; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.metamx.common.lifecycle.Lifecycle; + +/** + */ +public class DatabaseSegmentManagerProvider implements Provider +{ + private final ObjectMapper jsonMapper; + private final Supplier config; + private final Supplier dbTables; + private final DbConnector dbConnector; + private final Lifecycle lifecycle; + + @Inject + public DatabaseSegmentManagerProvider( + ObjectMapper jsonMapper, + Supplier config, + Supplier dbTables, + DbConnector dbConnector, + Lifecycle lifecycle + ) + { + this.jsonMapper = jsonMapper; + this.config = config; + this.dbTables = dbTables; + this.dbConnector = dbConnector; + this.lifecycle = lifecycle; + } + + @Override + public DatabaseSegmentManager get() + { + lifecycle.addHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + dbConnector.createSegmentTable(); + } + + @Override + public void stop() + { + + } + } + ); + + return new DatabaseSegmentManager( + jsonMapper, + config, + dbTables, + dbConnector.getDBI() + ); + } +} diff --git a/server/src/main/java/io/druid/guice/AWSModule.java b/server/src/main/java/io/druid/guice/AWSModule.java new file mode 100644 index 000000000000..9a9b6d3b0f0c --- /dev/null +++ b/server/src/main/java/io/druid/guice/AWSModule.java @@ -0,0 +1,76 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2Client; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.ProvisionException; +import io.druid.segment.loading.AWSCredentialsConfig; +import org.jets3t.service.S3ServiceException; +import org.jets3t.service.impl.rest.httpclient.RestS3Service; + +/** + */ +public class AWSModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.s3", AWSCredentialsConfig.class); + } + + @Provides + @LazySingleton + public AWSCredentials getAWSCredentials(AWSCredentialsConfig config) + { + return new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey()); + } + + @Provides + @LazySingleton + public org.jets3t.service.security.AWSCredentials getJets3tAWSCredentials(AWSCredentialsConfig config) + { + return new org.jets3t.service.security.AWSCredentials(config.getAccessKey(), config.getSecretKey()); + } + + @Provides + @LazySingleton + public RestS3Service getRestS3Service(org.jets3t.service.security.AWSCredentials credentials) + { + try { + return new RestS3Service(credentials); + } + catch (S3ServiceException e) { + throw new ProvisionException("Unable to create a RestS3Service", e); + } + } + + @Provides + @LazySingleton + public AmazonEC2 getEc2Client(AWSCredentials credentials) + { + return new AmazonEC2Client(credentials); + } +} diff --git a/server/src/main/java/io/druid/guice/AnnouncerModule.java b/server/src/main/java/io/druid/guice/AnnouncerModule.java new file mode 100644 index 000000000000..379cc29392e9 --- /dev/null +++ b/server/src/main/java/io/druid/guice/AnnouncerModule.java @@ -0,0 +1,54 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import io.druid.concurrent.Execs; +import io.druid.curator.announcement.Announcer; +import io.druid.server.coordination.BatchDataSegmentAnnouncer; +import io.druid.server.coordination.DataSegmentAnnouncer; +import io.druid.server.coordination.DataSegmentAnnouncerProvider; +import io.druid.server.coordination.SingleDataSegmentAnnouncer; +import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; +import org.apache.curator.framework.CuratorFramework; + +/** + */ +public class AnnouncerModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.announcer", BatchDataSegmentAnnouncerConfig.class); + JsonConfigProvider.bind(binder, "druid.announcer", DataSegmentAnnouncerProvider.class); + binder.bind(DataSegmentAnnouncer.class).toProvider(DataSegmentAnnouncerProvider.class); + binder.bind(BatchDataSegmentAnnouncer.class).in(ManageLifecycleLast.class); + binder.bind(SingleDataSegmentAnnouncer.class).in(ManageLifecycleLast.class); + } + + @Provides + @ManageLifecycle + public Announcer getAnnouncer(CuratorFramework curator) + { + return new Announcer(curator, Execs.singleThreaded("Announcer-%s")); + } +} diff --git a/server/src/main/java/io/druid/guice/BrokerModule.java b/server/src/main/java/io/druid/guice/BrokerModule.java new file mode 100644 index 000000000000..8ebc51f9832e --- /dev/null +++ b/server/src/main/java/io/druid/guice/BrokerModule.java @@ -0,0 +1,75 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.TypeLiteral; +import io.druid.client.BrokerServerView; +import io.druid.client.CachingClusteredClient; +import io.druid.client.TimelineServerView; +import io.druid.client.cache.Cache; +import io.druid.client.cache.CacheProvider; +import io.druid.collections.ResourceHolder; +import io.druid.collections.StupidPool; +import io.druid.guice.annotations.Global; +import io.druid.query.MapQueryToolChestWarehouse; +import io.druid.query.QueryToolChestWarehouse; + +import java.nio.ByteBuffer; + +/** + */ +public class BrokerModule implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(QueryToolChestWarehouse.class).to(MapQueryToolChestWarehouse.class); + + binder.bind(CachingClusteredClient.class).in(LazySingleton.class); + binder.bind(TimelineServerView.class).to(BrokerServerView.class).in(LazySingleton.class); + + binder.bind(Cache.class).toProvider(CacheProvider.class).in(ManageLifecycle.class); + JsonConfigProvider.bind(binder, "druid.broker.cache", CacheProvider.class); + + // This is a workaround and needs to be made better in the near future. + binder.bind( + new TypeLiteral>() + { + } + ).annotatedWith(Global.class).toInstance(new NoopStupidPool(null)); + } + + private static class NoopStupidPool extends StupidPool + { + public NoopStupidPool(Supplier generator) + { + super(generator); + } + + @Override + public ResourceHolder take() + { + throw new UnsupportedOperationException(); + } + } +} diff --git a/server/src/main/java/io/druid/guice/CassandraDataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/CassandraDataSegmentPusherProvider.java new file mode 100644 index 000000000000..7ce7e2219fa6 --- /dev/null +++ b/server/src/main/java/io/druid/guice/CassandraDataSegmentPusherProvider.java @@ -0,0 +1,47 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.cassandra.CassandraDataSegmentConfig; +import io.druid.segment.loading.cassandra.CassandraDataSegmentPusher; + +import javax.validation.constraints.NotNull; + +/** + */ +public class CassandraDataSegmentPusherProvider implements DataSegmentPusherProvider +{ + @JacksonInject + @NotNull + private CassandraDataSegmentConfig config = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public DataSegmentPusher get() + { + return new CassandraDataSegmentPusher(config, jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/guice/CoordinatorModule.java b/server/src/main/java/io/druid/guice/CoordinatorModule.java new file mode 100644 index 000000000000..df0e36a53c5e --- /dev/null +++ b/server/src/main/java/io/druid/guice/CoordinatorModule.java @@ -0,0 +1,80 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.metamx.common.concurrent.ScheduledExecutorFactory; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.db.DatabaseRuleManager; +import io.druid.db.DatabaseRuleManagerConfig; +import io.druid.db.DatabaseRuleManagerProvider; +import io.druid.db.DatabaseSegmentManager; +import io.druid.db.DatabaseSegmentManagerConfig; +import io.druid.db.DatabaseSegmentManagerProvider; +import io.druid.server.http.MasterRedirectInfo; +import io.druid.server.http.RedirectFilter; +import io.druid.server.http.RedirectInfo; +import io.druid.server.http.RedirectServlet; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.DruidMasterConfig; +import io.druid.server.master.LoadQueueTaskMaster; +import org.apache.curator.framework.CuratorFramework; + +/** + */ +public class CoordinatorModule implements Module +{ + @Override + public void configure(Binder binder) + { + ConfigProvider.bind(binder, DruidMasterConfig.class); + + JsonConfigProvider.bind(binder, "druid.manager.segment", DatabaseSegmentManagerConfig.class); + JsonConfigProvider.bind(binder, "druid.manager.rules", DatabaseRuleManagerConfig.class); + + binder.bind(RedirectServlet.class).in(LazySingleton.class); + binder.bind(RedirectFilter.class).in(LazySingleton.class); + + binder.bind(DatabaseSegmentManager.class) + .toProvider(DatabaseSegmentManagerProvider.class) + .in(ManageLifecycle.class); + + binder.bind(DatabaseRuleManager.class) + .toProvider(DatabaseRuleManagerProvider.class) + .in(ManageLifecycle.class); + + binder.bind(IndexingServiceClient.class).in(LazySingleton.class); + + binder.bind(RedirectInfo.class).to(MasterRedirectInfo.class).in(LazySingleton.class); + + binder.bind(DruidMaster.class); + } + + @Provides @LazySingleton + public LoadQueueTaskMaster getLoadQueueTaskMaster( + CuratorFramework curator, ObjectMapper jsonMapper, ScheduledExecutorFactory factory, DruidMasterConfig config + ) + { + return new LoadQueueTaskMaster(curator, jsonMapper, factory.create(1, "Master-PeonExec--%d"), config); + } +} diff --git a/server/src/main/java/io/druid/guice/DataSegmentPullerModule.java b/server/src/main/java/io/druid/guice/DataSegmentPullerModule.java new file mode 100644 index 000000000000..3fd70ceecab4 --- /dev/null +++ b/server/src/main/java/io/druid/guice/DataSegmentPullerModule.java @@ -0,0 +1,81 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.segment.loading.HdfsDataSegmentPuller; +import io.druid.segment.loading.LocalDataSegmentPuller; +import io.druid.segment.loading.OmniSegmentLoader; +import io.druid.segment.loading.S3DataSegmentPuller; +import io.druid.segment.loading.SegmentLoader; +import io.druid.segment.loading.cassandra.CassandraDataSegmentConfig; +import io.druid.segment.loading.cassandra.CassandraDataSegmentPuller; +import org.apache.hadoop.conf.Configuration; + +/** + */ +public class DataSegmentPullerModule implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(SegmentLoader.class).to(OmniSegmentLoader.class).in(LazySingleton.class); + + bindDeepStorageLocal(binder); + bindDeepStorageS3(binder); + bindDeepStorageHdfs(binder); + bindDeepStorageCassandra(binder); + } + + private static void bindDeepStorageLocal(Binder binder) + { + DruidBinders.dataSegmentPullerBinder(binder) + .addBinding("local") + .to(LocalDataSegmentPuller.class) + .in(LazySingleton.class); + } + + private static void bindDeepStorageS3(Binder binder) + { + DruidBinders.dataSegmentPullerBinder(binder) + .addBinding("s3_zip") + .to(S3DataSegmentPuller.class) + .in(LazySingleton.class); + } + + private static void bindDeepStorageHdfs(Binder binder) + { + DruidBinders.dataSegmentPullerBinder(binder) + .addBinding("hdfs") + .to(HdfsDataSegmentPuller.class) + .in(LazySingleton.class); + binder.bind(Configuration.class).toInstance(new Configuration()); + } + + private static void bindDeepStorageCassandra(Binder binder) + { + DruidBinders.dataSegmentPullerBinder(binder) + .addBinding("c*") + .to(CassandraDataSegmentPuller.class) + .in(LazySingleton.class); + ConfigProvider.bind(binder, CassandraDataSegmentConfig.class); + } +} diff --git a/server/src/main/java/io/druid/guice/DataSegmentPusherModule.java b/server/src/main/java/io/druid/guice/DataSegmentPusherModule.java new file mode 100644 index 000000000000..cd65b35b7ee4 --- /dev/null +++ b/server/src/main/java/io/druid/guice/DataSegmentPusherModule.java @@ -0,0 +1,48 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.HdfsDataSegmentPusherConfig; +import io.druid.segment.loading.S3DataSegmentPusherConfig; +import io.druid.segment.loading.cassandra.CassandraDataSegmentConfig; +import org.apache.hadoop.conf.Configuration; + +/** + */ +public class DataSegmentPusherModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.pusher", DataSegmentPusherProvider.class); + + JsonConfigProvider.bind(binder, "druid.pusher.s3", S3DataSegmentPusherConfig.class); + binder.bind(Configuration.class).toInstance(new Configuration()); + + JsonConfigProvider.bind(binder, "druid.pusher.hdfs", HdfsDataSegmentPusherConfig.class); + + JsonConfigProvider.bind(binder, "druid.pusher.cassandra", CassandraDataSegmentConfig.class); + + binder.bind(DataSegmentPusher.class).toProvider(DataSegmentPusherProvider.class); + } +} diff --git a/server/src/main/java/io/druid/guice/DataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/DataSegmentPusherProvider.java new file mode 100644 index 000000000000..564f5002e940 --- /dev/null +++ b/server/src/main/java/io/druid/guice/DataSegmentPusherProvider.java @@ -0,0 +1,37 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; +import io.druid.segment.loading.DataSegmentPusher; + +/** + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = LocalDataSegmentPusherProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "s3_zip", value = S3DataSegmentPusherProvider.class), + @JsonSubTypes.Type(name = "hdfs", value = HdfsDataSegmentPusherProvider.class), + @JsonSubTypes.Type(name = "c*", value = CassandraDataSegmentPusherProvider.class) +}) +public interface DataSegmentPusherProvider extends Provider +{ +} diff --git a/server/src/main/java/io/druid/guice/DruidBinders.java b/server/src/main/java/io/druid/guice/DruidBinders.java new file mode 100644 index 000000000000..5bc84227cd58 --- /dev/null +++ b/server/src/main/java/io/druid/guice/DruidBinders.java @@ -0,0 +1,58 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.MapBinder; +import io.druid.query.Query; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryToolChest; +import io.druid.segment.loading.DataSegmentPuller; + +/** + */ +public class DruidBinders +{ + public static MapBinder, QueryRunnerFactory> queryRunnerFactoryBinder(Binder binder) + { + return MapBinder.newMapBinder( + binder, new TypeLiteral>() + { + }, TypeLiteral.get(QueryRunnerFactory.class) + ); + } + + public static MapBinder, QueryToolChest> queryToolChestBinder(Binder binder) + { + return MapBinder.newMapBinder( + binder, new TypeLiteral>() + { + }, new TypeLiteral() + { + } + ); + } + + public static MapBinder dataSegmentPullerBinder(Binder binder) + { + return MapBinder.newMapBinder(binder, String.class, DataSegmentPuller.class); + } +} diff --git a/server/src/main/java/io/druid/guice/DruidProcessingModule.java b/server/src/main/java/io/druid/guice/DruidProcessingModule.java new file mode 100644 index 000000000000..cf0157de9e0d --- /dev/null +++ b/server/src/main/java/io/druid/guice/DruidProcessingModule.java @@ -0,0 +1,136 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.ProvisionException; +import com.metamx.common.concurrent.ExecutorServiceConfig; +import com.metamx.common.logger.Logger; +import com.metamx.emitter.service.ServiceEmitter; +import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.collections.StupidPool; +import io.druid.concurrent.Execs; +import io.druid.guice.annotations.Global; +import io.druid.guice.annotations.Processing; +import io.druid.query.MetricsEmittingExecutorService; +import io.druid.server.DruidProcessingConfig; + +import java.lang.reflect.InvocationTargetException; +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +/** + */ +public class DruidProcessingModule implements Module +{ + private static final Logger log = new Logger(DruidProcessingModule.class); + + @Override + public void configure(Binder binder) + { + ConfigProvider.bind(binder, DruidProcessingConfig.class, ImmutableMap.of("base_path", "druid.processing")); + binder.bind(ExecutorServiceConfig.class).to(DruidProcessingConfig.class); + } + + @Provides + @Processing + @ManageLifecycle + public ExecutorService getProcessingExecutorService(ExecutorServiceConfig config, ServiceEmitter emitter) + { + return new MetricsEmittingExecutorService( + Executors.newFixedThreadPool(config.getNumThreads(), Execs.makeThreadFactory(config.getFormatString())), + emitter, + new ServiceMetricEvent.Builder() + ); + } + + @Provides + @LazySingleton + @Global + public StupidPool getIntermediateResultsPool(DruidProcessingConfig config) + { + try { + Class vmClass = Class.forName("sun.misc.VM"); + Object maxDirectMemoryObj = vmClass.getMethod("maxDirectMemory").invoke(null); + + if (maxDirectMemoryObj == null || !(maxDirectMemoryObj instanceof Number)) { + log.info("Cannot determine maxDirectMemory from[%s]", maxDirectMemoryObj); + } else { + long maxDirectMemory = ((Number) maxDirectMemoryObj).longValue(); + + final long memoryNeeded = (long) config.intermediateComputeSizeBytes() * (config.getNumThreads() + 1); + if (maxDirectMemory < memoryNeeded) { + throw new ProvisionException( + String.format( + "Not enough direct memory. Please adjust -XX:MaxDirectMemorySize or druid.computation.buffer.size: " + + "maxDirectMemory[%,d], memoryNeeded[%,d], druid.computation.buffer.size[%,d], numThreads[%,d]", + maxDirectMemory, memoryNeeded, config.intermediateComputeSizeBytes(), config.getNumThreads() + ) + ); + } + } + } + catch (ClassNotFoundException e) { + log.info("No VM class, cannot do memory check."); + } + catch (NoSuchMethodException e) { + log.info("VM.maxDirectMemory doesn't exist, cannot do memory check."); + } + catch (InvocationTargetException e) { + log.warn(e, "static method shouldn't throw this"); + } + catch (IllegalAccessException e) { + log.warn(e, "public method, shouldn't throw this"); + } + + return new IntermediateProcessingBufferPool(config.intermediateComputeSizeBytes()); + } + + private static class IntermediateProcessingBufferPool extends StupidPool + { + private static final Logger log = new Logger(IntermediateProcessingBufferPool.class); + + public IntermediateProcessingBufferPool(final int computationBufferSize) + { + super( + new Supplier() + { + final AtomicLong count = new AtomicLong(0); + + @Override + public ByteBuffer get() + { + log.info( + "Allocating new intermediate processing buffer[%,d] of size[%,d]", + count.getAndIncrement(), computationBufferSize + ); + return ByteBuffer.allocateDirect(computationBufferSize); + } + } + ); + } + } +} diff --git a/server/src/main/java/io/druid/guice/HdfsDataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/HdfsDataSegmentPusherProvider.java new file mode 100644 index 000000000000..c6cbf50d16a2 --- /dev/null +++ b/server/src/main/java/io/druid/guice/HdfsDataSegmentPusherProvider.java @@ -0,0 +1,52 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.HdfsDataSegmentPusher; +import io.druid.segment.loading.HdfsDataSegmentPusherConfig; +import org.apache.hadoop.conf.Configuration; + +import javax.validation.constraints.NotNull; + +/** + */ +public class HdfsDataSegmentPusherProvider implements DataSegmentPusherProvider +{ + @JacksonInject + @NotNull + private HdfsDataSegmentPusherConfig hdfsDataSegmentPusherConfig = null; + + @JacksonInject + @NotNull + private Configuration config = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public DataSegmentPusher get() + { + return new HdfsDataSegmentPusher(hdfsDataSegmentPusherConfig, config, jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/guice/HistoricalModule.java b/server/src/main/java/io/druid/guice/HistoricalModule.java new file mode 100644 index 000000000000..0e00526351bd --- /dev/null +++ b/server/src/main/java/io/druid/guice/HistoricalModule.java @@ -0,0 +1,40 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.metamx.common.logger.Logger; +import io.druid.server.coordination.ServerManager; +import io.druid.server.coordination.ZkCoordinator; + +/** + */ +public class HistoricalModule implements Module +{ + private static final Logger log = new Logger(HistoricalModule.class); + + @Override + public void configure(Binder binder) + { + binder.bind(ServerManager.class).in(LazySingleton.class); + binder.bind(ZkCoordinator.class).in(ManageLifecycle.class); + } +} diff --git a/server/src/main/java/io/druid/guice/HttpClientModule.java b/server/src/main/java/io/druid/guice/HttpClientModule.java new file mode 100644 index 000000000000..8bf7c70b6980 --- /dev/null +++ b/server/src/main/java/io/druid/guice/HttpClientModule.java @@ -0,0 +1,171 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Binding; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.Provider; +import com.google.inject.TypeLiteral; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.http.client.HttpClient; +import com.metamx.http.client.HttpClientConfig; +import com.metamx.http.client.HttpClientInit; +import io.druid.guice.annotations.Global; +import org.joda.time.Duration; +import org.joda.time.Period; + +import javax.net.ssl.SSLContext; +import javax.validation.constraints.Min; +import java.lang.annotation.Annotation; + +/** + */ +public class HttpClientModule implements Module +{ + public static HttpClientModule global() + { + return new HttpClientModule("druid.global.http", Global.class); + } + + private final String propertyPrefix; + private Annotation annotation = null; + private Class annotationClazz = null; + + public HttpClientModule(String propertyPrefix) + { + this.propertyPrefix = propertyPrefix; + } + + public HttpClientModule(String propertyPrefix, Class annotation) + { + this.propertyPrefix = propertyPrefix; + this.annotationClazz = annotation; + } + + public HttpClientModule(String propertyPrefix, Annotation annotation) + { + this.propertyPrefix = propertyPrefix; + this.annotation = annotation; + } + + @Override + public void configure(Binder binder) + { + if (annotation != null) { + JsonConfigProvider.bind(binder, propertyPrefix, DruidHttpClientConfig.class, annotation); + binder.bind(HttpClient.class) + .annotatedWith(annotation) + .toProvider(new HttpClientProvider(annotation)) + .in(LazySingleton.class); + } + else if (annotationClazz != null) { + JsonConfigProvider.bind(binder, propertyPrefix, DruidHttpClientConfig.class, annotationClazz); + binder.bind(HttpClient.class) + .annotatedWith(annotationClazz) + .toProvider(new HttpClientProvider(annotationClazz)) + .in(LazySingleton.class); + } + else { + JsonConfigProvider.bind(binder, propertyPrefix, DruidHttpClientConfig.class); + binder.bind(HttpClient.class) + .toProvider(new HttpClientProvider()) + .in(LazySingleton.class); + } + } + + public static class DruidHttpClientConfig + { + @JsonProperty + @Min(0) + private int numConnections = 5; + + @JsonProperty + private Period readTimeout = null; + + public int getNumConnections() + { + return numConnections; + } + + public Duration getReadTimeout() + { + return readTimeout == null ? null : readTimeout.toStandardDuration(); + } + } + + public static class HttpClientProvider implements Provider + { + private final Key> configKey; + private final Key sslContextKey; + + private Provider> configProvider; + private Provider lifecycleProvider; + private Binding sslContextBinding; + + public HttpClientProvider() + { + configKey = Key.get(new TypeLiteral>(){}); + sslContextKey = Key.get(SSLContext.class); + } + + public HttpClientProvider(Annotation annotation) + { + configKey = Key.get(new TypeLiteral>(){}, annotation); + sslContextKey = Key.get(SSLContext.class, annotation); + } + + public HttpClientProvider(Class annotation) + { + configKey = Key.get(new TypeLiteral>(){}, annotation); + sslContextKey = Key.get(SSLContext.class, annotation); + } + + @Inject + public void configure(Injector injector) + { + configProvider = injector.getProvider(configKey); + sslContextBinding = injector.getExistingBinding(sslContextKey); + lifecycleProvider = injector.getProvider(Lifecycle.class); + } + + @Override + public HttpClient get() + { + final DruidHttpClientConfig config = configProvider.get().get(); + + final HttpClientConfig.Builder builder = HttpClientConfig + .builder() + .withNumConnections(config.getNumConnections()) + .withReadTimeout(config.getReadTimeout()); + + if (sslContextBinding != null) { + builder.withSslContext(sslContextBinding.getProvider().get()); + } + + return HttpClientInit.createClient(builder.build(), lifecycleProvider.get()); + } + } +} diff --git a/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java b/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java new file mode 100644 index 000000000000..60e0c15281b5 --- /dev/null +++ b/server/src/main/java/io/druid/guice/IndexingServiceDiscoveryModule.java @@ -0,0 +1,83 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.TypeLiteral; +import io.druid.client.indexing.IndexingService; +import io.druid.client.indexing.IndexingServiceSelector; +import io.druid.client.indexing.IndexingServiceSelectorConfig; +import io.druid.client.selector.DiscoverySelector; +import io.druid.client.selector.Server; +import org.apache.curator.x.discovery.ServiceDiscovery; +import org.apache.curator.x.discovery.ServiceInstance; +import org.apache.curator.x.discovery.ServiceProvider; + +import java.io.IOException; + +/** + */ +public class IndexingServiceDiscoveryModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.selectors.indexing", IndexingServiceSelectorConfig.class); + binder.bind(new TypeLiteral>(){}) + .annotatedWith(IndexingService.class) + .to(IndexingServiceSelector.class); + + binder.bind(IndexingServiceSelector.class).in(ManageLifecycle.class); + } + + @Provides + @LazySingleton @IndexingService + public ServiceProvider getServiceProvider( + IndexingServiceSelectorConfig config, + ServiceDiscovery serviceDiscovery + ) + { + if (config.getServiceName() == null) { + return new ServiceProvider() + { + @Override + public void start() throws Exception + { + + } + + @Override + public ServiceInstance getInstance() throws Exception + { + return null; + } + + @Override + public void close() throws IOException + { + + } + }; + } + return serviceDiscovery.serviceProviderBuilder().serviceName(config.getServiceName()).build(); + } +} diff --git a/server/src/main/java/io/druid/guice/LocalDataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/LocalDataSegmentPusherProvider.java new file mode 100644 index 000000000000..3af8510dd1d7 --- /dev/null +++ b/server/src/main/java/io/druid/guice/LocalDataSegmentPusherProvider.java @@ -0,0 +1,43 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.LocalDataSegmentPusher; +import io.druid.segment.loading.LocalDataSegmentPusherConfig; + +import javax.validation.constraints.NotNull; + +/** + */ +public class LocalDataSegmentPusherProvider extends LocalDataSegmentPusherConfig implements DataSegmentPusherProvider +{ + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public DataSegmentPusher get() + { + return new LocalDataSegmentPusher(this, jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java new file mode 100644 index 000000000000..0f9970a6f301 --- /dev/null +++ b/server/src/main/java/io/druid/guice/QueryRunnerFactoryModule.java @@ -0,0 +1,70 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.multibindings.MapBinder; +import io.druid.query.Query; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.groupby.GroupByQueryEngine; +import io.druid.query.groupby.GroupByQueryRunnerFactory; +import io.druid.query.metadata.SegmentMetadataQueryRunnerFactory; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; +import io.druid.query.search.SearchQueryRunnerFactory; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import io.druid.query.timeboundary.TimeBoundaryQueryRunnerFactory; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryRunnerFactory; + +import java.util.Map; + +/** + */ +public class QueryRunnerFactoryModule extends QueryToolChestModule +{ + private static final Map, Class> mappings = + ImmutableMap., Class>builder() + .put(TimeseriesQuery.class, TimeseriesQueryRunnerFactory.class) + .put(SearchQuery.class, SearchQueryRunnerFactory.class) + .put(TimeBoundaryQuery.class, TimeBoundaryQueryRunnerFactory.class) + .put(SegmentMetadataQuery.class, SegmentMetadataQueryRunnerFactory.class) + .put(GroupByQuery.class, GroupByQueryRunnerFactory.class) + .build(); + + @Override + public void configure(Binder binder) + { + super.configure(binder); + + final MapBinder, QueryRunnerFactory> queryFactoryBinder = DruidBinders.queryRunnerFactoryBinder( + binder + ); + + for (Map.Entry, Class> entry : mappings.entrySet()) { + queryFactoryBinder.addBinding(entry.getKey()).to(entry.getValue()); + binder.bind(entry.getValue()).in(LazySingleton.class); + } + + binder.bind(GroupByQueryEngine.class).in(LazySingleton.class); + } +} diff --git a/server/src/main/java/io/druid/guice/QueryToolChestModule.java b/server/src/main/java/io/druid/guice/QueryToolChestModule.java new file mode 100644 index 000000000000..4da4eb2537f8 --- /dev/null +++ b/server/src/main/java/io/druid/guice/QueryToolChestModule.java @@ -0,0 +1,69 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.multibindings.MapBinder; +import io.druid.query.Query; +import io.druid.query.QueryToolChest; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.groupby.GroupByQueryConfig; +import io.druid.query.groupby.GroupByQueryQueryToolChest; +import io.druid.query.metadata.SegmentMetadataQueryQueryToolChest; +import io.druid.query.metadata.metadata.SegmentMetadataQuery; +import io.druid.query.search.SearchQueryQueryToolChest; +import io.druid.query.search.search.SearchQuery; +import io.druid.query.search.search.SearchQueryConfig; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import io.druid.query.timeboundary.TimeBoundaryQueryQueryToolChest; +import io.druid.query.timeseries.TimeseriesQuery; +import io.druid.query.timeseries.TimeseriesQueryQueryToolChest; + +import java.util.Map; + +/** + */ +public class QueryToolChestModule implements Module +{ + public final Map, Class> mappings = + ImmutableMap., Class>builder() + .put(TimeseriesQuery.class, TimeseriesQueryQueryToolChest.class) + .put(SearchQuery.class, SearchQueryQueryToolChest.class) + .put(TimeBoundaryQuery.class, TimeBoundaryQueryQueryToolChest.class) + .put(SegmentMetadataQuery.class, SegmentMetadataQueryQueryToolChest.class) + .put(GroupByQuery.class, GroupByQueryQueryToolChest.class) + .build(); + + @Override + public void configure(Binder binder) + { + MapBinder, QueryToolChest> toolChests = DruidBinders.queryToolChestBinder(binder); + + for (Map.Entry, Class> entry : mappings.entrySet()) { + toolChests.addBinding(entry.getKey()).to(entry.getValue()); + binder.bind(entry.getValue()).in(LazySingleton.class); + } + + JsonConfigProvider.bind(binder, "druid.query.groupBy", GroupByQueryConfig.class); + JsonConfigProvider.bind(binder, "druid.query.search", SearchQueryConfig.class); + } +} diff --git a/server/src/main/java/io/druid/guice/QueryableModule.java b/server/src/main/java/io/druid/guice/QueryableModule.java new file mode 100644 index 000000000000..184f80a1b582 --- /dev/null +++ b/server/src/main/java/io/druid/guice/QueryableModule.java @@ -0,0 +1,66 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import io.druid.initialization.DruidModule; +import io.druid.query.QuerySegmentWalker; +import io.druid.server.QueryServlet; +import io.druid.server.log.EmittingRequestLoggerProvider; +import io.druid.server.log.FileRequestLoggerProvider; +import io.druid.server.log.RequestLogger; +import io.druid.server.log.RequestLoggerProvider; + +import java.util.Arrays; +import java.util.List; + +/** + */ +public class QueryableModule implements DruidModule +{ + private final Class walkerClass; + + public QueryableModule( + Class walkerClass + ) + { + this.walkerClass = walkerClass; + } + + @Override + public void configure(Binder binder) + { + binder.bind(QueryServlet.class).in(LazySingleton.class); + binder.bind(QuerySegmentWalker.class).to(walkerClass).in(LazySingleton.class); + binder.bind(RequestLogger.class).toProvider(RequestLoggerProvider.class).in(ManageLifecycle.class); + JsonConfigProvider.bind(binder, "druid.request.logging", RequestLoggerProvider.class); + } + + @Override + public List getJacksonModules() + { + return Arrays.asList( + new SimpleModule("QueryableModule") + .registerSubtypes(EmittingRequestLoggerProvider.class, FileRequestLoggerProvider.class) + ); + } +} diff --git a/server/src/main/java/io/druid/guice/S3DataSegmentPusherProvider.java b/server/src/main/java/io/druid/guice/S3DataSegmentPusherProvider.java new file mode 100644 index 000000000000..2cb9765cbf7f --- /dev/null +++ b/server/src/main/java/io/druid/guice/S3DataSegmentPusherProvider.java @@ -0,0 +1,52 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.S3DataSegmentPusher; +import io.druid.segment.loading.S3DataSegmentPusherConfig; +import org.jets3t.service.impl.rest.httpclient.RestS3Service; + +import javax.validation.constraints.NotNull; + +/** + */ +public class S3DataSegmentPusherProvider implements DataSegmentPusherProvider +{ + @JacksonInject + @NotNull + private RestS3Service restS3Service = null; + + @JacksonInject + @NotNull + private S3DataSegmentPusherConfig config = null; + + @JacksonInject + @NotNull + private ObjectMapper jsonMapper = null; + + @Override + public DataSegmentPusher get() + { + return new S3DataSegmentPusher(restS3Service, config, jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/guice/ServerModule.java b/server/src/main/java/io/druid/guice/ServerModule.java new file mode 100644 index 000000000000..3417ef3d82ec --- /dev/null +++ b/server/src/main/java/io/druid/guice/ServerModule.java @@ -0,0 +1,70 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.fasterxml.jackson.databind.jsontype.NamedType; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.metamx.common.concurrent.ScheduledExecutorFactory; +import com.metamx.common.concurrent.ScheduledExecutors; +import com.metamx.common.lifecycle.Lifecycle; +import io.druid.guice.annotations.Self; +import io.druid.initialization.DruidModule; +import io.druid.server.DruidNode; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.partition.LinearShardSpec; +import io.druid.timeline.partition.NumberedShardSpec; +import io.druid.timeline.partition.SingleDimensionShardSpec; + +import java.util.Arrays; +import java.util.List; + +/** + */ +public class ServerModule implements DruidModule +{ + @Override + public void configure(Binder binder) + { + ConfigProvider.bind(binder, ZkPathsConfig.class); + + JsonConfigProvider.bind(binder, "druid", DruidNode.class, Self.class); + } + + @Provides @LazySingleton + public ScheduledExecutorFactory getScheduledExecutorFactory(Lifecycle lifecycle) + { + return ScheduledExecutors.createFactory(lifecycle); + } + + @Override + public List getJacksonModules() + { + return Arrays.asList( + new SimpleModule() + .registerSubtypes( + new NamedType(SingleDimensionShardSpec.class, "single"), + new NamedType(LinearShardSpec.class, "linear"), + new NamedType(NumberedShardSpec.class, "numbered") + ) + ); + } +} diff --git a/realtime/src/main/java/com/metamx/druid/realtime/RealtimeMain.java b/server/src/main/java/io/druid/guice/ServerViewModule.java similarity index 51% rename from realtime/src/main/java/com/metamx/druid/realtime/RealtimeMain.java rename to server/src/main/java/io/druid/guice/ServerViewModule.java index c81ab548a2e5..9ccee0984e0e 100644 --- a/realtime/src/main/java/com/metamx/druid/realtime/RealtimeMain.java +++ b/server/src/main/java/io/druid/guice/ServerViewModule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,36 +17,25 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.realtime; +package io.druid.guice; -import com.metamx.common.lifecycle.Lifecycle; -import com.metamx.common.logger.Logger; -import com.metamx.druid.log.LogLevelAdjuster; +import com.google.inject.Binder; +import com.google.inject.Module; +import io.druid.client.InventoryView; +import io.druid.client.ServerInventoryView; +import io.druid.client.ServerInventoryViewProvider; +import io.druid.client.ServerView; /** */ -public class RealtimeMain +public class ServerViewModule implements Module { - private static final Logger log = new Logger(RealtimeMain.class); - - public static void main(String[] args) throws Exception + @Override + public void configure(Binder binder) { - LogLevelAdjuster.register(); - - Lifecycle lifecycle = new Lifecycle(); - - lifecycle.addManagedInstance( - RealtimeNode.builder().build() - ); - - try { - lifecycle.start(); - } - catch (Throwable t) { - log.info(t, "Throwable caught at startup, committing seppuku"); - System.exit(2); - } - - lifecycle.join(); + JsonConfigProvider.bind(binder, "druid.announcer", ServerInventoryViewProvider.class); + binder.bind(InventoryView.class).to(ServerInventoryView.class); + binder.bind(ServerView.class).to(ServerInventoryView.class); + binder.bind(ServerInventoryView.class).toProvider(ServerInventoryViewProvider.class).in(ManageLifecycle.class); } -} \ No newline at end of file +} diff --git a/server/src/main/java/io/druid/guice/StorageNodeModule.java b/server/src/main/java/io/druid/guice/StorageNodeModule.java new file mode 100644 index 000000000000..2ee270cc3300 --- /dev/null +++ b/server/src/main/java/io/druid/guice/StorageNodeModule.java @@ -0,0 +1,71 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import io.druid.client.DruidServerConfig; +import io.druid.guice.annotations.Self; +import io.druid.query.DefaultQueryRunnerFactoryConglomerate; +import io.druid.query.QueryRunnerFactoryConglomerate; +import io.druid.segment.loading.MMappedQueryableIndexFactory; +import io.druid.segment.loading.QueryableIndexFactory; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.server.DruidNode; +import io.druid.server.coordination.DruidServerMetadata; + +/** + */ +public class StorageNodeModule implements Module +{ + private final String nodeType; + + public StorageNodeModule(String nodeType) + { + this.nodeType = nodeType; + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.server", DruidServerConfig.class); + JsonConfigProvider.bind(binder, "druid.segmentCache", SegmentLoaderConfig.class); + + binder.bind(QueryableIndexFactory.class).to(MMappedQueryableIndexFactory.class).in(LazySingleton.class); + + binder.bind(QueryRunnerFactoryConglomerate.class) + .to(DefaultQueryRunnerFactoryConglomerate.class) + .in(LazySingleton.class); + } + + @Provides + @LazySingleton + public DruidServerMetadata getMetadata(@Self DruidNode node, DruidServerConfig config) + { + return new DruidServerMetadata( + node.getHost(), + node.getHost(), + config.getMaxSize(), + nodeType, + config.getTier() + ); + } +} diff --git a/server/src/main/java/io/druid/guice/annotations/Client.java b/server/src/main/java/io/druid/guice/annotations/Client.java new file mode 100644 index 000000000000..240a0a8bcd5f --- /dev/null +++ b/server/src/main/java/io/druid/guice/annotations/Client.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice.annotations; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + */ +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@BindingAnnotation +public @interface Client +{ +} diff --git a/server/src/main/java/io/druid/guice/annotations/Processing.java b/server/src/main/java/io/druid/guice/annotations/Processing.java new file mode 100644 index 000000000000..e386476c1c95 --- /dev/null +++ b/server/src/main/java/io/druid/guice/annotations/Processing.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.guice.annotations; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + */ +@BindingAnnotation +@Target({ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +public @interface Processing +{ +} diff --git a/server/src/main/java/io/druid/segment/loading/AWSCredentialsConfig.java b/server/src/main/java/io/druid/segment/loading/AWSCredentialsConfig.java new file mode 100644 index 000000000000..b9a032fd4ea5 --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/AWSCredentialsConfig.java @@ -0,0 +1,43 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +public class AWSCredentialsConfig +{ + @JsonProperty + private String accessKey = ""; + + @JsonProperty + private String secretKey = ""; + + public String getAccessKey() + { + return accessKey; + } + + public String getSecretKey() + { + return secretKey; + } +} diff --git a/server/src/main/java/com/metamx/druid/loading/DataSegmentPusherUtil.java b/server/src/main/java/io/druid/segment/loading/DataSegmentPusherUtil.java similarity index 93% rename from server/src/main/java/com/metamx/druid/loading/DataSegmentPusherUtil.java rename to server/src/main/java/io/druid/segment/loading/DataSegmentPusherUtil.java index 349194836d96..b64f26806ca3 100644 --- a/server/src/main/java/com/metamx/druid/loading/DataSegmentPusherUtil.java +++ b/server/src/main/java/io/druid/segment/loading/DataSegmentPusherUtil.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; - -import org.joda.time.format.ISODateTimeFormat; +package io.druid.segment.loading; import com.google.common.base.Joiner; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; +import org.joda.time.format.ISODateTimeFormat; /** */ diff --git a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPuller.java b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPuller.java similarity index 67% rename from server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPuller.java rename to server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPuller.java index fd0c93264ab2..d19c4cef7938 100644 --- a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPuller.java +++ b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPuller.java @@ -1,8 +1,28 @@ -package com.metamx.druid.loading; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; import com.google.common.io.Closeables; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.utils.CompressionUtils; +import com.google.inject.Inject; +import io.druid.common.utils.CompressionUtils; +import io.druid.timeline.DataSegment; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; @@ -17,6 +37,7 @@ public class HdfsDataSegmentPuller implements DataSegmentPuller { private final Configuration config; + @Inject public HdfsDataSegmentPuller(final Configuration config) { this.config = config; diff --git a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusher.java similarity index 74% rename from server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusher.java rename to server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusher.java index e232bbb8f087..eb6206986893 100644 --- a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusher.java @@ -1,14 +1,34 @@ -package com.metamx.druid.loading; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; import com.google.common.io.OutputSupplier; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.utils.CompressionUtils; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -28,6 +48,7 @@ public class HdfsDataSegmentPusher implements DataSegmentPusher private final Configuration hadoopConfig; private final ObjectMapper jsonMapper; + @Inject public HdfsDataSegmentPusher( HdfsDataSegmentPusherConfig config, Configuration hadoopConfig, diff --git a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusherConfig.java b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusherConfig.java similarity index 75% rename from server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusherConfig.java rename to server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusherConfig.java index b27d03672bc5..e329fdeb4c4d 100644 --- a/server/src/main/java/com/metamx/druid/loading/HdfsDataSegmentPusherConfig.java +++ b/server/src/main/java/io/druid/segment/loading/HdfsDataSegmentPusherConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; -import org.skife.config.Config; +import com.fasterxml.jackson.annotation.JsonProperty; /** */ public abstract class HdfsDataSegmentPusherConfig { - @Config("druid.pusher.hdfs.storageDirectory") - public abstract String getStorageDirectory(); + @JsonProperty + public String storageDirectory = ""; + + public String getStorageDirectory() + { + return storageDirectory; + } } diff --git a/server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPuller.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPuller.java similarity index 94% rename from server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPuller.java rename to server/src/main/java/io/druid/segment/loading/LocalDataSegmentPuller.java index 8cdb8e0a7a27..fd29d9192e92 100644 --- a/server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPuller.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPuller.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.google.common.io.Files; import com.metamx.common.MapUtils; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.utils.CompressionUtils; +import io.druid.common.utils.CompressionUtils; +import io.druid.timeline.DataSegment; import java.io.File; import java.io.IOException; diff --git a/server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java similarity index 92% rename from server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPusher.java rename to server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java index 1493b162572e..a252337499e2 100644 --- a/server/src/main/java/com/metamx/druid/loading/LocalDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusher.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Files; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.utils.CompressionUtils; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; import java.io.File; import java.io.IOException; @@ -40,6 +41,7 @@ public class LocalDataSegmentPusher implements DataSegmentPusher private final LocalDataSegmentPusherConfig config; private final ObjectMapper jsonMapper; + @Inject public LocalDataSegmentPusher( LocalDataSegmentPusherConfig config, ObjectMapper jsonMapper diff --git a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusherConfig.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusherConfig.java new file mode 100644 index 000000000000..823ffe38ee5f --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentPusherConfig.java @@ -0,0 +1,37 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.File; + +/** + */ +public class LocalDataSegmentPusherConfig +{ + @JsonProperty + public File storageDirectory; + + public File getStorageDirectory() + { + return storageDirectory; + } +} diff --git a/server/src/main/java/com/metamx/druid/loading/MMappedQueryableIndexFactory.java b/server/src/main/java/io/druid/segment/loading/MMappedQueryableIndexFactory.java similarity index 61% rename from server/src/main/java/com/metamx/druid/loading/MMappedQueryableIndexFactory.java rename to server/src/main/java/io/druid/segment/loading/MMappedQueryableIndexFactory.java index d2cef9b76b3c..717b0b513a0d 100644 --- a/server/src/main/java/com/metamx/druid/loading/MMappedQueryableIndexFactory.java +++ b/server/src/main/java/io/druid/segment/loading/MMappedQueryableIndexFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.metamx.common.logger.Logger; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.v1.IndexIO; +import io.druid.segment.IndexIO; +import io.druid.segment.QueryableIndex; import org.apache.commons.io.FileUtils; import java.io.File; @@ -37,25 +37,6 @@ public class MMappedQueryableIndexFactory implements QueryableIndexFactory public QueryableIndex factorize(File parentDir) throws SegmentLoadingException { try { - if (! IndexIO.canBeMapped(parentDir)) { - File canBeMappedDir = new File(parentDir, "forTheMapping"); - if (canBeMappedDir.exists()) { - FileUtils.deleteDirectory(canBeMappedDir); - } - canBeMappedDir.mkdirs(); - - IndexIO.storeLatest(IndexIO.readIndex(parentDir), canBeMappedDir); - if (! IndexIO.canBeMapped(canBeMappedDir)) { - throw new SegmentLoadingException("WTF!? newly written file[%s] cannot be mapped!?", canBeMappedDir); - } - for (File file : canBeMappedDir.listFiles()) { - if (! file.renameTo(new File(parentDir, file.getName()))) { - throw new SegmentLoadingException("Couldn't rename[%s] to [%s]", canBeMappedDir, parentDir); - } - } - FileUtils.deleteDirectory(canBeMappedDir); - } - return IndexIO.loadIndex(parentDir); } catch (IOException e) { diff --git a/server/src/main/java/com/metamx/druid/loading/SingleSegmentLoader.java b/server/src/main/java/io/druid/segment/loading/OmniSegmentLoader.java similarity index 52% rename from server/src/main/java/com/metamx/druid/loading/SingleSegmentLoader.java rename to server/src/main/java/io/druid/segment/loading/OmniSegmentLoader.java index 987939bb0691..31efc1b0ea9e 100644 --- a/server/src/main/java/com/metamx/druid/loading/SingleSegmentLoader.java +++ b/server/src/main/java/io/druid/segment/loading/OmniSegmentLoader.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,77 +17,52 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Sets; -import com.google.common.primitives.Longs; +import com.google.common.collect.Lists; import com.google.inject.Inject; -import com.metamx.common.IAE; import com.metamx.common.ISE; +import com.metamx.common.MapUtils; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.QueryableIndexSegment; -import com.metamx.druid.index.Segment; +import io.druid.segment.QueryableIndex; +import io.druid.segment.QueryableIndexSegment; +import io.druid.segment.Segment; +import io.druid.timeline.DataSegment; import org.apache.commons.io.FileUtils; import java.io.File; import java.io.IOException; import java.util.Iterator; import java.util.List; -import java.util.Set; +import java.util.Map; /** */ -public class SingleSegmentLoader implements SegmentLoader +public class OmniSegmentLoader implements SegmentLoader { - private static final Logger log = new Logger(SingleSegmentLoader.class); + private static final Logger log = new Logger(OmniSegmentLoader.class); - private final DataSegmentPuller dataSegmentPuller; + private final Map pullers; private final QueryableIndexFactory factory; + private final SegmentLoaderConfig config; private final List locations; @Inject - public SingleSegmentLoader( - DataSegmentPuller dataSegmentPuller, + public OmniSegmentLoader( + Map pullers, QueryableIndexFactory factory, SegmentLoaderConfig config ) { - this.dataSegmentPuller = dataSegmentPuller; + this.pullers = pullers; this.factory = factory; + this.config = config; - final ImmutableList.Builder locBuilder = ImmutableList.builder(); - - // This is a really, really stupid way of getting this information. Splitting on commas and bars is error-prone - // We should instead switch it up to be a JSON Array of JSON Object or something and cool stuff like that - // But, that'll have to wait for some other day. - for (String dirSpec : config.getCacheDirectory().split(",")) { - String[] dirSplit = dirSpec.split("\\|"); - if (dirSplit.length == 1) { - locBuilder.add(new StorageLocation(new File(dirSplit[0]), config.getServerMaxSize())); - } - else if (dirSplit.length == 2) { - final Long maxSize = Longs.tryParse(dirSplit[1]); - if (maxSize == null) { - throw new IAE("Size of a local segment storage location must be an integral number, got[%s]", dirSplit[1]); - } - locBuilder.add(new StorageLocation(new File(dirSplit[0]), maxSize)); - } - else { - throw new ISE( - "Unknown segment storage location[%s]=>[%s], config[%s].", - dirSplit.length, dirSpec, config.getCacheDirectory() - ); - } + this.locations = Lists.newArrayList(); + for (StorageLocationConfig locationConfig : config.getLocations()) { + locations.add(new StorageLocation(locationConfig.getPath(), locationConfig.getMaxSize())); } - locations = locBuilder.build(); - - Preconditions.checkArgument(locations.size() > 0, "Must have at least one segment cache directory."); - log.info("Using storage locations[%s]", locations); } @Override @@ -116,6 +91,7 @@ public Segment getSegment(DataSegment segment) throws SegmentLoadingException return new QueryableIndexSegment(segment.getIdentifier(), index); } + @Override public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException { StorageLocation loc = findStorageLocationIfLoaded(segment); @@ -141,7 +117,7 @@ public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException log.debug("Unable to make parent file[%s]", storageDir); } - dataSegmentPuller.getSegmentFiles(segment, storageDir); + getPuller(segment.getLoadSpec()).getSegmentFiles(segment, storageDir); loc.addSegment(segment); retVal = storageDir; @@ -158,6 +134,10 @@ public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException @Override public void cleanup(DataSegment segment) throws SegmentLoadingException { + if (!config.isDeleteOnRemove()) { + return; + } + StorageLocation loc = findStorageLocationIfLoaded(segment); if (loc == null) { @@ -176,71 +156,15 @@ public void cleanup(DataSegment segment) throws SegmentLoadingException } } - static class StorageLocation + private DataSegmentPuller getPuller(Map loadSpec) throws SegmentLoadingException { - private final File path; - private final long maxSize; - private final Set segments; - - private volatile long currSize = 0; - - StorageLocation( - File path, - long maxSize - ) - { - this.path = path; - this.maxSize = maxSize; - - this.segments = Sets.newHashSet(); - } + String type = MapUtils.getString(loadSpec, "type"); + DataSegmentPuller loader = pullers.get(type); - File getPath() - { - return path; + if (loader == null) { + throw new SegmentLoadingException("Unknown loader type[%s]. Known types are %s", type, pullers.keySet()); } - Long getMaxSize() - { - return maxSize; - } - - synchronized void addSegment(DataSegment segment) - { - if (segments.add(segment)) { - currSize += segment.getSize(); - } - } - - synchronized void removeSegment(DataSegment segment) - { - if (segments.remove(segment)) { - currSize -= segment.getSize(); - } - } - - boolean canHandle(long size) - { - return available() >= size; - } - - synchronized long available() - { - return maxSize - currSize; - } - - StorageLocation mostEmpty(StorageLocation other) - { - return available() > other.available() ? this : other; - } - - @Override - public String toString() - { - return "StorageLocation{" + - "path=" + path + - ", maxSize=" + maxSize + - '}'; - } + return loader; } } diff --git a/server/src/main/java/com/metamx/druid/loading/QueryableIndexFactory.java b/server/src/main/java/io/druid/segment/loading/QueryableIndexFactory.java similarity index 85% rename from server/src/main/java/com/metamx/druid/loading/QueryableIndexFactory.java rename to server/src/main/java/io/druid/segment/loading/QueryableIndexFactory.java index 535649baea4a..cf802b56e41b 100644 --- a/server/src/main/java/com/metamx/druid/loading/QueryableIndexFactory.java +++ b/server/src/main/java/io/druid/segment/loading/QueryableIndexFactory.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; -import com.metamx.druid.index.QueryableIndex; +import io.druid.segment.QueryableIndex; import java.io.File; -import java.io.IOException; /** */ diff --git a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentKiller.java b/server/src/main/java/io/druid/segment/loading/S3DataSegmentKiller.java similarity index 94% rename from server/src/main/java/com/metamx/druid/loading/S3DataSegmentKiller.java rename to server/src/main/java/io/druid/segment/loading/S3DataSegmentKiller.java index 63b815a7e85a..314822679aea 100644 --- a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentKiller.java +++ b/server/src/main/java/io/druid/segment/loading/S3DataSegmentKiller.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.google.inject.Inject; import com.metamx.common.MapUtils; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.jets3t.service.ServiceException; import org.jets3t.service.impl.rest.httpclient.RestS3Service; diff --git a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPuller.java b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPuller.java similarity index 96% rename from server/src/main/java/com/metamx/druid/loading/S3DataSegmentPuller.java rename to server/src/main/java/io/druid/segment/loading/S3DataSegmentPuller.java index c6a82ffa3ff3..49b7501ad512 100644 --- a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPuller.java +++ b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPuller.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.google.common.base.Throwables; import com.google.common.io.ByteStreams; @@ -27,9 +27,9 @@ import com.metamx.common.ISE; import com.metamx.common.MapUtils; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.common.s3.S3Utils; -import com.metamx.druid.utils.CompressionUtils; +import io.druid.common.utils.CompressionUtils; +import io.druid.storage.s3.S3Utils; +import io.druid.timeline.DataSegment; import org.apache.commons.io.FileUtils; import org.jets3t.service.ServiceException; import org.jets3t.service.impl.rest.httpclient.RestS3Service; diff --git a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPusher.java similarity index 94% rename from server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusher.java rename to server/src/main/java/io/druid/segment/loading/S3DataSegmentPusher.java index a5462a83eb79..a1469446f311 100644 --- a/server/src/main/java/com/metamx/druid/loading/S3DataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPusher.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -25,11 +25,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Files; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.common.s3.S3Utils; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.utils.CompressionUtils; +import com.google.inject.Inject; import com.metamx.emitter.EmittingLogger; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.IndexIO; +import io.druid.storage.s3.S3Utils; +import io.druid.timeline.DataSegment; import org.jets3t.service.ServiceException; import org.jets3t.service.acl.gs.GSAccessControlList; import org.jets3t.service.impl.rest.httpclient.RestS3Service; @@ -48,6 +49,7 @@ public class S3DataSegmentPusher implements DataSegmentPusher private final S3DataSegmentPusherConfig config; private final ObjectMapper jsonMapper; + @Inject public S3DataSegmentPusher( RestS3Service s3Client, S3DataSegmentPusherConfig config, diff --git a/indexing-hadoop/src/main/java/com/metamx/druid/indexer/updater/ZkUpdaterJobSpec.java b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPusherConfig.java similarity index 64% rename from indexing-hadoop/src/main/java/com/metamx/druid/indexer/updater/ZkUpdaterJobSpec.java rename to server/src/main/java/io/druid/segment/loading/S3DataSegmentPusherConfig.java index b79fab217473..b74d22597d7c 100644 --- a/indexing-hadoop/src/main/java/com/metamx/druid/indexer/updater/ZkUpdaterJobSpec.java +++ b/server/src/main/java/io/druid/segment/loading/S3DataSegmentPusherConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,34 +17,35 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.indexer.updater; +package io.druid.segment.loading; import com.fasterxml.jackson.annotation.JsonProperty; /** */ -public class ZkUpdaterJobSpec implements UpdaterJobSpec +public class S3DataSegmentPusherConfig { - @JsonProperty("zkHosts") - public String zkQuorum; + @JsonProperty + public String bucket = ""; - @JsonProperty("zkBasePath") - private String zkBasePath; + @JsonProperty + public String baseKey = ""; - public ZkUpdaterJobSpec() {} + @JsonProperty + public boolean disableAcl = false; - public String getZkQuorum() + public String getBucket() { - return zkQuorum; + return bucket; } - public String getZkBasePath() + public String getBaseKey() { - return zkBasePath; + return baseKey; } - public boolean postToZk() + public boolean getDisableAcl() { - return !(zkQuorum == null || zkBasePath == null); + return disableAcl; } } diff --git a/server/src/main/java/com/metamx/druid/loading/SegmentLoader.java b/server/src/main/java/io/druid/segment/loading/SegmentLoader.java similarity index 73% rename from server/src/main/java/com/metamx/druid/loading/SegmentLoader.java rename to server/src/main/java/io/druid/segment/loading/SegmentLoader.java index 75c7591a3955..d4c050b6044a 100644 --- a/server/src/main/java/com/metamx/druid/loading/SegmentLoader.java +++ b/server/src/main/java/io/druid/segment/loading/SegmentLoader.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.Segment; +import io.druid.segment.Segment; +import io.druid.timeline.DataSegment; + +import java.io.File; /** */ public interface SegmentLoader { public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException; - public Segment getSegment(DataSegment loadSpec) throws SegmentLoadingException; + public Segment getSegment(DataSegment segment) throws SegmentLoadingException; + public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException; public void cleanup(DataSegment loadSpec) throws SegmentLoadingException; } diff --git a/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java b/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java new file mode 100644 index 000000000000..f4034865022f --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/SegmentLoaderConfig.java @@ -0,0 +1,70 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.hibernate.validator.constraints.NotEmpty; + +import java.io.File; +import java.util.List; + +/** + */ +public class SegmentLoaderConfig +{ + @JsonProperty + @NotEmpty + private List locations = null; + + @JsonProperty("deleteOnRemove") + private boolean deleteOnRemove = true; + + @JsonProperty + private File infoDir = null; + + public List getLocations() + { + return locations; + } + + public boolean isDeleteOnRemove() + { + return deleteOnRemove; + } + + public File getInfoDir() + { + if (infoDir == null) { + infoDir = new File(locations.get(0).getPath(), "info_dir"); + } + + return infoDir; + } + + @Override + public String toString() + { + return "SegmentLoaderConfig{" + + "locations=" + getLocations() + + ", deleteOnRemove=" + isDeleteOnRemove() + + ", infoDir=" + getInfoDir() + + '}'; + } +} diff --git a/server/src/main/java/io/druid/segment/loading/StorageLocation.java b/server/src/main/java/io/druid/segment/loading/StorageLocation.java new file mode 100644 index 000000000000..339ed88058fa --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/StorageLocation.java @@ -0,0 +1,84 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.google.common.collect.Sets; +import io.druid.timeline.DataSegment; + +import java.io.File; +import java.util.Set; + +/** +*/ +class StorageLocation +{ + private final File path; + private final long maxSize; + private final Set segments; + + private volatile long currSize = 0; + + StorageLocation(File path, long maxSize) + { + this.path = path; + this.maxSize = maxSize; + + this.segments = Sets.newHashSet(); + } + + File getPath() + { + return path; + } + + long getMaxSize() + { + return maxSize; + } + + synchronized void addSegment(DataSegment segment) + { + if (segments.add(segment)) { + currSize += segment.getSize(); + } + } + + synchronized void removeSegment(DataSegment segment) + { + if (segments.remove(segment)) { + currSize -= segment.getSize(); + } + } + + boolean canHandle(long size) + { + return available() >= size; + } + + synchronized long available() + { + return maxSize - currSize; + } + + StorageLocation mostEmpty(StorageLocation other) + { + return available() > other.available() ? this : other; + } +} diff --git a/server/src/main/java/io/druid/segment/loading/StorageLocationConfig.java b/server/src/main/java/io/druid/segment/loading/StorageLocationConfig.java new file mode 100644 index 000000000000..2ab5920313d7 --- /dev/null +++ b/server/src/main/java/io/druid/segment/loading/StorageLocationConfig.java @@ -0,0 +1,58 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import java.io.File; + +/** + */ +public class StorageLocationConfig +{ + @JsonProperty + @NotNull + private File path = null; + + @JsonProperty + @Min(1) + private long maxSize = Long.MAX_VALUE; + + public File getPath() + { + return path; + } + + public long getMaxSize() + { + return maxSize; + } + + @Override + public String toString() + { + return "StorageLocationConfig{" + + "path=" + path + + ", maxSize=" + maxSize + + '}'; + } +} diff --git a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentConfig.java b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentConfig.java similarity index 71% rename from server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentConfig.java rename to server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentConfig.java index 6e7976f824d3..d9551a787822 100644 --- a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentConfig.java +++ b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading.cassandra; +package io.druid.segment.loading.cassandra; -import org.skife.config.Config; -import org.skife.config.Default; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Cassandra Config @@ -29,11 +28,19 @@ */ public abstract class CassandraDataSegmentConfig { - @Config("druid.pusher.cassandra.host") - @Default("") - public abstract String getHost(); + @JsonProperty + public String host = ""; - @Config("druid.pusher.cassandra.keyspace") - @Default("") - public abstract String getKeyspace(); + @JsonProperty + public String keyspace = ""; + + public String getKeyspace() + { + return keyspace; + } + + public String getHost() + { + return host; + } } diff --git a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPuller.java b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPuller.java similarity index 91% rename from server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPuller.java rename to server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPuller.java index 87e6105f1614..01f9264d3ccc 100644 --- a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPuller.java +++ b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPuller.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,25 +17,25 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading.cassandra; - -import java.io.File; -import java.io.OutputStream; - -import org.apache.commons.io.FileUtils; +package io.druid.segment.loading.cassandra; import com.google.common.io.Files; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.loading.DataSegmentPuller; -import com.metamx.druid.loading.SegmentLoadingException; -import com.metamx.druid.utils.CompressionUtils; import com.netflix.astyanax.connectionpool.OperationResult; import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; import com.netflix.astyanax.model.ColumnList; import com.netflix.astyanax.recipes.storage.ChunkedStorage; import com.netflix.astyanax.recipes.storage.ObjectMetadata; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.loading.DataSegmentPuller; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.timeline.DataSegment; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.OutputStream; /** * Cassandra Segment Puller @@ -48,6 +48,7 @@ public class CassandraDataSegmentPuller extends CassandraStorage implements Data private static final int CONCURRENCY = 10; private static final int BATCH_SIZE = 10; + @Inject public CassandraDataSegmentPuller(CassandraDataSegmentConfig config) { super(config); diff --git a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPusher.java b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPusher.java similarity index 68% rename from server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPusher.java rename to server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPusher.java index 57bc72b9124e..76b4ac52ddd3 100644 --- a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraDataSegmentPusher.java +++ b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraDataSegmentPusher.java @@ -1,20 +1,40 @@ -package com.metamx.druid.loading.cassandra; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; +package io.druid.segment.loading.cassandra; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.loading.DataSegmentPusher; -import com.metamx.druid.loading.DataSegmentPusherUtil; -import com.metamx.druid.utils.CompressionUtils; import com.netflix.astyanax.MutationBatch; import com.netflix.astyanax.recipes.storage.ChunkedStorage; +import io.druid.common.utils.CompressionUtils; +import io.druid.segment.IndexIO; +import io.druid.segment.loading.DataSegmentPusher; +import io.druid.segment.loading.DataSegmentPusherUtil; +import io.druid.timeline.DataSegment; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; /** * Cassandra Segment Pusher @@ -28,6 +48,7 @@ public class CassandraDataSegmentPusher extends CassandraStorage implements Data private static final Joiner JOINER = Joiner.on("/").skipNulls(); private final ObjectMapper jsonMapper; + @Inject public CassandraDataSegmentPusher( CassandraDataSegmentConfig config, ObjectMapper jsonMapper) diff --git a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraStorage.java b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraStorage.java similarity index 97% rename from server/src/main/java/com/metamx/druid/loading/cassandra/CassandraStorage.java rename to server/src/main/java/io/druid/segment/loading/cassandra/CassandraStorage.java index e9d046091353..3a693575c18f 100644 --- a/server/src/main/java/com/metamx/druid/loading/cassandra/CassandraStorage.java +++ b/server/src/main/java/io/druid/segment/loading/cassandra/CassandraStorage.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading.cassandra; +package io.druid.segment.loading.cassandra; import com.netflix.astyanax.AstyanaxContext; import com.netflix.astyanax.Keyspace; diff --git a/client/src/main/java/com/metamx/druid/http/ClientInfoResource.java b/server/src/main/java/io/druid/server/ClientInfoResource.java similarity index 94% rename from client/src/main/java/com/metamx/druid/http/ClientInfoResource.java rename to server/src/main/java/io/druid/server/ClientInfoResource.java index 8bcb8865e7c6..782029f29b5f 100644 --- a/client/src/main/java/com/metamx/druid/http/ClientInfoResource.java +++ b/server/src/main/java/io/druid/server/ClientInfoResource.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.inject.Inject; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.InventoryView; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.client.InventoryView; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/client/src/main/java/com/metamx/druid/http/ClientQuerySegmentWalker.java b/server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java similarity index 80% rename from client/src/main/java/com/metamx/druid/http/ClientQuerySegmentWalker.java rename to server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java index d053d6507a29..76d9d1aa8382 100644 --- a/client/src/main/java/com/metamx/druid/http/ClientQuerySegmentWalker.java +++ b/server/src/main/java/io/druid/server/ClientQuerySegmentWalker.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,20 +17,21 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.google.common.base.Function; -import com.metamx.druid.Query; -import com.metamx.druid.client.CachingClusteredClient; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.MetricsEmittingQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.metamx.druid.query.segment.SegmentDescriptor; +import com.google.inject.Inject; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.client.CachingClusteredClient; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.MetricsEmittingQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryToolChest; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.SegmentDescriptor; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -39,19 +40,20 @@ */ public class ClientQuerySegmentWalker implements QuerySegmentWalker { - private final QueryToolChestWarehouse warehouse; private final ServiceEmitter emitter; private final CachingClusteredClient baseClient; + private final QueryToolChestWarehouse warehouse; + @Inject public ClientQuerySegmentWalker( - QueryToolChestWarehouse warehouse, ServiceEmitter emitter, - CachingClusteredClient baseClient + CachingClusteredClient baseClient, + QueryToolChestWarehouse warehouse ) { - this.warehouse = warehouse; this.emitter = emitter; this.baseClient = baseClient; + this.warehouse = warehouse; } @Override diff --git a/client/src/main/java/com/metamx/druid/http/DirectClientQuerySegmentWalker.java b/server/src/main/java/io/druid/server/DirectClientQuerySegmentWalker.java similarity index 79% rename from client/src/main/java/com/metamx/druid/http/DirectClientQuerySegmentWalker.java rename to server/src/main/java/io/druid/server/DirectClientQuerySegmentWalker.java index e1cac8a90c3c..224f03691ead 100644 --- a/client/src/main/java/com/metamx/druid/http/DirectClientQuerySegmentWalker.java +++ b/server/src/main/java/io/druid/server/DirectClientQuerySegmentWalker.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; -import com.metamx.druid.Query; -import com.metamx.druid.client.DirectDruidClient; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryToolChestWarehouse; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.metamx.druid.query.segment.SegmentDescriptor; +import io.druid.client.DirectDruidClient; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryToolChestWarehouse; +import io.druid.query.SegmentDescriptor; import org.joda.time.Interval; /** diff --git a/server/src/main/java/io/druid/server/DruidNode.java b/server/src/main/java/io/druid/server/DruidNode.java new file mode 100644 index 000000000000..4c8528bd6ee6 --- /dev/null +++ b/server/src/main/java/io/druid/server/DruidNode.java @@ -0,0 +1,121 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.druid.common.utils.SocketUtil; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + */ +public class DruidNode +{ + private String hostNoPort; + + @JsonProperty("service") + @NotNull + private String serviceName = null; + + @JsonProperty + @NotNull + private String host = null; + + @JsonProperty + @Min(0) @Max(0xffff) + private int port = -1; + + @JsonCreator + public DruidNode( + @JsonProperty("service") String serviceName, + @JsonProperty("host") String host, + @JsonProperty("port") Integer port + ) + { + this.serviceName = serviceName; + + if (port == null) { + if (host == null) { + setHostAndPort(null, -1, null); + } + else if (host.contains(":")) { + final String[] hostParts = host.split(":"); + try { + setHostAndPort(host, Integer.parseInt(hostParts[1]), hostParts[0]); + } + catch (NumberFormatException e) { + setHostAndPort(host, -1, hostParts[0]); + } + } + else { + final int openPort = SocketUtil.findOpenPort(8080); + setHostAndPort(String.format("%s:%d", host, openPort), openPort, host); + } + } + else { + if (host == null || host.contains(":")) { + setHostAndPort(host, port, host == null ? null : host.split(":")[0]); + } + else { + setHostAndPort(String.format("%s:%d", host, port), port, host); + } + } + } + + private void setHostAndPort(String host, int port, String hostNoPort) + { + this.host = host; + this.port = port; + this.hostNoPort = hostNoPort; + } + + public String getServiceName() + { + return serviceName; + } + + public String getHost() + { + return host; + } + + public int getPort() + { + return port; + } + + public String getHostNoPort() + { + return hostNoPort; + } + + @Override + public String toString() + { + return "DruidNode{" + + "serviceName='" + serviceName + '\'' + + ", host='" + host + '\'' + + ", port=" + port + + '}'; + } +} diff --git a/server/src/main/java/com/metamx/druid/DruidProcessingConfig.java b/server/src/main/java/io/druid/server/DruidProcessingConfig.java similarity index 93% rename from server/src/main/java/com/metamx/druid/DruidProcessingConfig.java rename to server/src/main/java/io/druid/server/DruidProcessingConfig.java index b497ae8f6ce6..ada4eef609d3 100644 --- a/server/src/main/java/com/metamx/druid/DruidProcessingConfig.java +++ b/server/src/main/java/io/druid/server/DruidProcessingConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid; +package io.druid.server; import com.metamx.common.concurrent.ExecutorServiceConfig; import org.skife.config.Config; diff --git a/client/src/main/java/com/metamx/druid/http/GuiceServletConfig.java b/server/src/main/java/io/druid/server/GuiceServletConfig.java similarity index 93% rename from client/src/main/java/com/metamx/druid/http/GuiceServletConfig.java rename to server/src/main/java/io/druid/server/GuiceServletConfig.java index fcfac0a66f83..b7a52a4eb6bc 100644 --- a/client/src/main/java/com/metamx/druid/http/GuiceServletConfig.java +++ b/server/src/main/java/io/druid/server/GuiceServletConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.google.inject.Injector; import com.google.inject.servlet.GuiceServletContextListener; diff --git a/client/src/main/java/com/metamx/druid/http/QueryServlet.java b/server/src/main/java/io/druid/server/QueryServlet.java similarity index 92% rename from client/src/main/java/com/metamx/druid/http/QueryServlet.java rename to server/src/main/java/io/druid/server/QueryServlet.java index 8b871a49bcc6..49d32ebe0dd1 100644 --- a/client/src/main/java/com/metamx/druid/http/QueryServlet.java +++ b/server/src/main/java/io/druid/server/QueryServlet.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; @@ -25,17 +25,20 @@ import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteStreams; import com.google.common.io.Closeables; +import com.google.inject.Inject; import com.metamx.common.guava.Sequence; import com.metamx.common.guava.Sequences; import com.metamx.common.logger.Logger; -import com.metamx.druid.Query; -import com.metamx.druid.query.segment.QuerySegmentWalker; import com.metamx.emitter.service.AlertEvent; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; - +import io.druid.guice.annotations.Json; +import io.druid.guice.annotations.Smile; +import io.druid.query.Query; +import io.druid.query.QuerySegmentWalker; +import io.druid.server.log.RequestLogger; +import org.eclipse.jetty.server.Request; import org.joda.time.DateTime; -import org.mortbay.jetty.Request; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; @@ -58,9 +61,10 @@ public class QueryServlet extends HttpServlet private final ServiceEmitter emitter; private final RequestLogger requestLogger; + @Inject public QueryServlet( - ObjectMapper jsonMapper, - ObjectMapper smileMapper, + @Json ObjectMapper jsonMapper, + @Smile ObjectMapper smileMapper, QuerySegmentWalker texasRanger, ServiceEmitter emitter, RequestLogger requestLogger diff --git a/client/src/main/java/com/metamx/druid/http/RequestLogLine.java b/server/src/main/java/io/druid/server/RequestLogLine.java similarity index 94% rename from client/src/main/java/com/metamx/druid/http/RequestLogLine.java rename to server/src/main/java/io/druid/server/RequestLogLine.java index 0dbecb03ec8d..a52485ae2790 100644 --- a/client/src/main/java/com/metamx/druid/http/RequestLogLine.java +++ b/server/src/main/java/io/druid/server/RequestLogLine.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; -import com.metamx.druid.Query; +import io.druid.query.Query; import org.joda.time.DateTime; import java.util.Arrays; diff --git a/server/src/main/java/io/druid/server/StatusResource.java b/server/src/main/java/io/druid/server/StatusResource.java new file mode 100644 index 000000000000..a2b30268d4da --- /dev/null +++ b/server/src/main/java/io/druid/server/StatusResource.java @@ -0,0 +1,103 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; + +/** + */ +@Path("/{a:status|health}") +public class StatusResource +{ + @GET + @Produces("application/json") + public Status doGet() + { + return new Status( + StatusResource.class.getPackage().getImplementationVersion(), + new Memory(Runtime.getRuntime()) + ); + } + + public static class Status { + final String version; + final Memory memory; + + public Status(String version, Memory memory) + { + this.version = version; + this.memory = memory; + } + + @JsonProperty + public String getVersion() + { + return version; + } + + @JsonProperty + public Memory getMemory() + { + return memory; + } + } + + public static class Memory { + final long maxMemory; + final long totalMemory; + final long freeMemory; + final long usedMemory; + + public Memory(Runtime runtime) { + maxMemory = runtime.maxMemory(); + totalMemory = runtime.totalMemory(); + freeMemory = runtime.freeMemory(); + usedMemory = totalMemory - freeMemory; + } + + @JsonProperty + public long getMaxMemory() + { + return maxMemory; + } + + @JsonProperty + public long getTotalMemory() + { + return totalMemory; + } + + @JsonProperty + public long getFreeMemory() + { + return freeMemory; + } + + @JsonProperty + public long getUsedMemory() + { + return usedMemory; + } + } +} diff --git a/server/src/main/java/io/druid/server/ZkPathsModule.java b/server/src/main/java/io/druid/server/ZkPathsModule.java new file mode 100644 index 000000000000..346312213531 --- /dev/null +++ b/server/src/main/java/io/druid/server/ZkPathsModule.java @@ -0,0 +1,33 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server; + +import com.google.inject.Binder; +import com.google.inject.Module; + +/** + */ +public class ZkPathsModule implements Module +{ + @Override + public void configure(Binder binder) + { + } +} diff --git a/client/src/main/java/com/metamx/druid/coordination/AbstractDataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/AbstractDataSegmentAnnouncer.java similarity index 93% rename from client/src/main/java/com/metamx/druid/coordination/AbstractDataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/AbstractDataSegmentAnnouncer.java index 1bb0b8114eeb..ba73b74e9949 100644 --- a/client/src/main/java/com/metamx/druid/coordination/AbstractDataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/AbstractDataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,8 +25,8 @@ import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; import com.metamx.common.logger.Logger; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkPathsConfig; +import io.druid.curator.announcement.Announcer; +import io.druid.server.initialization.ZkPathsConfig; import org.apache.curator.utils.ZKPaths; /** diff --git a/client/src/main/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncer.java similarity index 83% rename from client/src/main/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncer.java index 46aedcee1626..d0b24ea56539 100644 --- a/client/src/main/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,18 +17,20 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig; +import io.druid.curator.announcement.Announcer; +import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.utils.ZKPaths; import org.joda.time.DateTime; @@ -45,7 +47,7 @@ public class BatchDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer { private static final Logger log = new Logger(BatchDataSegmentAnnouncer.class); - private final ZkDataSegmentAnnouncerConfig config; + private final BatchDataSegmentAnnouncerConfig config; private final Announcer announcer; private final ObjectMapper jsonMapper; private final String liveSegmentLocation; @@ -56,27 +58,29 @@ public class BatchDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer private final Set availableZNodes = new ConcurrentSkipListSet(); private final Map segmentLookup = Maps.newConcurrentMap(); + @Inject public BatchDataSegmentAnnouncer( DruidServerMetadata server, - ZkDataSegmentAnnouncerConfig config, + BatchDataSegmentAnnouncerConfig config, + ZkPathsConfig zkPaths, Announcer announcer, ObjectMapper jsonMapper ) { - super(server, config, announcer, jsonMapper); - + super(server, zkPaths, announcer, jsonMapper); this.config = config; this.announcer = announcer; this.jsonMapper = jsonMapper; - this.liveSegmentLocation = ZKPaths.makePath(config.getLiveSegmentsPath(), server.getName()); + + this.liveSegmentLocation = ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), server.getName()); } @Override public void announceSegment(DataSegment segment) throws IOException { int newBytesLen = jsonMapper.writeValueAsBytes(segment).length; - if (newBytesLen > config.getMaxNumBytes()) { - throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxNumBytes()); + if (newBytesLen > config.getMaxBytesPerNode()) { + throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxBytesPerNode()); } synchronized (lock) { @@ -85,17 +89,17 @@ public void announceSegment(DataSegment segment) throws IOException SegmentZNode availableZNode = new SegmentZNode(makeServedSegmentPath(new DateTime().toString())); availableZNode.addSegment(segment); - log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath()); - announcer.announce(availableZNode.getPath(), availableZNode.getBytes()); - segmentLookup.put(segment, availableZNode); - availableZNodes.add(availableZNode); - } else { // update existing batch - Iterator iter = availableZNodes.iterator(); - boolean done = false; - while (iter.hasNext() && !done) { - SegmentZNode availableZNode = iter.next(); - if (availableZNode.getBytes().length + newBytesLen < config.getMaxNumBytes()) { - availableZNode.addSegment(segment); + log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath()); + announcer.announce(availableZNode.getPath(), availableZNode.getBytes()); + segmentLookup.put(segment, availableZNode); + availableZNodes.add(availableZNode); + } else { // update existing batch + Iterator iter = availableZNodes.iterator(); + boolean done = false; + while (iter.hasNext() && !done) { + SegmentZNode availableZNode = iter.next(); + if (availableZNode.getBytes().length + newBytesLen < config.getMaxBytesPerNode()) { + availableZNode.addSegment(segment); log.info("Announcing segment[%s] at path[%s]", segment.getIdentifier(), availableZNode.getPath()); announcer.update(availableZNode.getPath(), availableZNode.getBytes()); @@ -145,11 +149,11 @@ public void announceSegments(Iterable segments) throws IOException for (DataSegment segment : segments) { int newBytesLen = jsonMapper.writeValueAsBytes(segment).length; - if (newBytesLen > config.getMaxNumBytes()) { - throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxNumBytes()); + if (newBytesLen > config.getMaxBytesPerNode()) { + throw new ISE("byte size %,d exceeds %,d", newBytesLen, config.getMaxBytesPerNode()); } - if (count >= config.getSegmentsPerNode() || byteSize + newBytesLen > config.getMaxNumBytes()) { + if (count >= config.getSegmentsPerNode() || byteSize + newBytesLen > config.getMaxBytesPerNode()) { segmentZNode.addSegments(batch); announcer.announce(segmentZNode.getPath(), segmentZNode.getBytes()); diff --git a/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncerProvider.java b/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncerProvider.java new file mode 100644 index 000000000000..e51de3a245c8 --- /dev/null +++ b/server/src/main/java/io/druid/server/coordination/BatchDataSegmentAnnouncerProvider.java @@ -0,0 +1,39 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.coordination; + +import com.fasterxml.jackson.annotation.JacksonInject; + +import javax.validation.constraints.NotNull; + +/** + */ +public class BatchDataSegmentAnnouncerProvider implements DataSegmentAnnouncerProvider +{ + @JacksonInject + @NotNull + private BatchDataSegmentAnnouncer batchAnnouncer = null; + + @Override + public DataSegmentAnnouncer get() + { + return batchAnnouncer; + } +} diff --git a/client/src/main/java/com/metamx/druid/coordination/DataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncer.java similarity index 89% rename from client/src/main/java/com/metamx/druid/coordination/DataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncer.java index 71eaaa37276b..8993551c1b43 100644 --- a/client/src/main/java/com/metamx/druid/coordination/DataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.io.IOException; diff --git a/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncerProvider.java b/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncerProvider.java new file mode 100644 index 000000000000..eeed92b045a7 --- /dev/null +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentAnnouncerProvider.java @@ -0,0 +1,35 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.coordination; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +/** + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = LegacyDataSegmentAnnouncerProvider.class) +@JsonSubTypes(value = { + @JsonSubTypes.Type(name = "legacy", value = LegacyDataSegmentAnnouncerProvider.class), + @JsonSubTypes.Type(name = "batch", value = BatchDataSegmentAnnouncerProvider.class) +}) +public interface DataSegmentAnnouncerProvider extends Provider +{ +} diff --git a/server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeHandler.java b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeHandler.java similarity index 87% rename from server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeHandler.java rename to server/src/main/java/io/druid/server/coordination/DataSegmentChangeHandler.java index bc88027bbb8c..460e4b4256f4 100644 --- a/server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeHandler.java +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeHandler.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,9 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; /** */ diff --git a/server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeRequest.java b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeRequest.java similarity index 90% rename from server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeRequest.java rename to server/src/main/java/io/druid/server/coordination/DataSegmentChangeRequest.java index 596d49428f6d..0c29d1e1c750 100644 --- a/server/src/main/java/com/metamx/druid/coordination/DataSegmentChangeRequest.java +++ b/server/src/main/java/io/druid/server/coordination/DataSegmentChangeRequest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.client.DataSegment; /** */ diff --git a/client/src/main/java/com/metamx/druid/coordination/DruidServerMetadata.java b/server/src/main/java/io/druid/server/coordination/DruidServerMetadata.java similarity index 92% rename from client/src/main/java/com/metamx/druid/coordination/DruidServerMetadata.java rename to server/src/main/java/io/druid/server/coordination/DruidServerMetadata.java index 232df085e07f..ecb7f33b2eaf 100644 --- a/client/src/main/java/com/metamx/druid/coordination/DruidServerMetadata.java +++ b/server/src/main/java/io/druid/server/coordination/DruidServerMetadata.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -37,7 +37,8 @@ public DruidServerMetadata( @JsonProperty("name") String name, @JsonProperty("host") String host, @JsonProperty("maxSize") long maxSize, - @JsonProperty("type") String type, @JsonProperty("tier") String tier + @JsonProperty("type") String type, + @JsonProperty("tier") String tier ) { this.name = name; diff --git a/server/src/main/java/io/druid/server/coordination/LegacyDataSegmentAnnouncerProvider.java b/server/src/main/java/io/druid/server/coordination/LegacyDataSegmentAnnouncerProvider.java new file mode 100644 index 000000000000..f5a5dc405451 --- /dev/null +++ b/server/src/main/java/io/druid/server/coordination/LegacyDataSegmentAnnouncerProvider.java @@ -0,0 +1,46 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.coordination; + +import com.fasterxml.jackson.annotation.JacksonInject; + +import javax.validation.constraints.NotNull; +import java.util.Arrays; + +/** + */ +public class LegacyDataSegmentAnnouncerProvider implements DataSegmentAnnouncerProvider +{ + @JacksonInject + @NotNull + private SingleDataSegmentAnnouncer singleAnnouncer = null; + + @JacksonInject + @NotNull + private BatchDataSegmentAnnouncer batchAnnouncer = null; + + @Override + public DataSegmentAnnouncer get() + { + return new MultipleDataSegmentAnnouncerDataSegmentAnnouncer( + Arrays.asList(singleAnnouncer, batchAnnouncer) + ); + } +} diff --git a/client/src/main/java/com/metamx/druid/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java similarity index 69% rename from client/src/main/java/com/metamx/druid/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java index 053a3d85fac2..59eb4312497a 100644 --- a/client/src/main/java/com/metamx/druid/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/MultipleDataSegmentAnnouncerDataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.metamx.common.lifecycle.LifecycleStart; -import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import java.io.IOException; @@ -33,31 +28,15 @@ */ public class MultipleDataSegmentAnnouncerDataSegmentAnnouncer implements DataSegmentAnnouncer { - private final Iterable dataSegmentAnnouncers; + private final Iterable dataSegmentAnnouncers; public MultipleDataSegmentAnnouncerDataSegmentAnnouncer( - Iterable dataSegmentAnnouncers + Iterable dataSegmentAnnouncers ) { this.dataSegmentAnnouncers = dataSegmentAnnouncers; } - @LifecycleStart - public void start() - { - for (AbstractDataSegmentAnnouncer dataSegmentAnnouncer : dataSegmentAnnouncers) { - dataSegmentAnnouncer.start(); - } - } - - @LifecycleStop - public void stop() - { - for (AbstractDataSegmentAnnouncer dataSegmentAnnouncer : dataSegmentAnnouncers) { - dataSegmentAnnouncer.stop(); - } - } - @Override public void announceSegment(DataSegment segment) throws IOException { diff --git a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestDrop.java b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java similarity index 92% rename from server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestDrop.java rename to server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java index 05d4121c0391..9b0fc755fb3b 100644 --- a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestDrop.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestDrop.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; /** */ diff --git a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestLoad.java b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java similarity index 92% rename from server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestLoad.java rename to server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java index 64bb64fc3ed7..2a66693b66f4 100644 --- a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestLoad.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestLoad.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonUnwrapped; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; /** */ diff --git a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestNoop.java b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestNoop.java similarity index 91% rename from server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestNoop.java rename to server/src/main/java/io/druid/server/coordination/SegmentChangeRequestNoop.java index e72154fd81d7..e713278b47c4 100644 --- a/server/src/main/java/com/metamx/druid/coordination/SegmentChangeRequestNoop.java +++ b/server/src/main/java/io/druid/server/coordination/SegmentChangeRequestNoop.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; /** */ diff --git a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java b/server/src/main/java/io/druid/server/coordination/ServerManager.java similarity index 89% rename from server/src/main/java/com/metamx/druid/coordination/ServerManager.java rename to server/src/main/java/io/druid/server/coordination/ServerManager.java index 965cb4e3de64..ee7135db0bb2 100644 --- a/server/src/main/java/com/metamx/druid/coordination/ServerManager.java +++ b/server/src/main/java/io/druid/server/coordination/ServerManager.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,41 +17,43 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.google.common.base.Function; import com.google.common.base.Predicates; import com.google.common.collect.Ordering; +import com.google.inject.Inject; import com.metamx.common.ISE; import com.metamx.common.guava.FunctionalIterable; -import com.metamx.druid.Query; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.collect.CountingMap; -import com.metamx.druid.index.ReferenceCountingSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.loading.SegmentLoader; -import com.metamx.druid.loading.SegmentLoadingException; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.partition.PartitionHolder; -import com.metamx.druid.query.BySegmentQueryRunner; -import com.metamx.druid.query.FinalizeResultsQueryRunner; -import com.metamx.druid.query.MetricsEmittingQueryRunner; -import com.metamx.druid.query.NoopQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryRunnerFactoryConglomerate; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.ReferenceCountingSegmentQueryRunner; -import com.metamx.druid.query.segment.QuerySegmentSpec; -import com.metamx.druid.query.segment.QuerySegmentWalker; -import com.metamx.druid.query.segment.SegmentDescriptor; -import com.metamx.druid.query.segment.SpecificSegmentQueryRunner; -import com.metamx.druid.query.segment.SpecificSegmentSpec; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.collections.CountingMap; +import io.druid.guice.annotations.Processing; +import io.druid.query.BySegmentQueryRunner; +import io.druid.query.FinalizeResultsQueryRunner; +import io.druid.query.MetricsEmittingQueryRunner; +import io.druid.query.NoopQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerFactoryConglomerate; +import io.druid.query.QuerySegmentWalker; +import io.druid.query.QueryToolChest; +import io.druid.query.ReferenceCountingSegmentQueryRunner; +import io.druid.query.SegmentDescriptor; +import io.druid.query.spec.QuerySegmentSpec; +import io.druid.query.spec.SpecificSegmentQueryRunner; +import io.druid.query.spec.SpecificSegmentSpec; +import io.druid.segment.ReferenceCountingSegment; +import io.druid.segment.Segment; +import io.druid.segment.loading.SegmentLoader; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.PartitionChunk; +import io.druid.timeline.partition.PartitionHolder; import org.joda.time.Interval; import javax.annotation.Nullable; @@ -78,11 +80,12 @@ public class ServerManager implements QuerySegmentWalker private final CountingMap dataSourceSizes = new CountingMap(); private final CountingMap dataSourceCounts = new CountingMap(); + @Inject public ServerManager( SegmentLoader segmentLoader, QueryRunnerFactoryConglomerate conglomerate, ServiceEmitter emitter, - ExecutorService exec + @Processing ExecutorService exec ) { this.segmentLoader = segmentLoader; diff --git a/client/src/main/java/com/metamx/druid/coordination/SingleDataSegmentAnnouncer.java b/server/src/main/java/io/druid/server/coordination/SingleDataSegmentAnnouncer.java similarity index 90% rename from client/src/main/java/com/metamx/druid/coordination/SingleDataSegmentAnnouncer.java rename to server/src/main/java/io/druid/server/coordination/SingleDataSegmentAnnouncer.java index 1757ccbe6af7..37107e0dfdbd 100644 --- a/client/src/main/java/com/metamx/druid/coordination/SingleDataSegmentAnnouncer.java +++ b/server/src/main/java/io/druid/server/coordination/SingleDataSegmentAnnouncer.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkPathsConfig; +import io.druid.curator.announcement.Announcer; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.utils.ZKPaths; import java.io.IOException; @@ -36,6 +37,7 @@ public class SingleDataSegmentAnnouncer extends AbstractDataSegmentAnnouncer private final ObjectMapper jsonMapper; private final String servedSegmentsLocation; + @Inject public SingleDataSegmentAnnouncer( DruidServerMetadata server, ZkPathsConfig config, diff --git a/server/src/main/java/com/metamx/druid/coordination/ZkCoordinator.java b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java similarity index 89% rename from server/src/main/java/com/metamx/druid/coordination/ZkCoordinator.java rename to server/src/main/java/io/druid/server/coordination/ZkCoordinator.java index 39799c812ee7..a55341a75a15 100644 --- a/server/src/main/java/com/metamx/druid/coordination/ZkCoordinator.java +++ b/server/src/main/java/io/druid/server/coordination/ZkCoordinator.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,18 +17,20 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.google.inject.Inject; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.loading.SegmentLoadingException; import com.metamx.emitter.EmittingLogger; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.PathChildrenCache; @@ -49,7 +51,7 @@ public class ZkCoordinator implements DataSegmentChangeHandler private final Object lock = new Object(); private final ObjectMapper jsonMapper; - private final ZkCoordinatorConfig config; + private final SegmentLoaderConfig config; private final ZkPathsConfig zkPaths; private final DruidServerMetadata me; private final DataSegmentAnnouncer announcer; @@ -59,9 +61,10 @@ public class ZkCoordinator implements DataSegmentChangeHandler private volatile PathChildrenCache loadQueueCache; private volatile boolean started; + @Inject public ZkCoordinator( ObjectMapper jsonMapper, - ZkCoordinatorConfig config, + SegmentLoaderConfig config, ZkPathsConfig zkPaths, DruidServerMetadata me, DataSegmentAnnouncer announcer, @@ -100,21 +103,13 @@ public void start() throws IOException ); try { - config.getSegmentInfoCacheDirectory().mkdirs(); + config.getInfoDir().mkdirs(); curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(servedSegmentsLocation).ensure(curator.getZookeeperClient()); curator.newNamespaceAwareEnsurePath(liveSegmentsLocation).ensure(curator.getZookeeperClient()); - if (config.isLoadFromSegmentCacheEnabled()) { - try { - loadCache(); - } - catch (Exception e) { - log.makeAlert(e, "Exception loading from cache") - .emit(); - } - } + loadCache(); loadQueueCache.getListenable().addListener( new PathChildrenCacheListener() @@ -198,7 +193,7 @@ public void stop() private void loadCache() { - File baseDir = config.getSegmentInfoCacheDirectory(); + File baseDir = config.getInfoDir(); if (!baseDir.exists()) { return; } @@ -213,7 +208,7 @@ private void loadCache() } else { log.warn("Unable to find cache file for %s. Deleting lookup entry", segment.getIdentifier()); - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.delete()) { log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); } @@ -243,7 +238,7 @@ public void addSegment(DataSegment segment) throw new SegmentLoadingException(e, "Exception loading segment[%s]", segment.getIdentifier()); } - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.exists()) { try { jsonMapper.writeValue(segmentInfoCacheFile, segment); @@ -290,7 +285,7 @@ public void addSegments(Iterable segments) continue; } - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.exists()) { try { jsonMapper.writeValue(segmentInfoCacheFile, segment); @@ -334,7 +329,7 @@ public void removeSegment(DataSegment segment) try { serverManager.dropSegment(segment); - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.delete()) { log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); } @@ -354,7 +349,7 @@ public void removeSegments(Iterable segments) for (DataSegment segment : segments) { serverManager.dropSegment(segment); - File segmentInfoCacheFile = new File(config.getSegmentInfoCacheDirectory(), segment.getIdentifier()); + File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier()); if (!segmentInfoCacheFile.delete()) { log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile); } diff --git a/server/src/main/java/com/metamx/druid/http/InfoResource.java b/server/src/main/java/io/druid/server/http/InfoResource.java similarity index 87% rename from server/src/main/java/com/metamx/druid/http/InfoResource.java rename to server/src/main/java/io/druid/server/http/InfoResource.java index 8fd17f315dea..670942d06cfb 100644 --- a/server/src/main/java/com/metamx/druid/http/InfoResource.java +++ b/server/src/main/java/io/druid/server/http/InfoResource.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; import com.google.common.base.Function; import com.google.common.collect.Collections2; @@ -26,19 +26,19 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.InventoryView; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.rules.Rule; +import com.google.inject.Inject; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.client.InventoryView; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.db.DatabaseRuleManager; +import io.druid.db.DatabaseSegmentManager; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.rules.Rule; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; import javax.annotation.Nullable; -import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -83,8 +83,8 @@ public Map apply(DataSegment segment) { return new ImmutableMap.Builder() .put("id", segment.getIdentifier()) - .put("dataSource", segment.getDimensions()) - .put("interval", segment.getInterval()) + .put("dataSource", segment.getDataSource()) + .put("interval", segment.getInterval().toString()) .put("version", segment.getVersion()) .put("size", segment.getSize()) .build(); @@ -143,9 +143,11 @@ public Response getClusterInfo( .build(); } return Response.ok( - Iterables.transform( - serverInventoryView.getInventory(), - simplifyClusterFn + Lists.newArrayList( + Iterables.transform( + serverInventoryView.getInventory(), + simplifyClusterFn + ) ) ).build(); } @@ -159,20 +161,22 @@ public Response getClusterServers( { Response.ResponseBuilder builder = Response.status(Response.Status.OK); if (full != null) { - return builder.entity(serverInventoryView.getInventory()).build(); + return builder.entity(Lists.newArrayList(serverInventoryView.getInventory())).build(); } return builder.entity( - Iterables.transform( - serverInventoryView.getInventory(), - new Function() - { - @Override - public String apply(@Nullable DruidServer druidServer) - { - return druidServer.getName(); - } - } + Lists.newArrayList( + Iterables.transform( + serverInventoryView.getInventory(), + new Function() + { + @Override + public String apply(DruidServer druidServer) + { + return druidServer.getHost(); + } + } + ) ) ).build(); } @@ -259,44 +263,48 @@ public Response getClusterSegments( Response.ResponseBuilder builder = Response.status(Response.Status.OK); if (full != null) { return builder.entity( - Iterables.concat( - Iterables.transform( - serverInventoryView.getInventory(), - new Function>() - { - @Override - public Iterable apply(@Nullable DruidServer druidServer) - { - return druidServer.getSegments().values(); - } - } + Lists.newArrayList( + Iterables.concat( + Iterables.transform( + serverInventoryView.getInventory(), + new Function>() + { + @Override + public Iterable apply(@Nullable DruidServer druidServer) + { + return druidServer.getSegments().values(); + } + } + ) ) ) ).build(); } return builder.entity( - Iterables.concat( - Iterables.transform( - serverInventoryView.getInventory(), - new Function>() - { - @Override - public Iterable apply(@Nullable DruidServer druidServer) - { - return Collections2.transform( - druidServer.getSegments().values(), - new Function() - { - @Override - public String apply(@Nullable DataSegment segment) - { - return segment.getIdentifier(); - } - } - ); - } - } + Lists.newArrayList( + Iterables.concat( + Iterables.transform( + serverInventoryView.getInventory(), + new Function>() + { + @Override + public Iterable apply(@Nullable DruidServer druidServer) + { + return Collections2.transform( + druidServer.getSegments().values(), + new Function() + { + @Override + public String apply(@Nullable DataSegment segment) + { + return segment.getIdentifier(); + } + } + ); + } + } + ) ) ) ).build(); @@ -388,16 +396,18 @@ public Response getQueryableDataSources( } return builder.entity( - Iterables.transform( - getDataSources(), - new Function() - { - @Override - public String apply(@Nullable DruidDataSource dataSource) - { - return dataSource.getName(); - } - } + Lists.newArrayList( + Iterables.transform( + getDataSources(), + new Function() + { + @Override + public String apply(@Nullable DruidDataSource dataSource) + { + return dataSource.getName(); + } + } + ) ) ).build(); } diff --git a/server/src/main/resources/static/js/.Rhistory b/server/src/main/java/io/druid/server/http/MasterMain.java similarity index 100% rename from server/src/main/resources/static/js/.Rhistory rename to server/src/main/java/io/druid/server/http/MasterMain.java diff --git a/server/src/main/java/io/druid/server/http/MasterRedirectInfo.java b/server/src/main/java/io/druid/server/http/MasterRedirectInfo.java new file mode 100644 index 000000000000..3857ea04d63a --- /dev/null +++ b/server/src/main/java/io/druid/server/http/MasterRedirectInfo.java @@ -0,0 +1,66 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.http; + +import com.google.common.base.Throwables; +import com.google.inject.Inject; +import io.druid.server.master.DruidMaster; + +import java.net.URL; + +/** +*/ +public class MasterRedirectInfo implements RedirectInfo +{ + private final DruidMaster master; + + @Inject + public MasterRedirectInfo(DruidMaster master) { + this.master = master; + } + + @Override + public boolean doLocal() + { + return master.isClusterMaster(); + } + + @Override + public URL getRedirectURL(String queryString, String requestURI) + { + try { + final String currentMaster = master.getCurrentMaster(); + if (currentMaster == null) { + return null; + } + + String location = String.format("http://%s%s", currentMaster, requestURI); + + if (queryString != null) { + location = String.format("%s?%s", location, queryString); + } + + return new URL(location); + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } +} diff --git a/server/src/main/java/com/metamx/druid/http/MasterResource.java b/server/src/main/java/io/druid/server/http/MasterResource.java similarity index 93% rename from server/src/main/java/com/metamx/druid/http/MasterResource.java rename to server/src/main/java/io/druid/server/http/MasterResource.java index 9bb59d79d435..794e48a1c243 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterResource.java +++ b/server/src/main/java/io/druid/server/http/MasterResource.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.LoadPeonCallback; +import com.google.inject.Inject; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.LoadPeonCallback; -import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; diff --git a/server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java b/server/src/main/java/io/druid/server/http/MasterSegmentSettingsResource.java similarity index 56% rename from server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java rename to server/src/main/java/io/druid/server/http/MasterSegmentSettingsResource.java index ea14c587f789..e86311ccaec9 100644 --- a/server/src/main/java/com/metamx/druid/http/MasterSegmentSettingsResource.java +++ b/server/src/main/java/io/druid/server/http/MasterSegmentSettingsResource.java @@ -1,25 +1,26 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.http; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.http; -import com.metamx.druid.config.JacksonConfigManager; -import com.metamx.druid.master.MasterSegmentSettings; +import io.druid.common.config.JacksonConfigManager; +import io.druid.server.master.MasterSegmentSettings; import javax.inject.Inject; import javax.ws.rs.Consumes; diff --git a/server/src/main/java/com/metamx/druid/http/RedirectFilter.java b/server/src/main/java/io/druid/server/http/RedirectFilter.java similarity index 86% rename from server/src/main/java/com/metamx/druid/http/RedirectFilter.java rename to server/src/main/java/io/druid/server/http/RedirectFilter.java index f6f4c30e7cf9..671a2cb207e5 100644 --- a/server/src/main/java/com/metamx/druid/http/RedirectFilter.java +++ b/server/src/main/java/io/druid/server/http/RedirectFilter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; -import com.metamx.http.client.response.HttpResponseHandler; import javax.servlet.Filter; import javax.servlet.FilterChain; @@ -39,15 +39,13 @@ public class RedirectFilter implements Filter { private static final Logger log = new Logger(RedirectFilter.class); - private final HttpResponseHandler responseHandler; private final RedirectInfo redirectInfo; + @Inject public RedirectFilter( - HttpResponseHandler responseHandler, RedirectInfo redirectInfo ) { - this.responseHandler = responseHandler; this.redirectInfo = redirectInfo; } @@ -73,7 +71,13 @@ public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) chain.doFilter(request, response); } else { URL url = redirectInfo.getRedirectURL(request.getQueryString(), request.getRequestURI()); - log.info("Forwarding request to [%s]", url); + log.debug("Forwarding request to [%s]", url); + + if (url == null) { + // We apparently have no master, so let's do a Service Unavailable + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + return; + } response.setStatus(HttpServletResponse.SC_MOVED_TEMPORARILY); response.setHeader("Location", url.toString()); diff --git a/server/src/main/java/com/metamx/druid/http/RedirectInfo.java b/server/src/main/java/io/druid/server/http/RedirectInfo.java similarity index 91% rename from server/src/main/java/com/metamx/druid/http/RedirectInfo.java rename to server/src/main/java/io/druid/server/http/RedirectInfo.java index d14839710fcf..3efea33a1462 100644 --- a/server/src/main/java/com/metamx/druid/http/RedirectInfo.java +++ b/server/src/main/java/io/druid/server/http/RedirectInfo.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; import java.net.URL; diff --git a/server/src/main/java/com/metamx/druid/http/RedirectServlet.java b/server/src/main/java/io/druid/server/http/RedirectServlet.java similarity index 94% rename from server/src/main/java/com/metamx/druid/http/RedirectServlet.java rename to server/src/main/java/io/druid/server/http/RedirectServlet.java index 2befa9cd4583..2226fed3d836 100644 --- a/server/src/main/java/com/metamx/druid/http/RedirectServlet.java +++ b/server/src/main/java/io/druid/server/http/RedirectServlet.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,8 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; +import com.google.inject.Inject; import com.metamx.common.logger.Logger; import org.mortbay.jetty.servlet.DefaultServlet; @@ -38,6 +39,7 @@ public class RedirectServlet extends DefaultServlet private final RedirectInfo redirectInfo; + @Inject public RedirectServlet( RedirectInfo redirectInfo ) diff --git a/server/src/main/java/com/metamx/druid/http/SegmentToDrop.java b/server/src/main/java/io/druid/server/http/SegmentToDrop.java similarity index 94% rename from server/src/main/java/com/metamx/druid/http/SegmentToDrop.java rename to server/src/main/java/io/druid/server/http/SegmentToDrop.java index 44e9d4a0d33f..e3fa092e6102 100644 --- a/server/src/main/java/com/metamx/druid/http/SegmentToDrop.java +++ b/server/src/main/java/io/druid/server/http/SegmentToDrop.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/main/java/com/metamx/druid/http/SegmentToMove.java b/server/src/main/java/io/druid/server/http/SegmentToMove.java similarity index 94% rename from server/src/main/java/com/metamx/druid/http/SegmentToMove.java rename to server/src/main/java/io/druid/server/http/SegmentToMove.java index 85a6ab7227ff..159afa0cda2d 100644 --- a/server/src/main/java/com/metamx/druid/http/SegmentToMove.java +++ b/server/src/main/java/io/druid/server/http/SegmentToMove.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.http; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/main/java/io/druid/server/initialization/BatchDataSegmentAnnouncerConfig.java b/server/src/main/java/io/druid/server/initialization/BatchDataSegmentAnnouncerConfig.java new file mode 100644 index 000000000000..5df79357c774 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/BatchDataSegmentAnnouncerConfig.java @@ -0,0 +1,49 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import javax.validation.constraints.Max; +import javax.validation.constraints.Min; + +/** + */ +public class BatchDataSegmentAnnouncerConfig +{ + @JsonProperty + @Min(1) + private int segmentsPerNode = 50; + + @JsonProperty + @Max(1024 * 1024) + @Min(1024) + private long maxBytesPerNode = 512 * 1024; + + public int getSegmentsPerNode() + { + return segmentsPerNode; + } + + public long getMaxBytesPerNode() + { + return maxBytesPerNode; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/ConfigModule.java b/server/src/main/java/io/druid/server/initialization/ConfigModule.java new file mode 100644 index 000000000000..62644146a05a --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/ConfigModule.java @@ -0,0 +1,50 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.metamx.common.config.Config; +import io.druid.guice.JsonConfigurator; +import io.druid.guice.LazySingleton; +import org.skife.config.ConfigurationObjectFactory; + +import javax.validation.Validation; +import javax.validation.Validator; +import java.util.Properties; + +/** + */ +public class ConfigModule implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(Validator.class).toInstance(Validation.buildDefaultValidatorFactory().getValidator()); + binder.bind(JsonConfigurator.class).in(LazySingleton.class); + } + + @Provides @LazySingleton + public ConfigurationObjectFactory makeFactory(Properties props) + { + return Config.createFactory(props); + } +} diff --git a/server/src/main/java/io/druid/server/initialization/CuratorDiscoveryConfig.java b/server/src/main/java/io/druid/server/initialization/CuratorDiscoveryConfig.java new file mode 100644 index 000000000000..e029112b1839 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/CuratorDiscoveryConfig.java @@ -0,0 +1,40 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + */ +public class CuratorDiscoveryConfig +{ + @JsonProperty + private String path = null; + + public String getPath() + { + return path; + } + + public boolean useDiscovery() + { + return path != null; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/EmitterModule.java b/server/src/main/java/io/druid/server/initialization/EmitterModule.java new file mode 100644 index 000000000000..c7b29d3af53d --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/EmitterModule.java @@ -0,0 +1,142 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Binding; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.Provider; +import com.google.inject.Provides; +import com.google.inject.TypeLiteral; +import com.google.inject.name.Named; +import com.google.inject.name.Names; +import com.metamx.common.ISE; +import com.metamx.common.logger.Logger; +import com.metamx.emitter.EmittingLogger; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.service.ServiceEmitter; +import io.druid.guice.LazySingleton; +import io.druid.guice.ManageLifecycle; +import io.druid.guice.annotations.Self; +import io.druid.server.DruidNode; + +import java.lang.annotation.Annotation; +import java.util.List; +import java.util.Properties; + +/** + */ +public class EmitterModule implements Module +{ + private static final Logger log = new Logger(EmitterModule.class); + private static final String EMITTER_PROPERTY = "druid.emitter"; + + private final Properties props; + + @Inject + public EmitterModule( + Properties props + ) + { + this.props = props; + } + + @Override + public void configure(Binder binder) + { + String emitterType = props.getProperty(EMITTER_PROPERTY, ""); + + binder.install(new LogEmitterModule()); + binder.install(new HttpEmitterModule()); + + binder.bind(Emitter.class).toProvider(new EmitterProvider(emitterType)).in(LazySingleton.class); + } + + @Provides + @ManageLifecycle + public ServiceEmitter getServiceEmitter(@Self Supplier configSupplier, Emitter emitter) + { + final DruidNode config = configSupplier.get(); + final ServiceEmitter retVal = new ServiceEmitter(config.getServiceName(), config.getHost(), emitter); + EmittingLogger.registerEmitter(retVal); + return retVal; + } + + private static class EmitterProvider implements Provider + { + private final String emitterType; + + private Emitter emitter = null; + + EmitterProvider( + String emitterType + ) + { + this.emitterType = emitterType; + } + + @Inject + public void inject(Injector injector) + { + final List> emitterBindings = injector.findBindingsByType(new TypeLiteral(){}); + + emitter = findEmitter(emitterType, emitterBindings); + + if (emitter == null) { + emitter = findEmitter(LogEmitterModule.EMITTER_TYPE, emitterBindings); + } + + if (emitter == null) { + List knownTypes = Lists.newArrayList(); + for (Binding binding : emitterBindings) { + final Annotation annotation = binding.getKey().getAnnotation(); + if (annotation != null) { + knownTypes.add(((Named) annotation).value()); + } + } + throw new ISE("Uknown emitter type[%s]=[%s], known types[%s]", EMITTER_PROPERTY, emitterType, knownTypes); + } + } + + private Emitter findEmitter(String emitterType, List> emitterBindings) + { + for (Binding binding : emitterBindings) { + if (Names.named(emitterType).equals(binding.getKey().getAnnotation())) { + return binding.getProvider().get(); + } + } + return null; + } + + + @Override + public Emitter get() + { + if (emitter == null) { + throw new ISE("Emitter was null, that's bad!"); + } + return emitter; + } + } +} diff --git a/server/src/main/java/io/druid/server/initialization/ExtensionsConfig.java b/server/src/main/java/io/druid/server/initialization/ExtensionsConfig.java new file mode 100644 index 000000000000..0973a2494346 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/ExtensionsConfig.java @@ -0,0 +1,61 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.ImmutableList; + +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + */ +public class ExtensionsConfig +{ + @JsonProperty + @NotNull + private List coordinates = ImmutableList.of(); + + @JsonProperty + @NotNull + private String localRepository = String.format("%s/%s", System.getProperty("user.home"), ".m2/repository"); + + @JsonProperty + @NotNull + private List remoteRepositories = ImmutableList.of( + "http://repo1.maven.org/maven2/", + "https://metamx.artifactoryonline.com/metamx/pub-libs-releases-local" + ); + + public List getCoordinates() + { + return coordinates; + } + + public String getLocalRepository() + { + return localRepository; + } + + public List getRemoteRepositories() + { + return remoteRepositories; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java b/server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java new file mode 100644 index 000000000000..9141f7316b18 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/HttpEmitterConfig.java @@ -0,0 +1,36 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +/** + */ +public class HttpEmitterConfig extends com.metamx.emitter.core.HttpEmitterConfig +{ + @JsonProperty + private Period timeOut = new Period("PT5M"); + + public Period getReadTimeout() + { + return timeOut; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java b/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java new file mode 100644 index 000000000000..fb460b7f3d02 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/HttpEmitterModule.java @@ -0,0 +1,67 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.google.inject.util.Providers; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.core.HttpPostEmitter; +import com.metamx.http.client.HttpClientConfig; +import com.metamx.http.client.HttpClientInit; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.guice.ManageLifecycle; + +import javax.annotation.Nullable; +import javax.net.ssl.SSLContext; + +/** + */ +public class HttpEmitterModule implements Module +{ + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter.http", HttpEmitterConfig.class); + + // Fix the injection of this if we want to enable ssl emission of events. + binder.bind(SSLContext.class).toProvider(Providers.of(null)).in(LazySingleton.class); + } + + @Provides @ManageLifecycle @Named("http") + public Emitter getEmitter(Supplier config, @Nullable SSLContext sslContext, Lifecycle lifecycle) + { + final HttpClientConfig.Builder builder = HttpClientConfig + .builder() + .withNumConnections(1) + .withReadTimeout(config.get().getReadTimeout().toStandardDuration()); + + if (sslContext != null) { + builder.withSslContext(sslContext); + } + + return new HttpPostEmitter(config.get(), HttpClientInit.createClient(builder.build(), lifecycle)); + } +} diff --git a/server/src/main/java/io/druid/server/initialization/Initialization.java b/server/src/main/java/io/druid/server/initialization/Initialization.java new file mode 100644 index 000000000000..96b20ae077ae --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/Initialization.java @@ -0,0 +1,245 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Throwables; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.io.ByteStreams; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.metamx.common.ISE; +import com.metamx.common.logger.Logger; +import io.druid.guice.DruidSecondaryModule; +import io.druid.guice.annotations.Json; +import io.druid.guice.annotations.Smile; +import io.druid.initialization.DruidModule; +import io.tesla.aether.TeslaAether; +import io.tesla.aether.internal.DefaultTeslaAether; +import org.eclipse.aether.artifact.Artifact; +import org.eclipse.aether.artifact.DefaultArtifact; +import org.eclipse.aether.collection.CollectRequest; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.graph.DependencyFilter; +import org.eclipse.aether.graph.DependencyNode; +import org.eclipse.aether.resolution.DependencyRequest; +import org.eclipse.aether.util.artifact.JavaScopes; +import org.eclipse.aether.util.filter.DependencyFilterUtils; + +import java.io.PrintStream; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.Set; + +/** + */ +public class Initialization +{ + private static final Logger log = new Logger(Initialization.class); + private static final Map loadersMap = Maps.newHashMap(); + + private static final Set exclusions = Sets.newHashSet( + "io.druid", + "com.metamx.druid" + ); + + public synchronized static List getFromExtensions(ExtensionsConfig config, Class clazz) + { + final TeslaAether aether = getAetherClient(config); + List retVal = Lists.newArrayList(); + + for (String coordinate : config.getCoordinates()) { + log.info("Loading extension[%s]", coordinate); + try { + ClassLoader loader = loadersMap.get(coordinate); + if (loader == null) { + final CollectRequest collectRequest = new CollectRequest(); + collectRequest.setRoot(new Dependency(new DefaultArtifact(coordinate), JavaScopes.RUNTIME)); + DependencyRequest dependencyRequest = new DependencyRequest( + collectRequest, + DependencyFilterUtils.andFilter( + DependencyFilterUtils.classpathFilter(JavaScopes.RUNTIME), + new DependencyFilter() + { + @Override + public boolean accept(DependencyNode node, List parents) + { + if (accept(node.getArtifact())) { + return false; + } + + for (DependencyNode parent : parents) { + if (accept(parent.getArtifact())) { + return false; + } + } + + return true; + } + + private boolean accept(final Artifact artifact) + { + return exclusions.contains(artifact.getGroupId()); + } + } + ) + ); + + final List artifacts = aether.resolveArtifacts(dependencyRequest); + List urls = Lists.newArrayListWithExpectedSize(artifacts.size()); + for (Artifact artifact : artifacts) { + if (!exclusions.contains(artifact.getGroupId())) { + urls.add(artifact.getFile().toURI().toURL()); + } + else { + log.error("Skipped Artifact[%s]", artifact); + } + } + + for (URL url : urls) { + log.error("Added URL[%s]", url); + } + + loader = new URLClassLoader(urls.toArray(new URL[urls.size()]), Initialization.class.getClassLoader()); + loadersMap.put(coordinate, loader); + } + + final ServiceLoader serviceLoader = ServiceLoader.load(clazz, loader); + + for (T module : serviceLoader) { + log.info("Adding extension module[%s]", module.getClass()); + retVal.add(module); + } + } + catch (Exception e) { + throw Throwables.propagate(e); + } + } + + return retVal; + } + + private static DefaultTeslaAether getAetherClient(ExtensionsConfig config) + { + /* + DefaultTeslaAether logs a bunch of stuff to System.out, which is annoying. We choose to disable that + unless debug logging is turned on. "Disabling" it, however, is kinda bass-ackwards. We copy out a reference + to the current System.out, and set System.out to a noop output stream. Then after DefaultTeslaAether has pulled + The reference we swap things back. + + This has implications for other things that are running in parallel to this. Namely, if anything else also grabs + a reference to System.out or tries to log to it while we have things adjusted like this, then they will also log + to nothingness. Fortunately, the code that calls this is single-threaded and shouldn't hopefully be running + alongside anything else that's grabbing System.out. But who knows. + */ + if (log.isTraceEnabled() || log.isDebugEnabled()) { + return new DefaultTeslaAether(config.getLocalRepository(), config.getRemoteRepositories()); + } + + PrintStream oldOut = System.out; + try { + System.setOut(new PrintStream(ByteStreams.nullOutputStream())); + return new DefaultTeslaAether(config.getLocalRepository(), config.getRemoteRepositories()); + } + finally { + System.setOut(oldOut); + } + } + + public static Injector makeInjectorWithModules(final Injector baseInjector, List modules) + { + ModuleList actualModules = new ModuleList(baseInjector); + actualModules.addModule(DruidSecondaryModule.class); + for (Object module : modules) { + actualModules.addModule(module); + } + + final ExtensionsConfig config = baseInjector.getInstance(ExtensionsConfig.class); + for (DruidModule module : Initialization.getFromExtensions(config, DruidModule.class)) { + actualModules.addModule(module); + } + + return Guice.createInjector(actualModules.getModules()); + } + + private static class ModuleList + { + private final Injector baseInjector; + private final ObjectMapper jsonMapper; + private final ObjectMapper smileMapper; + private final List modules; + + public ModuleList(Injector baseInjector) { + this.baseInjector = baseInjector; + this.jsonMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Json.class)); + this.smileMapper = baseInjector.getInstance(Key.get(ObjectMapper.class, Smile.class)); + this.modules = Lists.newArrayList(); + } + + private List getModules() + { + return Collections.unmodifiableList(modules); + } + + public void addModule(Object input) + { + if (input instanceof DruidModule) { + baseInjector.injectMembers(input); + modules.add(registerJacksonModules(((DruidModule) input))); + } + else if (input instanceof Module) { + baseInjector.injectMembers(input); + modules.add((Module) input); + } + else if (input instanceof Class) { + if (DruidModule.class.isAssignableFrom((Class) input)) { + modules.add(registerJacksonModules(baseInjector.getInstance((Class) input))); + } + else if (Module.class.isAssignableFrom((Class) input)) { + modules.add(baseInjector.getInstance((Class) input)); + return; + } + else { + throw new ISE("Class[%s] does not implement %s", input.getClass(), Module.class); + } + } + else { + throw new ISE("Unknown module type[%s]", input.getClass()); + } + } + + private DruidModule registerJacksonModules(DruidModule module) + { + for (com.fasterxml.jackson.databind.Module jacksonModule : module.getJacksonModules()) { + jsonMapper.registerModule(jacksonModule); + smileMapper.registerModule(jacksonModule); + } + return module; + } + } +} diff --git a/server/src/main/java/io/druid/server/initialization/JettyServerInitializer.java b/server/src/main/java/io/druid/server/initialization/JettyServerInitializer.java new file mode 100644 index 000000000000..44229e99dc78 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/JettyServerInitializer.java @@ -0,0 +1,30 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.inject.Injector; +import org.eclipse.jetty.server.Server; + +/** + */ +public interface JettyServerInitializer +{ + public void initialize(Server server, Injector injector); +} diff --git a/server/src/main/java/io/druid/server/initialization/JettyServerModule.java b/server/src/main/java/io/druid/server/initialization/JettyServerModule.java new file mode 100644 index 000000000000..80c6fb7cc4bd --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/JettyServerModule.java @@ -0,0 +1,175 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.primitives.Ints; +import com.google.inject.Binder; +import com.google.inject.ConfigurationException; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Provides; +import com.google.inject.ProvisionException; +import com.google.inject.Scopes; +import com.google.inject.name.Names; +import com.metamx.common.lifecycle.Lifecycle; +import com.metamx.common.logger.Logger; +import com.sun.jersey.api.core.DefaultResourceConfig; +import com.sun.jersey.api.core.ResourceConfig; +import com.sun.jersey.guice.JerseyServletModule; +import com.sun.jersey.guice.spi.container.servlet.GuiceContainer; +import com.sun.jersey.spi.container.servlet.WebConfig; +import io.druid.guice.Jerseys; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.LazySingleton; +import io.druid.guice.annotations.JSR311Resource; +import io.druid.guice.annotations.Self; +import io.druid.server.DruidNode; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.nio.SelectChannelConnector; +import org.eclipse.jetty.util.thread.QueuedThreadPool; + +import javax.servlet.ServletException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + */ +public class JettyServerModule extends JerseyServletModule +{ + private static final Logger log = new Logger(JettyServerModule.class); + + private final JettyServerInitializer initializer; + private final List> resources = Lists.newArrayList(); + + public JettyServerModule( + JettyServerInitializer initializer + ) + { + this.initializer = initializer; + } + + public JettyServerModule addResource(Class resource) + { + resources.add(resource); + return this; + } + + @Override + protected void configureServlets() + { + Binder binder = binder(); + + JsonConfigProvider.bind(binder, "druid.server.http", ServerConfig.class); + + binder.bind(GuiceContainer.class).to(DruidGuiceContainer.class); + binder.bind(DruidGuiceContainer.class).in(Scopes.SINGLETON); + serve("/*").with(DruidGuiceContainer.class); + + for (Class resource : resources) { + Jerseys.addResource(binder, resource); + binder.bind(resource).in(LazySingleton.class); + } + + binder.bind(Key.get(Server.class, Names.named("ForTheEagerness"))).to(Server.class).asEagerSingleton(); + } + + public static class DruidGuiceContainer extends GuiceContainer + { + private final Set> resources; + + @Inject + public DruidGuiceContainer( + Injector injector, + @JSR311Resource Set> resources + ) + { + super(injector); + this.resources = resources; + } + + @Override + protected ResourceConfig getDefaultResourceConfig( + Map props, WebConfig webConfig + ) throws ServletException + { + return new DefaultResourceConfig(resources); + } + } + + @Provides @LazySingleton + public Server getServer(Injector injector, Lifecycle lifecycle, @Self DruidNode node, ServerConfig config) + { + final Server server = makeJettyServer(node, config); + try { + initializer.initialize(server, injector); + } + catch (ConfigurationException e) { + throw new ProvisionException(Iterables.getFirst(e.getErrorMessages(), null).getMessage()); + } + + + lifecycle.addHandler( + new Lifecycle.Handler() + { + @Override + public void start() throws Exception + { + server.start(); + } + + @Override + public void stop() + { + try { + server.stop(); + } + catch (Exception e) { + log.warn(e, "Unable to stop Jetty server."); + } + } + } + ); + return server; + } + + private static Server makeJettyServer(@Self DruidNode node, ServerConfig config) + { + final QueuedThreadPool threadPool = new QueuedThreadPool(); + threadPool.setMinThreads(config.getNumThreads()); + threadPool.setMaxThreads(config.getNumThreads()); + + final Server server = new Server(); + server.setThreadPool(threadPool); + + SelectChannelConnector connector = new SelectChannelConnector(); + connector.setPort(node.getPort()); + connector.setMaxIdleTime(Ints.checkedCast(config.getMaxIdleTime().toStandardDuration().getMillis())); + connector.setStatsOn(true); + + server.setConnectors(new Connector[]{connector}); + + return server; + } +} diff --git a/server/src/main/java/io/druid/server/initialization/LogEmitterModule.java b/server/src/main/java/io/druid/server/initialization/LogEmitterModule.java new file mode 100644 index 000000000000..4759cf577589 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/LogEmitterModule.java @@ -0,0 +1,51 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Supplier; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.name.Named; +import com.metamx.emitter.core.Emitter; +import com.metamx.emitter.core.LoggingEmitter; +import com.metamx.emitter.core.LoggingEmitterConfig; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.ManageLifecycle; + +/** + */ +public class LogEmitterModule implements Module +{ + public static final String EMITTER_TYPE = "logging"; + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.emitter.logging", LoggingEmitterConfig.class); + } + + @Provides @ManageLifecycle @Named(EMITTER_TYPE) + public Emitter makeEmitter(Supplier config, ObjectMapper jsonMapper) + { + return new LoggingEmitter(config.get(), jsonMapper); + } +} diff --git a/server/src/main/java/io/druid/server/initialization/PropertiesModule.java b/server/src/main/java/io/druid/server/initialization/PropertiesModule.java new file mode 100644 index 000000000000..3aad005dd68f --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/PropertiesModule.java @@ -0,0 +1,88 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.google.common.base.Charsets; +import com.google.common.base.Throwables; +import com.google.common.io.Closeables; +import com.google.inject.Binder; +import com.google.inject.Module; +import com.metamx.common.logger.Logger; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Properties; + +/** + */ +public class PropertiesModule implements Module +{ + private static final Logger log = new Logger(PropertiesModule.class); + + private final String propertiesFile; + + public PropertiesModule(String propertiesFile) + { + this.propertiesFile = propertiesFile; + } + + @Override + public void configure(Binder binder) + { + final Properties fileProps = new Properties(); + Properties systemProps = System.getProperties(); + + Properties props = new Properties(fileProps); + props.putAll(systemProps); + + InputStream stream = ClassLoader.getSystemResourceAsStream(propertiesFile); + try { + if (stream == null) { + File workingDirectoryFile = new File(systemProps.getProperty("druid.properties.file", propertiesFile)); + if (workingDirectoryFile.exists()) { + stream = new BufferedInputStream(new FileInputStream(workingDirectoryFile)); + } + } + + if (stream != null) { + log.info("Loading properties from %s", propertiesFile); + try { + fileProps.load(new InputStreamReader(stream, Charsets.UTF_8)); + } + catch (IOException e) { + throw Throwables.propagate(e); + } + } + } + catch (FileNotFoundException e) { + log.wtf(e, "This can only happen if the .exists() call lied. That's f'd up."); + } + finally { + Closeables.closeQuietly(stream); + } + + binder.bind(Properties.class).toInstance(props); + } +} diff --git a/server/src/main/java/io/druid/server/initialization/ServerConfig.java b/server/src/main/java/io/druid/server/initialization/ServerConfig.java new file mode 100644 index 000000000000..6188e9681f50 --- /dev/null +++ b/server/src/main/java/io/druid/server/initialization/ServerConfig.java @@ -0,0 +1,49 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.initialization; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.joda.time.Period; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + */ +public class ServerConfig +{ + @JsonProperty + @Min(1) + private int numThreads = 10; + + @JsonProperty + @NotNull + private Period maxIdleTime = new Period("PT5m"); + + public int getNumThreads() + { + return numThreads; + } + + public Period getMaxIdleTime() + { + return maxIdleTime; + } +} diff --git a/client/src/main/java/com/metamx/druid/initialization/ZkPathsConfig.java b/server/src/main/java/io/druid/server/initialization/ZkPathsConfig.java similarity index 95% rename from client/src/main/java/com/metamx/druid/initialization/ZkPathsConfig.java rename to server/src/main/java/io/druid/server/initialization/ZkPathsConfig.java index 065829fd9ead..2169177b16c4 100644 --- a/client/src/main/java/com/metamx/druid/initialization/ZkPathsConfig.java +++ b/server/src/main/java/io/druid/server/initialization/ZkPathsConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.initialization; +package io.druid.server.initialization; import org.apache.curator.utils.ZKPaths; import org.skife.config.Config; -import org.skife.config.Default; public abstract class ZkPathsConfig { diff --git a/client/src/main/java/com/metamx/druid/http/EmittingRequestLogger.java b/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java similarity index 74% rename from client/src/main/java/com/metamx/druid/http/EmittingRequestLogger.java rename to server/src/main/java/io/druid/server/log/EmittingRequestLogger.java index 600eceae6f92..70df84450a61 100644 --- a/client/src/main/java/com/metamx/druid/http/EmittingRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2013 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,38 +17,35 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.log; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.Query; -import com.metamx.emitter.core.Emitter; import com.metamx.emitter.core.Event; +import com.metamx.emitter.service.ServiceEmitter; +import com.metamx.emitter.service.ServiceEventBuilder; +import io.druid.query.Query; +import io.druid.server.RequestLogLine; import org.joda.time.DateTime; import java.util.Map; public class EmittingRequestLogger implements RequestLogger { - - final String service; - final String host; - final Emitter emitter; + final ServiceEmitter emitter; final String feed; - public EmittingRequestLogger(String service, String host, Emitter emitter, String feed) + public EmittingRequestLogger(ServiceEmitter emitter, String feed) { this.emitter = emitter; - this.host = host; - this.service = service; this.feed = feed; } @Override public void log(final RequestLogLine requestLogLine) throws Exception { - emitter.emit(new RequestLogEvent(service, host, feed, requestLogLine)); + emitter.emit(new RequestLogEventBuilder(feed, requestLogLine)); } public static class RequestLogEvent implements Event @@ -119,4 +116,26 @@ public boolean isSafeToBuffer() return true; } } + + private class RequestLogEventBuilder implements ServiceEventBuilder + { + private final String feed; + private final RequestLogLine requestLogLine; + + public RequestLogEventBuilder( + String feed, + RequestLogLine requestLogLine + ) + { + this.feed = feed; + this.requestLogLine = requestLogLine; + } + + + @Override + public Event build(String service, String host) + { + return new RequestLogEvent(service, host, feed, requestLogLine); + } + } } diff --git a/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java new file mode 100644 index 000000000000..5509b7c38be2 --- /dev/null +++ b/server/src/main/java/io/druid/server/log/EmittingRequestLoggerProvider.java @@ -0,0 +1,55 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.log; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.metamx.emitter.service.ServiceEmitter; + +import javax.validation.constraints.NotNull; + +/** + */ +@JsonTypeName("emitter") +public class EmittingRequestLoggerProvider implements RequestLoggerProvider +{ + @JsonProperty + @NotNull + private String feed = null; + + @JacksonInject + @NotNull + private ServiceEmitter emitter = null; + + @Inject + public void injectMe(Injector injector) + { + System.out.println("YAYAYAYAYAYA!!!"); + } + + @Override + public RequestLogger get() + { + return new EmittingRequestLogger(emitter, feed); + } +} diff --git a/client/src/main/java/com/metamx/druid/http/FileRequestLogger.java b/server/src/main/java/io/druid/server/log/FileRequestLogger.java similarity index 96% rename from client/src/main/java/com/metamx/druid/http/FileRequestLogger.java rename to server/src/main/java/io/druid/server/log/FileRequestLogger.java index 32b245e6c60a..ba450ce9c5f1 100644 --- a/client/src/main/java/com/metamx/druid/http/FileRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/FileRequestLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.log; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Throwables; @@ -25,6 +25,7 @@ import com.metamx.common.concurrent.ScheduledExecutors; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; +import io.druid.server.RequestLogLine; import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.MutableDateTime; diff --git a/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java new file mode 100644 index 000000000000..c698e55daf7f --- /dev/null +++ b/server/src/main/java/io/druid/server/log/FileRequestLoggerProvider.java @@ -0,0 +1,56 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.log; + +import com.fasterxml.jackson.annotation.JacksonInject; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.metamx.common.concurrent.ScheduledExecutorFactory; +import io.druid.guice.annotations.Json; + +import javax.validation.constraints.NotNull; +import java.io.File; + +/** + */ +@JsonTypeName("file") +public class FileRequestLoggerProvider implements RequestLoggerProvider +{ + @JsonProperty + @NotNull + private File dir = null; + + @JacksonInject + @NotNull + private ScheduledExecutorFactory factory = null; + + + @JacksonInject + @NotNull + @Json + private ObjectMapper jsonMapper = null; + + @Override + public RequestLogger get() + { + return new FileRequestLogger(jsonMapper, factory.create(1, "RequestLogger-%s"), dir); + } +} diff --git a/client/src/main/java/com/metamx/druid/http/NoopRequestLogger.java b/server/src/main/java/io/druid/server/log/NoopRequestLogger.java similarity index 86% rename from client/src/main/java/com/metamx/druid/http/NoopRequestLogger.java rename to server/src/main/java/io/druid/server/log/NoopRequestLogger.java index ddc422c91be4..1088a13032a0 100644 --- a/client/src/main/java/com/metamx/druid/http/NoopRequestLogger.java +++ b/server/src/main/java/io/druid/server/log/NoopRequestLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.log; + +import io.druid.server.RequestLogLine; /** */ @@ -26,6 +28,6 @@ public class NoopRequestLogger implements RequestLogger @Override public void log(RequestLogLine requestLogLine) throws Exception { - // do nothing + // This is a no op! } } diff --git a/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java new file mode 100644 index 000000000000..86d41241ab21 --- /dev/null +++ b/server/src/main/java/io/druid/server/log/NoopRequestLoggerProvider.java @@ -0,0 +1,31 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.log; + +/** + */ +public class NoopRequestLoggerProvider implements RequestLoggerProvider +{ + @Override + public RequestLogger get() + { + return new NoopRequestLogger(); + } +} diff --git a/client/src/main/java/com/metamx/druid/http/RequestLogger.java b/server/src/main/java/io/druid/server/log/RequestLogger.java similarity index 87% rename from client/src/main/java/com/metamx/druid/http/RequestLogger.java rename to server/src/main/java/io/druid/server/log/RequestLogger.java index 7720938c4bc0..2d4c938252b8 100644 --- a/client/src/main/java/com/metamx/druid/http/RequestLogger.java +++ b/server/src/main/java/io/druid/server/log/RequestLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,9 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.http; +package io.druid.server.log; + +import io.druid.server.RequestLogLine; /** */ diff --git a/server/src/main/java/io/druid/server/log/RequestLoggerProvider.java b/server/src/main/java/io/druid/server/log/RequestLoggerProvider.java new file mode 100644 index 000000000000..236c2452034f --- /dev/null +++ b/server/src/main/java/io/druid/server/log/RequestLoggerProvider.java @@ -0,0 +1,32 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.log; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.google.inject.Provider; + +/** + * A Marker interface for things that can provide a RequestLogger. This can be combined with jackson polymorphic serde + * to provide new RequestLogger implementations as plugins. + */ +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type", defaultImpl = NoopRequestLoggerProvider.class) +public interface RequestLoggerProvider extends Provider +{ +} diff --git a/server/src/main/java/com/metamx/druid/master/BalancerSegmentHolder.java b/server/src/main/java/io/druid/server/master/BalancerSegmentHolder.java similarity index 88% rename from server/src/main/java/com/metamx/druid/master/BalancerSegmentHolder.java rename to server/src/main/java/io/druid/server/master/BalancerSegmentHolder.java index 7d62968e4b95..7eb01282b481 100644 --- a/server/src/main/java/com/metamx/druid/master/BalancerSegmentHolder.java +++ b/server/src/main/java/io/druid/server/master/BalancerSegmentHolder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; /** */ diff --git a/indexing-service/src/main/java/com/metamx/druid/indexing/worker/config/WorkerConfig.java b/server/src/main/java/io/druid/server/master/BalancerStrategy.java similarity index 55% rename from indexing-service/src/main/java/com/metamx/druid/indexing/worker/config/WorkerConfig.java rename to server/src/main/java/io/druid/server/master/BalancerStrategy.java index deca87c56f5a..0fb10eb2b6be 100644 --- a/indexing-service/src/main/java/com/metamx/druid/indexing/worker/config/WorkerConfig.java +++ b/server/src/main/java/io/druid/server/master/BalancerStrategy.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,30 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.indexing.worker.config; +package io.druid.server.master; -import org.skife.config.Config; -import org.skife.config.Default; +import io.druid.timeline.DataSegment; -/** - */ -public abstract class WorkerConfig -{ - @Config("druid.host") - public abstract String getHost(); +import java.util.List; - @Config("druid.worker.ip") - public abstract String getIp(); +public interface BalancerStrategy +{ + public ServerHolder findNewSegmentHomeBalancer(final DataSegment proposalSegment, final List serverHolders); - @Config("druid.worker.version") - public abstract String getVersion(); + public ServerHolder findNewSegmentHomeReplicator(final DataSegment proposalSegment, final List serverHolders); - @Config("druid.worker.masterService") - public abstract String getMasterService(); + public BalancerSegmentHolder pickSegmentToMove(final List serverHolders); - @Config("druid.worker.capacity") - public int getCapacity() - { - return Runtime.getRuntime().availableProcessors() - 1; - } + public void emitStats(String tier, MasterStats stats, List serverHolderList); } diff --git a/server/src/main/java/io/druid/server/master/BalancerStrategyFactory.java b/server/src/main/java/io/druid/server/master/BalancerStrategyFactory.java new file mode 100644 index 000000000000..abdb9e880201 --- /dev/null +++ b/server/src/main/java/io/druid/server/master/BalancerStrategyFactory.java @@ -0,0 +1,26 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package io.druid.server.master; + +import org.joda.time.DateTime; + +public interface BalancerStrategyFactory +{ + public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp); +} diff --git a/server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java b/server/src/main/java/io/druid/server/master/CostBalancerStrategy.java similarity index 89% rename from server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java rename to server/src/main/java/io/druid/server/master/CostBalancerStrategy.java index 0da949b9a7c1..06d5aa0836d1 100644 --- a/server/src/main/java/com/metamx/druid/master/CostBalancerStrategy.java +++ b/server/src/main/java/io/druid/server/master/CostBalancerStrategy.java @@ -1,32 +1,30 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -package com.metamx.druid.master; - -import com.google.common.collect.MinMaxPriorityQueue; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.master; + import com.metamx.common.Pair; -import com.metamx.druid.client.DataSegment; import com.metamx.emitter.EmittingLogger; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; -import java.util.Comparator; import java.util.List; public class CostBalancerStrategy implements BalancerStrategy diff --git a/server/src/main/java/io/druid/server/master/CostBalancerStrategyFactory.java b/server/src/main/java/io/druid/server/master/CostBalancerStrategyFactory.java new file mode 100644 index 000000000000..832ee8512774 --- /dev/null +++ b/server/src/main/java/io/druid/server/master/CostBalancerStrategyFactory.java @@ -0,0 +1,31 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package io.druid.server.master; + +import org.joda.time.DateTime; + +public class CostBalancerStrategyFactory implements BalancerStrategyFactory +{ + + @Override + public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp) + { + return new CostBalancerStrategy(referenceTimestamp); + } +} diff --git a/server/src/main/java/com/metamx/druid/master/DruidCluster.java b/server/src/main/java/io/druid/server/master/DruidCluster.java similarity index 95% rename from server/src/main/java/com/metamx/druid/master/DruidCluster.java rename to server/src/main/java/io/druid/server/master/DruidCluster.java index d7ef284ede0f..dd9d0c33bc58 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidCluster.java +++ b/server/src/main/java/io/druid/server/master/DruidCluster.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.google.common.collect.Ordering; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidServer; import java.util.Map; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMaster.java b/server/src/main/java/io/druid/server/master/DruidMaster.java similarity index 92% rename from server/src/main/java/com/metamx/druid/master/DruidMaster.java rename to server/src/main/java/io/druid/server/master/DruidMaster.java index fa1731be37e3..937c1c6bfa16 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMaster.java +++ b/server/src/main/java/io/druid/server/master/DruidMaster.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.base.Function; import com.google.common.base.Predicate; @@ -28,6 +28,7 @@ import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.io.Closeables; +import com.google.inject.Inject; import com.metamx.common.IAE; import com.metamx.common.Pair; import com.metamx.common.concurrent.ScheduledExecutorFactory; @@ -36,23 +37,25 @@ import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.lifecycle.LifecycleStart; import com.metamx.common.lifecycle.LifecycleStop; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.ServerInventoryView; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.config.JacksonConfigManager; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.initialization.ZkPathsConfig; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.client.ServerInventoryView; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.common.config.JacksonConfigManager; +import io.druid.concurrent.Execs; +import io.druid.db.DatabaseRuleManager; +import io.druid.db.DatabaseSegmentManager; +import io.druid.guice.ManageLifecycle; +import io.druid.segment.IndexIO; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.leader.LeaderLatch; import org.apache.curator.framework.recipes.leader.LeaderLatchListener; +import org.apache.curator.framework.recipes.leader.Participant; import org.apache.curator.utils.ZKPaths; import org.joda.time.DateTime; import org.joda.time.Duration; @@ -70,6 +73,7 @@ /** */ +@ManageLifecycle public class DruidMaster { public static final String MASTER_OWNER_NODE = "_MASTER"; @@ -96,6 +100,7 @@ public class DruidMaster private final AtomicReference leaderLatch; private volatile AtomicReference segmentSettingsAtomicReference; + @Inject public DruidMaster( DruidMasterConfig config, ZkPathsConfig zkPaths, @@ -155,8 +160,8 @@ public DruidMaster( this.exec = scheduledExecutorFactory.create(1, "Master-Exec--%d"); - this.leaderLatch = new AtomicReference(null); - this.segmentSettingsAtomicReference= new AtomicReference(null); + this.leaderLatch = new AtomicReference<>(null); + this.segmentSettingsAtomicReference= new AtomicReference<>(null); this.loadManagementPeons = loadQueuePeonMap; } @@ -210,16 +215,6 @@ public Map getLoadStatus() return loadStatus; } - public int lookupSegmentLifetime(DataSegment segment) - { - return serverInventoryView.lookupSegmentLifetime(segment); - } - - public void decrementRemovedSegmentsLifetime() - { - serverInventoryView.decrementRemovedSegmentsLifetime(); - } - public void removeSegment(DataSegment segment) { log.info("Removing Segment[%s]", segment); @@ -240,7 +235,17 @@ public String getCurrentMaster() { try { final LeaderLatch latch = leaderLatch.get(); - return latch == null ? null : latch.getLeader().getId(); + + if (latch == null) { + return null; + } + + Participant participant = latch.getLeader(); + if (participant.isLeader()) { + return participant.getId(); + } + + return null; } catch (Exception e) { throw Throwables.propagate(e); @@ -726,45 +731,28 @@ public boolean apply( SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(cluster); // Stop peons for servers that aren't there anymore. - for (String name : Sets.difference( - loadManagementPeons.keySet(), - Sets.newHashSet( - Iterables.transform( - servers, - new Function() - { - @Override - public String apply(@Nullable DruidServer input) - { - return input.getName(); - } - } - ) - ) - )) { + final Set disdappearedServers = Sets.newHashSet(loadManagementPeons.keySet()); + for (DruidServer server : servers) { + disdappearedServers.remove(server.getName()); + } + for (String name : disdappearedServers) { log.info("Removing listener for server[%s] which is no longer there.", name); LoadQueuePeon peon = loadManagementPeons.remove(name); peon.stop(); } - decrementRemovedSegmentsLifetime(); - return params.buildFromExisting() .withDruidCluster(cluster) .withDatabaseRuleManager(databaseRuleManager) .withLoadManagementPeons(loadManagementPeons) .withSegmentReplicantLookup(segmentReplicantLookup) .withBalancerReferenceTimestamp(DateTime.now()) - .withMasterSegmentSettings( - segmentSettingsAtomicReference.get() - ) + .withMasterSegmentSettings(segmentSettingsAtomicReference.get()) .build(); } }, new DruidMasterRuleRunner( - DruidMaster.this, - config.getReplicantLifetime(), - config.getReplicantThrottleLimit() + DruidMaster.this, config.getReplicantLifetime(), config.getReplicantThrottleLimit() ), new DruidMasterCleanup(DruidMaster.this), new DruidMasterBalancer(DruidMaster.this), diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java b/server/src/main/java/io/druid/server/master/DruidMasterBalancer.java similarity index 79% rename from server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java rename to server/src/main/java/io/druid/server/master/DruidMasterBalancer.java index bb547bae8d8d..28c966b4162b 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterBalancer.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterBalancer.java @@ -1,31 +1,31 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -package com.metamx.druid.master; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.master; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.metamx.common.guava.Comparators; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; import com.metamx.emitter.EmittingLogger; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.Comparator; @@ -156,27 +156,33 @@ protected void moveSegment( (toServer.getSegment(segmentName) == null) && new ServerHolder(toServer, toPeon).getAvailableSize() > segmentToMove.getSize()) { log.info("Moving [%s] from [%s] to [%s]", segmentName, fromServerName, toServerName); + + LoadPeonCallback callback = null; try { + currentlyMovingSegments.get(toServer.getTier()).put(segmentName, segment); + callback = new LoadPeonCallback() + { + @Override + protected void execute() + { + Map movingSegments = currentlyMovingSegments.get(toServer.getTier()); + if (movingSegments != null) { + movingSegments.remove(segmentName); + } + } + }; master.moveSegment( fromServerName, toServerName, segmentToMove.getIdentifier(), - new LoadPeonCallback() - { - @Override - protected void execute() - { - Map movingSegments = currentlyMovingSegments.get(toServer.getTier()); - if (movingSegments != null) { - movingSegments.remove(segmentName); - } - } - } + callback ); - currentlyMovingSegments.get(toServer.getTier()).put(segmentName, segment); } catch (Exception e) { log.makeAlert(e, String.format("[%s] : Moving exception", segmentName)).emit(); + if (callback != null) { + callback.execute(); + } } } else { currentlyMovingSegments.get(toServer.getTier()).remove(segmentName); diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterCleanup.java b/server/src/main/java/io/druid/server/master/DruidMasterCleanup.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/DruidMasterCleanup.java rename to server/src/main/java/io/druid/server/master/DruidMasterCleanup.java index 6ba31a772493..e31b1ae45c79 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterCleanup.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterCleanup.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.metamx.common.guava.Comparators; import com.metamx.common.logger.Logger; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; import java.util.Map; import java.util.Set; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java b/server/src/main/java/io/druid/server/master/DruidMasterConfig.java similarity index 91% rename from server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java rename to server/src/main/java/io/druid/server/master/DruidMasterConfig.java index 3e2b27198707..d279b91d5c1f 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterConfig.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import org.joda.time.Duration; import org.skife.config.Config; @@ -54,12 +54,6 @@ public boolean isConvertSegments() return false; } - @Config("druid.master.merger.service") - public String getMergerServiceName() - { - return null; - } - @Config("druid.master.replicant.lifetime") @Default("15") public abstract int getReplicantLifetime(); diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterHelper.java b/server/src/main/java/io/druid/server/master/DruidMasterHelper.java similarity index 91% rename from server/src/main/java/com/metamx/druid/master/DruidMasterHelper.java rename to server/src/main/java/io/druid/server/master/DruidMasterHelper.java index d218fa46f281..860a8546e7ac 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterHelper.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterHelper.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; /** */ diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java b/server/src/main/java/io/druid/server/master/DruidMasterLogger.java similarity index 96% rename from server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java rename to server/src/main/java/io/druid/server/master/DruidMasterLogger.java index 1b226933a527..49a771177d33 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterLogger.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterLogger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.collect.CountingMap; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.client.DruidDataSource; +import io.druid.client.DruidServer; +import io.druid.collections.CountingMap; +import io.druid.timeline.DataSegment; import java.util.Map; import java.util.Set; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterRuleRunner.java b/server/src/main/java/io/druid/server/master/DruidMasterRuleRunner.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/DruidMasterRuleRunner.java rename to server/src/main/java/io/druid/server/master/DruidMasterRuleRunner.java index 04a3ce55da6d..6f23976f1270 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterRuleRunner.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterRuleRunner.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.master.rules.Rule; import com.metamx.emitter.EmittingLogger; +import io.druid.db.DatabaseRuleManager; +import io.druid.server.master.rules.Rule; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.List; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java b/server/src/main/java/io/druid/server/master/DruidMasterRuntimeParams.java similarity index 97% rename from server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java rename to server/src/main/java/io/druid/server/master/DruidMasterRuntimeParams.java index 0dfe9afca4f7..9f89ad1f05b3 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterRuntimeParams.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterRuntimeParams.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.metamx.common.guava.Comparators; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidDataSource; -import com.metamx.druid.db.DatabaseRuleManager; import com.metamx.emitter.service.ServiceEmitter; +import io.druid.client.DruidDataSource; +import io.druid.db.DatabaseRuleManager; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.Collection; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentInfoLoader.java b/server/src/main/java/io/druid/server/master/DruidMasterSegmentInfoLoader.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/DruidMasterSegmentInfoLoader.java rename to server/src/main/java/io/druid/server/master/DruidMasterSegmentInfoLoader.java index 2158328c1d8a..b80eaf8b42dd 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentInfoLoader.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterSegmentInfoLoader.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.util.Set; diff --git a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java b/server/src/main/java/io/druid/server/master/DruidMasterSegmentMerger.java similarity index 96% rename from server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java rename to server/src/main/java/io/druid/server/master/DruidMasterSegmentMerger.java index d178cf1a8ad7..0ca304ffe915 100644 --- a/server/src/main/java/com/metamx/druid/master/DruidMasterSegmentMerger.java +++ b/server/src/main/java/io/druid/server/master/DruidMasterSegmentMerger.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -32,12 +32,12 @@ import com.metamx.common.Pair; import com.metamx.common.guava.FunctionalIterable; import com.metamx.common.logger.Logger; -import com.metamx.druid.TimelineObjectHolder; -import com.metamx.druid.VersionedIntervalTimeline; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.timeline.DataSegment; +import io.druid.timeline.TimelineObjectHolder; +import io.druid.timeline.VersionedIntervalTimeline; +import io.druid.timeline.partition.NoneShardSpec; +import io.druid.timeline.partition.PartitionChunk; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/main/java/com/metamx/druid/master/LoadPeonCallback.java b/server/src/main/java/io/druid/server/master/LoadPeonCallback.java similarity index 91% rename from server/src/main/java/com/metamx/druid/master/LoadPeonCallback.java rename to server/src/main/java/io/druid/server/master/LoadPeonCallback.java index 309a9d26b1ee..50a513961e9a 100644 --- a/server/src/main/java/com/metamx/druid/master/LoadPeonCallback.java +++ b/server/src/main/java/io/druid/server/master/LoadPeonCallback.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; /** */ diff --git a/server/src/main/java/com/metamx/druid/master/LoadQueuePeon.java b/server/src/main/java/io/druid/server/master/LoadQueuePeon.java similarity index 97% rename from server/src/main/java/com/metamx/druid/master/LoadQueuePeon.java rename to server/src/main/java/io/druid/server/master/LoadQueuePeon.java index 2b5c96cb1873..ece30058a3f9 100644 --- a/server/src/main/java/com/metamx/druid/master/LoadQueuePeon.java +++ b/server/src/main/java/io/druid/server/master/LoadQueuePeon.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; @@ -25,12 +25,12 @@ import com.google.common.collect.Lists; import com.metamx.common.ISE; import com.metamx.common.guava.Comparators; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.coordination.DataSegmentChangeRequest; -import com.metamx.druid.coordination.SegmentChangeRequestDrop; -import com.metamx.druid.coordination.SegmentChangeRequestLoad; -import com.metamx.druid.coordination.SegmentChangeRequestNoop; import com.metamx.emitter.EmittingLogger; +import io.druid.server.coordination.DataSegmentChangeRequest; +import io.druid.server.coordination.SegmentChangeRequestDrop; +import io.druid.server.coordination.SegmentChangeRequestLoad; +import io.druid.server.coordination.SegmentChangeRequestNoop; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.CuratorWatcher; import org.apache.curator.utils.ZKPaths; diff --git a/server/src/main/java/com/metamx/druid/master/LoadQueueTaskMaster.java b/server/src/main/java/io/druid/server/master/LoadQueueTaskMaster.java similarity index 92% rename from server/src/main/java/com/metamx/druid/master/LoadQueueTaskMaster.java rename to server/src/main/java/io/druid/server/master/LoadQueueTaskMaster.java index 2547127bc5af..1d1d114d6dc1 100644 --- a/server/src/main/java/com/metamx/druid/master/LoadQueueTaskMaster.java +++ b/server/src/main/java/io/druid/server/master/LoadQueueTaskMaster.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.inject.Inject; import org.apache.curator.framework.CuratorFramework; -import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; /** @@ -35,6 +35,7 @@ public class LoadQueueTaskMaster private final ScheduledExecutorService peonExec; private final DruidMasterConfig config; + @Inject public LoadQueueTaskMaster( CuratorFramework curator, ObjectMapper jsonMapper, diff --git a/server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java b/server/src/main/java/io/druid/server/master/MasterSegmentSettings.java similarity index 80% rename from server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java rename to server/src/main/java/io/druid/server/master/MasterSegmentSettings.java index 1bafb38ecc83..a31f9b8dce8d 100644 --- a/server/src/main/java/com/metamx/druid/master/MasterSegmentSettings.java +++ b/server/src/main/java/io/druid/server/master/MasterSegmentSettings.java @@ -1,22 +1,22 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package io.druid.server.master; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/server/src/main/java/com/metamx/druid/master/MasterStats.java b/server/src/main/java/io/druid/server/master/MasterStats.java similarity index 95% rename from server/src/main/java/com/metamx/druid/master/MasterStats.java rename to server/src/main/java/io/druid/server/master/MasterStats.java index 7cdbe5103954..ed23b9f0b794 100644 --- a/server/src/main/java/com/metamx/druid/master/MasterStats.java +++ b/server/src/main/java/io/druid/server/master/MasterStats.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Maps; -import com.metamx.druid.collect.CountingMap; +import io.druid.collections.CountingMap; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; diff --git a/server/src/main/java/com/metamx/druid/master/MergerWhitelist.java b/server/src/main/java/io/druid/server/master/MergerWhitelist.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/MergerWhitelist.java rename to server/src/main/java/io/druid/server/master/MergerWhitelist.java index 8bfa7293b00a..e84d91eead90 100644 --- a/server/src/main/java/com/metamx/druid/master/MergerWhitelist.java +++ b/server/src/main/java/io/druid/server/master/MergerWhitelist.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; diff --git a/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategy.java b/server/src/main/java/io/druid/server/master/RandomBalancerStrategy.java similarity index 58% rename from server/src/main/java/com/metamx/druid/master/RandomBalancerStrategy.java rename to server/src/main/java/io/druid/server/master/RandomBalancerStrategy.java index d953b69b3db8..cc8e62afec63 100644 --- a/server/src/main/java/com/metamx/druid/master/RandomBalancerStrategy.java +++ b/server/src/main/java/io/druid/server/master/RandomBalancerStrategy.java @@ -1,25 +1,25 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.util.List; import java.util.Random; diff --git a/server/src/main/java/io/druid/server/master/RandomBalancerStrategyFactory.java b/server/src/main/java/io/druid/server/master/RandomBalancerStrategyFactory.java new file mode 100644 index 000000000000..e2f4641ed562 --- /dev/null +++ b/server/src/main/java/io/druid/server/master/RandomBalancerStrategyFactory.java @@ -0,0 +1,30 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package io.druid.server.master; + +import org.joda.time.DateTime; + +public class RandomBalancerStrategyFactory implements BalancerStrategyFactory +{ + @Override + public BalancerStrategy createBalancerStrategy(DateTime referenceTimestamp) + { + return new RandomBalancerStrategy(); + } +} diff --git a/server/src/main/java/com/metamx/druid/master/ReplicationThrottler.java b/server/src/main/java/io/druid/server/master/ReplicationThrottler.java similarity index 98% rename from server/src/main/java/com/metamx/druid/master/ReplicationThrottler.java rename to server/src/main/java/io/druid/server/master/ReplicationThrottler.java index fc2774374bdf..9584bf46f8c0 100644 --- a/server/src/main/java/com/metamx/druid/master/ReplicationThrottler.java +++ b/server/src/main/java/io/druid/server/master/ReplicationThrottler.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/server/src/main/java/com/metamx/druid/master/ReservoirSegmentSampler.java b/server/src/main/java/io/druid/server/master/ReservoirSegmentSampler.java similarity index 50% rename from server/src/main/java/com/metamx/druid/master/ReservoirSegmentSampler.java rename to server/src/main/java/io/druid/server/master/ReservoirSegmentSampler.java index 4db994b821af..b50cfc8ace00 100644 --- a/server/src/main/java/com/metamx/druid/master/ReservoirSegmentSampler.java +++ b/server/src/main/java/io/druid/server/master/ReservoirSegmentSampler.java @@ -1,25 +1,25 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -package com.metamx.druid.master; +package io.druid.server.master; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import java.util.List; import java.util.Random; diff --git a/server/src/main/java/com/metamx/druid/master/SegmentReplicantLookup.java b/server/src/main/java/io/druid/server/master/SegmentReplicantLookup.java similarity index 95% rename from server/src/main/java/com/metamx/druid/master/SegmentReplicantLookup.java rename to server/src/main/java/io/druid/server/master/SegmentReplicantLookup.java index 560787247f5c..d455a89c5f93 100644 --- a/server/src/main/java/com/metamx/druid/master/SegmentReplicantLookup.java +++ b/server/src/main/java/io/druid/server/master/SegmentReplicantLookup.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.HashBasedTable; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; import com.google.common.collect.Table; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; import java.util.Map; diff --git a/server/src/main/java/com/metamx/druid/master/ServerHolder.java b/server/src/main/java/io/druid/server/master/ServerHolder.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/ServerHolder.java rename to server/src/main/java/io/druid/server/master/ServerHolder.java index 5fba424f429c..1a6d40b57b5b 100644 --- a/server/src/main/java/com/metamx/druid/master/ServerHolder.java +++ b/server/src/main/java/io/druid/server/master/ServerHolder.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; /** */ diff --git a/server/src/main/java/com/metamx/druid/master/rules/DropRule.java b/server/src/main/java/io/druid/server/master/rules/DropRule.java similarity index 81% rename from server/src/main/java/com/metamx/druid/master/rules/DropRule.java rename to server/src/main/java/io/druid/server/master/rules/DropRule.java index 5806031d7c61..f9040cb46b7b 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/DropRule.java +++ b/server/src/main/java/io/druid/server/master/rules/DropRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterRuntimeParams; -import com.metamx.druid.master.MasterStats; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.DruidMasterRuntimeParams; +import io.druid.server.master.MasterStats; +import io.druid.timeline.DataSegment; /** * DropRules indicate when segments should be completely removed from the cluster. diff --git a/server/src/main/java/com/metamx/druid/master/rules/IntervalDropRule.java b/server/src/main/java/io/druid/server/master/rules/IntervalDropRule.java similarity index 91% rename from server/src/main/java/com/metamx/druid/master/rules/IntervalDropRule.java rename to server/src/main/java/io/druid/server/master/rules/IntervalDropRule.java index 1363c88b9da8..f5b570ca626c 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/IntervalDropRule.java +++ b/server/src/main/java/io/druid/server/master/rules/IntervalDropRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.client.DataSegment; - +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/main/java/com/metamx/druid/master/rules/IntervalLoadRule.java b/server/src/main/java/io/druid/server/master/rules/IntervalLoadRule.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/rules/IntervalLoadRule.java rename to server/src/main/java/io/druid/server/master/rules/IntervalLoadRule.java index 0cfe603e2b13..436c2e98e0a0 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/IntervalLoadRule.java +++ b/server/src/main/java/io/druid/server/master/rules/IntervalLoadRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; - +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/main/java/com/metamx/druid/master/rules/LoadRule.java b/server/src/main/java/io/druid/server/master/rules/LoadRule.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/rules/LoadRule.java rename to server/src/main/java/io/druid/server/master/rules/LoadRule.java index b133af1799e0..6caafeb15c32 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/LoadRule.java +++ b/server/src/main/java/io/druid/server/master/rules/LoadRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,19 +17,19 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.google.common.collect.Lists; import com.google.common.collect.MinMaxPriorityQueue; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.master.BalancerStrategy; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterRuntimeParams; -import com.metamx.druid.master.LoadPeonCallback; -import com.metamx.druid.master.MasterStats; -import com.metamx.druid.master.ReplicationThrottler; -import com.metamx.druid.master.ServerHolder; import com.metamx.emitter.EmittingLogger; +import io.druid.server.master.BalancerStrategy; +import io.druid.server.master.DruidMaster; +import io.druid.server.master.DruidMasterRuntimeParams; +import io.druid.server.master.LoadPeonCallback; +import io.druid.server.master.MasterStats; +import io.druid.server.master.ReplicationThrottler; +import io.druid.server.master.ServerHolder; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import java.util.ArrayList; diff --git a/server/src/main/java/com/metamx/druid/master/rules/PeriodDropRule.java b/server/src/main/java/io/druid/server/master/rules/PeriodDropRule.java similarity index 92% rename from server/src/main/java/com/metamx/druid/master/rules/PeriodDropRule.java rename to server/src/main/java/io/druid/server/master/rules/PeriodDropRule.java index 5fa6affcf9e6..7a490bccf910 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/PeriodDropRule.java +++ b/server/src/main/java/io/druid/server/master/rules/PeriodDropRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.client.DataSegment; - +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/server/src/main/java/com/metamx/druid/master/rules/PeriodLoadRule.java b/server/src/main/java/io/druid/server/master/rules/PeriodLoadRule.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/rules/PeriodLoadRule.java rename to server/src/main/java/io/druid/server/master/rules/PeriodLoadRule.java index 632b6acbc67c..e75734a07376 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/PeriodLoadRule.java +++ b/server/src/main/java/io/druid/server/master/rules/PeriodLoadRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; - +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/server/src/main/java/com/metamx/druid/master/rules/Rule.java b/server/src/main/java/io/druid/server/master/rules/Rule.java similarity index 86% rename from server/src/main/java/com/metamx/druid/master/rules/Rule.java rename to server/src/main/java/io/druid/server/master/rules/Rule.java index 1c77a0ebc8ff..c576dc3a9fd3 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/Rule.java +++ b/server/src/main/java/io/druid/server/master/rules/Rule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterRuntimeParams; -import com.metamx.druid.master.MasterStats; - +import io.druid.server.master.DruidMaster; +import io.druid.server.master.DruidMasterRuntimeParams; +import io.druid.server.master.MasterStats; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; /** diff --git a/server/src/main/java/com/metamx/druid/master/rules/RuleMap.java b/server/src/main/java/io/druid/server/master/rules/RuleMap.java similarity index 93% rename from server/src/main/java/com/metamx/druid/master/rules/RuleMap.java rename to server/src/main/java/io/druid/server/master/rules/RuleMap.java index 8354a5e110f5..a8f80ee1b2d7 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/RuleMap.java +++ b/server/src/main/java/io/druid/server/master/rules/RuleMap.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.google.common.collect.Lists; diff --git a/server/src/main/java/com/metamx/druid/master/rules/SizeDropRule.java b/server/src/main/java/io/druid/server/master/rules/SizeDropRule.java similarity index 92% rename from server/src/main/java/com/metamx/druid/master/rules/SizeDropRule.java rename to server/src/main/java/io/druid/server/master/rules/SizeDropRule.java index 0bd9f94cd006..f7ad948a54bd 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/SizeDropRule.java +++ b/server/src/main/java/io/druid/server/master/rules/SizeDropRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Range; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; /** diff --git a/server/src/main/java/com/metamx/druid/master/rules/SizeLoadRule.java b/server/src/main/java/io/druid/server/master/rules/SizeLoadRule.java similarity index 94% rename from server/src/main/java/com/metamx/druid/master/rules/SizeLoadRule.java rename to server/src/main/java/io/druid/server/master/rules/SizeLoadRule.java index 421432ec6b98..df33cb0f50d7 100644 --- a/server/src/main/java/com/metamx/druid/master/rules/SizeLoadRule.java +++ b/server/src/main/java/io/druid/server/master/rules/SizeLoadRule.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,12 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.collect.Range; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.joda.time.DateTime; /** diff --git a/server/src/main/java/com/metamx/druid/db/DatabaseRuleManagerConfig.java b/server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java similarity index 59% rename from server/src/main/java/com/metamx/druid/db/DatabaseRuleManagerConfig.java rename to server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java index 41e21a0943b0..9de045854de5 100644 --- a/server/src/main/java/com/metamx/druid/db/DatabaseRuleManagerConfig.java +++ b/server/src/main/java/io/druid/server/metrics/DruidMonitorSchedulerConfig.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,24 +17,29 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.server.metrics; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.metamx.metrics.MonitorSchedulerConfig; import org.joda.time.Duration; -import org.skife.config.Config; -import org.skife.config.Default; +import org.joda.time.Period; /** */ -public abstract class DatabaseRuleManagerConfig +public class DruidMonitorSchedulerConfig extends MonitorSchedulerConfig { - @Config("druid.database.ruleTable") - public abstract String getRuleTable(); + @JsonProperty + private Period emissionPeriod = new Period("PT1M"); - @Config("druid.database.defaultDatasource") - @Default("_default") - public abstract String getDefaultDatasource(); + @JsonProperty + public Period getEmissionPeriod() + { + return emissionPeriod; + } - @Config("druid.database.rules.poll.duration") - @Default("PT1M") - public abstract Duration getRulesPollDuration(); + @Override + public Duration getEmitterPeriod() + { + return emissionPeriod.toStandardDuration(); + } } diff --git a/server/src/main/java/io/druid/server/metrics/MetricsModule.java b/server/src/main/java/io/druid/server/metrics/MetricsModule.java new file mode 100644 index 000000000000..ee7dd20c2f8f --- /dev/null +++ b/server/src/main/java/io/druid/server/metrics/MetricsModule.java @@ -0,0 +1,122 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.metrics; + +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; +import com.google.inject.Binder; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.name.Names; +import com.metamx.common.logger.Logger; +import com.metamx.emitter.service.ServiceEmitter; +import com.metamx.metrics.Monitor; +import com.metamx.metrics.MonitorScheduler; +import io.druid.concurrent.Execs; +import io.druid.guice.JsonConfigProvider; +import io.druid.guice.JsonConfigurator; +import io.druid.guice.LazySingleton; +import io.druid.guice.ManageLifecycle; + +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * Sets up the {@link MonitorScheduler} to monitor things on a regular schedule. {@link Monitor}s must be explicitly + * bound in order to be loaded. + */ +public class MetricsModule implements Module +{ + private static final Logger log = new Logger(MetricsModule.class); + + private final List> monitors = new CopyOnWriteArrayList>(); + public boolean configured = false; + + public MetricsModule register(Class monitorClazz) + { + synchronized (monitors) { + Preconditions.checkState(!configured, "Cannot register monitor[%s] after configuration.", monitorClazz); + } + monitors.add(monitorClazz); + return this; + } + + @Inject + public void setProperties(Properties props, JsonConfigurator configurator) + { + final MonitorsConfig config = configurator.configurate( + props, + "druid.monitoring", + MonitorsConfig.class + ); + + for (Class monitorClazz : config.getMonitors()) { + register(monitorClazz); + } + } + + @Override + public void configure(Binder binder) + { + JsonConfigProvider.bind(binder, "druid.monitoring", DruidMonitorSchedulerConfig.class); + + for (Class monitor : monitors) { + binder.bind(monitor).in(LazySingleton.class); + } + + // Instantiate eagerly so that we get everything registered and put into the Lifecycle + binder.bind(Key.get(MonitorScheduler.class, Names.named("ForTheEagerness"))) + .to(MonitorScheduler.class) + .asEagerSingleton(); + } + + @Provides + @ManageLifecycle + public MonitorScheduler getMonitorScheduler( + Supplier config, + ServiceEmitter emitter, + Injector injector + ) + { + List monitors = Lists.newArrayList(); + + for (Key key : injector.getBindings().keySet()) { + if (Monitor.class.isAssignableFrom(key.getTypeLiteral().getRawType())) { + final Monitor monitor = (Monitor) injector.getInstance(key); + + log.info("Adding monitor[%s]", monitor); + + monitors.add(monitor); + } + } + + return new MonitorScheduler( + config.get(), + Execs.scheduledSingleThreaded("MonitorScheduler-%s"), + emitter, + monitors + ); + } +} diff --git a/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java b/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java new file mode 100644 index 000000000000..1bf1e78bbace --- /dev/null +++ b/server/src/main/java/io/druid/server/metrics/MonitorsConfig.java @@ -0,0 +1,49 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.metrics; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.collect.Lists; +import com.metamx.metrics.Monitor; + +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + */ +public class MonitorsConfig +{ + @JsonProperty("monitors") + @NotNull + private List> monitors = Lists.newArrayList(); + + public List> getMonitors() + { + return monitors; + } + + @Override + public String toString() + { + return "MonitorsConfig{" + + "monitors=" + monitors + + '}'; + } +} diff --git a/server/src/main/java/com/metamx/druid/metrics/ServerMonitor.java b/server/src/main/java/io/druid/server/metrics/ServerMonitor.java similarity index 60% rename from server/src/main/java/com/metamx/druid/metrics/ServerMonitor.java rename to server/src/main/java/io/druid/server/metrics/ServerMonitor.java index 7db7e3ba9227..684068fa4535 100644 --- a/server/src/main/java/com/metamx/druid/metrics/ServerMonitor.java +++ b/server/src/main/java/io/druid/server/metrics/ServerMonitor.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,52 +17,53 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.metrics; +package io.druid.server.metrics; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.coordination.DruidServerMetadata; -import com.metamx.druid.coordination.ServerManager; +import com.google.inject.Inject; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceMetricEvent; import com.metamx.metrics.AbstractMonitor; +import io.druid.client.DruidServerConfig; +import io.druid.server.coordination.ServerManager; import java.util.Map; public class ServerMonitor extends AbstractMonitor { - private final DruidServerMetadata druidServer; + private final DruidServerConfig serverConfig; private final ServerManager serverManager; + @Inject public ServerMonitor( - DruidServerMetadata druidServer, + DruidServerConfig serverConfig, ServerManager serverManager ) { - this.druidServer = druidServer; + this.serverConfig = serverConfig; this.serverManager = serverManager; } @Override public boolean doMonitor(ServiceEmitter emitter) { - emitter.emit(new ServiceMetricEvent.Builder().build("server/segment/max", druidServer.getMaxSize())); + emitter.emit(new ServiceMetricEvent.Builder().build("server/segment/max", serverConfig.getMaxSize())); for (Map.Entry entry : serverManager.getDataSourceSizes().entrySet()) { String dataSource = entry.getKey(); long used = entry.getValue(); - emitter.emit( - new ServiceMetricEvent.Builder() - .setUser1(dataSource) - .build("server/segment/used", used) - ); + final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setUser1(dataSource) + .setUser2(serverConfig.getTier()); + + emitter.emit(builder.build("server/segment/used", used)); + emitter.emit(builder.build("server/segment/usedPercent", used / (double) serverConfig.getMaxSize())); } + for (Map.Entry entry : serverManager.getDataSourceCounts().entrySet()) { String dataSource = entry.getKey(); long count = entry.getValue(); - emitter.emit( - new ServiceMetricEvent.Builder() - .setUser1(dataSource) - .build("server/segment/count", count) - ); + final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder().setUser1(dataSource) + .setUser2(serverConfig.getTier()); + + emitter.emit(builder.build("server/segment/count", count)); } return true; diff --git a/client/src/main/java/com/metamx/druid/sql/SQLRunner.java b/server/src/main/java/io/druid/server/sql/SQLRunner.java similarity index 85% rename from client/src/main/java/com/metamx/druid/sql/SQLRunner.java rename to server/src/main/java/io/druid/server/sql/SQLRunner.java index e2111060b014..561b10f26186 100644 --- a/client/src/main/java/com/metamx/druid/sql/SQLRunner.java +++ b/server/src/main/java/io/druid/server/sql/SQLRunner.java @@ -1,4 +1,23 @@ -package com.metamx.druid.sql; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.sql; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -8,17 +27,17 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.io.Closeables; -import com.metamx.druid.Druids; -import com.metamx.druid.Query; -import com.metamx.druid.aggregation.AggregatorFactory; -import com.metamx.druid.input.Row; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.query.dimension.DimensionSpec; -import com.metamx.druid.query.group.GroupByQuery; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.TimeseriesResultValue; -import com.metamx.druid.sql.antlr4.DruidSQLLexer; -import com.metamx.druid.sql.antlr4.DruidSQLParser; +import io.druid.data.input.Row; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Druids; +import io.druid.query.Query; +import io.druid.query.Result; +import io.druid.query.aggregation.AggregatorFactory; +import io.druid.query.dimension.DimensionSpec; +import io.druid.query.groupby.GroupByQuery; +import io.druid.query.timeseries.TimeseriesResultValue; +import io.druid.sql.antlr4.DruidSQLLexer; +import io.druid.sql.antlr4.DruidSQLParser; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CharStream; import org.antlr.v4.runtime.CommonTokenStream; diff --git a/indexing-common/src/main/java/com/metamx/druid/common/s3/S3Utils.java b/server/src/main/java/io/druid/storage/s3/S3Utils.java similarity index 97% rename from indexing-common/src/main/java/com/metamx/druid/common/s3/S3Utils.java rename to server/src/main/java/io/druid/storage/s3/S3Utils.java index 84ce35df9478..044b3fa76c9b 100644 --- a/indexing-common/src/main/java/com/metamx/druid/common/s3/S3Utils.java +++ b/server/src/main/java/io/druid/storage/s3/S3Utils.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.common.s3; +package io.druid.storage.s3; import com.google.common.base.Throwables; import com.metamx.common.logger.Logger; diff --git a/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java new file mode 100644 index 000000000000..4dd8c1002a0f --- /dev/null +++ b/server/src/main/java/io/druid/timeline/partition/LinearShardSpec.java @@ -0,0 +1,61 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.timeline.partition; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import io.druid.data.input.InputRow; + +import java.util.Map; + +public class LinearShardSpec implements ShardSpec +{ + private int partitionNum; + + @JsonCreator + public LinearShardSpec( + @JsonProperty("partitionNum") Integer partitionNum + ) + { + this.partitionNum = Preconditions.checkNotNull(partitionNum, "Must set partitionNum on LinearShardSpec"); + } + + @JsonProperty("partitionNum") + @Override + public int getPartitionNum() { + return partitionNum; + } + + @Override + public PartitionChunk createChunk(T obj) { + return new LinearPartitionChunk(partitionNum, obj); + } + + @Override + public boolean isInChunk(Map dimensions) { + return true; + } + + @Override + public boolean isInChunk(InputRow inputRow) { + return true; + } +} diff --git a/client/src/main/java/com/metamx/druid/shard/NumberedShardSpec.java b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java similarity index 90% rename from client/src/main/java/com/metamx/druid/shard/NumberedShardSpec.java rename to server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java index 325446cb25bc..e0e09c9fa76c 100644 --- a/client/src/main/java/com/metamx/druid/shard/NumberedShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/NumberedShardSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,15 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.shard; +package io.druid.timeline.partition; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.partition.NumberedPartitionChunk; -import com.metamx.druid.partition.PartitionChunk; +import io.druid.data.input.InputRow; import java.util.Map; diff --git a/client/src/main/java/com/metamx/druid/shard/SingleDimensionShardSpec.java b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java similarity index 93% rename from client/src/main/java/com/metamx/druid/shard/SingleDimensionShardSpec.java rename to server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java index 98ff7f2eae19..e6b41f5a62a4 100644 --- a/client/src/main/java/com/metamx/druid/shard/SingleDimensionShardSpec.java +++ b/server/src/main/java/io/druid/timeline/partition/SingleDimensionShardSpec.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,12 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.shard; +package io.druid.timeline.partition; import com.fasterxml.jackson.annotation.JsonProperty; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.partition.PartitionChunk; -import com.metamx.druid.partition.StringPartitionChunk; +import io.druid.data.input.InputRow; import java.util.List; import java.util.Map; diff --git a/server/src/test/java/com/metamx/druid/master/LoadQueuePeonTester.java b/server/src/test/java/com/metamx/druid/master/LoadQueuePeonTester.java deleted file mode 100644 index 366cde87e1d5..000000000000 --- a/server/src/test/java/com/metamx/druid/master/LoadQueuePeonTester.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.metamx.druid.master; - -import com.metamx.druid.client.DataSegment; - -import java.util.concurrent.ConcurrentSkipListSet; - -public class LoadQueuePeonTester extends LoadQueuePeon -{ - private final ConcurrentSkipListSet segmentsToLoad = new ConcurrentSkipListSet(); - - public LoadQueuePeonTester() - { - super(null, null, null, null, null); - } - - @Override - public void loadSegment( - DataSegment segment, - LoadPeonCallback callback - ) - { - segmentsToLoad.add(segment); - } - - public ConcurrentSkipListSet getSegmentsToLoad() - { - return segmentsToLoad; - } -} diff --git a/server/src/test/java/io/druid/TestUtil.java b/server/src/test/java/io/druid/TestUtil.java new file mode 100644 index 000000000000..e184c77998fc --- /dev/null +++ b/server/src/test/java/io/druid/TestUtil.java @@ -0,0 +1,41 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid; + +import com.fasterxml.jackson.databind.Module; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.druid.guice.ServerModule; +import io.druid.jackson.DefaultObjectMapper; + +import java.util.List; + +/** + */ +public class TestUtil +{ + public static final ObjectMapper MAPPER = new DefaultObjectMapper(); + + static { + final List list = new ServerModule().getJacksonModules(); + for (Module module : list) { + MAPPER.registerModule(module); + } + } +} diff --git a/client/src/test/java/com/metamx/druid/client/DataSegmentTest.java b/server/src/test/java/io/druid/client/DataSegmentTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/client/DataSegmentTest.java rename to server/src/test/java/io/druid/client/DataSegmentTest.java index 63f3326a92ea..c79a965d47e0 100644 --- a/client/src/test/java/com/metamx/druid/client/DataSegmentTest.java +++ b/server/src/test/java/io/druid/client/DataSegmentTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,11 +25,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.shard.NoneShardSpec; -import com.metamx.druid.shard.SingleDimensionShardSpec; - +import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; +import io.druid.timeline.partition.SingleDimensionShardSpec; import org.joda.time.DateTime; import org.joda.time.Interval; import org.junit.Assert; diff --git a/client/src/test/java/com/metamx/druid/client/cache/ByteCountingLRUMapTest.java b/server/src/test/java/io/druid/client/cache/ByteCountingLRUMapTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/client/cache/ByteCountingLRUMapTest.java rename to server/src/test/java/io/druid/client/cache/ByteCountingLRUMapTest.java index fac858d21927..5edf6a669e67 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/ByteCountingLRUMapTest.java +++ b/server/src/test/java/io/druid/client/cache/ByteCountingLRUMapTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import org.junit.Assert; import org.junit.Before; diff --git a/client/src/test/java/com/metamx/druid/client/cache/MapCacheTest.java b/server/src/test/java/io/druid/client/cache/MapCacheTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/client/cache/MapCacheTest.java rename to server/src/test/java/io/druid/client/cache/MapCacheTest.java index 23a3bd1d6413..fbff6f9fe2f3 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/MapCacheTest.java +++ b/server/src/test/java/io/druid/client/cache/MapCacheTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.primitives.Ints; import org.junit.Assert; diff --git a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBenchmark.java b/server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java similarity index 81% rename from client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBenchmark.java rename to server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java index c63e98e8988a..7ef6047d9ac2 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheBenchmark.java +++ b/server/src/test/java/io/druid/client/cache/MemcachedCacheBenchmark.java @@ -1,4 +1,23 @@ -package com.metamx.druid.client.cache; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.client.cache; import com.google.caliper.Param; import com.google.caliper.Runner; diff --git a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheTest.java b/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java similarity index 99% rename from client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheTest.java rename to server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java index 23ca0ea96930..55c85f967f0f 100644 --- a/client/src/test/java/com/metamx/druid/client/cache/MemcachedCacheTest.java +++ b/server/src/test/java/io/druid/client/cache/MemcachedCacheTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.cache; +package io.druid.client.cache; import com.google.common.collect.Lists; import com.google.common.collect.Maps; diff --git a/client/src/test/java/com/metamx/druid/client/BatchServerInventoryViewTest.java b/server/src/test/java/io/druid/client/client/BatchServerInventoryViewTest.java similarity index 83% rename from client/src/test/java/com/metamx/druid/client/BatchServerInventoryViewTest.java rename to server/src/test/java/io/druid/client/client/BatchServerInventoryViewTest.java index 7690b0ab06b0..38864822eb8d 100644 --- a/client/src/test/java/com/metamx/druid/client/BatchServerInventoryViewTest.java +++ b/server/src/test/java/io/druid/client/client/BatchServerInventoryViewTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client; +package io.druid.client.client; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Joiner; @@ -26,13 +26,16 @@ import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; import com.metamx.common.ISE; -import com.metamx.druid.coordination.BatchDataSegmentAnnouncer; -import com.metamx.druid.coordination.DruidServerMetadata; -import com.metamx.druid.curator.PotentiallyGzippedCompressionProvider; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.client.BatchServerInventoryView; +import io.druid.client.DruidServer; +import io.druid.curator.PotentiallyGzippedCompressionProvider; +import io.druid.curator.announcement.Announcer; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.server.coordination.BatchDataSegmentAnnouncer; +import io.druid.server.coordination.DruidServerMetadata; +import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import junit.framework.Assert; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -45,7 +48,6 @@ import org.junit.Test; import java.util.Set; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; /** @@ -93,30 +95,20 @@ public void setUp() throws Exception "type", "tier" ), - new ZkDataSegmentAnnouncerConfig() + new BatchDataSegmentAnnouncerConfig() { - @Override - public String getZkBasePath() - { - return testBasePath; - } - @Override public int getSegmentsPerNode() { return 50; } - - @Override - public long getMaxNumBytes() - { - return 100000; - } - + }, + new ZkPathsConfig() + { @Override - public String getAnnouncerType() + public String getZkBasePath() { - return "batch"; + return testBasePath; } }, announcer, @@ -130,20 +122,6 @@ public String getAnnouncerType() } batchServerInventoryView = new BatchServerInventoryView( - new ServerInventoryViewConfig() - { - @Override - public int getRemovedSegmentLifetime() - { - return 0; - } - - @Override - public String getAnnouncerType() - { - return "batch"; - } - }, new ZkPathsConfig() { @Override @@ -153,7 +131,6 @@ public String getZkBasePath() } }, cf, - Executors.newSingleThreadExecutor(), jsonMapper ); diff --git a/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java b/server/src/test/java/io/druid/client/selector/ServerSelectorTest.java similarity index 89% rename from client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java rename to server/src/test/java/io/druid/client/selector/ServerSelectorTest.java index 9009ec3d2463..d66648beb68a 100644 --- a/client/src/test/java/com/metamx/druid/client/selector/ServerSelectorTest.java +++ b/server/src/test/java/io/druid/client/selector/ServerSelectorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,22 +17,22 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.client.selector; +package io.druid.client.selector; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.util.concurrent.SettableFuture; -import com.metamx.druid.Druids; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DirectDruidClient; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.query.ReflectionQueryToolChestWarehouse; -import com.metamx.druid.query.timeboundary.TimeBoundaryQuery; -import com.metamx.druid.shard.NoneShardSpec; import com.metamx.http.client.HttpClient; import com.metamx.http.client.Request; import com.metamx.http.client.RequestBuilder; +import io.druid.client.DirectDruidClient; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.Druids; +import io.druid.query.ReflectionQueryToolChestWarehouse; +import io.druid.query.timeboundary.TimeBoundaryQuery; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import junit.framework.Assert; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpMethod; diff --git a/client/src/test/java/com/metamx/druid/curator/CuratorTestBase.java b/server/src/test/java/io/druid/curator/CuratorTestBase.java similarity index 54% rename from client/src/test/java/com/metamx/druid/curator/CuratorTestBase.java rename to server/src/test/java/io/druid/curator/CuratorTestBase.java index dcc0cfda60e5..a850d756ea33 100644 --- a/client/src/test/java/com/metamx/druid/curator/CuratorTestBase.java +++ b/server/src/test/java/io/druid/curator/CuratorTestBase.java @@ -1,4 +1,23 @@ -package com.metamx.druid.curator; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator; import com.google.common.io.Closeables; import org.apache.curator.framework.CuratorFramework; diff --git a/client/src/test/java/com/metamx/druid/curator/announcement/AnnouncerTest.java b/server/src/test/java/io/druid/curator/announcement/AnnouncerTest.java similarity index 96% rename from client/src/test/java/com/metamx/druid/curator/announcement/AnnouncerTest.java rename to server/src/test/java/io/druid/curator/announcement/AnnouncerTest.java index 9f4276e164a0..528b0fd77f41 100644 --- a/client/src/test/java/com/metamx/druid/curator/announcement/AnnouncerTest.java +++ b/server/src/test/java/io/druid/curator/announcement/AnnouncerTest.java @@ -17,11 +17,11 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.curator.announcement; +package io.druid.curator.announcement; import com.google.common.collect.Sets; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.curator.CuratorTestBase; +import io.druid.concurrent.Execs; +import io.druid.curator.CuratorTestBase; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.CuratorEvent; import org.apache.curator.framework.api.CuratorEventType; @@ -114,8 +114,6 @@ public void eventReceived(CuratorFramework client, CuratorEvent event) throws Ex @Test public void testSessionKilled() throws Exception { - final ExecutorService exec = Execs.singleThreaded("test-announcer-sanity-%s"); - curator.start(); Announcer announcer = new Announcer(curator, exec); try { diff --git a/client/src/test/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerTest.java b/server/src/test/java/io/druid/curator/inventory/CuratorInventoryManagerTest.java similarity index 88% rename from client/src/test/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerTest.java rename to server/src/test/java/io/druid/curator/inventory/CuratorInventoryManagerTest.java index da72db060483..1c13475912cc 100644 --- a/client/src/test/java/com/metamx/druid/curator/inventory/CuratorInventoryManagerTest.java +++ b/server/src/test/java/io/druid/curator/inventory/CuratorInventoryManagerTest.java @@ -1,10 +1,28 @@ -package com.metamx.druid.curator.inventory; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.curator.inventory; import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.primitives.Ints; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.curator.CuratorTestBase; +import io.druid.concurrent.Execs; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.api.CuratorEvent; import org.apache.curator.framework.api.CuratorEventType; @@ -21,7 +39,7 @@ /** */ -public class CuratorInventoryManagerTest extends CuratorTestBase +public class CuratorInventoryManagerTest extends io.druid.curator.CuratorTestBase { private ExecutorService exec; diff --git a/server/src/test/java/com/metamx/druid/db/DatabaseSegmentManagerTest.java b/server/src/test/java/io/druid/db/DatabaseSegmentManagerTest.java similarity index 85% rename from server/src/test/java/com/metamx/druid/db/DatabaseSegmentManagerTest.java rename to server/src/test/java/io/druid/db/DatabaseSegmentManagerTest.java index 751cad29a130..267f86a8a56c 100644 --- a/server/src/test/java/com/metamx/druid/db/DatabaseSegmentManagerTest.java +++ b/server/src/test/java/io/druid/db/DatabaseSegmentManagerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,52 +17,38 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.db; +package io.druid.db; +import com.google.common.base.Suppliers; import com.google.common.collect.Maps; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.jackson.DefaultObjectMapper; import org.easymock.EasyMock; -import org.joda.time.Duration; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.skife.jdbi.v2.DBI; +import org.skife.jdbi.v2.IDBI; import org.skife.jdbi.v2.tweak.HandleCallback; import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.concurrent.ScheduledExecutorService; /** */ public class DatabaseSegmentManagerTest { private DatabaseSegmentManager manager; - private DBI dbi; + private IDBI dbi; private List> testRows; @Before public void setUp() throws Exception { - dbi = EasyMock.createMock(DBI.class); + dbi = EasyMock.createMock(IDBI.class); manager = new DatabaseSegmentManager( new DefaultObjectMapper(), - EasyMock.createMock(ScheduledExecutorService.class), - new DatabaseSegmentManagerConfig() - { - @Override - public String getSegmentTable() - { - return null; - } - - @Override - public Duration getPollDuration() - { - return null; - } - }, + Suppliers.ofInstance(new DatabaseSegmentManagerConfig()), + Suppliers.ofInstance(DbTablesConfig.fromBase("test")), dbi ); @@ -108,10 +94,7 @@ public Duration getPollDuration() + "\"twitterstream_2012-01-05T00:00:00.000Z_2012-01-06T00:00:00.000Z_2012-01-06T22:19:12.565Z\"}" ); - testRows = Arrays.>asList( - map1, - map2 - ); + testRows = Arrays.>asList(map1, map2); } @After @@ -128,5 +111,6 @@ public void testPoll() manager.start(); manager.poll(); + manager.stop(); } } diff --git a/server/src/test/java/com/metamx/druid/loading/CacheTestSegmentLoader.java b/server/src/test/java/io/druid/segment/loading/CacheTestSegmentLoader.java similarity index 83% rename from server/src/test/java/com/metamx/druid/loading/CacheTestSegmentLoader.java rename to server/src/test/java/io/druid/segment/loading/CacheTestSegmentLoader.java index ebf6ae09b10b..42f16fd21c15 100644 --- a/server/src/test/java/com/metamx/druid/loading/CacheTestSegmentLoader.java +++ b/server/src/test/java/io/druid/segment/loading/CacheTestSegmentLoader.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,13 +17,13 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.loading; +package io.druid.segment.loading; import com.metamx.common.MapUtils; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.Segment; +import io.druid.segment.QueryableIndex; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; import java.io.File; @@ -77,6 +77,12 @@ public void close() throws IOException }; } + @Override + public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException + { + throw new UnsupportedOperationException(); + } + @Override public void cleanup(DataSegment loadSpec) throws SegmentLoadingException { diff --git a/server/src/test/java/com/metamx/druid/loading/DataSegmentPusherUtilTest.java b/server/src/test/java/io/druid/segment/loading/DataSegmentPusherUtilTest.java similarity index 51% rename from server/src/test/java/com/metamx/druid/loading/DataSegmentPusherUtilTest.java rename to server/src/test/java/io/druid/segment/loading/DataSegmentPusherUtilTest.java index 2337f882098b..d58c210eb4be 100644 --- a/server/src/test/java/com/metamx/druid/loading/DataSegmentPusherUtilTest.java +++ b/server/src/test/java/io/druid/segment/loading/DataSegmentPusherUtilTest.java @@ -1,9 +1,28 @@ -package com.metamx.druid.loading; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/loading/SingleSegmentLoaderTest.java b/server/src/test/java/io/druid/segment/loading/StorageLocationTest.java similarity index 59% rename from server/src/test/java/com/metamx/druid/loading/SingleSegmentLoaderTest.java rename to server/src/test/java/io/druid/segment/loading/StorageLocationTest.java index f82311609acb..ca499d041e55 100644 --- a/server/src/test/java/com/metamx/druid/loading/SingleSegmentLoaderTest.java +++ b/server/src/test/java/io/druid/segment/loading/StorageLocationTest.java @@ -1,7 +1,26 @@ -package com.metamx.druid.loading; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.segment.loading; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.client.DataSegment; +import io.druid.timeline.DataSegment; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; @@ -11,13 +30,13 @@ /** */ -public class SingleSegmentLoaderTest +public class StorageLocationTest { @Test public void testStorageLocation() throws Exception { long expectedAvail = 1000l; - SingleSegmentLoader.StorageLocation loc = new SingleSegmentLoader.StorageLocation(new File("/tmp"), expectedAvail); + StorageLocation loc = new StorageLocation(new File("/tmp"), expectedAvail); verifyLoc(expectedAvail, loc); @@ -46,7 +65,7 @@ public void testStorageLocation() throws Exception verifyLoc(expectedAvail, loc); } - private void verifyLoc(long maxSize, SingleSegmentLoader.StorageLocation loc) + private void verifyLoc(long maxSize, StorageLocation loc) { Assert.assertEquals(maxSize, loc.available()); for (int i = 0; i <= maxSize; ++i) { diff --git a/server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestDropTest.java b/server/src/test/java/io/druid/server/coordination/SegmentChangeRequestDropTest.java similarity index 90% rename from server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestDropTest.java rename to server/src/test/java/io/druid/server/coordination/SegmentChangeRequestDropTest.java index 92026a533e6c..1d87114032e9 100644 --- a/server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestDropTest.java +++ b/server/src/test/java/io/druid/server/coordination/SegmentChangeRequestDropTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.shard.NoneShardSpec; - +import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestLoadTest.java b/server/src/test/java/io/druid/server/coordination/SegmentChangeRequestLoadTest.java similarity index 90% rename from server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestLoadTest.java rename to server/src/test/java/io/druid/server/coordination/SegmentChangeRequestLoadTest.java index 626889741809..fb2745df0428 100644 --- a/server/src/test/java/com/metamx/druid/coordination/SegmentChangeRequestLoadTest.java +++ b/server/src/test/java/io/druid/server/coordination/SegmentChangeRequestLoadTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,16 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.shard.NoneShardSpec; - +import io.druid.jackson.DefaultObjectMapper; +import io.druid.segment.IndexIO; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/coordination/ServerManagerTest.java b/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java similarity index 94% rename from server/src/test/java/com/metamx/druid/coordination/ServerManagerTest.java rename to server/src/test/java/io/druid/server/coordination/ServerManagerTest.java index f2cd54f45517..f3483487c1d9 100644 --- a/server/src/test/java/com/metamx/druid/coordination/ServerManagerTest.java +++ b/server/src/test/java/io/druid/server/coordination/ServerManagerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; @@ -35,36 +35,37 @@ import com.metamx.common.guava.Yielder; import com.metamx.common.guava.YieldingAccumulator; import com.metamx.common.guava.YieldingSequenceBase; -import com.metamx.druid.Druids; -import com.metamx.druid.Query; -import com.metamx.druid.QueryGranularity; -import com.metamx.druid.StorageAdapter; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.index.QueryableIndex; -import com.metamx.druid.index.ReferenceCountingSegment; -import com.metamx.druid.index.Segment; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.loading.SegmentLoader; -import com.metamx.druid.loading.SegmentLoadingException; -import com.metamx.druid.metrics.NoopServiceEmitter; -import com.metamx.druid.query.ConcatQueryRunner; -import com.metamx.druid.query.MetricManipulationFn; -import com.metamx.druid.query.NoopQueryRunner; -import com.metamx.druid.query.QueryRunner; -import com.metamx.druid.query.QueryRunnerFactory; -import com.metamx.druid.query.QueryRunnerFactoryConglomerate; -import com.metamx.druid.query.QueryToolChest; -import com.metamx.druid.query.search.SearchQuery; -import com.metamx.druid.result.Result; -import com.metamx.druid.result.SearchResultValue; -import com.metamx.druid.shard.NoneShardSpec; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceMetricEvent; +import io.druid.granularity.QueryGranularity; +import io.druid.query.ConcatQueryRunner; +import io.druid.query.Druids; +import io.druid.query.NoopQueryRunner; +import io.druid.query.Query; +import io.druid.query.QueryRunner; +import io.druid.query.QueryRunnerFactory; +import io.druid.query.QueryRunnerFactoryConglomerate; +import io.druid.query.QueryToolChest; +import io.druid.query.Result; +import io.druid.query.aggregation.MetricManipulationFn; +import io.druid.query.search.SearchResultValue; +import io.druid.query.search.search.SearchQuery; +import io.druid.segment.IndexIO; +import io.druid.segment.QueryableIndex; +import io.druid.segment.ReferenceCountingSegment; +import io.druid.segment.Segment; +import io.druid.segment.StorageAdapter; +import io.druid.segment.loading.SegmentLoader; +import io.druid.segment.loading.SegmentLoadingException; +import io.druid.server.metrics.NoopServiceEmitter; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.io.File; import java.io.IOException; import java.util.Arrays; import java.util.Iterator; @@ -114,6 +115,12 @@ public Segment getSegment(final DataSegment segment) ); } + @Override + public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException + { + throw new UnsupportedOperationException(); + } + @Override public void cleanup(DataSegment segment) throws SegmentLoadingException { diff --git a/server/src/test/java/com/metamx/druid/coordination/ZkCoordinatorTest.java b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java similarity index 81% rename from server/src/test/java/com/metamx/druid/coordination/ZkCoordinatorTest.java rename to server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java index 746e7035cbb3..e1d78741b7c0 100644 --- a/server/src/test/java/com/metamx/druid/coordination/ZkCoordinatorTest.java +++ b/server/src/test/java/io/druid/server/coordination/ZkCoordinatorTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,24 +17,25 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; import com.metamx.common.logger.Logger; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.concurrent.Execs; -import com.metamx.druid.curator.CuratorTestBase; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.loading.CacheTestSegmentLoader; -import com.metamx.druid.metrics.NoopServiceEmitter; -import com.metamx.druid.query.NoopQueryRunnerFactoryConglomerate; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.concurrent.Execs; +import io.druid.curator.CuratorTestBase; +import io.druid.curator.announcement.Announcer; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.query.NoopQueryRunnerFactoryConglomerate; +import io.druid.segment.IndexIO; +import io.druid.segment.loading.CacheTestSegmentLoader; +import io.druid.segment.loading.SegmentLoaderConfig; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.server.metrics.NoopServiceEmitter; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.Interval; import org.junit.After; import org.junit.Assert; @@ -54,7 +55,7 @@ public class ZkCoordinatorTest extends CuratorTestBase private ZkCoordinator zkCoordinator; private ServerManager serverManager; private DataSegmentAnnouncer announcer; - private File cacheDir; + private File infoDir; private final ObjectMapper jsonMapper = new DefaultObjectMapper(); private static final Logger log = new Logger(ZkCoordinatorTest.class); @@ -64,12 +65,12 @@ public void setUp() throws Exception setupServerAndCurator(); curator.start(); try { - cacheDir = new File(File.createTempFile("blah", "blah2").getParent(), "ZkCoordinatorTest"); - cacheDir.mkdirs(); - for (File file : cacheDir.listFiles()) { + infoDir = new File(File.createTempFile("blah", "blah2").getParent(), "ZkCoordinatorTest"); + infoDir.mkdirs(); + for (File file : infoDir.listFiles()) { file.delete(); } - log.info("Creating tmp test files in [%s]", cacheDir); + log.info("Creating tmp test files in [%s]", infoDir); } catch (IOException e) { throw new RuntimeException(e); @@ -99,12 +100,11 @@ me, zkPaths, new Announcer(curator, Execs.singleThreaded("blah")), jsonMapper zkCoordinator = new ZkCoordinator( jsonMapper, - new ZkCoordinatorConfig() - { + new SegmentLoaderConfig(){ @Override - public File getSegmentInfoCacheDirectory() + public File getInfoDir() { - return cacheDir; + return infoDir; } }, zkPaths, @@ -155,8 +155,8 @@ public void testLoadCache() throws Exception deleteSegmentFromCache(segment); } - Assert.assertEquals(0, cacheDir.listFiles().length); - Assert.assertTrue(cacheDir.delete()); + Assert.assertEquals(0, infoDir.listFiles().length); + Assert.assertTrue(infoDir.delete()); } private DataSegment makeSegment(String dataSource, String version, Interval interval) @@ -165,7 +165,7 @@ private DataSegment makeSegment(String dataSource, String version, Interval inte dataSource, interval, version, - ImmutableMap.of("version", version, "interval", interval, "cacheDir", cacheDir), + ImmutableMap.of("version", version, "interval", interval, "cacheDir", infoDir), Arrays.asList("dim1", "dim2", "dim3"), Arrays.asList("metric1", "metric2"), new NoneShardSpec(), @@ -176,12 +176,12 @@ private DataSegment makeSegment(String dataSource, String version, Interval inte private void writeSegmentToCache(final DataSegment segment) throws IOException { - if (!cacheDir.exists()) { - cacheDir.mkdir(); + if (!infoDir.exists()) { + infoDir.mkdir(); } File segmentInfoCacheFile = new File( - cacheDir, + infoDir, segment.getIdentifier() ); try { @@ -197,7 +197,7 @@ private void writeSegmentToCache(final DataSegment segment) throws IOException private void deleteSegmentFromCache(final DataSegment segment) throws IOException { File segmentInfoCacheFile = new File( - cacheDir, + infoDir, segment.getIdentifier() ); if (segmentInfoCacheFile.exists()) { @@ -209,8 +209,8 @@ private void deleteSegmentFromCache(final DataSegment segment) throws IOExceptio private void checkCache(List segments) throws IOException { - Assert.assertTrue(cacheDir.exists()); - File[] files = cacheDir.listFiles(); + Assert.assertTrue(infoDir.exists()); + File[] files = infoDir.listFiles(); List sortedFiles = Lists.newArrayList(files); Collections.sort(sortedFiles); diff --git a/client/src/test/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncerTest.java b/server/src/test/java/io/druid/server/coordination/coordination/BatchDataSegmentAnnouncerTest.java similarity index 91% rename from client/src/test/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncerTest.java rename to server/src/test/java/io/druid/server/coordination/coordination/BatchDataSegmentAnnouncerTest.java index 608fe850a74e..245a3573c5f6 100644 --- a/client/src/test/java/com/metamx/druid/coordination/BatchDataSegmentAnnouncerTest.java +++ b/server/src/test/java/io/druid/server/coordination/coordination/BatchDataSegmentAnnouncerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.coordination; +package io.druid.server.coordination.coordination; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -25,11 +25,14 @@ import com.google.common.base.Throwables; import com.google.common.collect.Sets; import com.google.common.util.concurrent.MoreExecutors; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.curator.PotentiallyGzippedCompressionProvider; -import com.metamx.druid.curator.announcement.Announcer; -import com.metamx.druid.initialization.ZkDataSegmentAnnouncerConfig; -import com.metamx.druid.jackson.DefaultObjectMapper; +import io.druid.curator.PotentiallyGzippedCompressionProvider; +import io.druid.curator.announcement.Announcer; +import io.druid.jackson.DefaultObjectMapper; +import io.druid.server.coordination.BatchDataSegmentAnnouncer; +import io.druid.server.coordination.DruidServerMetadata; +import io.druid.server.initialization.BatchDataSegmentAnnouncerConfig; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.timeline.DataSegment; import junit.framework.Assert; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -92,30 +95,20 @@ public void setUp() throws Exception "type", "tier" ), - new ZkDataSegmentAnnouncerConfig() + new BatchDataSegmentAnnouncerConfig() { - @Override - public String getZkBasePath() - { - return testBasePath; - } - @Override public int getSegmentsPerNode() { return 50; } - - @Override - public long getMaxNumBytes() - { - return 100000; - } - + }, + new ZkPathsConfig() + { @Override - public String getAnnouncerType() + public String getZkBasePath() { - return "batch"; + return testBasePath; } }, announcer, diff --git a/server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java b/server/src/test/java/io/druid/server/master/DruidMasterBalancerProfiler.java similarity index 85% rename from server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java rename to server/src/test/java/io/druid/server/master/DruidMasterBalancerProfiler.java index 482274c0b281..9830f3c71ef8 100644 --- a/server/src/test/java/com/metamx/druid/utils/DruidMasterBalancerProfiler.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterBalancerProfiler.java @@ -1,23 +1,23 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -package com.metamx.druid.utils; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.master; import com.google.common.base.Stopwatch; import com.google.common.collect.ImmutableList; @@ -25,26 +25,14 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.master.DruidCluster; -import com.metamx.druid.master.DruidMaster; -import com.metamx.druid.master.DruidMasterBalancerTester; -import com.metamx.druid.master.DruidMasterRuleRunner; -import com.metamx.druid.master.DruidMasterRuntimeParams; -import com.metamx.druid.master.LoadPeonCallback; -import com.metamx.druid.master.LoadQueuePeon; -import com.metamx.druid.master.LoadQueuePeonTester; -import com.metamx.druid.master.MasterSegmentSettings; -import com.metamx.druid.master.ReplicationThrottler; -import com.metamx.druid.master.SegmentReplicantLookup; -import com.metamx.druid.master.ServerHolder; -import com.metamx.druid.master.rules.PeriodLoadRule; -import com.metamx.druid.master.rules.Rule; -import com.metamx.druid.shard.NoneShardSpec; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; +import io.druid.client.DruidServer; +import io.druid.db.DatabaseRuleManager; +import io.druid.server.master.rules.PeriodLoadRule; +import io.druid.server.master.rules.Rule; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.easymock.EasyMock; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java b/server/src/test/java/io/druid/server/master/DruidMasterBalancerTest.java similarity index 94% rename from server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java rename to server/src/test/java/io/druid/server/master/DruidMasterBalancerTest.java index 8513c157de3d..a284bbe9931c 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTest.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterBalancerTest.java @@ -1,31 +1,31 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ - -package com.metamx.druid.master; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.master; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.MinMaxPriorityQueue; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import junit.framework.Assert; import org.easymock.EasyMock; import org.joda.time.DateTime; diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTester.java b/server/src/test/java/io/druid/server/master/DruidMasterBalancerTester.java similarity index 63% rename from server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTester.java rename to server/src/test/java/io/druid/server/master/DruidMasterBalancerTester.java index 3b5865fb361b..e2afe47c9092 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterBalancerTester.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterBalancerTester.java @@ -1,7 +1,26 @@ -package com.metamx.druid.master; +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; +package io.druid.server.master; + +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; public class DruidMasterBalancerTester extends DruidMasterBalancer { diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java b/server/src/test/java/io/druid/server/master/DruidMasterRuleRunnerTest.java similarity index 98% rename from server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java rename to server/src/test/java/io/druid/server/master/DruidMasterRuleRunnerTest.java index 27dc46636acd..937430d5d152 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterRuleRunnerTest.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterRuleRunnerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; @@ -25,17 +25,17 @@ import com.google.common.collect.MinMaxPriorityQueue; import com.google.common.collect.Ordering; import com.google.common.collect.Sets; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.db.DatabaseRuleManager; -import com.metamx.druid.index.v1.IndexIO; -import com.metamx.druid.master.rules.IntervalDropRule; -import com.metamx.druid.master.rules.IntervalLoadRule; -import com.metamx.druid.master.rules.Rule; -import com.metamx.druid.shard.NoneShardSpec; import com.metamx.emitter.EmittingLogger; import com.metamx.emitter.service.ServiceEmitter; import com.metamx.emitter.service.ServiceEventBuilder; +import io.druid.client.DruidServer; +import io.druid.db.DatabaseRuleManager; +import io.druid.segment.IndexIO; +import io.druid.server.master.rules.IntervalDropRule; +import io.druid.server.master.rules.IntervalLoadRule; +import io.druid.server.master.rules.Rule; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.easymock.EasyMock; import org.joda.time.DateTime; import org.joda.time.Interval; diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java b/server/src/test/java/io/druid/server/master/DruidMasterSegmentMergerTest.java similarity index 98% rename from server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java rename to server/src/test/java/io/druid/server/master/DruidMasterSegmentMergerTest.java index 04caa13a8a4d..37392ac2e9ab 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterSegmentMergerTest.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterSegmentMergerTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,14 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.indexing.IndexingServiceClient; -import com.metamx.druid.shard.LinearShardSpec; +import io.druid.client.indexing.IndexingServiceClient; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.LinearShardSpec; import junit.framework.Assert; import org.joda.time.Interval; import org.junit.Test; diff --git a/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java b/server/src/test/java/io/druid/server/master/DruidMasterTest.java similarity index 89% rename from server/src/test/java/com/metamx/druid/master/DruidMasterTest.java rename to server/src/test/java/io/druid/server/master/DruidMasterTest.java index 80993ed62f22..a1334718e947 100644 --- a/server/src/test/java/com/metamx/druid/master/DruidMasterTest.java +++ b/server/src/test/java/io/druid/server/master/DruidMasterTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,17 +17,17 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master; +package io.druid.server.master; import com.google.common.collect.MapMaker; import com.metamx.common.concurrent.ScheduledExecutorFactory; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.client.SingleServerInventoryView; -import com.metamx.druid.curator.inventory.InventoryManagerConfig; -import com.metamx.druid.db.DatabaseSegmentManager; -import com.metamx.druid.initialization.ZkPathsConfig; -import com.metamx.druid.metrics.NoopServiceEmitter; +import io.druid.client.DruidServer; +import io.druid.client.SingleServerInventoryView; +import io.druid.curator.inventory.InventoryManagerConfig; +import io.druid.db.DatabaseSegmentManager; +import io.druid.server.initialization.ZkPathsConfig; +import io.druid.server.metrics.NoopServiceEmitter; +import io.druid.timeline.DataSegment; import org.apache.curator.framework.CuratorFramework; import org.easymock.EasyMock; import org.joda.time.Duration; @@ -94,12 +94,6 @@ public Duration getMasterSegmentMergerPeriod() return null; } - @Override - public String getMergerServiceName() - { - return ""; - } - @Override public int getReplicantLifetime() { diff --git a/server/src/test/java/io/druid/server/master/LoadQueuePeonTester.java b/server/src/test/java/io/druid/server/master/LoadQueuePeonTester.java new file mode 100644 index 000000000000..a8e53d1504d9 --- /dev/null +++ b/server/src/test/java/io/druid/server/master/LoadQueuePeonTester.java @@ -0,0 +1,48 @@ +/* + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.master; + +import io.druid.timeline.DataSegment; + +import java.util.concurrent.ConcurrentSkipListSet; + +public class LoadQueuePeonTester extends LoadQueuePeon +{ + private final ConcurrentSkipListSet segmentsToLoad = new ConcurrentSkipListSet(); + + public LoadQueuePeonTester() + { + super(null, null, null, null, null); + } + + @Override + public void loadSegment( + DataSegment segment, + LoadPeonCallback callback + ) + { + segmentsToLoad.add(segment); + } + + public ConcurrentSkipListSet getSegmentsToLoad() + { + return segmentsToLoad; + } +} diff --git a/server/src/test/java/com/metamx/druid/master/ReservoirSegmentSamplerTest.java b/server/src/test/java/io/druid/server/master/ReservoirSegmentSamplerTest.java similarity index 88% rename from server/src/test/java/com/metamx/druid/master/ReservoirSegmentSamplerTest.java rename to server/src/test/java/io/druid/server/master/ReservoirSegmentSamplerTest.java index 7b768e111726..9561e5feda70 100644 --- a/server/src/test/java/com/metamx/druid/master/ReservoirSegmentSamplerTest.java +++ b/server/src/test/java/io/druid/server/master/ReservoirSegmentSamplerTest.java @@ -1,29 +1,30 @@ /* -* Druid - a distributed column store. -* Copyright (C) 2012 Metamarkets Group Inc. -* -* This program is free software; you can redistribute it and/or -* modify it under the terms of the GNU General Public License -* as published by the Free Software Foundation; either version 2 -* of the License, or (at your option) any later version. -* -* This program is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with this program; if not, write to the Free Software -* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -*/ -package com.metamx.druid.master; + * Druid - a distributed column store. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ + +package io.druid.server.master; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.client.DruidServer; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.client.DruidServer; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import junit.framework.Assert; import org.easymock.EasyMock; import org.joda.time.DateTime; diff --git a/server/src/test/java/com/metamx/druid/master/rules/PeriodDropRuleTest.java b/server/src/test/java/io/druid/server/master/rules/PeriodDropRuleTest.java similarity index 94% rename from server/src/test/java/com/metamx/druid/master/rules/PeriodDropRuleTest.java rename to server/src/test/java/io/druid/server/master/rules/PeriodDropRuleTest.java index c6bceb08e5e2..3b4967a1c7d1 100644 --- a/server/src/test/java/com/metamx/druid/master/rules/PeriodDropRuleTest.java +++ b/server/src/test/java/io/druid/server/master/rules/PeriodDropRuleTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/server/src/test/java/com/metamx/druid/master/rules/PeriodLoadRuleTest.java b/server/src/test/java/io/druid/server/master/rules/PeriodLoadRuleTest.java similarity index 93% rename from server/src/test/java/com/metamx/druid/master/rules/PeriodLoadRuleTest.java rename to server/src/test/java/io/druid/server/master/rules/PeriodLoadRuleTest.java index 3944d96ecb99..de10902c6f2c 100644 --- a/server/src/test/java/com/metamx/druid/master/rules/PeriodLoadRuleTest.java +++ b/server/src/test/java/io/druid/server/master/rules/PeriodLoadRuleTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,10 +17,10 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.master.rules; +package io.druid.server.master.rules; -import com.metamx.druid.client.DataSegment; -import com.metamx.druid.shard.NoneShardSpec; +import io.druid.timeline.DataSegment; +import io.druid.timeline.partition.NoneShardSpec; import org.joda.time.DateTime; import org.joda.time.Interval; import org.joda.time.Period; diff --git a/server/src/test/java/com/metamx/druid/metrics/NoopServiceEmitter.java b/server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java similarity index 92% rename from server/src/test/java/com/metamx/druid/metrics/NoopServiceEmitter.java rename to server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java index f980bc4885cd..8a9a59440996 100644 --- a/server/src/test/java/com/metamx/druid/metrics/NoopServiceEmitter.java +++ b/server/src/test/java/io/druid/server/metrics/NoopServiceEmitter.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.metrics; +package io.druid.server.metrics; import com.metamx.emitter.core.Event; import com.metamx.emitter.service.ServiceEmitter; diff --git a/client/src/test/java/com/metamx/druid/shard/NumberedShardSpecTest.java b/server/src/test/java/io/druid/server/shard/NumberedShardSpecTest.java similarity index 85% rename from client/src/test/java/com/metamx/druid/shard/NumberedShardSpecTest.java rename to server/src/test/java/io/druid/server/shard/NumberedShardSpecTest.java index 15a9c8989220..cc5533245625 100644 --- a/client/src/test/java/com/metamx/druid/shard/NumberedShardSpecTest.java +++ b/server/src/test/java/io/druid/server/shard/NumberedShardSpecTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,14 +17,15 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.shard; +package io.druid.server.shard; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; -import com.metamx.druid.jackson.DefaultObjectMapper; -import com.metamx.druid.partition.PartitionChunk; +import io.druid.TestUtil; +import io.druid.timeline.partition.NumberedShardSpec; +import io.druid.timeline.partition.PartitionChunk; +import io.druid.timeline.partition.ShardSpec; import junit.framework.Assert; import org.junit.Test; @@ -35,9 +36,8 @@ public class NumberedShardSpecTest @Test public void testSerdeRoundTrip() throws Exception { - final ObjectMapper jsonMapper = new DefaultObjectMapper(); - final ShardSpec spec = jsonMapper.readValue( - jsonMapper.writeValueAsBytes(new NumberedShardSpec(1, 2)), + final ShardSpec spec = TestUtil.MAPPER.readValue( + TestUtil.MAPPER.writeValueAsBytes(new NumberedShardSpec(1, 2)), ShardSpec.class ); Assert.assertEquals(1, spec.getPartitionNum()); @@ -47,8 +47,7 @@ public void testSerdeRoundTrip() throws Exception @Test public void testSerdeBackwardsCompat() throws Exception { - final ObjectMapper jsonMapper = new DefaultObjectMapper(); - final ShardSpec spec = jsonMapper.readValue( + final ShardSpec spec = TestUtil.MAPPER.readValue( "{\"type\": \"numbered\", \"partitions\": 2, \"partitionNum\": 1}", ShardSpec.class ); diff --git a/client/src/test/java/com/metamx/druid/shard/SingleDimensionShardSpecTest.java b/server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java similarity index 94% rename from client/src/test/java/com/metamx/druid/shard/SingleDimensionShardSpecTest.java rename to server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java index 217b18d9be7f..e262bab1f988 100644 --- a/client/src/test/java/com/metamx/druid/shard/SingleDimensionShardSpecTest.java +++ b/server/src/test/java/io/druid/server/shard/SingleDimensionShardSpecTest.java @@ -1,6 +1,6 @@ /* * Druid - a distributed column store. - * Copyright (C) 2012 Metamarkets Group Inc. + * Copyright (C) 2012, 2013 Metamarkets Group Inc. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -17,7 +17,7 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ -package com.metamx.druid.shard; +package io.druid.server.shard; import com.google.common.base.Function; import com.google.common.base.Preconditions; @@ -26,12 +26,12 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.metamx.common.Pair; -import com.metamx.druid.input.InputRow; -import com.metamx.druid.input.MapBasedInputRow; +import io.druid.data.input.InputRow; +import io.druid.data.input.MapBasedInputRow; +import io.druid.timeline.partition.SingleDimensionShardSpec; import org.junit.Assert; import org.junit.Test; -import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/services/pom.xml b/services/pom.xml index de247ffba4bc..0f202976a75f 100644 --- a/services/pom.xml +++ b/services/pom.xml @@ -1,7 +1,7 @@