From 977e867ad89c4ad45554544143a07641980535ff Mon Sep 17 00:00:00 2001 From: Gian Merlino Date: Fri, 25 Mar 2016 14:24:58 -0700 Subject: [PATCH] Downgrade geoip2, exclude com.google.http-client. Reverts "Update com.maxmind.geoip2 to 2.6.0" and exclude the google http client from com.maxmind.geoip2. This should satisfy the original need from #2646 (wanting to run Druid along with an upgraded com.google.http-client) while preventing Jackson conflicts pointed out in #2717. Fixes #2717. This reverts commit 21b7572533592f1700f86379483d87e9e340f2a7. --- .../storage/hdfs/HdfsDataSegmentPusherTest.java | 4 ++-- .../druid/storage/s3/S3DataSegmentPusherTest.java | 4 ++-- .../indexer/hadoop/DatasourceInputFormatTest.java | 3 +-- pom.xml | 6 +++++- .../segment/loading/LocalDataSegmentFinder.java | 2 +- .../realtime/firehose/WikipediaIrcDecoder.java | 7 +++---- .../lookup/cache/LookupCoordinatorManager.java | 15 ++++++++++++--- .../FiniteAppenderatorDriverTest.java | 4 ++-- 8 files changed, 28 insertions(+), 17 deletions(-) diff --git a/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java index 7bcf4db16f44..16969774a672 100644 --- a/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java +++ b/extensions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java @@ -19,8 +19,8 @@ package io.druid.storage.hdfs; -import com.google.api.client.util.Lists; -import com.google.api.client.util.Maps; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.io.Files; import io.druid.jackson.DefaultObjectMapper; import io.druid.timeline.DataSegment; diff --git a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java index 62de4a622407..6e26d87eeb86 100644 --- a/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java +++ b/extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java @@ -19,8 +19,8 @@ package io.druid.storage.s3; -import com.google.api.client.util.Lists; -import com.google.api.client.util.Maps; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import com.google.common.io.Files; import io.druid.jackson.DefaultObjectMapper; import io.druid.timeline.DataSegment; diff --git a/indexing-hadoop/src/test/java/io/druid/indexer/hadoop/DatasourceInputFormatTest.java b/indexing-hadoop/src/test/java/io/druid/indexer/hadoop/DatasourceInputFormatTest.java index 71b95a5986c1..2f1293a57936 100644 --- a/indexing-hadoop/src/test/java/io/druid/indexer/hadoop/DatasourceInputFormatTest.java +++ b/indexing-hadoop/src/test/java/io/druid/indexer/hadoop/DatasourceInputFormatTest.java @@ -19,10 +19,10 @@ package io.druid.indexer.hadoop; -import com.google.api.client.util.Maps; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import com.google.common.collect.Sets; import io.druid.indexer.JobHelper; import io.druid.jackson.DefaultObjectMapper; @@ -46,7 +46,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.Arrays; import java.util.List; import java.util.Map; diff --git a/pom.xml b/pom.xml index d21f977bde1d..a3754af84d9c 100644 --- a/pom.xml +++ b/pom.xml @@ -612,8 +612,12 @@ com.maxmind.geoip2 geoip2 - 2.6.0 + 0.4.0 + + com.google.http-client + google-http-client + org.apache.httpcomponents httpclient diff --git a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java index 54e651ac43e5..479eaedbef97 100644 --- a/server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java +++ b/server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java @@ -20,7 +20,7 @@ package io.druid.segment.loading; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.util.Sets; +import com.google.common.collect.Sets; import com.google.inject.Inject; import com.metamx.common.logger.Logger; import io.druid.guice.LocalDataStorageDruidModule; diff --git a/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java b/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java index 1ef270b572ec..9ddfabaf961f 100644 --- a/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java +++ b/server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java @@ -24,10 +24,9 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.maxmind.db.CHMCache; import com.maxmind.geoip2.DatabaseReader; import com.maxmind.geoip2.exception.GeoIp2Exception; -import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.Omni; import com.metamx.common.logger.Logger; import io.druid.data.input.InputRow; import io.druid.data.input.Row; @@ -124,7 +123,7 @@ private DatabaseReader openDefaultGeoIpDb(File geoDb) { private DatabaseReader openGeoIpDb(File geoDb) { try { - DatabaseReader reader = new DatabaseReader.Builder(geoDb).withCache(new CHMCache()).build(); + DatabaseReader reader = new DatabaseReader(geoDb); log.info("Using geo ip database at [%s].", geoDb); return reader; } catch (IOException e) { @@ -197,7 +196,7 @@ public InputRow decodeMessage(final DateTime timestamp, String channel, String m if (anonymous) { try { final InetAddress ip = InetAddress.getByName(ipMatch.group()); - final CityResponse lookup = geoLookup.city(ip); + final Omni lookup = geoLookup.omni(ip); dimensions.put("continent", lookup.getContinent().getName()); dimensions.put("country", lookup.getCountry().getName()); diff --git a/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java b/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java index a108ab097ddd..05617fef62ac 100644 --- a/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java +++ b/server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes; -import com.google.api.client.http.HttpStatusCodes; import com.google.common.base.Function; import com.google.common.base.Predicates; import com.google.common.base.Throwables; @@ -160,7 +159,7 @@ void deleteOnHost(final URL url) lookupCoordinatorManagerConfig.getHostDeleteTimeout() ).get()) { // 404 is ok here, that means it was already deleted - if (!HttpStatusCodes.isSuccess(returnCode.get()) || HttpStatusCodes.STATUS_CODE_NOT_FOUND != returnCode.get()) { + if (!httpStatusIsSuccess(returnCode.get()) || !httpStatusIsNotFound(returnCode.get())) { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { StreamUtils.copyAndClose(result, baos); @@ -210,7 +209,7 @@ void updateAllOnHost(final URL url, Map> knownLookup makeResponseHandler(returnCode, reasonString), lookupCoordinatorManagerConfig.getHostUpdateTimeout() ).get()) { - if (!HttpStatusCodes.isSuccess(returnCode.get())) { + if (!httpStatusIsSuccess(returnCode.get())) { final ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { StreamUtils.copyAndClose(result, baos); @@ -614,4 +613,14 @@ static URL getLookupsURL(HostAndPort druidNode) throws MalformedURLException ListenerResource.BASE_PATH + "/" + LOOKUP_LISTEN_ANNOUNCE_KEY ); } + + private static boolean httpStatusIsSuccess(int statusCode) + { + return statusCode >= 200 && statusCode < 300; + } + + private static boolean httpStatusIsNotFound(int statusCode) + { + return statusCode == 404; + } } diff --git a/server/src/test/java/io/druid/segment/realtime/appenderator/FiniteAppenderatorDriverTest.java b/server/src/test/java/io/druid/segment/realtime/appenderator/FiniteAppenderatorDriverTest.java index 2ed509a5d697..8211d2520bd8 100644 --- a/server/src/test/java/io/druid/segment/realtime/appenderator/FiniteAppenderatorDriverTest.java +++ b/server/src/test/java/io/druid/segment/realtime/appenderator/FiniteAppenderatorDriverTest.java @@ -20,15 +20,15 @@ package io.druid.segment.realtime.appenderator; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.util.Maps; -import com.google.api.client.util.Sets; import com.google.common.base.Function; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; +import com.google.common.collect.Maps; import com.google.common.collect.Ordering; +import com.google.common.collect.Sets; import com.metamx.common.Granularity; import io.druid.data.input.Committer; import io.druid.data.input.InputRow;