Skip to content

Downgrade geoip2, exclude com.google.http-client. #2741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fjy merged 1 commit into from
Mar 26, 2016
Merged

Downgrade geoip2, exclude com.google.http-client. #2741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...ions-core/hdfs-storage/src/test/java/io/druid/storage/hdfs/HdfsDataSegmentPusherTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@

package io.druid.storage.hdfs;

import com.google.api.client.util.Lists;
import com.google.api.client.util.Maps;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
import io.druid.jackson.DefaultObjectMapper;
import io.druid.timeline.DataSegment;
Expand Down
4 changes: 2 additions & 2 deletions extensions-core/s3-extensions/src/test/java/io/druid/storage/s3/S3DataSegmentPusherTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@

package io.druid.storage.s3;

import com.google.api.client.util.Lists;
import com.google.api.client.util.Maps;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.Files;
import io.druid.jackson.DefaultObjectMapper;
import io.druid.timeline.DataSegment;
Expand Down
3 changes: 1 addition & 2 deletions indexing-hadoop/src/test/java/io/druid/indexer/hadoop/DatasourceInputFormatTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@

package io.druid.indexer.hadoop;

import com.google.api.client.util.Maps;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import io.druid.indexer.JobHelper;
import io.druid.jackson.DefaultObjectMapper;
Expand All @@ -46,7 +46,6 @@
import org.junit.Test;

import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

Expand Down
6 changes: 5 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -612,8 +612,12 @@
<dependency>
<groupId>com.maxmind.geoip2</groupId>
<artifactId>geoip2</artifactId>
<version>2.6.0</version>
<version>0.4.0</version>
<exclusions>
<exclusion>
<groupId>com.google.http-client</groupId>
<artifactId>google-http-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/io/druid/segment/loading/LocalDataSegmentFinder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
package io.druid.segment.loading;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.api.client.util.Sets;
import com.google.common.collect.Sets;
import com.google.inject.Inject;
import com.metamx.common.logger.Logger;
import io.druid.guice.LocalDataStorageDruidModule;
Expand Down
7 changes: 3 additions & 4 deletions server/src/main/java/io/druid/segment/realtime/firehose/WikipediaIrcDecoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,9 @@
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.maxmind.db.CHMCache;
import com.maxmind.geoip2.DatabaseReader;
import com.maxmind.geoip2.exception.GeoIp2Exception;
import com.maxmind.geoip2.model.CityResponse;
import com.maxmind.geoip2.model.Omni;
import com.metamx.common.logger.Logger;
import io.druid.data.input.InputRow;
import io.druid.data.input.Row;
Expand Down Expand Up @@ -124,7 +123,7 @@ private DatabaseReader openDefaultGeoIpDb(File geoDb) {

private DatabaseReader openGeoIpDb(File geoDb) {
try {
DatabaseReader reader = new DatabaseReader.Builder(geoDb).withCache(new CHMCache()).build();
DatabaseReader reader = new DatabaseReader(geoDb);
log.info("Using geo ip database at [%s].", geoDb);
return reader;
} catch (IOException e) {
Expand Down Expand Up @@ -197,7 +196,7 @@ public InputRow decodeMessage(final DateTime timestamp, String channel, String m
if (anonymous) {
try {
final InetAddress ip = InetAddress.getByName(ipMatch.group());
final CityResponse lookup = geoLookup.city(ip);
final Omni lookup = geoLookup.omni(ip);

dimensions.put("continent", lookup.getContinent().getName());
dimensions.put("country", lookup.getCountry().getName());
Expand Down
15 changes: 12 additions & 3 deletions server/src/main/java/io/druid/server/lookup/cache/LookupCoordinatorManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes;
import com.google.api.client.http.HttpStatusCodes;
import com.google.common.base.Function;
import com.google.common.base.Predicates;
import com.google.common.base.Throwables;
Expand Down Expand Up @@ -160,7 +159,7 @@ void deleteOnHost(final URL url)
lookupCoordinatorManagerConfig.getHostDeleteTimeout()
).get()) {
// 404 is ok here, that means it was already deleted
if (!HttpStatusCodes.isSuccess(returnCode.get()) || HttpStatusCodes.STATUS_CODE_NOT_FOUND != returnCode.get()) {
if (!httpStatusIsSuccess(returnCode.get()) || !httpStatusIsNotFound(returnCode.get())) {
Copy link
Copy Markdown
Contributor Author

@gianm gianm Mar 25, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@drcrallen is this right? I think the || should be &&

Copy link
Copy Markdown
Contributor

@drcrallen drcrallen Mar 25, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Or is correct.

Normal logic is success only.

Here are 404 is also a "success"

Copy link
Copy Markdown
Contributor

@drcrallen drcrallen Mar 25, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh wai, let me look.

Copy link
Copy Markdown
Contributor

@drcrallen drcrallen Mar 25, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, this is supposed to be any of 200 202 or 404 essentially. Looking into why unit tests didn't report weird stuff here

Copy link
Copy Markdown
Contributor

@drcrallen drcrallen Mar 25, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the catch. Will have unit test fix in soon

Copy link
Copy Markdown
Contributor

@drcrallen drcrallen Mar 26, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test Fixed in #2743

final ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
StreamUtils.copyAndClose(result, baos);
Expand Down Expand Up @@ -210,7 +209,7 @@ void updateAllOnHost(final URL url, Map<String, Map<String, Object>> knownLookup
makeResponseHandler(returnCode, reasonString),
lookupCoordinatorManagerConfig.getHostUpdateTimeout()
).get()) {
if (!HttpStatusCodes.isSuccess(returnCode.get())) {
if (!httpStatusIsSuccess(returnCode.get())) {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
try {
StreamUtils.copyAndClose(result, baos);
Expand Down Expand Up @@ -614,4 +613,14 @@ static URL getLookupsURL(HostAndPort druidNode) throws MalformedURLException
ListenerResource.BASE_PATH + "/" + LOOKUP_LISTEN_ANNOUNCE_KEY
);
}

private static boolean httpStatusIsSuccess(int statusCode)
{
return statusCode >= 200 && statusCode < 300;
}

private static boolean httpStatusIsNotFound(int statusCode)
{
return statusCode == 404;
}
}
4 changes: 2 additions & 2 deletions ...er/src/test/java/io/druid/segment/realtime/appenderator/FiniteAppenderatorDriverTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@
package io.druid.segment.realtime.appenderator;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.api.client.util.Maps;
import com.google.api.client.util.Sets;
import com.google.common.base.Function;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Ordering;
import com.google.common.collect.Sets;
import com.metamx.common.Granularity;
import io.druid.data.input.Committer;
import io.druid.data.input.InputRow;
Expand Down