diff --git a/.gitignore b/.gitignore index a1c2a23..5ddaba9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,31 @@ -# Compiled class file -*.class +target/ +!.mvn/wrapper/maven-wrapper.jar +.checkstyle -# Log file -*.log +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans -# BlueJ files -*.ctxt +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr -# Mobile Tools for Java (J2ME) -.mtj.tmp/ +### NetBeans ### +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ -# Package Files # -*.jar -*.war -*.nar -*.ear -*.zip -*.tar.gz -*.rar +### Log file ### +*.log -# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +### virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml ### hs_err_pid* diff --git a/README.md b/README.md index aa68257..4030142 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# github +# GitHub A collection of Github connectors and plugins diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..07209ea --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,406 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/GithubBatchSource-batchsource.md b/docs/GithubBatchSource-batchsource.md new file mode 100644 index 0000000..2c7dee0 --- /dev/null +++ b/docs/GithubBatchSource-batchsource.md @@ -0,0 +1,28 @@ +# GitHub batch source + +Description +----------- +This plugin is used to query GitHub API. + +Using this plugin users can select the data sets associated with the specified repository and collect raw level data. + +Properties +---------- +### Basic + +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Repository owner name:** GitHub username who owns the repository from which the data is retrieved. + +**Repository name:** Repository name from which the data is retrieved. + +**Dataset name:** Dataset name that you would like to retrieve. + +### Advanced + +**GitHub API hostname:** GitHub API hostname from which the data is retrieved. Optional, for GitHub Enterprise only. +By default, _api.github.com_ + +### Credentials + +**Authorization token:** Authorization token to be used to authenticate to GitHub API. diff --git a/icons/GithubBatchSource-batchsource.png b/icons/GithubBatchSource-batchsource.png new file mode 100644 index 0000000..4790adf Binary files /dev/null and b/icons/GithubBatchSource-batchsource.png differ diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..960e81c --- /dev/null +++ b/pom.xml @@ -0,0 +1,366 @@ + + + 4.0.0 + + io.cdap.plugin + github-plugin + GitHub plugin + 1.0.0-SNAPSHOT + A collection of gitHub connectors and plugins + https://github.com/data-integrations/github + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + + + CDAP + cdap-dev@googlegroups.com + CDAP + http://www.cdap.io + + + + + + sonatype + https://oss.sonatype.org/content/groups/public + + + sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots + + + eclipse + https://repo.eclipse.org/content/groups/releases + + + + + https://issues.cask.co/browse/CDAP + + + + UTF-8 + + widgets + docs + + ${project.basedir} + 6.1.0-SNAPSHOT + 2.3.0 + 2.3.0-SNAPSHOT + 4.11 + 1.10.19 + 2.1.3 + 3.11.1 + 3.9.0 + 1.32.1 + + + + + io.cdap.cdap + cdap-etl-api + ${cdap.version} + + + io.cdap.plugin + hydrator-common + ${hydrator.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + commons-logging + commons-logging + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.avro + avro + + + org.apache.zookeeper + zookeeper + + + com.google.guava + guava + + + jersey-core + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + servlet-api + javax.servlet + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + jasper-compiler + tomcat + + + jasper-runtime + tomcat + + + jsp-api + javax.servlet.jsp + + + slf4j-api + org.slf4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.slf4j + slf4j-log4j12 + + + com.google.inject.extensions + guice-servlet + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey.contribs + jersey-guice + + + javax.servlet + servlet-api + + + com.google.guava + guava + + + + + com.google.http-client + google-http-client-gson + ${google-http-client-gson.version} + + + + + io.cdap.cdap + cdap-data-pipeline + ${cdap.version} + test + + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-all + ${mockito.version} + test + + + org.assertj + assertj-core + ${assertj.version} + test + + + io.github.benas + random-beans + test + ${random-beans.version} + + + io.cdap.cdap + hydrator-test + ${cdap.version} + test + + + com.fasterxml.jackson.core + jackson-core + 2.9.8 + test + + + com.fasterxml.jackson.core + jackson-annotations + 2.9.0 + test + + + com.google.inject + guice + 4.2.2 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.14.1 + + -Xmx5000m -Djava.awt.headless=true -XX:+UseG1GC -XX:OnOutOfMemoryError="kill -9 %p" + -Djava.net.preferIPv4Stack=true + + false + plain + + ${project.build.directory} + + + **/*TestsSuite.java + **/*TestSuite.java + **/Test*.java + **/*Test.java + **/*TestCase.java + + + **/*TestRun.java + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + validate + process-test-classes + + checkstyle.xml + suppressions.xml + UTF-8 + true + true + true + **/org/apache/cassandra/**,**/org/apache/hadoop/** + + + check + + + + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + + org.apache.felix + maven-bundle-plugin + 3.3.0 + true + + + *;inline=false;scope=compile + true + + <_exportcontents>io.cdap.plugin.github.* + lib + + + + + package + + bundle + + + + + + io.cdap + cdap-maven-plugin + 1.1.0 + + + system:cdap-data-pipeline[6.1.0-SNAPSHOT,7.0.0-SNAPSHOT) + + + + + create-artifact-config + prepare-package + + create-plugin-json + + + + + + + + diff --git a/src/main/java/io/cdap/plugin/github/source/batch/GithubBatchSource.java b/src/main/java/io/cdap/plugin/github/source/batch/GithubBatchSource.java new file mode 100644 index 0000000..92e79e4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/batch/GithubBatchSource.java @@ -0,0 +1,85 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.batch; + +import com.google.common.base.Preconditions; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.github.source.common.DatasetTransformer; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import org.apache.hadoop.io.NullWritable; + +import java.util.stream.Collectors; + +/** + * Plugin returns records from Github API V3. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(GithubBatchSource.NAME) +@Description(GithubBatchSource.DESCRIPTION) +public class GithubBatchSource extends BatchSource { + + public static final String NAME = "GithubBatchSource"; + public static final String DESCRIPTION = "Reads data from Github API."; + + private final GithubBatchSourceConfig config; + + public GithubBatchSource(GithubBatchSourceConfig config) { + this.config = config; + } + + public void prepareRun(BatchSourceContext batchSourceContext) { + validateConfiguration(batchSourceContext.getFailureCollector()); + LineageRecorder lineageRecorder = new LineageRecorder(batchSourceContext, config.referenceName); + lineageRecorder.createExternalDataset(config.getSchema()); + lineageRecorder.recordRead("Read", "Reading Github data", + Preconditions.checkNotNull(config.getSchema().getFields()).stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + + batchSourceContext.setInput(Input.of(config.referenceName, new GithubFormatProvider(config))); + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + FailureCollector failureCollector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); + IdUtils.validateReferenceName(config.referenceName, failureCollector); + validateConfiguration(failureCollector); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void transform(KeyValue input, Emitter emitter) { + emitter.emit(DatasetTransformer.transform(input.getValue(), config.getSchema())); + } + + private void validateConfiguration(FailureCollector failureCollector) { + config.validate(failureCollector); + failureCollector.getOrThrowException(); + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/batch/GithubBatchSourceConfig.java b/src/main/java/io/cdap/plugin/github/source/batch/GithubBatchSourceConfig.java new file mode 100644 index 0000000..7ae7a68 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/batch/GithubBatchSourceConfig.java @@ -0,0 +1,188 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.github.source.batch; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.github.source.common.SchemaBuilder; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import io.cdap.plugin.github.source.common.model.impl.Branch; +import io.cdap.plugin.github.source.common.model.impl.Collaborator; +import io.cdap.plugin.github.source.common.model.impl.Comment; +import io.cdap.plugin.github.source.common.model.impl.Commit; +import io.cdap.plugin.github.source.common.model.impl.Content; +import io.cdap.plugin.github.source.common.model.impl.DeployKey; +import io.cdap.plugin.github.source.common.model.impl.Deployment; +import io.cdap.plugin.github.source.common.model.impl.Fork; +import io.cdap.plugin.github.source.common.model.impl.Invitation; +import io.cdap.plugin.github.source.common.model.impl.Page; +import io.cdap.plugin.github.source.common.model.impl.Release; +import io.cdap.plugin.github.source.common.model.impl.TrafficReferrer; +import io.cdap.plugin.github.source.common.model.impl.Webhook; + +import javax.annotation.Nullable; + +/** + * Provides all required configuration for reading Github data from Batch Source. + */ +public class GithubBatchSourceConfig extends ReferencePluginConfig { + + public static final String AUTHORIZATION_TOKEN = "authorizationToken"; + public static final String AUTHORIZATION_TOKEN_DISPLAY_NAME = "Authorization token"; + public static final String REPOSITORY_OWNER = "repoOwner"; + public static final String REPOSITORY_OWNER_DISPLAY_NAME = "Repository owner name"; + public static final String REPOSITORY_NAME = "repoName"; + public static final String REPOSITORY_NAME_DISPLAY_NAME = "Repository name"; + public static final String DATASET_NAME = "datasetName"; + public static final String DATASET_NAME_DISPLAY_NAME = "Dataset name"; + public static final String HOSTNAME = "hostname"; + + @Name(AUTHORIZATION_TOKEN) + @Description("Authorization token to access GitHub API") + @Macro + protected String authorizationToken; + + @Name(REPOSITORY_OWNER) + @Description("GitHub repository owner") + @Macro + protected String repoOwner; + + @Name(REPOSITORY_NAME) + @Description("GitHub repository name") + @Macro + protected String repoName; + + @Name(DATASET_NAME) + @Description("Dataset name that you would like to retrieve") + @Macro + protected String datasetName; + + @Name(HOSTNAME) + @Description("GitHub API hostname") + @Nullable + @Macro + protected String hostname; + + private transient Schema schema = null; + + public GithubBatchSourceConfig(String referenceName) { + super(referenceName); + } + + public Schema getSchema() { + if (schema == null) { + schema = SchemaBuilder.buildSchema(datasetName, getDatasetClass()); + } + return schema; + } + + public String getAuthorizationToken() { + return authorizationToken; + } + + public String getRepoOwner() { + return repoOwner; + } + + public String getRepoName() { + return repoName; + } + + public String getDatasetName() { + return datasetName; + } + + public Class getDatasetClass() { + switch (datasetName) { + case "Branches": { + return Branch.class; + } + case "Collaborators": { + return Collaborator.class; + } + case "Comments": { + return Comment.class; + } + case "Commits": { + return Commit.class; + } + case "Contents": { + return Content.class; + } + case "Deploy Keys": { + return DeployKey.class; + } + case "Deployments": { + return Deployment.class; + } + case "Forks": { + return Fork.class; + } + case "Invitations": { + return Invitation.class; + } + case "Pages": { + return Page.class; + } + case "Releases": { + return Release.class; + } + case "Traffic:Referrers": { + return TrafficReferrer.class; + } + case "Webhooks": { + return Webhook.class; + } + default: { + throw new IllegalArgumentException("Unsupported dataset name!"); + } + } + } + + @Nullable + public String getHostname() { + return hostname; + } + + public void validate(FailureCollector failureCollector) { + if (!containsMacro(authorizationToken) && Strings.isNullOrEmpty(authorizationToken)) { + failureCollector + .addFailure(String.format("%s must be specified.", AUTHORIZATION_TOKEN_DISPLAY_NAME), null) + .withConfigProperty(AUTHORIZATION_TOKEN_DISPLAY_NAME); + } + if (!containsMacro(repoOwner) && Strings.isNullOrEmpty(repoOwner)) { + failureCollector + .addFailure(String.format("%s must be specified.", REPOSITORY_OWNER_DISPLAY_NAME), null) + .withConfigProperty(REPOSITORY_OWNER_DISPLAY_NAME); + } + if (!containsMacro(repoName) && Strings.isNullOrEmpty(repoName)) { + failureCollector + .addFailure(String.format("%s must be specified.", REPOSITORY_NAME_DISPLAY_NAME), null) + .withConfigProperty(REPOSITORY_NAME_DISPLAY_NAME); + } + if (!containsMacro(datasetName) && Strings.isNullOrEmpty(datasetName)) { + failureCollector + .addFailure(String.format("%s must be specified.", DATASET_NAME_DISPLAY_NAME), null) + .withConfigProperty(DATASET_NAME_DISPLAY_NAME); + } + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/batch/GithubFormatProvider.java b/src/main/java/io/cdap/plugin/github/source/batch/GithubFormatProvider.java new file mode 100644 index 0000000..4ca1157 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/batch/GithubFormatProvider.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.batch; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.InputFormatProvider; + +import java.util.Map; + +/** + * InputFormatProvider used by cdap to provide configurations to mapreduce job + */ +public class GithubFormatProvider implements InputFormatProvider { + + public static final String PROPERTY_CONFIG_JSON = "cdap.github.config"; + private static final Gson GSON = new GsonBuilder().create(); + + private final Map conf; + + public GithubFormatProvider(GithubBatchSourceConfig config) { + this.conf = new ImmutableMap.Builder() + .put(PROPERTY_CONFIG_JSON, GSON.toJson(config)) + .build(); + } + + @Override + public String getInputFormatClassName() { + return GithubInputFormat.class.getName(); + } + + @Override + public Map getInputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/batch/GithubInputFormat.java b/src/main/java/io/cdap/plugin/github/source/batch/GithubInputFormat.java new file mode 100644 index 0000000..01d8e2d --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/batch/GithubInputFormat.java @@ -0,0 +1,110 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.batch; + +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.github.source.common.GitHubRequestFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * InputFormat for mapreduce job, which provides a single split of data. + */ +public class GithubInputFormat extends InputFormat { + + private static final Gson GSON = new GsonBuilder().create(); + + @Override + public List getSplits(JobContext jobContext) throws IOException { + Configuration conf = jobContext.getConfiguration(); + String configJson = conf.get(GithubFormatProvider.PROPERTY_CONFIG_JSON); + GithubBatchSourceConfig config = GSON.fromJson(configJson, GithubBatchSourceConfig.class); + + String url = GitHubRequestFactory.generateFirstCallUrl(config); + HttpRequest httpRequest = GitHubRequestFactory.buildRequest(url, config.getAuthorizationToken()); + HttpResponse response = httpRequest.execute(); + Object paginationMetadata = response.getHeaders().get("Link"); + if (Objects.nonNull(paginationMetadata)) { + String paginationUrls = (String) ((List) paginationMetadata).get(0); + String lastLink = getLastLink(paginationUrls); + + Integer totalPagesCount = getTotalPagesCount(lastLink); + return IntStream.range(1, totalPagesCount) + .mapToObj(pageNumber -> { + String pageLink = transformLink(lastLink, pageNumber); + return new GithubSplit(pageLink); + }) + .collect(Collectors.toList()); + } else { + return Collections.singletonList(new GithubSplit(url)); + } + } + + + @Override + public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) { + Configuration conf = taskAttemptContext.getConfiguration(); + String configJson = conf.get(GithubFormatProvider.PROPERTY_CONFIG_JSON); + GithubBatchSourceConfig config = GSON.fromJson(configJson, GithubBatchSourceConfig.class); + return new GithubRecordReader(config, ((GithubSplit) inputSplit).getLink()); + } + + private String transformLink(String link, Integer pageNumber) { + int indexOfLastParamValue = link.lastIndexOf("="); + String substring = link.substring(0, indexOfLastParamValue + 1); + return substring + pageNumber; + } + + private String getLastLink(String paginationHeader) { + String[] links = paginationHeader.split(","); + for (String link : links) { + if (link.contains("last")) { + String[] parts = link.split(";"); + String url = parts[0]; + return url.substring(2, url.length() - 1); + } + } + return null; + } + + private Integer getTotalPagesCount(String lastLink) { + String[] split = lastLink.split("\\?"); + String paramsString = split[1]; + String[] params = paramsString.split("&"); + + for (String param : params) { + String[] keyValueString = param.split("="); + if ("page".equals(keyValueString[0])) { + return Integer.valueOf(keyValueString[1]); + } + } + return null; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/batch/GithubRecordReader.java b/src/main/java/io/cdap/plugin/github/source/batch/GithubRecordReader.java new file mode 100644 index 0000000..81276f6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/batch/GithubRecordReader.java @@ -0,0 +1,89 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.batch; + +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpResponse; +import io.cdap.plugin.github.source.common.GitHubRequestFactory; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.lang.reflect.Array; +import java.util.Arrays; +import java.util.Iterator; + +import static io.cdap.plugin.github.source.common.GitHubRequestFactory.DEFAULT_PAGE_SIZE; + +/** + * RecordReader implementation, which reads {@link GitHubModel} instances from GitHub repository API + */ +public class GithubRecordReader extends RecordReader { + + private final GithubBatchSourceConfig config; + private final String link; + + private Iterator currentPage; + private GitHubModel currentRow; + private Integer currentRowIndex = 0; + + public GithubRecordReader(GithubBatchSourceConfig config, String link) { + this.config = config; + this.link = link; + } + + @Override + public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { + HttpRequest httpRequest = GitHubRequestFactory.buildRequest(link, config.getAuthorizationToken()); + HttpResponse response = httpRequest.execute(); + Class datasetClass = (Class) + Array.newInstance(config.getDatasetClass(), 0).getClass(); + currentPage = Arrays.stream(response.parseAs(datasetClass)).iterator(); + } + + @Override + public boolean nextKeyValue() { + if (!currentPage.hasNext()) { + return false; + } + currentRowIndex++; + currentRow = currentPage.next(); + return true; + } + + @Override + public NullWritable getCurrentKey() { + return null; + } + + @Override + public GitHubModel getCurrentValue() { + return currentRow; + } + + @Override + public float getProgress() { + return currentRowIndex / (float) DEFAULT_PAGE_SIZE; + } + + @Override + public void close() { + + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/batch/GithubSplit.java b/src/main/java/io/cdap/plugin/github/source/batch/GithubSplit.java new file mode 100644 index 0000000..361c6ee --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/batch/GithubSplit.java @@ -0,0 +1,64 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.github.source.batch; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * A no-op split. + */ +public class GithubSplit extends InputSplit implements Writable { + + private String link; + + public GithubSplit() { + // For serialization + } + + public GithubSplit(String link) { + this.link = link; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeUTF(link); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + this.link = dataInput.readUTF(); + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String[] getLocations() { + return new String[0]; + } + + public String getLink() { + return link; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/DatasetTransformer.java b/src/main/java/io/cdap/plugin/github/source/common/DatasetTransformer.java new file mode 100644 index 0000000..b27b85a --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/DatasetTransformer.java @@ -0,0 +1,80 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.github.source.common.model.GitHubModel; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * This is helper class for transforming {@link GitHubModel} instance to {@link StructuredRecord}. + */ +public class DatasetTransformer { + + /** + * Transforms {@link GitHubModel} instance to {@link StructuredRecord} instance accordingly to given schema. + */ + public static StructuredRecord transform(Object model, Schema schema) { + try { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + Class clazz = model.getClass(); + List schemaFields = schema.getFields(); + for (Schema.Field schemaField : schemaFields) { + Schema.Type schemaType = schemaField.getSchema().getType(); + Field field = getFieldByName(schemaField.getName(), clazz); + if (schemaType.isSimpleType() || Schema.Type.UNION.equals(schemaType)) { + builder.set(schemaField.getName(), field.get(model)); + } else if (Schema.Type.ARRAY.equals(schemaType)) { + Schema componentSchema = schemaField.getSchema().getComponentSchema(); + Object result = field.get(model); + if (Objects.nonNull(componentSchema) && !componentSchema.isSimpleOrNullableSimple()) { + result = ((List) result).stream() + .map(arrItem -> transform(arrItem, componentSchema)) + .collect(Collectors.toList()); + } + builder.set(schemaField.getName(), result); + } else { + StructuredRecord structuredRecord = transform(field.get(model), schemaField.getSchema()); + builder.set(schemaField.getName(), structuredRecord); + } + } + return builder.build(); + } catch (IllegalAccessException e) { + throw new IllegalStateException(String.format("Exception when transforming %s to StructuredRecord", + model.getClass().getSimpleName()), e); + } + } + + private static Field getFieldByName(String fieldName, Class clazz) { + Field declaredField = null; + Class currentClass = clazz; + while (currentClass != Object.class) { + try { + declaredField = currentClass.getDeclaredField(fieldName); + declaredField.setAccessible(true); + return declaredField; + } catch (NoSuchFieldException e) { + currentClass = currentClass.getSuperclass(); + } + } + return declaredField; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/GitHubRequestFactory.java b/src/main/java/io/cdap/plugin/github/source/common/GitHubRequestFactory.java new file mode 100644 index 0000000..5eccf85 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/GitHubRequestFactory.java @@ -0,0 +1,103 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common; + +import com.google.api.client.http.GenericUrl; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.JsonObjectParser; +import com.google.api.client.json.gson.GsonFactory; +import io.cdap.plugin.github.source.batch.GithubBatchSourceConfig; + +import java.io.IOException; + +/** + * Helper class to create GitHub data requests. + */ +public class GitHubRequestFactory { + + public static final Integer DEFAULT_PAGE_SIZE = 100; + + private static HttpRequestFactory requestFactory = new NetHttpTransport() + .createRequestFactory((HttpRequest request) -> + request.setParser(new JsonObjectParser(GsonFactory.getDefaultInstance()))); + + public static String generateFirstCallUrl(GithubBatchSourceConfig config) { + String host = config.getHostname() != null ? config.getHostname() : "https://api.github.com"; + return host + "/repos" + "/" + config.getRepoOwner() + "/" + config.getRepoName() + "/" + + getPathByDatasetName(config.getDatasetName()) + "?per_page=" + DEFAULT_PAGE_SIZE; + } + + public static HttpRequest buildRequest(String url, String authToken) throws IOException { + HttpRequest httpRequest = requestFactory.buildGetRequest(new GenericUrl(url)); + addHeaders(httpRequest, authToken); + return httpRequest; + } + + private static void addHeaders(HttpRequest httpRequest, String authToken) { + httpRequest.getHeaders().setAuthorization("token " + authToken); + httpRequest.getHeaders().setUserAgent("curl/7.37.0"); + } + + private static String getPathByDatasetName(String dataset) { + + switch (dataset) { + case "Branches": { + return "branches"; + } + case "Collaborators": { + return "collaborators"; + } + case "Comments": { + return "comments"; + } + case "Commits": { + return "commits"; + } + case "Contents": { + return "contents"; + } + case "Deploy Keys": { + return "keys"; + } + case "Deployments": { + return "deployments"; + } + case "Forks": { + return "forks"; + } + case "Invitations": { + return "invitations"; + } + case "Pages": { + return "pages"; + } + case "Releases": { + return "releases"; + } + case "Traffic:Referrers": { + return "traffic/popular/referrers"; + } + case "Webhooks": { + return "hooks"; + } + default: { + throw new IllegalArgumentException(String.format("Unsupported dataset name %s!", dataset)); + } + } + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/SchemaBuilder.java b/src/main/java/io/cdap/plugin/github/source/common/SchemaBuilder.java new file mode 100644 index 0000000..936513a --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/SchemaBuilder.java @@ -0,0 +1,88 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common; + +import io.cdap.cdap.api.data.schema.Schema; + +import java.lang.reflect.Field; +import java.lang.reflect.ParameterizedType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Helper class to map GitHub repository fields sets to final {@link Schema}. + */ +public class SchemaBuilder { + + public static Schema buildSchema(String schemaName, Class model) { + List fields = getModelFields(model); + List schemaFields = new ArrayList<>(); + for (Field field : fields) { + Schema.Type schemaType = getSchemaType(field.getType()); + if (schemaType.isSimpleType()) { + schemaFields.add(getSimpleSchemaField(field)); + } else if (Schema.Type.ARRAY.equals(schemaType)) { + ParameterizedType parameterizedType = (ParameterizedType) field.getGenericType(); + Class clazz = (Class) parameterizedType.getActualTypeArguments()[0]; + Schema.Type arraySchemaType = getSchemaType(clazz); + if (!arraySchemaType.isSimpleType()) { + Schema schema = buildSchema(field.getName(), clazz); + schemaFields.add(Schema.Field.of(field.getName(), Schema.arrayOf(schema))); + } else { + schemaFields.add(Schema.Field.of(field.getName(), + Schema.arrayOf(Schema.of(getSchemaType(clazz))))); + } + } else { + Schema schema = buildSchema(field.getName(), field.getType()); + schemaFields.add(Schema.Field.of(field.getName(), schema)); + } + } + return Schema.recordOf(schemaName, schemaFields); + } + + public static List getModelFields(Class clazz) { + List fields = new ArrayList<>(); + Class currentClass = clazz; + while (currentClass != Object.class) { + Field[] declaredFields = currentClass.getDeclaredFields(); + fields.addAll(Arrays.asList(declaredFields)); + currentClass = currentClass.getSuperclass(); + } + return fields; + } + + private static Schema.Field getSimpleSchemaField(Field field) { + return Schema.Field.of(field.getName(), Schema.nullableOf(Schema.of( + getSchemaType(field.getType())))); + } + + private static Schema.Type getSchemaType(Class clazz) { + if (String.class.equals(clazz)) { + return Schema.Type.STRING; + } else if (Integer.class.equals(clazz)) { + return Schema.Type.INT; + } else if (Long.class.equals(clazz)) { + return Schema.Type.LONG; + } else if (Boolean.class.equals(clazz)) { + return Schema.Type.BOOLEAN; + } else if (List.class.equals(clazz)) { + return Schema.Type.ARRAY; + } else { + return Schema.Type.RECORD; + } + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/GitHubModel.java b/src/main/java/io/cdap/plugin/github/source/common/model/GitHubModel.java new file mode 100644 index 0000000..742121f --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/GitHubModel.java @@ -0,0 +1,22 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model; + +/** + * Generic GitHub model interface + */ +public interface GitHubModel { +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Branch.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Branch.java new file mode 100644 index 0000000..fc82f93 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Branch.java @@ -0,0 +1,68 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; + +import java.util.List; + +/** + * Branch model + */ +public class Branch implements GitHubModel { + + @Key + private String name; + @Key + private Commit commit; + @Key("protected") + private Boolean isProtected; + @Key + private Protection protection; + @Key("protection_url") + private String protectionUrl; + + /** + * Branch.Commit model + */ + public static class Commit { + @Key + private String sha; + @Key + private String url; + } + + /** + * Branch.Protection model + */ + public static class Protection { + @Key + private Boolean enabled; + @Key("required_status_checks") + private RequiredStatusChecks requiredStatusChecks; + + /** + * Branch.Protection.RequiredStatusChecks model + */ + public static class RequiredStatusChecks { + @Key("enforcement_level") + private String enforcementLevel; + @Key + private List contexts; + } + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Collaborator.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Collaborator.java new file mode 100644 index 0000000..c349529 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Collaborator.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.impl.user.User; + +/** + * Collaborator model + */ +public class Collaborator extends User { + + @Key + private Permissions permissions; + + /** + * Collaborator.Permissions model + */ + public static class Permissions { + @Key + private Boolean pull; + @Key + private Boolean push; + @Key + private Boolean admin; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Comment.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Comment.java new file mode 100644 index 0000000..f3c8036 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Comment.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import io.cdap.plugin.github.source.common.model.impl.user.User; + +/** + * Comment model + */ +public class Comment implements GitHubModel { + + @Key("html_url") + private String htmlUrl; + @Key + private String url; + @Key + private Long id; + @Key("node_id") + private String nodeId; + @Key + private String body; + @Key + private String path; + @Key + private Integer position; + @Key + private Integer line; + @Key("commit_id") + private String commitId; + @Key + private User user; + @Key("created_at") + private String createdAt; + @Key("updated_at") + private String updatedAt; +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Commit.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Commit.java new file mode 100644 index 0000000..d0994a0 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Commit.java @@ -0,0 +1,103 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import io.cdap.plugin.github.source.common.model.impl.user.User; + +import java.util.List; + +/** + * Commit model + */ +public class Commit implements GitHubModel { + + @Key + private String url; + @Key + private String sha; + @Key("node_id") + private String nodeId; + @Key("html_url") + private String htmlUrl; + @Key("comments_url") + private String commentsUrl; + @Key + private CommitData commit; + @Key("author") + private User mainAuthor; + @Key("committer") + private User mainCommitter; + @Key + private List parents; + + /** + * Commit.CommitData model + */ + public static class CommitData { + @Key + private String url; + @Key + private CommitUser author; + @Key + private CommitUser committer; + @Key + private String message; + @Key + private Tree tree; + @Key("comment_count") + private Integer commentCount; + @Key + private Verification verification; + + /** + * Commit.CommitData.CommitUser model + */ + public static class CommitUser { + @Key + private String name; + @Key + private String email; + @Key + private String date; + } + + /** + * Commit.CommitData.Tree model + */ + public static class Tree { + @Key + private String url; + @Key + private String sha; + } + + /** + * Commit.CommitData.Verification model + */ + public static class Verification { + @Key + private Boolean verified; + @Key + private String reason; + @Key + private String signature; + @Key + private String payload; + } + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Content.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Content.java new file mode 100644 index 0000000..45061ea --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Content.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; + +/** + * Content model + */ +public class Content implements GitHubModel { + + @Key + private String type; + @Key + private String encoding; + @Key + private Long size; + @Key + private String name; + @Key + private String path; + @Key + private String content; + @Key + private String sha; + @Key + private String url; + @Key("git_url") + private String gitUrl; + @Key("html_url") + private String htmlUrl; + @Key("download_url") + private String downloadUrl; + @Key("_links") + private Link links; + + /** + * Content.Link model + */ + public static class Link { + @Key + private String git; + @Key + private String self; + @Key + private String html; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/DeployKey.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/DeployKey.java new file mode 100644 index 0000000..0ac7264 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/DeployKey.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; + +/** + * DeployKey model + */ +public class DeployKey implements GitHubModel { + + @Key + private Long id; + @Key + private String key; + @Key + private String url; + @Key + private String title; + @Key + private Boolean verified; + @Key("created_at") + private String createdAt; + @Key("read_only") + private Boolean readOnly; +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Deployment.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Deployment.java new file mode 100644 index 0000000..87ce0be --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Deployment.java @@ -0,0 +1,69 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import io.cdap.plugin.github.source.common.model.impl.user.User; + +/** + * Deployment model + */ +public class Deployment implements GitHubModel { + + @Key + private String url; + @Key + private Long id; + @Key("node_id") + private String nodeId; + @Key + private String sha; + @Key + private String ref; + @Key + private String task; + @Key + private Payload payload; + @Key("original_environment") + private String originalEnvironment; + @Key + private String environment; + @Key + private String description; + @Key + private User creator; + @Key("created_at") + private String createdAt; + @Key("updated_at") + private String updatedAt; + @Key("statuses_url") + private String statusesUrl; + @Key("repository_url") + private String repositoryUrl; + @Key("transient_environment") + private Boolean transientEnvironment; + @Key("production_environment") + private Boolean productionEnvironment; + + /** + * Deployment.Payload model + */ + public static class Payload { + @Key + private String deploy; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Fork.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Fork.java new file mode 100644 index 0000000..2e7c9f2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Fork.java @@ -0,0 +1,209 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import io.cdap.plugin.github.source.common.model.impl.user.User; + +import java.util.List; + +/** + * Fork model + */ +public class Fork implements GitHubModel { + + @Key + private Long id; + @Key("node_id") + private String nodeId; + @Key + private String name; + @Key("full_name") + private String fullName; + @Key + private User owner; + @Key("private") + private Boolean isPrivate; + @Key("html_url") + private String htmlUrl; + @Key + private String description; + @Key + private Boolean fork; + @Key + private String url; + @Key("archive_url") + private String archiveUrl; + @Key("assignees_url") + private String assigneesUrl; + @Key("blobs_url") + private String blobsUrl; + @Key("branches_url") + private String branchesUrl; + @Key("collaborators_url") + private String collaboratorsUrl; + @Key("comments_url") + private String commentsUrl; + @Key("commits_url") + private String commitsUrl; + @Key("compare_url") + private String compareUrl; + @Key("contents_url") + private String contentsUrl; + @Key("contributors_url") + private String contributorsUrl; + @Key("deployments_url") + private String deploymentsUrl; + @Key("downloads_url") + private String downloadsUrl; + @Key("events_url") + private String eventsUrl; + @Key("forks_url") + private String forksUrl; + @Key("git_commits_url") + private String gitCommitsUrl; + @Key("git_refs_url") + private String gitRefsUrl; + @Key("git_tags_url") + private String gitTagsUrl; + @Key("git_url") + private String gitUrl; + @Key("issue_comment_url") + private String issueCommentUrl; + @Key("issue_events_url") + private String issueEventsUrl; + @Key("issues_url") + private String issuesUrl; + @Key("keys_url") + private String keysUrl; + @Key("labels_url") + private String labelsUrl; + @Key("languages_url") + private String languagesUrl; + @Key("merges_url") + private String mergesUrl; + @Key("milestones_url") + private String milestonesUrl; + @Key("notifications_url") + private String notificationsUrl; + @Key("pulls_url") + private String pullsUrl; + @Key("releases_url") + private String releasesUrl; + @Key("ssh_url") + private String sshUrl; + @Key("stargazers_url") + private String stargazersUrl; + @Key("statuses_url") + private String statusesUrl; + @Key("subscribers_url") + private String subscribersUrl; + @Key("subscription_url") + private String subscriptionUrl; + @Key("tags_url") + private String tagsUrl; + @Key("teams_url") + private String teamsUrl; + @Key("trees_url") + private String treesUrl; + @Key("clone_url") + private String cloneUrl; + @Key("mirror_url") + private String mirrorUrl; + @Key("hooks_url") + private String hooksUrl; + @Key("svn_url") + private String svnUrl; + @Key + private String homepage; + @Key + private String language; + @Key("forks_count") + private Integer forksCount; + @Key("stargazers_count") + private Integer stargazersCount; + @Key("watchers_count") + private Integer watchersCount; + @Key + private Long size; + @Key("default_branch") + private String defaultBranch; + @Key("open_issues_count") + private Integer openIssuesCount; + @Key("is_template") + private Boolean isTemplate; + @Key + private List topics; + @Key("has_issues") + private Boolean hasIssues; + @Key("has_projects") + private Boolean hasProjects; + @Key("has_wiki") + private Boolean hasWiki; + @Key("has_pages") + private Boolean hasPages; + @Key("has_downloads") + private Boolean hasDownloads; + @Key + private Boolean archived; + @Key + private Boolean disabled; + @Key("pushed_at") + private String pushedAt; + @Key("created_at") + private String createdAt; + @Key("updated_at") + private String updatedAt; + @Key + private Permissions permissions; + @Key("template_repository") + private String templateRepository; + @Key("subscribers_count") + private Integer subscribersCount; + @Key("network_count") + private Integer networkCount; + @Key + private License license; + + /** + * Fork.License model + */ + public static class License { + @Key + private String key; + @Key + private String name; + @Key("spdx_id") + private String spdxId; + @Key + private String url; + @Key("node_id") + private String nodeId; + } + + /** + * Fork.Permissions model + */ + public static class Permissions { + @Key + private Boolean pull; + @Key + private Boolean push; + @Key + private Boolean admin; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Invitation.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Invitation.java new file mode 100644 index 0000000..6c10091 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Invitation.java @@ -0,0 +1,141 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import io.cdap.plugin.github.source.common.model.impl.user.User; + +/** + * Invitation model + */ +public class Invitation implements GitHubModel { + + @Key + private Long id; + @Key + private Repository repository; + @Key + private User invitee; + @Key + private User inviter; + @Key + private String permissions; + @Key("created_at") + private String createdAt; + @Key + private String url; + @Key("html_url") + private String htmlUrl; + + /** + * Invitation.Repository model + */ + public static class Repository { + @Key + private Long id; + @Key("node_id") + private String nodeId; + @Key + private String name; + @Key("full_name") + private String fullName; + @Key + private User owner; + @Key("private") + private Boolean isPrivate; + @Key("html_url") + private String htmlUrl; + @Key + private String description; + @Key + private Boolean fork; + @Key + private String url; + @Key("archive_url") + private String archiveUrl; + @Key("assignees_url") + private String assigneesUrl; + @Key("blobs_url") + private String blobsUrl; + @Key("branches_url") + private String branchesUrl; + @Key("collaborators_url") + private String collaboratorsUrl; + @Key("comments_url") + private String commentsUrl; + @Key("commits_url") + private String commitsUrl; + @Key("compare_url") + private String compareUrl; + @Key("contents_url") + private String contentsUrl; + @Key("contributors_url") + private String contributorsUrl; + @Key("deployments_url") + private String deploymentsUrl; + @Key("downloads_url") + private String downloadsUrl; + @Key("events_url") + private String eventsUrl; + @Key("forks_url") + private String forksUrl; + @Key("git_commits_url") + private String gitCommitsUrl; + @Key("git_refs_url") + private String gitRefsUrl; + @Key("git_tags_url") + private String gitTagsUrl; + @Key("git_url") + private String gitUrl; + @Key("issue_comment_url") + private String issueCommentUrl; + @Key("issue_events_url") + private String issueEventsUrl; + @Key("issues_url") + private String issuesUrl; + @Key("labels_url") + private String labelsUrl; + @Key("languages_url") + private String languagesUrl; + @Key("merges_url") + private String mergesUrl; + @Key("milestones_url") + private String milestonesUrl; + @Key("notifications_url") + private String notificationsUrl; + @Key("pulls_url") + private String pullsUrl; + @Key("releases_url") + private String releasesUrl; + @Key("ssh_url") + private String sshUrl; + @Key("stargazers_url") + private String stargazersUrl; + @Key("statuses_url") + private String statusesUrl; + @Key("subscribers_url") + private String subscribersUrl; + @Key("subscription_url") + private String subscriptionUrl; + @Key("tags_url") + private String tagsUrl; + @Key("teams_url") + private String teamsUrl; + @Key("trees_url") + private String treesUrl; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Page.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Page.java new file mode 100644 index 0000000..6cc64c6 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Page.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; + +/** + * Page model + */ +public class Page implements GitHubModel { + + @Key + private String url; + @Key + private String status; + @Key + private String cname; + @Key("custom_404") + private Boolean custom404; + @Key("html_url") + private String htmlUrl; + @Key + private Source source; + + /** + * Page.Source model + */ + public static class Source { + @Key + private String branch; + @Key + private String directory; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Release.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Release.java new file mode 100644 index 0000000..242e291 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Release.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; +import io.cdap.plugin.github.source.common.model.impl.user.User; + +import java.util.List; + + +/** + * Release model + */ +public class Release implements GitHubModel { + + @Key + private String url; + @Key("html_url") + private String htmlUrl; + @Key("assets_url") + private String assetsUrl; + @Key("upload_url") + private String uploadUrl; + @Key("tarball_url") + private String tarballUrl; + @Key("zipball_url") + private String zipballUrl; + @Key + private Long id; + @Key("node_id") + private String nodeId; + @Key("tag_name") + private String tagName; + @Key("target_commitish") + private String targetCommitish; + @Key + private String name; + @Key + private String body; + @Key + private Boolean draft; + @Key + private Boolean prerelease; + @Key("created_at") + private String createdAt; + @Key("published_at") + private String publishedAt; + @Key + private User author; + @Key + private List assets; + + /** + * Release.Assert model + */ + public static class Assert { + @Key + private String url; + @Key("browser_download_url") + private String browserDownloadUrl; + @Key + private Long id; + @Key("node_id") + private String nodeId; + @Key + private String name; + @Key + private String label; + @Key + private String state; + @Key("content_type") + private String contentType; + @Key + private Long size; + @Key("download_count") + private Integer downloadCount; + @Key("created_at") + private String createdAt; + @Key("updated_at") + private String updatedAt; + @Key + private User uploader; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/TrafficReferrer.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/TrafficReferrer.java new file mode 100644 index 0000000..67261a2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/TrafficReferrer.java @@ -0,0 +1,32 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; + +/** + * TrafficReferrer model + */ +public class TrafficReferrer implements GitHubModel { + + @Key + private String referrer; + @Key + private Integer count; + @Key + private Integer uniques; +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/Webhook.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Webhook.java new file mode 100644 index 0000000..38fc246 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/Webhook.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; + +import java.util.List; + +/** + * Webhook model + */ +public class Webhook implements GitHubModel { + + @Key + private String type; + @Key + private Long id; + @Key + private String name; + @Key + private Boolean active; + @Key + private List events; + @Key + private Config config; + @Key("updated_at") + private String updatedAt; + @Key("created_at") + private String createdAt; + @Key + private String url; + @Key("test_url") + private String testUrl; + @Key("ping_url") + private String pingUrl; + @Key + private LastResponse lastResponse; + + /** + * Webhook.Config model + */ + public static class Config { + @Key("content_type") + private String contentType; + @Key("insecure_ssl") + private String insecureSsl; + @Key + private String url; + } + + /** + * Webhook.LastResponse model + */ + public static class LastResponse { + @Key + private String code; + @Key + private String status; + @Key + private String message; + } +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/user/OwnerUser.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/user/OwnerUser.java new file mode 100644 index 0000000..3f4e9f7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/user/OwnerUser.java @@ -0,0 +1,27 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl.user; + +import com.google.api.client.util.Key; + +/** + * OwnerUser model + */ +public class OwnerUser extends User { + + @Key + private String owner; +} diff --git a/src/main/java/io/cdap/plugin/github/source/common/model/impl/user/User.java b/src/main/java/io/cdap/plugin/github/source/common/model/impl/user/User.java new file mode 100644 index 0000000..539716a --- /dev/null +++ b/src/main/java/io/cdap/plugin/github/source/common/model/impl/user/User.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.common.model.impl.user; + +import com.google.api.client.util.Key; +import io.cdap.plugin.github.source.common.model.GitHubModel; + +/** + * User model + */ +public class User implements GitHubModel { + + @Key + private Long id; + @Key("node_id") + private String nodeId; + @Key("avatar_url") + private String avatarUrl; + @Key("gravatar_id") + private String gravatarId; + @Key + private String url; + @Key("html_url") + private String htmlUrl; + @Key("followers_url") + private String followersUrl; + @Key("following_url") + private String followingUrl; + @Key("gists_url") + private String gistsUrl; + @Key("starred_url") + private String starredUrl; + @Key("subscriptions_url") + private String subscriptionsUrl; + @Key("organizations_url") + private String organizationsUrl; + @Key("repos_url") + private String reposUrl; + @Key("events_url") + private String eventsUrl; + @Key("received_events_url") + private String receivedEventsUrl; + @Key + private String type; + @Key("site_admin") + private Boolean siteAdmin; +} diff --git a/src/test/java/io/cdap/plugin/github/source/batch/GithubBatchSourceConfigTest.java b/src/test/java/io/cdap/plugin/github/source/batch/GithubBatchSourceConfigTest.java new file mode 100644 index 0000000..710b053 --- /dev/null +++ b/src/test/java/io/cdap/plugin/github/source/batch/GithubBatchSourceConfigTest.java @@ -0,0 +1,136 @@ +package io.cdap.plugin.github.source.batch; + +import io.cdap.cdap.etl.api.validation.ValidationFailure; +import io.cdap.cdap.etl.mock.validation.MockFailureCollector; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Collection; + +import static io.cdap.plugin.github.source.batch.GithubBatchSourceConfig.AUTHORIZATION_TOKEN; +import static io.cdap.plugin.github.source.batch.GithubBatchSourceConfig.DATASET_NAME; +import static io.cdap.plugin.github.source.batch.GithubBatchSourceConfig.REPOSITORY_NAME; +import static io.cdap.plugin.github.source.batch.GithubBatchSourceConfig.REPOSITORY_OWNER; + +public class GithubBatchSourceConfigTest { + + private MockFailureCollector failureCollector; + + @Before + public void setUp() { + failureCollector = new MockFailureCollector(); + } + + @Test + public void testValidateFieldsCaseCorrectFields() { + //given + GithubBatchSourceConfig config = new GithubBatchSourceConfig("ref"); + config.authorizationToken = "token"; + config.repoOwner = "owner"; + config.repoName = "repo"; + config.datasetName = "dataset"; + + //when + config.validate(failureCollector); + + //then + Assert.assertTrue(failureCollector.getValidationFailures().isEmpty()); + } + + @Test + public void testValidateFieldsCaseEmptyFields() { + //given + GithubBatchSourceConfig config = new GithubBatchSourceConfig("ref"); + + //when + config.validate(failureCollector); + + //then + Assert.assertEquals(4, failureCollector.getValidationFailures().size()); + } + + @Test + public void testValidateConfigCaseAuthTokenNull() { + //given + GithubBatchSourceConfig config = new GithubBatchSourceConfig("ref"); + MockFailureCollector failureCollector = new MockFailureCollector(); + config.repoOwner = "owner"; + config.repoName = "repo"; + config.datasetName = "dataset"; + + //when + config.validate(failureCollector); + + boolean isStartDateFailure = failureCollector.getValidationFailures().stream() + .map(ValidationFailure::getCauses) + .flatMap(Collection::stream) + .anyMatch(cause -> cause.getAttributes().containsValue(AUTHORIZATION_TOKEN)); + + //then + Assert.assertTrue(isStartDateFailure); + } + + @Test + public void testValidateConfigCaseRepoOwnerNull() { + //given + GithubBatchSourceConfig config = new GithubBatchSourceConfig("ref"); + MockFailureCollector failureCollector = new MockFailureCollector(); + config.authorizationToken = "token"; + config.repoName = "repo"; + config.datasetName = "dataset"; + + //when + config.validate(failureCollector); + + boolean isStartDateFailure = failureCollector.getValidationFailures().stream() + .map(ValidationFailure::getCauses) + .flatMap(Collection::stream) + .anyMatch(cause -> cause.getAttributes().containsValue(REPOSITORY_OWNER)); + + //then + Assert.assertTrue(isStartDateFailure); + } + + @Test + public void testValidateConfigCaseRepoNameNull() { + //given + GithubBatchSourceConfig config = new GithubBatchSourceConfig("ref"); + MockFailureCollector failureCollector = new MockFailureCollector(); + config.authorizationToken = "token"; + config.repoOwner = "owner"; + config.datasetName = "dataset"; + + //when + config.validate(failureCollector); + + boolean isStartDateFailure = failureCollector.getValidationFailures().stream() + .map(ValidationFailure::getCauses) + .flatMap(Collection::stream) + .anyMatch(cause -> cause.getAttributes().containsValue(REPOSITORY_NAME)); + + //then + Assert.assertTrue(isStartDateFailure); + } + + @Test + public void testValidateConfigCaseDatasetNameNull() { + //given + GithubBatchSourceConfig config = new GithubBatchSourceConfig("ref"); + MockFailureCollector failureCollector = new MockFailureCollector(); + config.authorizationToken = "token"; + config.repoOwner = "owner"; + config.repoName = "repo"; + + //when + config.validate(failureCollector); + + boolean isStartDateFailure = failureCollector.getValidationFailures().stream() + .map(ValidationFailure::getCauses) + .flatMap(Collection::stream) + .anyMatch(cause -> cause.getAttributes().containsValue(DATASET_NAME)); + + //then + Assert.assertTrue(isStartDateFailure); + } +} diff --git a/src/test/java/io/cdap/plugin/github/source/common/DatasetTransformerTest.java b/src/test/java/io/cdap/plugin/github/source/common/DatasetTransformerTest.java new file mode 100644 index 0000000..b86b771 --- /dev/null +++ b/src/test/java/io/cdap/plugin/github/source/common/DatasetTransformerTest.java @@ -0,0 +1,61 @@ +package io.cdap.plugin.github.source.common; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.github.source.common.model.impl.Commit; +import io.github.benas.randombeans.api.EnhancedRandom; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; + +import static io.github.benas.randombeans.EnhancedRandomBuilder.aNewEnhancedRandom; + +@RunWith(Parameterized.class) +public class DatasetTransformerTest { + + private static final EnhancedRandom random = aNewEnhancedRandom(); + + private Class clazz; + private Object model; + + public DatasetTransformerTest(Class clazz) { + this.clazz = clazz; + this.model = random.nextObject(clazz); + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][]{ +// {Branch.class}, +// {Collaborator.class}, +// {Comment.class}, + {Commit.class}, +// {Content.class}, +// {DeployKey.class}, +// {Deployment.class}, +// {Fork.class}, +// {Invitation.class}, +// {Page.class}, +// {Release.class}, +// {TrafficReferrer.class}, +// {Webhook.class} + }); + } + + @Test + public void testTransformModel() throws NoSuchFieldException, IllegalAccessException { + //given + Schema schema = SchemaBuilder.buildSchema(clazz.getSimpleName(), clazz); + + //when + StructuredRecord output = DatasetTransformer.transform(model, schema); + + //then + Assert.assertNotNull(schema.getFields()); + schema.getFields().forEach(field -> Assert.assertNotNull(output.get(field.getName()))); + } +} diff --git a/src/test/java/io/cdap/plugin/github/source/common/SchemaBuilderTest.java b/src/test/java/io/cdap/plugin/github/source/common/SchemaBuilderTest.java new file mode 100644 index 0000000..3463160 --- /dev/null +++ b/src/test/java/io/cdap/plugin/github/source/common/SchemaBuilderTest.java @@ -0,0 +1,70 @@ +package io.cdap.plugin.github.source.common; + +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.github.source.common.model.impl.Branch; +import io.cdap.plugin.github.source.common.model.impl.Collaborator; +import io.cdap.plugin.github.source.common.model.impl.Comment; +import io.cdap.plugin.github.source.common.model.impl.Commit; +import io.cdap.plugin.github.source.common.model.impl.Content; +import io.cdap.plugin.github.source.common.model.impl.DeployKey; +import io.cdap.plugin.github.source.common.model.impl.Deployment; +import io.cdap.plugin.github.source.common.model.impl.Fork; +import io.cdap.plugin.github.source.common.model.impl.Invitation; +import io.cdap.plugin.github.source.common.model.impl.Page; +import io.cdap.plugin.github.source.common.model.impl.Release; +import io.cdap.plugin.github.source.common.model.impl.TrafficReferrer; +import io.cdap.plugin.github.source.common.model.impl.Webhook; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Parameterized.class) +public class SchemaBuilderTest { + + private Class clazz; + + public SchemaBuilderTest(Class clazz) { + this.clazz = clazz; + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][]{ + {Branch.class}, + {Collaborator.class}, + {Comment.class}, + {Commit.class}, + {Content.class}, + {DeployKey.class}, + {Deployment.class}, + {Fork.class}, + {Invitation.class}, + {Page.class}, + {Release.class}, + {TrafficReferrer.class}, + {Webhook.class} + }); + } + + @Test + public void testBuildSchema() { + //given + int fieldsCount = clazz.getDeclaredFields().length; + Class superclass = clazz.getSuperclass(); + if (superclass != null) { + fieldsCount += superclass.getDeclaredFields().length; + } + + //when + Schema schema = SchemaBuilder.buildSchema(clazz.getSimpleName(), clazz); + + //then + Assert.assertNotNull(schema); + Assert.assertNotNull(schema.getFields()); + Assert.assertEquals(schema.getFields().size(), fieldsCount); + } +} diff --git a/src/test/java/io/cdap/plugin/github/source/common/model/GitHubModelTest.java b/src/test/java/io/cdap/plugin/github/source/common/model/GitHubModelTest.java new file mode 100644 index 0000000..bd68bc8 --- /dev/null +++ b/src/test/java/io/cdap/plugin/github/source/common/model/GitHubModelTest.java @@ -0,0 +1,106 @@ +package io.cdap.plugin.github.source.common.model; + +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestFactory; +import com.google.api.client.http.HttpResponse; +import com.google.api.client.http.LowLevelHttpRequest; +import com.google.api.client.http.LowLevelHttpResponse; +import com.google.api.client.json.Json; +import com.google.api.client.json.JsonObjectParser; +import com.google.api.client.json.gson.GsonFactory; +import com.google.api.client.testing.http.HttpTesting; +import com.google.api.client.testing.http.MockHttpTransport; +import com.google.api.client.testing.http.MockLowLevelHttpRequest; +import com.google.api.client.testing.http.MockLowLevelHttpResponse; +import com.google.common.io.ByteStreams; +import io.cdap.plugin.github.source.common.model.impl.Branch; +import io.cdap.plugin.github.source.common.model.impl.Collaborator; +import io.cdap.plugin.github.source.common.model.impl.Comment; +import io.cdap.plugin.github.source.common.model.impl.Commit; +import io.cdap.plugin.github.source.common.model.impl.Content; +import io.cdap.plugin.github.source.common.model.impl.DeployKey; +import io.cdap.plugin.github.source.common.model.impl.Deployment; +import io.cdap.plugin.github.source.common.model.impl.Fork; +import io.cdap.plugin.github.source.common.model.impl.Invitation; +import io.cdap.plugin.github.source.common.model.impl.Page; +import io.cdap.plugin.github.source.common.model.impl.Release; +import io.cdap.plugin.github.source.common.model.impl.TrafficReferrer; +import io.cdap.plugin.github.source.common.model.impl.Webhook; +import org.assertj.core.api.AssertionsForClassTypes; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Parameterized.class) +public class GitHubModelTest { + + private Class clazz; + private String fileName; + + private HttpRequestFactory requestFactory; + + public GitHubModelTest(Class clazz, String fileName) { + this.clazz = clazz; + this.fileName = fileName; + } + + @Before + public void setUp() { + requestFactory = new MockHttpTransport() { + @Override + public LowLevelHttpRequest buildRequest(String method, String url) { + return new MockLowLevelHttpRequest() { + @Override + public LowLevelHttpResponse execute() throws IOException { + MockLowLevelHttpResponse response = new MockLowLevelHttpResponse(); + response.setContentType(Json.MEDIA_TYPE); + response.setContent(getFile(fileName)); + return response; + } + }; + } + }.createRequestFactory((HttpRequest request) -> + request.setParser(new JsonObjectParser(GsonFactory.getDefaultInstance()))); + } + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][]{ + {Branch[].class, "branches.json"}, + {Collaborator[].class, "collaborators.json"}, + {Comment[].class, "comments.json"}, + {Commit[].class, "commits.json"}, + {Content[].class, "contents.json"}, + {DeployKey[].class, "deploy_keys.json"}, + {Deployment[].class, "deployments.json"}, + {Fork[].class, "forks.json"}, + {Invitation[].class, "invitations.json"}, + {Page[].class, "pages.json"}, + {Release[].class, "releases.json"}, + {TrafficReferrer[].class, "traffic_referrers.json"}, + {Webhook[].class, "webhooks.json"} + }); + } + + @Test + public void validateModelFields() throws IOException { + HttpRequest request = requestFactory.buildGetRequest(HttpTesting.SIMPLE_GENERIC_URL); + HttpResponse response = request.execute(); + + Object result = response.parseAs(clazz); + + AssertionsForClassTypes.assertThat(result).hasNoNullFieldsOrProperties(); + } + + byte[] getFile(String fileName) throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + InputStream stream = classLoader.getResourceAsStream(fileName); + return ByteStreams.toByteArray(stream); + } +} diff --git a/src/test/java/io/cdap/plugin/github/source/etl/GitHubETLTest.java b/src/test/java/io/cdap/plugin/github/source/etl/GitHubETLTest.java new file mode 100644 index 0000000..12afe46 --- /dev/null +++ b/src/test/java/io/cdap/plugin/github/source/etl/GitHubETLTest.java @@ -0,0 +1,123 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.github.source.etl; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.artifact.ArtifactSummary; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.dataset.table.Table; +import io.cdap.cdap.datapipeline.DataPipelineApp; +import io.cdap.cdap.datapipeline.SmartWorkflow; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.mock.batch.MockSink; +import io.cdap.cdap.etl.mock.test.HydratorTestBase; +import io.cdap.cdap.etl.proto.v2.ETLBatchConfig; +import io.cdap.cdap.etl.proto.v2.ETLPlugin; +import io.cdap.cdap.etl.proto.v2.ETLStage; +import io.cdap.cdap.proto.ProgramRunStatus; +import io.cdap.cdap.proto.artifact.AppRequest; +import io.cdap.cdap.proto.id.ApplicationId; +import io.cdap.cdap.proto.id.ArtifactId; +import io.cdap.cdap.proto.id.NamespaceId; +import io.cdap.cdap.test.ApplicationManager; +import io.cdap.cdap.test.DataSetManager; +import io.cdap.cdap.test.WorkflowManager; +import io.cdap.plugin.github.source.batch.GithubBatchSource; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +public class GitHubETLTest extends HydratorTestBase { + + private static final ArtifactSummary APP_ARTIFACT = new ArtifactSummary("data-pipeline", "3.2.0"); + + private static String authorizationToken; + private static String repoName; + private static String repoOwner; + private static String datasetName; + + @BeforeClass + public static void setupTestClass() throws Exception { + authorizationToken = System.getProperty("github.authorization.token"); + if (Strings.isNullOrEmpty(authorizationToken)) { + throw new IllegalArgumentException("github.authorization.token system property must not be empty."); + } + repoName = System.getProperty("github.repo.name"); + if (Strings.isNullOrEmpty(repoName)) { + throw new IllegalArgumentException("github.repo.name system property must not be empty."); + } + repoOwner = System.getProperty("github.repo.owner"); + if (Strings.isNullOrEmpty(repoOwner)) { + throw new IllegalArgumentException("github.repo.owner system property must not be empty."); + } + datasetName = System.getProperty("github.repo.dataset"); + if (Strings.isNullOrEmpty(datasetName)) { + throw new IllegalArgumentException("github.repo.dataset system property must not be empty."); + } + + ArtifactId parentArtifact = NamespaceId.DEFAULT.artifact(APP_ARTIFACT.getName(), APP_ARTIFACT.getVersion()); + + // add the artifact and mock plugins + setupBatchArtifacts(parentArtifact, DataPipelineApp.class); + + // add our plugins artifact with the artifact as its parent. + // this will make our plugins available. + addPluginArtifact(NamespaceId.DEFAULT.artifact("example-plugins", "1.0.0"), + parentArtifact, GithubBatchSource.class); + } + + @Test + public void testGitHubBatchSource() throws Exception { + + ETLStage source = new ETLStage("GitHubETLTest", new ETLPlugin(GithubBatchSource.NAME, BatchSource.PLUGIN_TYPE, + getSourceMinimalDefaultConfigs(), null)); + + String outputDatasetName = "output-batchsourcetest_github"; + ETLStage sink = new ETLStage("sink", MockSink.getPlugin(outputDatasetName)); + + ETLBatchConfig etlConfig = ETLBatchConfig.builder() + .addStage(source) + .addStage(sink) + .addConnection(source.getName(), sink.getName()) + .build(); + + ApplicationId pipelineId = NamespaceId.DEFAULT.app("GitHubBatchTest"); + ApplicationManager appManager = deployApplication(pipelineId, new AppRequest<>(APP_ARTIFACT, etlConfig)); + + WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME); + workflowManager.startAndWaitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES); + + DataSetManager dataset = getDataset(outputDatasetName); + List outputRecords = MockSink.readOutput(dataset); + + Assert.assertNotNull(outputRecords); + } + + public Map getSourceMinimalDefaultConfigs() { + Map sourceProps = new HashMap<>(); + sourceProps.put("referenceName", "ref"); + sourceProps.put("authorizationToken", authorizationToken); + sourceProps.put("repoName", repoName); + sourceProps.put("repoOwner", repoOwner); + sourceProps.put("datasetName", datasetName); + return sourceProps; + } +} diff --git a/src/test/resources/branches.json b/src/test/resources/branches.json new file mode 100644 index 0000000..ac4f710 --- /dev/null +++ b/src/test/resources/branches.json @@ -0,0 +1,21 @@ +[ + { + "name": "master", + "commit": { + "sha": "c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc", + "url": "https://api.github.com/repos/octocat/Hello-World/commits/c5b97d5ae6c19d5c5df71a34c7fbeeda2479ccbc" + }, + "protected": true, + "protection": { + "enabled": true, + "required_status_checks": { + "enforcement_level": "non_admins", + "contexts": [ + "ci-test", + "linter" + ] + } + }, + "protection_url": "https://api.github.com/repos/octocat/hello-world/branches/master/protection" + } +] diff --git a/src/test/resources/collaborators.json b/src/test/resources/collaborators.json new file mode 100644 index 0000000..8fbc040 --- /dev/null +++ b/src/test/resources/collaborators.json @@ -0,0 +1,27 @@ +[ + { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false, + "permissions": { + "pull": true, + "push": true, + "admin": false + } + } +] diff --git a/src/test/resources/comments.json b/src/test/resources/comments.json new file mode 100644 index 0000000..3f35ab8 --- /dev/null +++ b/src/test/resources/comments.json @@ -0,0 +1,35 @@ +[ + { + "html_url": "https://github.com/octocat/Hello-World/commit/6dcb09b5b57875f334f61aebed695e2e4193db5e#commitcomment-1", + "url": "https://api.github.com/repos/octocat/Hello-World/comments/1", + "id": 1, + "node_id": "MDEzOkNvbW1pdENvbW1lbnQx", + "body": "Great stuff", + "path": "file1.txt", + "position": 4, + "line": 14, + "commit_id": "6dcb09b5b57875f334f61aebed695e2e4193db5e", + "user": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "created_at": "2011-04-14T16:00:49Z", + "updated_at": "2011-04-14T16:00:49Z" + } +] diff --git a/src/test/resources/commits.json b/src/test/resources/commits.json new file mode 100644 index 0000000..1c39d6e --- /dev/null +++ b/src/test/resources/commits.json @@ -0,0 +1,80 @@ +[ + { + "url": "https://api.github.com/repos/octocat/Hello-World/commits/6dcb09b5b57875f334f61aebed695e2e4193db5e", + "sha": "6dcb09b5b57875f334f61aebed695e2e4193db5e", + "node_id": "MDY6Q29tbWl0NmRjYjA5YjViNTc4NzVmMzM0ZjYxYWViZWQ2OTVlMmU0MTkzZGI1ZQ==", + "html_url": "https://github.com/octocat/Hello-World/commit/6dcb09b5b57875f334f61aebed695e2e4193db5e", + "comments_url": "https://api.github.com/repos/octocat/Hello-World/commits/6dcb09b5b57875f334f61aebed695e2e4193db5e/comments", + "commit": { + "url": "https://api.github.com/repos/octocat/Hello-World/git/commits/6dcb09b5b57875f334f61aebed695e2e4193db5e", + "author": { + "name": "Monalisa Octocat", + "email": "support@github.com", + "date": "2011-04-14T16:00:49Z" + }, + "committer": { + "name": "Monalisa Octocat", + "email": "support@github.com", + "date": "2011-04-14T16:00:49Z" + }, + "message": "Fix all the bugs", + "tree": { + "url": "https://api.github.com/repos/octocat/Hello-World/tree/6dcb09b5b57875f334f61aebed695e2e4193db5e", + "sha": "6dcb09b5b57875f334f61aebed695e2e4193db5e" + }, + "comment_count": 0, + "verification": { + "verified": false, + "reason": "unsigned", + "signature": null, + "payload": null + } + }, + "author": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "committer": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "parents": [ + { + "url": "https://api.github.com/repos/octocat/Hello-World/commits/6dcb09b5b57875f334f61aebed695e2e4193db5e", + "sha": "6dcb09b5b57875f334f61aebed695e2e4193db5e" + } + ] + } +] diff --git a/src/test/resources/contents.json b/src/test/resources/contents.json new file mode 100644 index 0000000..83ae599 --- /dev/null +++ b/src/test/resources/contents.json @@ -0,0 +1,20 @@ +[ + { + "type": "file", + "encoding": "base64", + "size": 5362, + "name": "README.md", + "path": "README.md", + "content": "encoded content ...", + "sha": "3d21ec53a331a6f037a91c368710b99387d012c1", + "url": "https://api.github.com/repos/octokit/octokit.rb/contents/README.md", + "git_url": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "html_url": "https://github.com/octokit/octokit.rb/blob/master/README.md", + "download_url": "https://raw.githubusercontent.com/octokit/octokit.rb/master/README.md", + "_links": { + "git": "https://api.github.com/repos/octokit/octokit.rb/git/blobs/3d21ec53a331a6f037a91c368710b99387d012c1", + "self": "https://api.github.com/repos/octokit/octokit.rb/contents/README.md", + "html": "https://github.com/octokit/octokit.rb/blob/master/README.md" + } + } +] diff --git a/src/test/resources/deploy_keys.json b/src/test/resources/deploy_keys.json new file mode 100644 index 0000000..e0d7b5b --- /dev/null +++ b/src/test/resources/deploy_keys.json @@ -0,0 +1,11 @@ +[ + { + "id": 1, + "key": "ssh-rsa AAA...", + "url": "https://api.github.com/repos/octocat/Hello-World/keys/1", + "title": "octocat@octomac", + "verified": true, + "created_at": "2014-12-10T15:53:42Z", + "read_only": true + } +] diff --git a/src/test/resources/deployments.json b/src/test/resources/deployments.json new file mode 100644 index 0000000..c99cbf4 --- /dev/null +++ b/src/test/resources/deployments.json @@ -0,0 +1,42 @@ +[ + { + "url": "https://api.github.com/repos/octocat/example/deployments/1", + "id": 1, + "node_id": "MDEwOkRlcGxveW1lbnQx", + "sha": "a84d88e7554fc1fa21bcbc4efae3c782a70d2b9d", + "ref": "topic-branch", + "task": "deploy", + "payload": { + "deploy": "migrate" + }, + "original_environment": "staging", + "environment": "production", + "description": "Deploy request from hubot", + "creator": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "created_at": "2012-07-20T01:19:13Z", + "updated_at": "2012-07-20T01:19:13Z", + "statuses_url": "https://api.github.com/repos/octocat/example/deployments/1/statuses", + "repository_url": "https://api.github.com/repos/octocat/example", + "transient_environment": false, + "production_environment": true + } +] diff --git a/src/test/resources/forks.json b/src/test/resources/forks.json new file mode 100644 index 0000000..ca33d47 --- /dev/null +++ b/src/test/resources/forks.json @@ -0,0 +1,114 @@ +[ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": true, + "url": "https://api.github.com/repos/octocat/Hello-World", + "archive_url": "http://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}", + "assignees_url": "http://api.github.com/repos/octocat/Hello-World/assignees{/user}", + "blobs_url": "http://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}", + "branches_url": "http://api.github.com/repos/octocat/Hello-World/branches{/branch}", + "collaborators_url": "http://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}", + "comments_url": "http://api.github.com/repos/octocat/Hello-World/comments{/number}", + "commits_url": "http://api.github.com/repos/octocat/Hello-World/commits{/sha}", + "compare_url": "http://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}", + "contents_url": "http://api.github.com/repos/octocat/Hello-World/contents/{+path}", + "contributors_url": "http://api.github.com/repos/octocat/Hello-World/contributors", + "deployments_url": "http://api.github.com/repos/octocat/Hello-World/deployments", + "downloads_url": "http://api.github.com/repos/octocat/Hello-World/downloads", + "events_url": "http://api.github.com/repos/octocat/Hello-World/events", + "forks_url": "http://api.github.com/repos/octocat/Hello-World/forks", + "git_commits_url": "http://api.github.com/repos/octocat/Hello-World/git/commits{/sha}", + "git_refs_url": "http://api.github.com/repos/octocat/Hello-World/git/refs{/sha}", + "git_tags_url": "http://api.github.com/repos/octocat/Hello-World/git/tags{/sha}", + "git_url": "git:github.com/octocat/Hello-World.git", + "issue_comment_url": "http://api.github.com/repos/octocat/Hello-World/issues/comments{/number}", + "issue_events_url": "http://api.github.com/repos/octocat/Hello-World/issues/events{/number}", + "issues_url": "http://api.github.com/repos/octocat/Hello-World/issues{/number}", + "keys_url": "http://api.github.com/repos/octocat/Hello-World/keys{/key_id}", + "labels_url": "http://api.github.com/repos/octocat/Hello-World/labels{/name}", + "languages_url": "http://api.github.com/repos/octocat/Hello-World/languages", + "merges_url": "http://api.github.com/repos/octocat/Hello-World/merges", + "milestones_url": "http://api.github.com/repos/octocat/Hello-World/milestones{/number}", + "notifications_url": "http://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}", + "pulls_url": "http://api.github.com/repos/octocat/Hello-World/pulls{/number}", + "releases_url": "http://api.github.com/repos/octocat/Hello-World/releases{/id}", + "ssh_url": "git@github.com:octocat/Hello-World.git", + "stargazers_url": "http://api.github.com/repos/octocat/Hello-World/stargazers", + "statuses_url": "http://api.github.com/repos/octocat/Hello-World/statuses/{sha}", + "subscribers_url": "http://api.github.com/repos/octocat/Hello-World/subscribers", + "subscription_url": "http://api.github.com/repos/octocat/Hello-World/subscription", + "tags_url": "http://api.github.com/repos/octocat/Hello-World/tags", + "teams_url": "http://api.github.com/repos/octocat/Hello-World/teams", + "trees_url": "http://api.github.com/repos/octocat/Hello-World/git/trees{/sha}", + "clone_url": "https://github.com/octocat/Hello-World.git", + "mirror_url": "git:git.example.com/octocat/Hello-World", + "hooks_url": "http://api.github.com/repos/octocat/Hello-World/hooks", + "svn_url": "https://svn.github.com/octocat/Hello-World", + "homepage": "https://github.com", + "language": null, + "forks_count": 9, + "stargazers_count": 80, + "watchers_count": 80, + "size": 108, + "default_branch": "master", + "open_issues_count": 0, + "is_template": true, + "topics": [ + "octocat", + "atom", + "electron", + "api" + ], + "has_issues": true, + "has_projects": true, + "has_wiki": true, + "has_pages": false, + "has_downloads": true, + "archived": false, + "disabled": false, + "pushed_at": "2011-01-26T19:06:43Z", + "created_at": "2011-01-26T19:01:12Z", + "updated_at": "2011-01-26T19:14:43Z", + "permissions": { + "admin": false, + "push": false, + "pull": true + }, + "template_repository": null, + "subscribers_count": 42, + "network_count": 0, + "license": { + "key": "mit", + "name": "MIT License", + "spdx_id": "MIT", + "url": "https://api.github.com/licenses/mit", + "node_id": "MDc6TGljZW5zZW1pdA==" + } + } +] diff --git a/src/test/resources/invitations.json b/src/test/resources/invitations.json new file mode 100644 index 0000000..84026ea --- /dev/null +++ b/src/test/resources/invitations.json @@ -0,0 +1,117 @@ +[ + { + "id": 1, + "repository": { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": false, + "url": "https://api.github.com/repos/octocat/Hello-World", + "archive_url": "http://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}", + "assignees_url": "http://api.github.com/repos/octocat/Hello-World/assignees{/user}", + "blobs_url": "http://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}", + "branches_url": "http://api.github.com/repos/octocat/Hello-World/branches{/branch}", + "collaborators_url": "http://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}", + "comments_url": "http://api.github.com/repos/octocat/Hello-World/comments{/number}", + "commits_url": "http://api.github.com/repos/octocat/Hello-World/commits{/sha}", + "compare_url": "http://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}", + "contents_url": "http://api.github.com/repos/octocat/Hello-World/contents/{+path}", + "contributors_url": "http://api.github.com/repos/octocat/Hello-World/contributors", + "deployments_url": "http://api.github.com/repos/octocat/Hello-World/deployments", + "downloads_url": "http://api.github.com/repos/octocat/Hello-World/downloads", + "events_url": "http://api.github.com/repos/octocat/Hello-World/events", + "forks_url": "http://api.github.com/repos/octocat/Hello-World/forks", + "git_commits_url": "http://api.github.com/repos/octocat/Hello-World/git/commits{/sha}", + "git_refs_url": "http://api.github.com/repos/octocat/Hello-World/git/refs{/sha}", + "git_tags_url": "http://api.github.com/repos/octocat/Hello-World/git/tags{/sha}", + "git_url": "git:github.com/octocat/Hello-World.git", + "issue_comment_url": "http://api.github.com/repos/octocat/Hello-World/issues/comments{/number}", + "issue_events_url": "http://api.github.com/repos/octocat/Hello-World/issues/events{/number}", + "issues_url": "http://api.github.com/repos/octocat/Hello-World/issues{/number}", + "keys_url": "http://api.github.com/repos/octocat/Hello-World/keys{/key_id}", + "labels_url": "http://api.github.com/repos/octocat/Hello-World/labels{/name}", + "languages_url": "http://api.github.com/repos/octocat/Hello-World/languages", + "merges_url": "http://api.github.com/repos/octocat/Hello-World/merges", + "milestones_url": "http://api.github.com/repos/octocat/Hello-World/milestones{/number}", + "notifications_url": "http://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}", + "pulls_url": "http://api.github.com/repos/octocat/Hello-World/pulls{/number}", + "releases_url": "http://api.github.com/repos/octocat/Hello-World/releases{/id}", + "ssh_url": "git@github.com:octocat/Hello-World.git", + "stargazers_url": "http://api.github.com/repos/octocat/Hello-World/stargazers", + "statuses_url": "http://api.github.com/repos/octocat/Hello-World/statuses/{sha}", + "subscribers_url": "http://api.github.com/repos/octocat/Hello-World/subscribers", + "subscription_url": "http://api.github.com/repos/octocat/Hello-World/subscription", + "tags_url": "http://api.github.com/repos/octocat/Hello-World/tags", + "teams_url": "http://api.github.com/repos/octocat/Hello-World/teams", + "trees_url": "http://api.github.com/repos/octocat/Hello-World/git/trees{/sha}" + }, + "invitee": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "inviter": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "permissions": "write", + "created_at": "2016-06-13T14:52:50-05:00", + "url": "https://api.github.com/user/repository_invitations/1296269", + "html_url": "https://github.com/octocat/Hello-World/invitations" + } +] diff --git a/src/test/resources/pages.json b/src/test/resources/pages.json new file mode 100644 index 0000000..a1c1d35 --- /dev/null +++ b/src/test/resources/pages.json @@ -0,0 +1,13 @@ +[ + { + "url": "https://api.github.com/repos/github/developer.github.com/pages", + "status": "built", + "cname": "developer.github.com", + "custom_404": false, + "html_url": "https://developer.github.com", + "source": { + "branch": "master", + "directory": "/" + } + } +] diff --git a/src/test/resources/releases.json b/src/test/resources/releases.json new file mode 100644 index 0000000..e586e73 --- /dev/null +++ b/src/test/resources/releases.json @@ -0,0 +1,76 @@ +[ + { + "url": "https://api.github.com/repos/octocat/Hello-World/releases/1", + "html_url": "https://github.com/octocat/Hello-World/releases/v1.0.0", + "assets_url": "https://api.github.com/repos/octocat/Hello-World/releases/1/assets", + "upload_url": "https://uploads.github.com/repos/octocat/Hello-World/releases/1/assets{?name,label}", + "tarball_url": "https://api.github.com/repos/octocat/Hello-World/tarball/v1.0.0", + "zipball_url": "https://api.github.com/repos/octocat/Hello-World/zipball/v1.0.0", + "id": 1, + "node_id": "MDc6UmVsZWFzZTE=", + "tag_name": "v1.0.0", + "target_commitish": "master", + "name": "v1.0.0", + "body": "Description of the release", + "draft": false, + "prerelease": false, + "created_at": "2013-02-27T19:35:32Z", + "published_at": "2013-02-27T19:35:32Z", + "author": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "assets": [ + { + "url": "https://api.github.com/repos/octocat/Hello-World/releases/assets/1", + "browser_download_url": "https://github.com/octocat/Hello-World/releases/download/v1.0.0/example.zip", + "id": 1, + "node_id": "MDEyOlJlbGVhc2VBc3NldDE=", + "name": "example.zip", + "label": "short description", + "state": "uploaded", + "content_type": "application/zip", + "size": 1024, + "download_count": 42, + "created_at": "2013-02-27T19:35:32Z", + "updated_at": "2013-02-27T19:35:32Z", + "uploader": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + } + } + ] + } +] diff --git a/src/test/resources/traffic_referrers.json b/src/test/resources/traffic_referrers.json new file mode 100644 index 0000000..00b31e8 --- /dev/null +++ b/src/test/resources/traffic_referrers.json @@ -0,0 +1,22 @@ +[ + { + "referrer": "Google", + "count": 4, + "uniques": 3 + }, + { + "referrer": "stackoverflow.com", + "count": 2, + "uniques": 2 + }, + { + "referrer": "eggsonbread.com", + "count": 1, + "uniques": 1 + }, + { + "referrer": "yandex.ru", + "count": 1, + "uniques": 1 + } +] diff --git a/src/test/resources/webhooks.json b/src/test/resources/webhooks.json new file mode 100644 index 0000000..8b88f8a --- /dev/null +++ b/src/test/resources/webhooks.json @@ -0,0 +1,27 @@ +[ + { + "type": "Repository", + "id": 12345678, + "name": "web", + "active": true, + "events": [ + "push", + "pull_request" + ], + "config": { + "content_type": "json", + "insecure_ssl": "0", + "url": "https://example.com/webhook" + }, + "updated_at": "2019-06-03T00:57:16Z", + "created_at": "2019-06-03T00:57:16Z", + "url": "https://api.github.com/repos/octocat/Hello-World/hooks/12345678", + "test_url": "https://api.github.com/repos/octocat/Hello-World/hooks/12345678/test", + "ping_url": "https://api.github.com/repos/octocat/Hello-World/hooks/12345678/pings", + "last_response": { + "code": null, + "status": "unused", + "message": null + } + } +] diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..2898910 --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + diff --git a/widgets/GithubBatchSource-batchsource.json b/widgets/GithubBatchSource-batchsource.json new file mode 100644 index 0000000..850d01f --- /dev/null +++ b/widgets/GithubBatchSource-batchsource.json @@ -0,0 +1,98 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName", + "widget-attributes": { + "placeholder": "Name used to identify this source for lineage" + } + }, + { + "widget-type": "textbox", + "label": "Repository owner name", + "name": "repoOwner", + "widget-attributes": { + "placeholder": "GitHub repository owner name" + } + }, + { + "widget-type": "textbox", + "label": "Repository name", + "name": "repoName", + "widget-attributes": { + "placeholder": "GitHub repository name" + } + }, + { + "widget-type": "select", + "label": "Dataset name", + "name": "datasetName", + "widget-attributes": { + "values": [ + "Branches", + "Collaborators", + "Comments", + "Commits", + "Contents", + "Deploy Keys", + "Deployments", + "Forks", + "Invitations", + "Pages", + "Releases", + "Traffic:Referrers", + "Webhooks" + ] + } + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "textbox", + "label": "GitHub API hostname", + "name": "hostname", + "widget-attributes": { + "placeholder": "GitHub API hostname. For GitHub Enterprise only" + } + } + ] + }, + { + "label": "Credentials", + "properties": [ + { + "widget-type": "textbox", + "label": "Authorization token", + "name": "authorizationToken", + "widget-attributes": { + "placeholder": "Authorization token to access GitHub API" + } + } + ] + } + ], + "jump-config": { + "datasets": [ + { + "ref-property-name": "referenceName" + } + ] + }, + "outputs": [ + { + "widget-type": "non-editable-schema-editor", + "schema": { + } + } + ] +}