Skip to content

Call stopGracefully() when batch ingestion is killed explicitly #8075

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ankit0811 wants to merge 1 commit into from
Closed

Call stopGracefully() when batch ingestion is killed explicitly #8075

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions indexing-service/src/main/java/org/apache/druid/indexing/common/task/HadoopIndexTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ public class HadoopIndexTask extends HadoopTask implements ChatHandler
{
private static final Logger log = new Logger(HadoopIndexTask.class);
private static final String HADOOP_JOB_ID_FILENAME = "mapReduceJobId.json";
private TaskConfig taskConfig = null;

private static String getTheDataSource(HadoopIngestionSpec spec)
{
Expand Down Expand Up @@ -223,7 +222,7 @@ public String getClasspathPrefix()
return classpathPrefix;
}

public String getHadoopJobIdFileName()
private String getHadoopJobIdFileName(TaskConfig taskConfig)
{
return new File(taskConfig.getTaskDir(getId()), HADOOP_JOB_ID_FILENAME).getAbsolutePath();
}
Expand All @@ -232,7 +231,6 @@ public String getHadoopJobIdFileName()
public TaskStatus run(TaskToolbox toolbox)
{
try {
taskConfig = toolbox.getConfig();
if (chatHandlerProvider.isPresent()) {
log.info("Found chat handler of class[%s]", chatHandlerProvider.get().getClass().getName());
chatHandlerProvider.get().register(getId(), this, false);
Expand Down Expand Up @@ -270,7 +268,7 @@ public TaskStatus run(TaskToolbox toolbox)
@SuppressWarnings("unchecked")
private TaskStatus runInternal(TaskToolbox toolbox) throws Exception
{
String hadoopJobIdFile = getHadoopJobIdFileName();
String hadoopJobIdFile = getHadoopJobIdFileName(toolbox.getConfig());
final ClassLoader loader = buildClassLoader(toolbox);
boolean determineIntervals = !spec.getDataSchema().getGranularitySpec().bucketIntervals().isPresent();

Expand Down Expand Up @@ -432,7 +430,7 @@ public void stopGracefully(TaskConfig taskConfig)
// To avoid issue of kill command once the ingestion task is actually completed
if (!ingestionState.equals(IngestionState.COMPLETED)) {
final ClassLoader oldLoader = Thread.currentThread().getContextClassLoader();
String hadoopJobIdFile = getHadoopJobIdFileName();
String hadoopJobIdFile = getHadoopJobIdFileName(taskConfig);

try {
ClassLoader loader = HadoopTask.buildClassLoader(getHadoopDependencyCoordinates(),
Expand Down
3 changes: 2 additions & 1 deletion indexing-service/src/main/java/org/apache/druid/indexing/overlord/ForkingTaskRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -596,8 +596,9 @@ public void shutdown(final String taskid, String reason)
}

if (taskInfo.processHolder != null) {
// Will trigger normal failure mechanisms due to process exit
// Will cleanup the underlying running task if any and trigger normal failure mechanisms due to process exit
log.info("Killing process for task: %s", taskid);
taskInfo.getTask().stopGracefully(taskConfig);
taskInfo.processHolder.process.destroy();
}
}
Expand Down