Skip to content

Fix more tests that launch ClusterStartup #5224

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
labkey-jeckels merged 2 commits into from
Feb 16, 2024
Merged

Fix more tests that launch ClusterStartup #5224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f2d2fd
Fix more tests that launch ClusterStartup
labkey-jeckels Feb 12, 2024
fdec8ff
Merge branch 'develop' into fb_embedded_moreClusterStartupTests
labkey-jeckels Feb 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions api/src/org/labkey/api/pipeline/PipelineService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,9 @@ static void setInstance(PipelineService instance)

boolean isEnterprisePipeline();

/** Generate command-line arguments to launch org.labkey.pipeline.cluster.ClusterStartup. Intended for automated tests */
List<String> getClusterStartupArguments() throws IOException;

enum JmsType { none, inProcess, external, unknown }
@NotNull
JmsType getJmsType();
Expand Down
101 changes: 101 additions & 0 deletions pipeline/src/org/labkey/pipeline/api/PipelineServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@

import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.activemq.ActiveMQConnectionFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
Expand All @@ -39,6 +41,7 @@
import org.labkey.api.data.TableSelector;
import org.labkey.api.exp.api.ExpRun;
import org.labkey.api.files.FileContentService;
import org.labkey.api.module.ModuleLoader;
import org.labkey.api.pipeline.AnalyzeForm;
import org.labkey.api.pipeline.ParamParser;
import org.labkey.api.pipeline.PipeRoot;
Expand Down Expand Up @@ -68,6 +71,7 @@
import org.labkey.api.util.FileUtil;
import org.labkey.api.util.JsonUtil;
import org.labkey.api.util.NetworkDrive;
import org.labkey.api.util.Pair;
import org.labkey.api.util.TestContext;
import org.labkey.api.util.logging.LogHelper;
import org.labkey.api.view.ActionURL;
Expand All @@ -77,6 +81,7 @@
import org.labkey.api.view.UnauthorizedException;
import org.labkey.api.view.ViewBackgroundInfo;
import org.labkey.api.view.ViewContext;
import org.labkey.bootstrap.ClusterBootstrap;
import org.labkey.pipeline.PipelineController;
import org.labkey.pipeline.analysis.ProtocolManagementAuditProvider;
import org.labkey.pipeline.importer.FolderImportJob;
Expand All @@ -95,7 +100,10 @@
import javax.naming.InitialContext;
import javax.naming.NamingException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
Expand All @@ -113,6 +121,9 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.zip.ZipEntry;

import static org.labkey.api.pipeline.PipelineJobNotificationProvider.DefaultPipelineJobNotificationProvider.DEFAULT_PIPELINE_JOB_NOTIFICATION_PROVIDER;
import static org.labkey.api.pipeline.file.AbstractFileAnalysisJob.ANALYSIS_PARAMETERS_ROLE_NAME;
Expand Down Expand Up @@ -446,6 +457,96 @@ public boolean isEnterprisePipeline()
return (getPipelineQueue() instanceof EPipelineQueueImpl);
}

/** @return first is labkeyBootstrap.jar, second is servletApi.jar */
private Pair<File, File> extractBootstrapFromEmbedded() throws IOException
{
File bootstrap = null;
File servlet = null;
// Look through the JAR files in the working directory, which is expected to contain the Spring Boot
// entrypoint and the Servlet API
File pwd = new File(".");
File[] jars = pwd.listFiles(f -> f.getName().toLowerCase().endsWith(".jar"));
for (File jar : jars)
{
try (JarFile j = new JarFile(jar))
{
// Look inside the JAR for a labkeyBootstrap*.jar file
Iterator<JarEntry> entries = j.entries().asIterator();
while (entries.hasNext())
{
JarEntry entry = entries.next();
if (entry.getName().contains("labkeyBootstrap") && entry.getName().toLowerCase().endsWith(".jar"))
{
bootstrap = extractEntry(j, entry, "labkeyBootstrap.jar");
}
if (entry.getName().contains("tomcat-servlet-api") && entry.getName().toLowerCase().endsWith(".jar"))
{
servlet = extractEntry(j, entry, "servletApi.jar");
}
}
}
}
return Pair.of(bootstrap, servlet);
}

private File extractEntry(JarFile jar, ZipEntry entry, String name) throws IOException
{
File result = FileUtil.getAbsoluteCaseSensitiveFile(new File(name));
try (InputStream in = jar.getInputStream(entry);
OutputStream out = new FileOutputStream(result))
{
IOUtils.copy(in, out);
}
return result;
}


@Override
public List<String> getClusterStartupArguments() throws IOException
{
List<String> args = new ArrayList<>();
args.add(System.getProperty("java.home") + "/bin/java" + (SystemUtils.IS_OS_WINDOWS ? ".exe" : ""));
File labkeyBootstrap = new File(new File(new File(System.getProperty("catalina.home")), "lib"), "labkeyBootstrap.jar");
File servletApi = null;

if (!labkeyBootstrap.exists())
{
Pair<File, File> extracted = extractBootstrapFromEmbedded();
labkeyBootstrap = extracted.first;
if (labkeyBootstrap == null || !labkeyBootstrap.exists())
{
throw new IllegalStateException("Couldn't find labkeyBootstrap.jar");
}
servletApi = extracted.second;
if (servletApi == null || !servletApi.exists())
{
throw new IllegalStateException("Couldn't find servletApi.jar");
}
}

// Uncomment this line if you want to debug the forked process
// args.add("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:5005");
args.add("-cp");
args.add(labkeyBootstrap.getPath());
args.add(ClusterBootstrap.class.getName());

for (String sysProp : new String[]{"labkey.externalModulesDir", "labkey.modulesDir", "cpas.modulesDir"})
{
String sysPropValue = StringUtils.trimToNull(System.getProperty(sysProp));
if (sysPropValue != null)
{
args.add("-D" + sysProp +"=" + sysPropValue);
}
}

args.add("-webappdir=" + ModuleLoader.getServletContext().getRealPath(""));
if (servletApi != null)
{
args.add("-pipelinelibdir=" + servletApi.getParent());
}
return args;
}

@Override
public @NotNull JmsType getJmsType()
{
Expand Down
91 changes: 3 additions & 88 deletions pipeline/src/org/labkey/pipeline/cluster/ClusterStartup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,49 +16,37 @@

package org.labkey.pipeline.cluster;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.SystemUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.labkey.api.module.ModuleLoader;
import org.labkey.api.pipeline.PipelineJob;
import org.labkey.api.pipeline.PipelineJobException;
import org.labkey.api.pipeline.PipelineJobService;
import org.labkey.api.pipeline.PipelineService;
import org.labkey.api.reader.Readers;
import org.labkey.api.util.ContextListener;
import org.labkey.api.util.FileUtil;
import org.labkey.api.util.JunitUtil;
import org.labkey.api.util.PageFlowUtil;
import org.labkey.api.util.Pair;
import org.labkey.api.util.TestContext;
import org.labkey.bootstrap.ClusterBootstrap;
import org.labkey.pipeline.AbstractPipelineStartup;
import org.labkey.pipeline.mule.test.DummyPipelineJob;
import org.mule.umo.manager.UMOManager;
import org.springframework.beans.factory.BeanFactory;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.zip.ZipEntry;

/**
* Entry point for pipeline jobs that are invoked on a cluster node. After completion of the job, the process
Expand Down Expand Up @@ -273,91 +261,18 @@ protected String executeJobRemote(List<String> args, int expectedExitCode) throw
@NotNull
private List<String> createArgs(@Nullable PipelineJob job) throws IOException
{
List<String> args = new ArrayList<>();
args.add(System.getProperty("java.home") + "/bin/java" + (SystemUtils.IS_OS_WINDOWS ? ".exe" : ""));
File labkeyBootstrap = new File(new File(new File(System.getProperty("catalina.home")), "lib"), "labkeyBootstrap.jar");
File servletApi = null;

if (!labkeyBootstrap.exists())
{
Pair<File, File> extracted = extractBootstrapFromEmbedded();
labkeyBootstrap = extracted.first;
if (labkeyBootstrap == null || !labkeyBootstrap.exists())
{
throw new IllegalStateException("Couldn't find labkeyBootstrap.jar");
}
servletApi = extracted.second;
if (servletApi == null || !servletApi.exists())
{
throw new IllegalStateException("Couldn't find servletApi.jar");
}
}

// Uncomment this line if you want to debug the forked process
// args.add("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=*:5005");
args.add("-cp");
args.add(labkeyBootstrap.getPath());
args.add(ClusterBootstrap.class.getName());
args.add("-webappdir=" + ModuleLoader.getServletContext().getRealPath(""));
if (servletApi != null)
{
args.add("-pipelinelibdir=" + servletApi.getParent());
}

List<String> args = PipelineService.get().getClusterStartupArguments();
if (job != null)
{
// Serialize to a file
File serializedJob = new File(_tempDir, "job.json");
File log = new File(_tempDir, "job.log");
job.setLogFile(log);
job.setLogFile(log.toPath());
job.writeToFile(serializedJob);
args.add(serializedJob.toURI().toString());
}

return args;
}

/** @return first is labkeyBootstrap.jar, second is servletApi.jar */
private Pair<File, File> extractBootstrapFromEmbedded() throws IOException
{
File bootstrap = null;
File servlet = null;
// Look through the JAR files in the working directory, which is expected to contain the Spring Boot
// entrypoint and the Servlet API
File pwd = new File(".");
File[] jars = pwd.listFiles(f -> f.getName().toLowerCase().endsWith(".jar"));
for (File jar : jars)
{
try (JarFile j = new JarFile(jar))
{
// Look inside the JAR for a labkeyBootstrap*.jar file
Iterator<JarEntry> entries = j.entries().asIterator();
while (entries.hasNext())
{
JarEntry entry = entries.next();
if (entry.getName().contains("labkeyBootstrap") && entry.getName().toLowerCase().endsWith(".jar"))
{
bootstrap = extractEntry(j, entry, "labkeyBootstrap.jar");
}
if (entry.getName().contains("tomcat-servlet-api") && entry.getName().toLowerCase().endsWith(".jar"))
{
servlet = extractEntry(j, entry, "servletApi.jar");
}
}
}
}
return Pair.of(bootstrap, servlet);
}

private File extractEntry(JarFile jar, ZipEntry entry, String name) throws IOException
{
File result = FileUtil.getAbsoluteCaseSensitiveFile(new File(name));
try (InputStream in = jar.getInputStream(entry);
OutputStream out = new FileOutputStream(result))
{
IOUtils.copy(in, out);
}
return result;
}
}
}