diff --git a/cloudfoundry-operations/src/main/java/org/cloudfoundry/operations/applications/ApplicationManifestUtils.java b/cloudfoundry-operations/src/main/java/org/cloudfoundry/operations/applications/ApplicationManifestUtils.java index b8f9360c01..c5d6afae8c 100644 --- a/cloudfoundry-operations/src/main/java/org/cloudfoundry/operations/applications/ApplicationManifestUtils.java +++ b/cloudfoundry-operations/src/main/java/org/cloudfoundry/operations/applications/ApplicationManifestUtils.java @@ -34,14 +34,20 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.util.Collections.emptyMap; +import static java.util.stream.Collectors.toMap; + /** * Utilities for dealing with {@link ApplicationManifest}s. Includes the functionality to transform to and from standard CLI YAML files. */ @@ -59,6 +65,8 @@ public final class ApplicationManifestUtils { YAML = new Yaml(dumperOptions); } + private static final Pattern FIND_VARIABLE_REGEX = Pattern.compile("\\(\\(([a-zA-Z]\\w+)\\)\\)"); + private ApplicationManifestUtils() { } @@ -70,7 +78,24 @@ private ApplicationManifestUtils() { * @return the resolved manifests */ public static List read(Path path) { - return doRead(path.toAbsolutePath()); + return doRead(path.toAbsolutePath(), emptyMap()); + } + + /** + * Reads a YAML manifest file (defined by the CLI) from a {@link Path} and converts it into a collection of {@link + * ApplicationManifest}s. Note that all resolution (both inheritance and common) is performed during read. + * + * @param path the path to read from + * @param variablesPath use variable substitution (described in Add Variables to a Manifest) + * @return the resolved manifests + */ + public static List read(Path path, Path variablesPath) { + Map variables = deserialize(variablesPath.toAbsolutePath()) + .entrySet() + .stream() + .collect(toMap(Map.Entry::getKey,e -> String.valueOf(e.getValue()))); + + return doRead(path.toAbsolutePath(), variables); } /** @@ -125,21 +150,33 @@ public static void write(OutputStream out, List application } } - private static void as(Map payload, String key, Function mapper, Consumer consumer) { + private static void as(Map payload, String key, Map variables, Function mapper, Consumer consumer) { Optional.ofNullable(payload.get(key)) + .map(o -> { + if(o instanceof String) { + Matcher m = FIND_VARIABLE_REGEX.matcher((String) o); + StringBuffer stringBuffer = new StringBuffer(); + while(m.find()){ + m.appendReplacement(stringBuffer, Optional.ofNullable(variables.get(m.group(1))) + .orElseThrow(() -> new NoSuchElementException("Expected to find variable: "+m.group(1)))); + } + m.appendTail(stringBuffer); + return stringBuffer.toString(); + } + return o; + }) .map(mapper) .ifPresent(consumer); } - private static void asBoolean(Map payload, String key, Consumer consumer) { - as(payload, key, Boolean.class::cast, consumer); + private static void asBoolean(Map payload, String key, Map variables, Consumer consumer) { + as(payload, key, variables, Boolean.class::cast, consumer); } @SuppressWarnings("unchecked") - private static void asDocker(Map payload, String key, Consumer consumer) { - as(payload, key, value -> { + private static void asDocker(Map payload, String key, Map variables, Consumer consumer) { + as(payload, key, variables, value -> { Map docker = ((Map) value); - return Docker.builder() .image(docker.get("image")) .password(docker.get("password")) @@ -148,35 +185,40 @@ private static void asDocker(Map payload, String key, Consumer payload, String key, Consumer consumer) { - as(payload, key, Integer.class::cast, consumer); + private static void asInteger(Map payload, String key, Map variables, Consumer consumer) { + as(payload, key, variables, (e) -> { + if(e instanceof String) { + return Integer.parseInt((String)e); + } + return (Integer) e; + }, consumer); } @SuppressWarnings("unchecked") - private static void asList(Map payload, String key, Function mapper, Consumer consumer) { - as(payload, key, value -> ((List) value).stream(), + private static void asList(Map payload, String key, Map variables, Function mapper, Consumer consumer) { + as(payload, key, variables, value -> ((List) value).stream(), values -> values .map(mapper) .forEach(consumer)); } - private static void asListOfString(Map payload, String key, Consumer consumer) { - asList(payload, key, String.class::cast, consumer); + private static void asListOfString(Map payload, String key, Map variables, Consumer consumer) { + asList(payload, key, variables, String.class::cast, consumer); } @SuppressWarnings("unchecked") - private static void asMap(Map payload, String key, Function valueMapper, Consumer2 consumer) { - as(payload, key, value -> ((Map) value), + private static void asMap(Map payload, String key, Map variables, Function valueMapper, Consumer2 consumer) { + as(payload, key, variables, value -> ((Map) value), values -> values.forEach((k, v) -> consumer.accept(k, valueMapper.apply(v)))); } - private static void asMapOfStringString(Map payload, String key, Consumer2 consumer) { - asMap(payload, key, String::valueOf, consumer); + private static void asMapOfStringString(Map payload, String key, Map variables, Consumer2 consumer) { + asMap(payload, key, variables, String::valueOf, consumer); } @SuppressWarnings("unchecked") - private static void asMemoryInteger(Map payload, String key, Consumer consumer) { - as(payload, key, raw -> { + private static void asMemoryInteger(Map payload, String key, Map variables, Consumer consumer) { + as(payload, key, variables, raw -> { if (raw instanceof Integer) { return (Integer) raw; } else if (raw instanceof String) { @@ -199,8 +241,8 @@ private static void asMemoryInteger(Map payload, String key, Con }, consumer); } - private static void asString(Map payload, String key, Consumer consumer) { - as(payload, key, String.class::cast, consumer); + private static void asString(Map payload, String key, Map variables, Consumer consumer) { + as(payload, key, variables, String.class::cast, consumer); } @SuppressWarnings("unchecked") @@ -213,7 +255,7 @@ private static Map deserialize(Path path) { throw Exceptions.propagate(e); } - asString(root.get(), "inherit", inherit -> { + asString(root.get(), "inherit", emptyMap(), inherit -> { Map inherited = deserialize(path.getParent().resolve(inherit)); merge(inherited, root.get()); root.set(inherited); @@ -223,17 +265,17 @@ private static Map deserialize(Path path) { } @SuppressWarnings("unchecked") - private static List doRead(Path path) { + private static List doRead(Path path, Map variables) { Map root = deserialize(path); - ApplicationManifest template = getTemplate(path, root); + ApplicationManifest template = getTemplate(path, root, variables); return Optional.ofNullable(root.get("applications")) .map(value -> ((List>) value).stream()) .orElseGet(Stream::empty) .map(application -> { String name = getName(application); - return toApplicationManifest(application, ApplicationManifest.builder().from(template), path) + return toApplicationManifest(application, variables, ApplicationManifest.builder().from(template), path) .name(name) .build(); }) @@ -265,8 +307,8 @@ private static Route getRoute(Map raw) { return Route.builder().route(route).build(); } - private static ApplicationManifest getTemplate(Path path, Map root) { - return toApplicationManifest(root, ApplicationManifest.builder(), path) + private static ApplicationManifest getTemplate(Path path, Map root, Map variables) { + return toApplicationManifest(root, variables, ApplicationManifest.builder(), path) .name("template") .build(); } @@ -322,30 +364,30 @@ private static void putIfPresent(Map yaml, String key, T val } @SuppressWarnings("unchecked") - private static ApplicationManifest.Builder toApplicationManifest(Map application, ApplicationManifest.Builder builder, Path root) { - asListOfString(application, "buildpacks", builder::buildpacks); - asString(application, "buildpack", builder::buildpacks); - asString(application, "command", builder::command); - asMemoryInteger(application, "disk_quota", builder::disk); - asDocker(application, "docker", builder::docker); - asString(application, "domain", builder::domain); - asListOfString(application, "domains", builder::domain); - asMapOfStringString(application, "env", builder::environmentVariable); - asString(application, "health-check-http-endpoint", builder::healthCheckHttpEndpoint); - asString(application, "health-check-type", healthCheckType -> builder.healthCheckType(ApplicationHealthCheck.from(healthCheckType))); - asString(application, "host", builder::host); - asListOfString(application, "hosts", builder::host); - asInteger(application, "instances", builder::instances); - asMemoryInteger(application, "memory", builder::memory); - asString(application, "name", builder::name); - asBoolean(application, "no-hostname", builder::noHostname); - asBoolean(application, "no-route", builder::noRoute); - asString(application, "path", path -> builder.path(root.getParent().resolve(path))); - asBoolean(application, "random-route", builder::randomRoute); - asList(application, "routes", raw -> getRoute((Map) raw), builder::route); - asListOfString(application, "services", builder::service); - asString(application, "stack", builder::stack); - asInteger(application, "timeout", builder::timeout); + private static ApplicationManifest.Builder toApplicationManifest(Map application, Map variables, ApplicationManifest.Builder builder, Path root) { + asListOfString(application, "buildpacks", variables, builder::buildpacks); + asString(application, "buildpack", variables, builder::buildpacks); + asString(application, "command", variables, builder::command); + asMemoryInteger(application, "disk_quota", variables, builder::disk); + asDocker(application, "docker", variables, builder::docker); + asString(application, "domain", variables, builder::domain); + asListOfString(application, "domains", variables, builder::domain); + asMapOfStringString(application, "env", variables, builder::environmentVariable); + asString(application, "health-check-http-endpoint", variables, builder::healthCheckHttpEndpoint); + asString(application, "health-check-type", variables, healthCheckType -> builder.healthCheckType(ApplicationHealthCheck.from(healthCheckType))); + asString(application, "host", variables, builder::host); + asListOfString(application, "hosts", variables, builder::host); + asInteger(application, "instances", variables, builder::instances); + asMemoryInteger(application, "memory", variables, builder::memory); + asString(application, "name", variables, builder::name); + asBoolean(application, "no-hostname", variables, builder::noHostname); + asBoolean(application, "no-route", variables, builder::noRoute); + asString(application, "path", variables, path -> builder.path(root.getParent().resolve(path))); + asBoolean(application, "random-route", variables, builder::randomRoute); + asList(application, "routes", variables, raw -> getRoute((Map) raw), builder::route); + asListOfString(application, "services", variables, builder::service); + asString(application, "stack", variables, builder::stack); + asInteger(application, "timeout", variables, builder::timeout); return builder; } diff --git a/cloudfoundry-operations/src/test/java/org/cloudfoundry/operations/applications/ApplicationManifestUtilsTest.java b/cloudfoundry-operations/src/test/java/org/cloudfoundry/operations/applications/ApplicationManifestUtilsTest.java index b6047f0a87..8a3cab6ade 100644 --- a/cloudfoundry-operations/src/test/java/org/cloudfoundry/operations/applications/ApplicationManifestUtilsTest.java +++ b/cloudfoundry-operations/src/test/java/org/cloudfoundry/operations/applications/ApplicationManifestUtilsTest.java @@ -27,10 +27,14 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.NoSuchElementException; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatExceptionOfType; import static org.cloudfoundry.operations.applications.ApplicationHealthCheck.NONE; import static org.cloudfoundry.operations.applications.ApplicationHealthCheck.PORT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assume.assumeTrue; public final class ApplicationManifestUtilsTest { @@ -455,6 +459,107 @@ public void readSingleBuildpack() throws IOException { assertThat(actual).isEqualTo(expected); } + @Test + public void readWithVariableSubstitution() throws IOException { + List expected = Collections.singletonList( + ApplicationManifest.builder() + .name("papa-1-application") + .buildpack("papa-buildpack") + .instances(2) + .memory(1024) + .build()); + + List actual = ApplicationManifestUtils.read( + new ClassPathResource("fixtures/manifest-papa-1.yml").getFile().toPath(), + new ClassPathResource("fixtures/vars-papa-1.yml").getFile().toPath()); + + assertThat(actual).isEqualTo(expected); + } + + @Test + public void readWithVariableSubstitution_throwExceptionOnMissing() throws IOException { + assertThatExceptionOfType(NoSuchElementException.class) + .isThrownBy(() -> { + ApplicationManifestUtils.read( + new ClassPathResource("fixtures/manifest-papa-2.yml").getFile().toPath(), + new ClassPathResource("fixtures/vars-papa-2.yml").getFile().toPath()); + }).withMessageMatching("Expected to find variable: abcdef"); + } + @Test + public void readWithVariableSubstitution_dontEvaluateRegex() throws IOException { + List expected = Collections.singletonList( + ApplicationManifest.builder() + .name("papa-7-application") + .buildpack("((regex*))") + .build()); + + List actual = ApplicationManifestUtils.read( + new ClassPathResource("fixtures/manifest-papa-7.yml").getFile().toPath(), + new ClassPathResource("fixtures/vars-papa-7.yml").getFile().toPath()); + + assertThat(actual).isEqualTo(expected); + } + @Test + public void readWithVariableSubstitution_avoidEndlessSubstitution() throws IOException { + List expected = Collections.singletonList( + ApplicationManifest.builder() + .name("papa-3-application") + .buildpack("((endless_2))") + .build()); + + List actual = ApplicationManifestUtils.read( + new ClassPathResource("fixtures/manifest-papa-3.yml").getFile().toPath(), + new ClassPathResource("fixtures/vars-papa-3.yml").getFile().toPath()); + + assertThat(actual).isEqualTo(expected); + } + + @Test + public void readWithVariableSubstitution_dontAllowInjectionTest() throws IOException { + List expected = Collections.singletonList( + ApplicationManifest.builder() + .name("papa-4-application") + .buildpack("((test))") + .build()); + + List actual = ApplicationManifestUtils.read( + new ClassPathResource("fixtures/manifest-papa-4.yml").getFile().toPath(), + new ClassPathResource("fixtures/vars-papa-4.yml").getFile().toPath()); + + assertThat(actual).isEqualTo(expected); + } + + @Test + public void readWithVariableSubstitution_addMultipleVariablesInOneField() throws IOException { + List expected = Collections.singletonList( + ApplicationManifest.builder() + .name("papa-5-application") + .buildpack("one and two is a very nice buildpack name for three") + .build()); + + List actual = ApplicationManifestUtils.read( + new ClassPathResource("fixtures/manifest-papa-5.yml").getFile().toPath(), + new ClassPathResource("fixtures/vars-papa-5.yml").getFile().toPath()); + + assertThat(actual).isEqualTo(expected); + } + + @Test + public void readWithVariableSubstitution_noSubstitutionAtAll() throws IOException { + List expected = Collections.singletonList( + ApplicationManifest.builder() + .name("papa-6-application") + .buildpack("buildpack_papa_6") + .build()); + + List actual = ApplicationManifestUtils.read( + new ClassPathResource("fixtures/manifest-papa-6.yml").getFile().toPath(), + new ClassPathResource("fixtures/vars-papa-6.yml").getFile().toPath()); + + assertThat(actual).isEqualTo(expected); + } + + @Test public void unixRead() throws IOException { assumeTrue(SystemUtils.IS_OS_UNIX); diff --git a/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-1.yml b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-1.yml new file mode 100644 index 0000000000..b49efa0485 --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-1.yml @@ -0,0 +1,6 @@ +--- +applications: +- name: papa-1-application + buildpack: papa-buildpack + instances: ((instances)) + memory: ((memory)) \ No newline at end of file diff --git a/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-2.yml b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-2.yml new file mode 100644 index 0000000000..5daa75e2d4 --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-2.yml @@ -0,0 +1,4 @@ +--- +applications: +- name: papa-2-application + buildpack: ((abcdef)) diff --git a/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-3.yml b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-3.yml new file mode 100644 index 0000000000..4cfc1a703c --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-3.yml @@ -0,0 +1,4 @@ +--- +applications: +- name: papa-3-application + buildpack: ((endless_1)) diff --git a/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-4.yml b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-4.yml new file mode 100644 index 0000000000..04a660142b --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-4.yml @@ -0,0 +1,4 @@ +--- +applications: +- name: papa-4-application + buildpack: ((((sub)) diff --git a/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-5.yml b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-5.yml new file mode 100644 index 0000000000..f979de5982 --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-5.yml @@ -0,0 +1,4 @@ +--- +applications: +- name: papa-5-application + buildpack: ((replace_1)) and ((replace_2)) is a very nice buildpack name for ((replace_3)) diff --git a/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-6.yml b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-6.yml new file mode 100644 index 0000000000..2cf62b2e96 --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-6.yml @@ -0,0 +1,4 @@ +--- +applications: +- name: papa-6-application + buildpack: buildpack_papa_6 diff --git a/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-7.yml b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-7.yml new file mode 100644 index 0000000000..4b34f3e072 --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/manifest-papa-7.yml @@ -0,0 +1,4 @@ +--- +applications: +- name: papa-7-application + buildpack: ((regex*)) diff --git a/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-1.yml b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-1.yml new file mode 100644 index 0000000000..69eb22a8fc --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-1.yml @@ -0,0 +1,2 @@ +instances: 2 +memory: 1G diff --git a/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-2.yml b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-2.yml new file mode 100644 index 0000000000..e034648103 --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-2.yml @@ -0,0 +1,2 @@ +abc.+: test + diff --git a/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-3.yml b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-3.yml new file mode 100644 index 0000000000..0bc4e50462 --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-3.yml @@ -0,0 +1,2 @@ +endless_1: ((endless_2)) +endless_2: ((endless_1)) diff --git a/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-4.yml b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-4.yml new file mode 100644 index 0000000000..9bdb5b86e4 --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-4.yml @@ -0,0 +1,2 @@ +sub: test)) +test: injected diff --git a/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-5.yml b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-5.yml new file mode 100644 index 0000000000..75397d176d --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-5.yml @@ -0,0 +1,3 @@ +replace_1: one +replace_2: two +replace_3: three diff --git a/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-6.yml b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-6.yml new file mode 100644 index 0000000000..75397d176d --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-6.yml @@ -0,0 +1,3 @@ +replace_1: one +replace_2: two +replace_3: three diff --git a/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-7.yml b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-7.yml new file mode 100644 index 0000000000..75397d176d --- /dev/null +++ b/cloudfoundry-operations/src/test/resources/fixtures/vars-papa-7.yml @@ -0,0 +1,3 @@ +replace_1: one +replace_2: two +replace_3: three