From 854ed5bd06f88860443dfd0e313cbc6f985c1a6e Mon Sep 17 00:00:00 2001 From: David Cavazos Date: Tue, 29 Sep 2020 08:52:58 -0700 Subject: [PATCH 0001/2127] [BEAM-10983] Add getting started from Spark page --- .../site/content/en/get-started/from-spark.md | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 website/www/site/content/en/get-started/from-spark.md diff --git a/website/www/site/content/en/get-started/from-spark.md b/website/www/site/content/en/get-started/from-spark.md new file mode 100644 index 000000000000..baa5ea10b773 --- /dev/null +++ b/website/www/site/content/en/get-started/from-spark.md @@ -0,0 +1,245 @@ +--- +title: "Getting started from Apache Spark" +--- + + +# Getting started from Apache Spark + +{{< localstorage language language-py >}} + +If you already know [_Apache Spark_](http://spark.apache.org/), +learning _Apache Beam_ is easy. +Beam and Spark are mostly equivalent, so you already know the basic concepts. + +A collection of elements in Spark is called a _Resilient Distributed Dataset_ (RDD), +while in Beam it's called a _Parallel Collection_ (PCollection). +A PCollection in Beam does _not_ have any ordering guarantees. + +Likewise, a transform in Beam is called a _Parallel Transform_ (PTransform). + +Here are some examples of common operations and their equivalent between PySpark and Beam. + +## Overview + +Here's a simple example of a PySpark pipeline that takes the numbers from one to four, +multiplies them by two, adds all the values together, and prints the result. + +{{< highlight py >}} +import pyspark + +with pyspark.SparkContext() as sc: + result = ( + sc.parallelize([1, 2, 3, 4]) + .map(lambda x: x * 2) + .reduce(lambda x, y: x + y) + ) + print(result) +{{< /highlight >}} + +In Beam you _pipe_ your data through the pipeline using the +_pipe operator_ `|` like `data | beam.Map(...)` instead of chaining +methods like `data.map(...)`, but they're doing the same thing. + +Here's how an equivalent pipeline looks like in Beam. + +{{< highlight py >}} +import apache_beam as beam + +with beam.Pipeline() as pipeline: + result = ( + pipeline + | beam.Create([1, 2, 3, 4]) + | beam.Map(lambda x: x * 2) + | beam.CombineGlobally(sum) + | beam.Map(print) + ) +{{< /highlight >}} + +> ℹ️ Note that we called `print` inside a `Map` transform. +> That's because we can only access the elements of a PCollection +> from within a PTransform. + +A label can optionally be added to a transform using the +_right shift operator_ `>>` like `data | 'My description' >> beam.Map(...)`. +This serves both as comments and makes your pipeline easier to debug. + +This is how the pipeline looks after adding labels. + +{{< highlight py >}} +import apache_beam as beam + +with beam.Pipeline() as pipeline: + result = ( + pipeline + | 'Create numbers' >> beam.Create([1, 2, 3, 4]) + | 'Multiply by two' >> beam.Map(lambda x: x * 2) + | 'Sum everything' >> beam.CombineGlobally(sum) + | beam.Map(print) + ) +{{< /highlight >}} + +## Setup + +Here's a comparison on how to get started both in PySpark and Beam. + +{{< table >}} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PySparkBeam
Install$ pip install pyspark$ pip install apache-beam
Importsimport pysparkimport apache_beam as beam
Creating a
local pipeline		 + with pyspark.SparkContext() as sc:
+     # Your pipeline code here. + 		+ with beam.Pipeline() as pipeline:
+     # Your pipeline code here. +
Creating valuesvalues = sc.parallelize([1, 2, 3, 4])values = pipeline | beam.Create([1, 2, 3, 4])
Creating
key-value pairs		 + pairs = sc.parallelize([
+     ('key1', 'value1'),
+     ('key2', 'value2'),
+     ('key3', 'value3'),
+ ]) + 		+ pairs = pipeline | beam.Create([
+     ('key1', 'value1'),
+     ('key2', 'value2'),
+     ('key3', 'value3'),
+ ]) +
Running a
local pipeline		$ spark-submit spark_pipeline.py$ python beam_pipeline.py
+{{< /table >}} + +## Transforms + +Here are the equivalents of some common transforms in both PySpark and Beam. + +{{< table >}} +| | PySpark | Beam | +|-------------------|---------------------------------------|---------------------------------------------------------| +| **Map** | `values.map(lambda x: x * 2)` | `values | beam.Map(lambda x: x * 2)` | +| **Filter** | `values.filter(lambda x: x % 2 == 0)` | `values | beam.Filter(lambda x: x % 2 == 0)` | +| **FlatMap** | `values.flatMap(lambda x: range(x))` | `values | beam.FlatMap(lambda x: range(x))` | +| **Group by key** | `pairs.groupByKey()` | `pairs | beam.GroupByKey()` | +| **Reduce** | `values.reduce(lambda x, y: x+y)` | `values | beam.CombineGlobally(sum)` | +| **Reduce by key** | `pairs.reduceByKey(lambda x, y: x+y)` | `pairs | beam.CombinePerKey(sum)` | +| **Distinct** | `values.distinct()` | `values | beam.Distinct()` | +| **Count** | `values.count()` | `values | beam.combiners.Count.Globally()` | +| **Count by key** | `pairs.countByKey()` | `pairs | beam.combiners.Count.PerKey()` | +| **Take smallest** | `values.takeOrdered(3)` | `values | beam.combiners.Top.Smallest(3)` | +| **Take largest** | `values.takeOrdered(3, lambda x: -x)` | `values | beam.combiners.Top.Largest(3)` | +| **Random sample** | `values.takeSample(False, 3)` | `values | beam.combiners.Sample.FixedSizeGlobally(3)` | +| **Union** | `values.union(otherValues)` | `(values, otherValues) | beam.Flatten()` | +| **Co-group** | `pairs.cogroup(otherPairs)` | `{'Xs': pairs, 'Ys': otherPairs} | beam.CoGroupByKey()` | +{{< /table >}} + +> ℹ️ To learn more about the transforms available in Beam, check the +> [Python transform gallery](/documentation/transforms/python/overview). + +## Using calculated values + +Since we are working in potentially distributed environments, +we can't guarantee that the results we've calculated are available at any given machine. + +In PySpark, we can get a result from a collection of elements (RDD) by using +`data.collect()`, or other aggregations such as `reduce()`, `count()` and more. + +Here's an example to scale numbers into a range between zero and one. + +{{< highlight py >}} +import pyspark + +with pyspark.SparkContext() as sc: + values = sc.parallelize([1, 2, 3, 4]) + total = values.reduce(lambda x, y: x + y) + + # We can simply use `total` since it's already a Python value from `reduce`. + scaled_values = values.map(lambda x: x / total) + + # But to access `scaled_values`, we need to call `collect`. + print(scaled_values.collect()) +{{< /highlight >}} + +In Beam the results from _all_ transforms result in a PCollection. +We use _side inputs_ to feed a PCollection into a transform and access its values. + +Any transform that accepts a function, like +[`Map`](/documentation/transforms/python/elementwise/map), +can take side inputs. +If we only need a single value, we can use +[`beam.pvalue.AsSingleton`](https://beam.apache.org/releases/pydoc/current/apache_beam.pvalue.html#apache_beam.pvalue.AsSingleton) and access them as a Python value. +If we need multiple values, we can use +[`beam.pvalue.AsIter`](https://beam.apache.org/releases/pydoc/current/apache_beam.pvalue.html#apache_beam.pvalue.AsIter) +and access them as an [`iterable`](https://docs.python.org/3/glossary.html#term-iterable). + +{{< highlight py >}} +import apache_beam as beam + +with beam.Pipeline() as pipeline: + values = pipeline | beam.Create([1, 2, 3, 4]) + total = values | beam.CombineGlobally(sum) + + # To access `total`, we need to pass it as a side input. + scaled_values = values | beam.Map( + lambda x, total: x / total, + total=beam.pvalue.AsSingleton(total)) + + scaled_values | beam.Map(print) +{{< /highlight >}} + +> ℹ️ In Beam we need to pass a side input explicitly, but we get the +> benefit that a reduction or aggregation does _not_ have to fit into memory. + +## Next Steps + +* Take a look at all the available transforms in the [Python transform gallery](/documentation/transforms/python/overview). +* Learn how to read from and write to files in the [_Pipeline I/O_ section of the _Programming guide_](/documentation/programming-guide/#pipeline-io) +* Walk through additional WordCount examples in the [WordCount Example Walkthrough](/get-started/wordcount-example). +* Take a self-paced tour through our [Learning Resources](/documentation/resources/learning-resources). +* Dive in to some of our favorite [Videos and Podcasts](/documentation/resources/videos-and-podcasts). +* Join the Beam [users@](/community/contact-us) mailing list. +* If you're interested in contributing to the Apache Beam codebase, see the [Contribution Guide](/contribute). + +Please don't hesitate to [reach out](/community/contact-us) if you encounter any issues! From c484e68489db6551aff70f6dde765ec0ca89d58d Mon Sep 17 00:00:00 2001 From: David Cavazos Date: Tue, 29 Sep 2020 10:17:00 -0700 Subject: [PATCH 0002/2127] Add "From Apache Spark" entry to side menu --- .../site/layouts/partials/section-menu/en/get-started.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/website/www/site/layouts/partials/section-menu/en/get-started.html b/website/www/site/layouts/partials/section-menu/en/get-started.html index 7bc4a54dd4ca..2c6c850a307b 100644 --- a/website/www/site/layouts/partials/section-menu/en/get-started.html +++ b/website/www/site/layouts/partials/section-menu/en/get-started.html @@ -22,12 +22,13 @@
  • Quickstart - Go
    • +
  • From Apache Spark
  • Example Walkthroughs +
  • WordCount
    • +
  • Mobile Gaming
    • +
  • Downloads
  • Security
    • From 01b75962042e98ae09d6745136e44ea2a2393a3f Mon Sep 17 00:00:00 2001 From: fpopic Date: Tue, 13 Oct 2020 12:43:53 +0200 Subject: [PATCH 0003/2127] Add example snippets to read fromQuery using BQ Storage API. --- ...ryReadFromQueryWithBigQueryStorageAPI.java | 56 +++++++++++++++++++ .../io/built-in/google-bigquery.md | 2 +- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java diff --git a/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java b/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java new file mode 100644 index 000000000000..599afe635483 --- /dev/null +++ b/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.examples.snippets.transforms.io.gcp.bigquery; + +// [START bigquery_read_from_query_with_bigquery_storage_api] + +import java.util.Arrays; +import org.apache.beam.examples.snippets.transforms.io.gcp.bigquery.BigQueryMyData.MyData; +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; +import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO.TypedRead.Method; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; + +class BigQueryReadFromQueryWithBigQueryStorageAPI { + public static PCollection readFromQueryWithBigQueryStorageAPI( + String project, String dataset, String table, Pipeline pipeline) { + + // String project = "my-project-id"; + // String dataset = "my_bigquery_dataset_id"; + // String table = "my_bigquery_table_id"; + + // Pipeline pipeline = Pipeline.create(); + + PCollection rows = + pipeline + .apply( + "Read from BigQuery table", + BigQueryIO.readTableRows() + .fromQuery(String.format("SELECT * FROM `%s.%s.%s`", project, dataset, table)) + .usingStandardSql() + .withMethod(Method.DIRECT_READ)) + .apply( + "TableRows to MyData", + MapElements.into(TypeDescriptor.of(MyData.class)).via(MyData::fromTableRow)); + + return rows; + } +} +// [END bigquery_read_from_query_with_bigquery_storage_api] diff --git a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md index bf252d7d1742..67c96b26ab75 100644 --- a/website/www/site/content/en/documentation/io/built-in/google-bigquery.md +++ b/website/www/site/content/en/documentation/io/built-in/google-bigquery.md @@ -360,7 +360,7 @@ GitHub](https://github.com/apache/beam/blob/master/examples/java/src/main/java/o The following code snippet reads with a query string. {{< highlight java >}} -// Snippet not yet available (BEAM-7034). +{{< code_sample "examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java" bigquery_read_from_query_with_bigquery_storage_api >}} {{< /highlight >}} {{< highlight py >}} From 393bbdeef9148266082b0211f87869656213a942 Mon Sep 17 00:00:00 2001 From: fpopic Date: Tue, 13 Oct 2020 14:15:49 +0200 Subject: [PATCH 0004/2127] Make the query example consistent with the previous one for the table. --- ...ryReadFromQueryWithBigQueryStorageAPI.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java b/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java index 599afe635483..f24737f0c23d 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java +++ b/examples/java/src/main/java/org/apache/beam/examples/snippets/transforms/io/gcp/bigquery/BigQueryReadFromQueryWithBigQueryStorageAPI.java @@ -30,7 +30,7 @@ class BigQueryReadFromQueryWithBigQueryStorageAPI { public static PCollection readFromQueryWithBigQueryStorageAPI( - String project, String dataset, String table, Pipeline pipeline) { + String project, String dataset, String table, String query, Pipeline pipeline) { // String project = "my-project-id"; // String dataset = "my_bigquery_dataset_id"; @@ -38,12 +38,31 @@ public static PCollection readFromQueryWithBigQueryStorageAPI( // Pipeline pipeline = Pipeline.create(); + /* + String query = String.format("SELECT\n" + + " string_field,\n" + + " int64_field,\n" + + " float64_field,\n" + + " numeric_field,\n" + + " bool_field,\n" + + " bytes_field,\n" + + " date_field,\n" + + " datetime_field,\n" + + " time_field,\n" + + " timestamp_field,\n" + + " geography_field,\n" + + " array_field,\n" + + " struct_field\n" + + "FROM\n" + + " `%s:%s.%s`", project, dataset, table) + */ + PCollection rows = pipeline .apply( "Read from BigQuery table", BigQueryIO.readTableRows() - .fromQuery(String.format("SELECT * FROM `%s.%s.%s`", project, dataset, table)) + .fromQuery(query) .usingStandardSql() .withMethod(Method.DIRECT_READ)) .apply( From 17aed03f86421d228d40c6e55201a341643edb40 Mon Sep 17 00:00:00 2001 From: David Cavazos Date: Mon, 19 Oct 2020 15:03:23 -0700 Subject: [PATCH 0005/2127] Add links to transform catalog --- .../site/content/en/get-started/from-spark.md | 79 +++++++++++-------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/website/www/site/content/en/get-started/from-spark.md b/website/www/site/content/en/get-started/from-spark.md index baa5ea10b773..0d4668f302da 100644 --- a/website/www/site/content/en/get-started/from-spark.md +++ b/website/www/site/content/en/get-started/from-spark.md @@ -21,7 +21,7 @@ limitations under the License. If you already know [_Apache Spark_](http://spark.apache.org/), learning _Apache Beam_ is easy. -Beam and Spark are mostly equivalent, so you already know the basic concepts. +The Beam and Spark APIs are similar, so you already know the basic concepts. A collection of elements in Spark is called a _Resilient Distributed Dataset_ (RDD), while in Beam it's called a _Parallel Collection_ (PCollection). @@ -39,13 +39,13 @@ multiplies them by two, adds all the values together, and prints the result. {{< highlight py >}} import pyspark -with pyspark.SparkContext() as sc: - result = ( - sc.parallelize([1, 2, 3, 4]) - .map(lambda x: x * 2) - .reduce(lambda x, y: x + y) - ) - print(result) +sc = pyspark.SparkContext() +result = ( + sc.parallelize([1, 2, 3, 4]) + .map(lambda x: x * 2) + .reduce(lambda x, y: x + y) +) +print(result) {{< /highlight >}} In Beam you _pipe_ your data through the pipeline using the @@ -71,6 +71,17 @@ with beam.Pipeline() as pipeline: > That's because we can only access the elements of a PCollection > from within a PTransform. +Another thing to note is that Beam pipelines are constructed _lazily_. +This means that when you _pipe_ `|` data you're only _decalring_ the +transformations and the order you want them to happen, +but the actual computation doesn't happen. +The pipeline is run _after_ the `with beam.Pipeline() as pipeline` context has +closed. +The pipeline is then sent to your runner of choice and it processes the data. + +> ℹ️ When the `with beam.Pipeline() as pipeline` context closes, +> it implicitly calls `pipeline.run()` which triggers the computation to happen. + A label can optionally be added to a transform using the _right shift operator_ `>>` like `data | 'My description' >> beam.Map(...)`. This serves both as comments and makes your pipeline easier to debug. @@ -86,7 +97,7 @@ with beam.Pipeline() as pipeline: | 'Create numbers' >> beam.Create([1, 2, 3, 4]) | 'Multiply by two' >> beam.Map(lambda x: x * 2) | 'Sum everything' >> beam.CombineGlobally(sum) - | beam.Map(print) + | 'Print results' >> beam.Map(print) ) {{< /highlight >}} @@ -114,8 +125,8 @@ Here's a comparison on how to get started both in PySpark and Beam. Creating a
    local pipeline     - with pyspark.SparkContext() as sc:
    -     # Your pipeline code here. + sc = pyspark.SparkContext() as sc:
    + # Your pipeline code here. with beam.Pipeline() as pipeline:
    @@ -157,22 +168,22 @@ Here's a comparison on how to get started both in PySpark and Beam. Here are the equivalents of some common transforms in both PySpark and Beam. {{< table >}} -| | PySpark | Beam | -|-------------------|---------------------------------------|---------------------------------------------------------| -| **Map** | `values.map(lambda x: x * 2)` | `values | beam.Map(lambda x: x * 2)` | -| **Filter** | `values.filter(lambda x: x % 2 == 0)` | `values | beam.Filter(lambda x: x % 2 == 0)` | -| **FlatMap** | `values.flatMap(lambda x: range(x))` | `values | beam.FlatMap(lambda x: range(x))` | -| **Group by key** | `pairs.groupByKey()` | `pairs | beam.GroupByKey()` | -| **Reduce** | `values.reduce(lambda x, y: x+y)` | `values | beam.CombineGlobally(sum)` | -| **Reduce by key** | `pairs.reduceByKey(lambda x, y: x+y)` | `pairs | beam.CombinePerKey(sum)` | -| **Distinct** | `values.distinct()` | `values | beam.Distinct()` | -| **Count** | `values.count()` | `values | beam.combiners.Count.Globally()` | -| **Count by key** | `pairs.countByKey()` | `pairs | beam.combiners.Count.PerKey()` | -| **Take smallest** | `values.takeOrdered(3)` | `values | beam.combiners.Top.Smallest(3)` | -| **Take largest** | `values.takeOrdered(3, lambda x: -x)` | `values | beam.combiners.Top.Largest(3)` | -| **Random sample** | `values.takeSample(False, 3)` | `values | beam.combiners.Sample.FixedSizeGlobally(3)` | -| **Union** | `values.union(otherValues)` | `(values, otherValues) | beam.Flatten()` | -| **Co-group** | `pairs.cogroup(otherPairs)` | `{'Xs': pairs, 'Ys': otherPairs} | beam.CoGroupByKey()` | +| | PySpark | Beam | +|----------------------------------------------------------------------------------|---------------------------------------|---------------------------------------------------------| +| [**Map**](/documentation/transforms/python/elementwise/map/) | `values.map(lambda x: x * 2)` | `values | beam.Map(lambda x: x * 2)` | +| [**Filter**](/documentation/transforms/python/elementwise/filter/) | `values.filter(lambda x: x % 2 == 0)` | `values | beam.Filter(lambda x: x % 2 == 0)` | +| [**FlatMap**](/documentation/transforms/python/elementwise/flatmap/) | `values.flatMap(lambda x: range(x))` | `values | beam.FlatMap(lambda x: range(x))` | +| [**Group by key**](/documentation/transforms/python/aggregation/groupbykey/) | `pairs.groupByKey()` | `pairs | beam.GroupByKey()` | +| [**Reduce**](/documentation/transforms/python/aggregation/combineglobally/) | `values.reduce(lambda x, y: x+y)` | `values | beam.CombineGlobally(sum)` | +| [**Reduce by key**](/documentation/transforms/python/aggregation/combineperkey/) | `pairs.reduceByKey(lambda x, y: x+y)` | `pairs | beam.CombinePerKey(sum)` | +| [**Distinct**](/documentation/transforms/python/aggregation/distinct/) | `values.distinct()` | `values | beam.Distinct()` | +| [**Count**](/documentation/transforms/python/aggregation/count/) | `values.count()` | `values | beam.combiners.Count.Globally()` | +| [**Count by key**](/documentation/transforms/python/aggregation/count/) | `pairs.countByKey()` | `pairs | beam.combiners.Count.PerKey()` | +| [**Take smallest**](/documentation/transforms/python/aggregation/top/) | `values.takeOrdered(3)` | `values | beam.combiners.Top.Smallest(3)` | +| [**Take largest**](/documentation/transforms/python/aggregation/top/) | `values.takeOrdered(3, lambda x: -x)` | `values | beam.combiners.Top.Largest(3)` | +| [**Random sample**](/documentation/transforms/python/aggregation/sample/) | `values.takeSample(False, 3)` | `values | beam.combiners.Sample.FixedSizeGlobally(3)` | +| [**Union**](/documentation/transforms/python/other/flatten/) | `values.union(otherValues)` | `(values, otherValues) | beam.Flatten()` | +| [**Co-group**](/documentation/transforms/python/aggregation/cogroupbykey/) | `pairs.cogroup(otherPairs)` | `{'Xs': pairs, 'Ys': otherPairs} | beam.CoGroupByKey()` | {{< /table >}} > ℹ️ To learn more about the transforms available in Beam, check the @@ -191,15 +202,15 @@ Here's an example to scale numbers into a range between zero and one. {{< highlight py >}} import pyspark -with pyspark.SparkContext() as sc: - values = sc.parallelize([1, 2, 3, 4]) - total = values.reduce(lambda x, y: x + y) +sc = pyspark.SparkContext() +values = sc.parallelize([1, 2, 3, 4]) +total = values.reduce(lambda x, y: x + y) - # We can simply use `total` since it's already a Python value from `reduce`. - scaled_values = values.map(lambda x: x / total) +# We can simply use `total` since it's already a Python value from `reduce`. +scaled_values = values.map(lambda x: x / total) - # But to access `scaled_values`, we need to call `collect`. - print(scaled_values.collect()) +# But to access `scaled_values`, we need to call `collect`. +print(scaled_values.collect()) {{< /highlight >}} In Beam the results from _all_ transforms result in a PCollection. From 26a2a04d04cc2c4030357291a09f8d262d9c67f3 Mon Sep 17 00:00:00 2001 From: David Cavazos Date: Mon, 19 Oct 2020 15:07:42 -0700 Subject: [PATCH 0006/2127] Add more links --- website/www/site/content/en/get-started/from-spark.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/www/site/content/en/get-started/from-spark.md b/website/www/site/content/en/get-started/from-spark.md index 0d4668f302da..a964e2781bf6 100644 --- a/website/www/site/content/en/get-started/from-spark.md +++ b/website/www/site/content/en/get-started/from-spark.md @@ -206,7 +206,7 @@ sc = pyspark.SparkContext() values = sc.parallelize([1, 2, 3, 4]) total = values.reduce(lambda x, y: x + y) -# We can simply use `total` since it's already a Python value from `reduce`. +# We can simply use `total` since it's already a Python `int` value from `reduce`. scaled_values = values.map(lambda x: x / total) # But to access `scaled_values`, we need to call `collect`. @@ -214,7 +214,8 @@ print(scaled_values.collect()) {{< /highlight >}} In Beam the results from _all_ transforms result in a PCollection. -We use _side inputs_ to feed a PCollection into a transform and access its values. +We use [_side inputs_](/documentation/programming-guide/#side-inputs) +to feed a PCollection into a transform and access its values. Any transform that accepts a function, like [`Map`](/documentation/transforms/python/elementwise/map), From 2f397998454a12bb04ae028519e47da950db9690 Mon Sep 17 00:00:00 2001 From: David Cavazos Date: Tue, 20 Oct 2020 09:49:43 -0700 Subject: [PATCH 0007/2127] Add RDD/DataFrame clarification --- website/www/site/content/en/get-started/from-spark.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/website/www/site/content/en/get-started/from-spark.md b/website/www/site/content/en/get-started/from-spark.md index a964e2781bf6..6c77c711e81d 100644 --- a/website/www/site/content/en/get-started/from-spark.md +++ b/website/www/site/content/en/get-started/from-spark.md @@ -2,14 +2,14 @@ title: "Getting started from Apache Spark" --- + + + + + + + diff --git a/website/www/site/assets/icons/github-icon.svg b/website/www/site/assets/icons/github-icon.svg new file mode 100644 index 000000000000..ba80c8261586 --- /dev/null +++ b/website/www/site/assets/icons/github-icon.svg @@ -0,0 +1,29 @@ + + + + + + + + + + + + diff --git a/website/www/site/assets/icons/open-source-icon.svg b/website/www/site/assets/icons/open-source-icon.svg new file mode 100644 index 000000000000..d6afab7898e8 --- /dev/null +++ b/website/www/site/assets/icons/open-source-icon.svg @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + diff --git a/website/www/site/assets/icons/portable-icon.svg b/website/www/site/assets/icons/portable-icon.svg new file mode 100644 index 000000000000..77189163e141 --- /dev/null +++ b/website/www/site/assets/icons/portable-icon.svg @@ -0,0 +1,27 @@ + + + + + + + + + + diff --git a/website/www/site/assets/icons/twitter-icon.svg b/website/www/site/assets/icons/twitter-icon.svg new file mode 100644 index 000000000000..e23e8210ae5b --- /dev/null +++ b/website/www/site/assets/icons/twitter-icon.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + diff --git a/website/www/site/assets/icons/unified-icon.svg b/website/www/site/assets/icons/unified-icon.svg new file mode 100644 index 000000000000..6dc75d26e576 --- /dev/null +++ b/website/www/site/assets/icons/unified-icon.svg @@ -0,0 +1,26 @@ + + + + + + + + + diff --git a/website/www/site/assets/icons/youtube-icon.svg b/website/www/site/assets/icons/youtube-icon.svg new file mode 100644 index 000000000000..3cf47d8631f5 --- /dev/null +++ b/website/www/site/assets/icons/youtube-icon.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/scss/_global.sass b/website/www/site/assets/scss/_global.sass index db0deb28e9de..a4dd35517e26 100644 --- a/website/www/site/assets/scss/_global.sass +++ b/website/www/site/assets/scss/_global.sass @@ -25,7 +25,6 @@ body .body background: #fff - max-width: 1440px margin: 0 auto padding-top: 130px diff --git a/website/www/site/assets/scss/_pillars.sass b/website/www/site/assets/scss/_media.scss similarity index 63% rename from website/www/site/assets/scss/_pillars.sass rename to website/www/site/assets/scss/_media.scss index 220e7ce8c6a0..b1146a520163 100644 --- a/website/www/site/assets/scss/_pillars.sass +++ b/website/www/site/assets/scss/_media.scss @@ -14,30 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -.pillars - margin: $pad-xl 0 - text-align: center - - .pillars__title - +type-h2 - margin-bottom: $pad - - .pillars__cols - +type-body - +md - display: flex - justify-content: center - - .pillars__cols__col - .pillars__cols__col__title - font-weight: 600 - margin-bottom: $pad/2 - - .pillars__cols__col__body - max-width: 350px - margin: 0 auto $pad-sm - - +md - padding: 0 $pad - margin: 0 auto + +$mobile: 640px; +$tablet: 1280px; +$fullhd: 1920px; diff --git a/website/www/site/assets/scss/_pillars.scss b/website/www/site/assets/scss/_pillars.scss new file mode 100644 index 000000000000..47c95d19ba0e --- /dev/null +++ b/website/www/site/assets/scss/_pillars.scss @@ -0,0 +1,151 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.pillars { + padding: $pad-l $pad; + + .pillars-title { + @extend .component-title; + + text-align: center; + border: none; + } + + .pillars-content { + display: grid; + grid-template-columns: 443px 443px; + grid-gap: 50px 89px; + justify-content: center; + margin-top: 84px; + + .pillars-item { + display: flex; + align-items: center; + + .pillars-item-icon { + margin-right: 47px; + } + + .pillars-item-description { + width: 100%; + max-width: 284px; + + .pillars-item-header { + @extend .component-header; + } + + .pillars-item-text { + @extend .component-text; + } + } + } + } + + .pillars-social { + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + margin-top: 117px; + + .pillars-social-icons { + display: flex; + align-items: center; + margin-bottom: 45px; + + svg { + height: 41px; + width: auto; + } + + #about-twitter-icon { + height: 45px; + } + + a { + filter: grayscale(100%); + opacity: 0.7; + + &:hover { + filter: grayscale(0); + opacity: 1; + } + } + + .pillars-youtube-icon { + margin: 0 80px; + } + } + + .pillars-social-text { + @extend .component-text; + max-width: 285px; + } + } +} + +@media (max-width: $tablet) { + .pillars { + padding: $pad-md $pad-s; + + .pillars-content { + grid-template-columns: 330px; + grid-column-gap: 47px; + margin-top: 62px; + + .pillars-item { + align-items: flex-start; + + .pillars-item-icon { + margin-right: 17px; + margin-top: 12px; + } + + svg { + width: 64px; + height: 64px; + } + } + } + + .pillars-social { + margin-top: 91px; + + .pillars-social-icons { + svg { + height: 34px; + width: auto; + } + + #about-twitter-icon { + height: 37px; + } + + a { + filter: none; + opacity: 1; + } + + .pillars-youtube-icon { + margin: 0 60px; + } + } + } + } +} diff --git a/website/www/site/assets/scss/_typography.scss b/website/www/site/assets/scss/_typography.scss new file mode 100644 index 000000000000..d093140ace72 --- /dev/null +++ b/website/www/site/assets/scss/_typography.scss @@ -0,0 +1,54 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + @import "media"; + +.component-title { + font-size: 36px; + font-weight: normal; + font-style: normal; + line-height: 1.1; + letter-spacing: normal; + color: $color-gray; +} + +.component-header { + font-size: 22px; + font-weight: 500; + font-style: normal; + line-height: 1.45; + letter-spacing: 0.43px; + color: $color-gray; +} + +.component-text { + font-size: 16px; + font-weight: normal; + font-style: normal; + line-height: 1.63; + letter-spacing: 0.43px; + color: $color-gray; +} + +@media (max-width: $tablet) { + .component-title { + font-size: 28px; + } + .component-header { + font-weight: normal; + } +} \ No newline at end of file diff --git a/website/www/site/assets/scss/_vars.sass b/website/www/site/assets/scss/_vars.sass index a5dd998b14c2..6a98df42306a 100644 --- a/website/www/site/assets/scss/_vars.sass +++ b/website/www/site/assets/scss/_vars.sass @@ -20,10 +20,13 @@ $color-dark: #37424B $color-white: #FFF $color-light-gray: #F7F7F7 $color-dark-gray: #555 +$color-gray: #333333 $pad-sm: 15px +$pad-s: 24px $pad: 30px $pad-md: 60px +$pad-l: 84px $pad-xl: 100px $box-shadow: 0px 3px 20px 0 rgba(0,0,0,0.075) diff --git a/website/www/site/assets/scss/main.scss b/website/www/site/assets/scss/main.scss index a2983ad59473..8007760cbbbd 100644 --- a/website/www/site/assets/scss/main.scss +++ b/website/www/site/assets/scss/main.scss @@ -24,6 +24,8 @@ @import "_type.sass"; @import "_global.sass"; @import "_navbar.sass"; +@import "_typography.scss"; +@import "_media.scss"; // Components. @import "_button.sass"; @@ -36,7 +38,7 @@ @import "_header.sass"; @import "_hero.sass"; @import "_logos.sass"; -@import "_pillars.sass"; +@import "_pillars.scss"; @import "_section-nav.sass"; @import "_page-nav.sass"; @import "_table-wrapper.sass"; diff --git a/website/www/site/data/en/pillars.yaml b/website/www/site/data/en/pillars.yaml index d2138bdd8b38..de864bcaaac8 100644 --- a/website/www/site/data/en/pillars.yaml +++ b/website/www/site/data/en/pillars.yaml @@ -12,7 +12,13 @@ - title: Unified body: Use a single programming model for both batch and streaming use cases. -- title: Portable - body: Execute pipelines on multiple execution environments. + icon: icons/unified-icon.svg - title: Extensible body: Write and share new SDKs, IO connectors, and transformation libraries. + icon: icons/extensive-icon.svg +- title: Portable + body: Execute pipelines on multiple execution environments. + icon: icons/portable-icon.svg +- title: Open Source + body: Use a single programming model for both batch and streaming use cases. + icon: icons/open-source-icon.svg diff --git a/website/www/site/data/en/pillars_social.yaml b/website/www/site/data/en/pillars_social.yaml new file mode 100644 index 000000000000..2ed3ff6c03a9 --- /dev/null +++ b/website/www/site/data/en/pillars_social.yaml @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: pillars-github-icon + icon: icons/github-icon.svg + url: https://github.com/apache/beam +- name: pillars-youtube-icon + icon: icons/youtube-icon.svg + url: https://www.youtube.com/channel/UChNnb_YO_7B0HlW6FhAXZZQ +- name: pillars-twitter-icon + icon: icons/twitter-icon.svg + url: https://twitter.com/apachebeam \ No newline at end of file diff --git a/website/www/site/i18n/home/en.yaml b/website/www/site/i18n/home/en.yaml index cbb3f23a78c3..43a3fea9bbc8 100644 --- a/website/www/site/i18n/home/en.yaml +++ b/website/www/site/i18n/home/en.yaml @@ -28,8 +28,6 @@ translation: "Go Quickstart" - id: home-hero-blog-title translation: "The latest from the blog" -- id: home-pillars-title - translation: "All about Apache Beam" - id: home-logos-title translation: "Works with" - id: home-cards-title diff --git a/website/www/site/i18n/home/pillars/en.yaml b/website/www/site/i18n/home/pillars/en.yaml new file mode 100644 index 000000000000..ab8a6d1ff002 --- /dev/null +++ b/website/www/site/i18n/home/pillars/en.yaml @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- id: home-pillars-title + translation: "All about Apache Beam" +- id: home-pillars-social-text + translation: "Check out our social media to learn more about the community!" \ No newline at end of file diff --git a/website/www/site/layouts/index.html b/website/www/site/layouts/index.html index e1f69be4c2f2..f173e4103a71 100644 --- a/website/www/site/layouts/index.html +++ b/website/www/site/layouts/index.html @@ -57,23 +57,27 @@ {{ end }} {{ define "pillars-section" }} -
    -
    +
    +

    {{ T "home-pillars-title" }} -

    -
    + +
    {{ $data := index $.Site.Data .Site.Language.Lang }} {{ range $pillar := $data.pillars }} -
    -
    - {{ $pillar.title }} -
    -
    - {{ $pillar.body }} -
    -
    + {{ partial "pillars/pillars-item" (dict "logo" $pillar.icon "header" $pillar.title "text" $pillar.body) }} {{ end }}
    +
    +
    + {{ $data := index $.Site.Data .Site.Language.Lang }} + {{ range $pillars_social := $data.pillars_social }} + {{ partial "pillars/pillars-social" (dict "icon" $pillars_social.icon "url" $pillars_social.url "name" $pillars_social.name) }} + {{ end }} +
    +

    + {{ T "home-pillars-social-text" }} +

    +
    {{ end }} diff --git a/website/www/site/layouts/partials/head.html b/website/www/site/layouts/partials/head.html index 1430d9ad4dfa..b3773320be6d 100644 --- a/website/www/site/layouts/partials/head.html +++ b/website/www/site/layouts/partials/head.html @@ -17,7 +17,7 @@ {{ if .Title }}{{ .Title }}{{ else }}{{ .Site.Title }}{{ end }} - + {{ $scssMain := "scss/main.scss"}} {{ if .Site.IsServer }} diff --git a/website/www/site/layouts/partials/pillars/pillars-item.html b/website/www/site/layouts/partials/pillars/pillars-item.html new file mode 100644 index 000000000000..84be04b0edbf --- /dev/null +++ b/website/www/site/layouts/partials/pillars/pillars-item.html @@ -0,0 +1,23 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +*/}} + +
    +
    + {{ with resources.Get .logo }} + {{ .Content | safeHTML }} + {{ end }} +
    +
    +
    {{ .header | markdownify }}
    +

    {{ .text | markdownify }}

    +
    +
    diff --git a/website/www/site/layouts/partials/pillars/pillars-social.html b/website/www/site/layouts/partials/pillars/pillars-social.html new file mode 100644 index 000000000000..76f6a5099a36 --- /dev/null +++ b/website/www/site/layouts/partials/pillars/pillars-social.html @@ -0,0 +1,19 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +*/}} + +
    + + {{ with resources.Get .icon }} + {{ .Content | safeHTML }} + {{ end }} + +
    \ No newline at end of file From 88b18deeb55f78806fe0eb854bcac6d2412ef401 Mon Sep 17 00:00:00 2001 From: Sruthi Sree Kumar Date: Thu, 19 Nov 2020 16:58:22 +0100 Subject: [PATCH 0014/2127] updated the output format --- .../apache/beam/validate/runner/service/TestService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java index 00e23ef89748..b94bb5c761a5 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java @@ -20,12 +20,12 @@ public interface TestService { - default List getAllTests(TestResult testResult) { + default List getAllTests(TestResult testResult) { List caseResults = new ArrayList<>(); Optional.ofNullable(testResult.getSuites()).ifPresent(suites -> suites.forEach(item -> caseResults.addAll(item.getCases()))); - List tests = new ArrayList<>(); - Optional.ofNullable(caseResults).ifPresent(cases -> cases.forEach(item -> tests.add(item.getClassName() + "." + item.getName() + " : " + item.getStatus()))); - return tests; +// List tests = new ArrayList<>(); +// Optional.ofNullable(caseResults).ifPresent(cases -> cases.forEach(item -> tests.add(item.getClassName() + "." + item.getName() + " : " + item.getStatus()))); + return caseResults; } default URL getUrl(Map job, Configuration configuration) throws URISyntaxException, IOException { From c4b1359c275e2f753021c0bdf3ccd45db2ac7df6 Mon Sep 17 00:00:00 2001 From: Kyle Weaver Date: Wed, 11 Nov 2020 14:31:16 -0800 Subject: [PATCH 0015/2127] [BEAM-10925] Create ZetaSQL-specific subclass of ScalarFunctionImpl that knows which function group it belongs to. This will be important for differentiating ZetaSQL built-in functions from UDFs. Also resolves the nullability error in findMethod. --- .../sql/impl/ScalarFunctionImpl.java | 47 ++-------- .../extensions/sql/zetasql/SqlAnalyzer.java | 6 ++ .../translation/ExpressionConverter.java | 7 +- .../sql/zetasql/translation/SqlOperators.java | 77 +++++++++++------ .../ZetaSqlScalarFunctionImpl.java | 86 +++++++++++++++++++ 5 files changed, 156 insertions(+), 67 deletions(-) create mode 100644 sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ZetaSqlScalarFunctionImpl.java diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/ScalarFunctionImpl.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/ScalarFunctionImpl.java index da8cb269748d..df725a72a808 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/ScalarFunctionImpl.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/ScalarFunctionImpl.java @@ -61,8 +61,7 @@ public class ScalarFunctionImpl extends UdfImplReflectiveFunctionBase private final CallImplementor implementor; - /** Private constructor. */ - private ScalarFunctionImpl(Method method, CallImplementor implementor) { + protected ScalarFunctionImpl(Method method, CallImplementor implementor) { super(method); this.implementor = implementor; } @@ -86,24 +85,6 @@ public static ImmutableMultimap createAll(Class clazz) { return builder.build(); } - /** - * Creates {@link org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.Function} from - * given class. - * - *

    If a method of the given name is not found or it does not suit, returns {@code null}. - * - * @param clazz class that is used to implement the function - * @param methodName Method name (typically "eval") - * @return created {@link ScalarFunction} or null - */ - public static Function create(Class clazz, String methodName) { - final Method method = findMethod(clazz, methodName); - if (method == null) { - return null; - } - return create(method); - } - /** * Creates {@link org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.Function} from * given method. When {@code eval} method does not suit, {@code null} is returned. @@ -112,6 +93,12 @@ public static Function create(Class clazz, String methodName) { * @return created {@link Function} or null */ public static Function create(Method method) { + validateMethod(method); + CallImplementor implementor = createImplementor(method); + return new ScalarFunctionImpl(method, implementor); + } + + protected static void validateMethod(Method method) { if (!Modifier.isStatic(method.getModifiers())) { Class clazz = method.getDeclaringClass(); if (!classHasPublicZeroArgsConstructor(clazz)) { @@ -121,9 +108,6 @@ public static Function create(Method method) { if (method.getExceptionTypes().length != 0) { throw new RuntimeException(method.getName() + " must not throw checked exception"); } - - CallImplementor implementor = createImplementor(method); - return new ScalarFunctionImpl(method, implementor); } @Override @@ -191,7 +175,7 @@ public Expression implement( } } - private static CallImplementor createImplementor(Method method) { + protected static CallImplementor createImplementor(Method method) { final NullPolicy nullPolicy = getNullPolicy(method); return RexImpTable.createImplementor( new ScalarReflectiveCallNotNullImplementor(method), nullPolicy, false); @@ -247,21 +231,6 @@ static boolean classHasPublicZeroArgsConstructor(Class clazz) { } return false; } - - /* - * Finds a method in a given class by name. - * @param clazz class to search method in - * @param name name of the method to find - * @return the first method with matching name or null when no method found - */ - static Method findMethod(Class clazz, String name) { - for (Method method : clazz.getMethods()) { - if (method.getName().equals(name) && !method.isBridge()) { - return method; - } - } - return null; - } } // End ScalarFunctionImpl.java diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java index b4666cd11405..f4db1f194a94 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SqlAnalyzer.java @@ -69,8 +69,14 @@ "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402) }) public class SqlAnalyzer { + // ZetaSQL function group identifiers. Different function groups may have divergent translation + // paths. public static final String PRE_DEFINED_WINDOW_FUNCTIONS = "pre_defined_window_functions"; public static final String USER_DEFINED_FUNCTIONS = "user_defined_functions"; + /** + * Same as {@link Function}.ZETASQL_FUNCTION_GROUP_NAME. Identifies built-in ZetaSQL functions. + */ + public static final String ZETASQL_FUNCTION_GROUP_NAME = "ZetaSQL"; private static final ImmutableSet SUPPORTED_STATEMENT_KINDS = ImmutableSet.of( diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java index 24a5e1c74c09..a4d0f0335bed 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ExpressionConverter.java @@ -26,6 +26,7 @@ import static com.google.zetasql.ZetaSQLType.TypeKind.TYPE_TIMESTAMP; import static org.apache.beam.sdk.extensions.sql.zetasql.SqlAnalyzer.PRE_DEFINED_WINDOW_FUNCTIONS; import static org.apache.beam.sdk.extensions.sql.zetasql.SqlAnalyzer.USER_DEFINED_FUNCTIONS; +import static org.apache.beam.sdk.extensions.sql.zetasql.SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME; import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; import com.google.common.base.Ascii; @@ -608,7 +609,7 @@ private RexNode convertResolvedFunctionCall( SqlOperator op = SqlOperatorMappingTable.ZETASQL_FUNCTION_TO_CALCITE_SQL_OPERATOR.get(funName); List operands = new ArrayList<>(); - if (funGroup.equals(PRE_DEFINED_WINDOW_FUNCTIONS)) { + if (PRE_DEFINED_WINDOW_FUNCTIONS.equals(funGroup)) { switch (funName) { case FIXED_WINDOW: case SESSION_WINDOW: @@ -646,7 +647,7 @@ private RexNode convertResolvedFunctionCall( throw new UnsupportedOperationException( "Unsupported function: " + funName + ". Only support TUMBLE, HOP, and SESSION now."); } - } else if (funGroup.equals("ZetaSQL")) { + } else if (ZETASQL_FUNCTION_GROUP_NAME.equals(funGroup)) { if (op == null) { Type returnType = functionCall.getSignature().getResultType().getType(); if (returnType != null) { @@ -664,7 +665,7 @@ private RexNode convertResolvedFunctionCall( operands.add( convertRexNodeFromResolvedExpr(expr, columnList, fieldList, outerFunctionArguments)); } - } else if (funGroup.equals(USER_DEFINED_FUNCTIONS)) { + } else if (USER_DEFINED_FUNCTIONS.equals(funGroup)) { ResolvedCreateFunctionStmt createFunctionStmt = userFunctionDefinitions.sqlScalarFunctions.get(functionCall.getFunction().getNamePath()); ResolvedExpr functionExpression = createFunctionStmt.getFunctionExpression(); diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java index 1c0835c4d96d..4564c1139f1a 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java @@ -26,6 +26,7 @@ import org.apache.beam.sdk.extensions.sql.impl.planner.BeamRelDataTypeSystem; import org.apache.beam.sdk.extensions.sql.impl.udaf.StringAgg; import org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils; +import org.apache.beam.sdk.extensions.sql.zetasql.SqlAnalyzer; import org.apache.beam.sdk.extensions.sql.zetasql.translation.impl.BeamBuiltinMethods; import org.apache.beam.sdk.extensions.sql.zetasql.translation.impl.CastFunctionImpl; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.jdbc.JavaTypeFactoryImpl; @@ -82,36 +83,57 @@ public class SqlOperators { new UdafImpl<>(new StringAgg.StringAggString())); public static final SqlOperator START_WITHS = - createUdfOperator("STARTS_WITH", BeamBuiltinMethods.STARTS_WITH_METHOD); + createUdfOperator( + "STARTS_WITH", + BeamBuiltinMethods.STARTS_WITH_METHOD, + SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator CONCAT = - createUdfOperator("CONCAT", BeamBuiltinMethods.CONCAT_METHOD); + createUdfOperator( + "CONCAT", BeamBuiltinMethods.CONCAT_METHOD, SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator REPLACE = - createUdfOperator("REPLACE", BeamBuiltinMethods.REPLACE_METHOD); + createUdfOperator( + "REPLACE", BeamBuiltinMethods.REPLACE_METHOD, SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); - public static final SqlOperator TRIM = createUdfOperator("TRIM", BeamBuiltinMethods.TRIM_METHOD); + public static final SqlOperator TRIM = + createUdfOperator( + "TRIM", BeamBuiltinMethods.TRIM_METHOD, SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator LTRIM = - createUdfOperator("LTRIM", BeamBuiltinMethods.LTRIM_METHOD); + createUdfOperator( + "LTRIM", BeamBuiltinMethods.LTRIM_METHOD, SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator RTRIM = - createUdfOperator("RTRIM", BeamBuiltinMethods.RTRIM_METHOD); + createUdfOperator( + "RTRIM", BeamBuiltinMethods.RTRIM_METHOD, SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator SUBSTR = - createUdfOperator("SUBSTR", BeamBuiltinMethods.SUBSTR_METHOD); + createUdfOperator( + "SUBSTR", BeamBuiltinMethods.SUBSTR_METHOD, SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator REVERSE = - createUdfOperator("REVERSE", BeamBuiltinMethods.REVERSE_METHOD); + createUdfOperator( + "REVERSE", BeamBuiltinMethods.REVERSE_METHOD, SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator CHAR_LENGTH = - createUdfOperator("CHAR_LENGTH", BeamBuiltinMethods.CHAR_LENGTH_METHOD); + createUdfOperator( + "CHAR_LENGTH", + BeamBuiltinMethods.CHAR_LENGTH_METHOD, + SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator ENDS_WITH = - createUdfOperator("ENDS_WITH", BeamBuiltinMethods.ENDS_WITH_METHOD); + createUdfOperator( + "ENDS_WITH", + BeamBuiltinMethods.ENDS_WITH_METHOD, + SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator LIKE = - createUdfOperator("LIKE", BeamBuiltinMethods.LIKE_METHOD, SqlSyntax.BINARY); + createUdfOperator( + "LIKE", + BeamBuiltinMethods.LIKE_METHOD, + SqlSyntax.BINARY, + SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator VALIDATE_TIMESTAMP = createUdfOperator( @@ -119,7 +141,8 @@ public class SqlOperators { DateTimeUtils.class, "validateTimestamp", x -> NULLABLE_TIMESTAMP, - ImmutableList.of(TIMESTAMP)); + ImmutableList.of(TIMESTAMP), + SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator VALIDATE_TIME_INTERVAL = createUdfOperator( @@ -127,13 +150,18 @@ public class SqlOperators { DateTimeUtils.class, "validateTimeInterval", x -> NULLABLE_BIGINT, - ImmutableList.of(BIGINT, OTHER)); + ImmutableList.of(BIGINT, OTHER), + SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator TIMESTAMP_OP = - createUdfOperator("TIMESTAMP", BeamBuiltinMethods.TIMESTAMP_METHOD); + createUdfOperator( + "TIMESTAMP", + BeamBuiltinMethods.TIMESTAMP_METHOD, + SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlOperator DATE_OP = - createUdfOperator("DATE", BeamBuiltinMethods.DATE_METHOD); + createUdfOperator( + "DATE", BeamBuiltinMethods.DATE_METHOD, SqlAnalyzer.ZETASQL_FUNCTION_GROUP_NAME); public static final SqlUserDefinedFunction CAST_OP = new SqlUserDefinedFunction( @@ -158,7 +186,7 @@ public static SqlFunction createZetaSqlFunction(String name, SqlTypeName returnT SqlFunctionCategory.USER_DEFINED_FUNCTION); } - private static SqlUserDefinedAggFunction createUdafOperator( + public static SqlUserDefinedAggFunction createUdafOperator( String name, SqlReturnTypeInference returnTypeInference, AggregateFunction function) { return new SqlUserDefinedAggFunction( new SqlIdentifier(name, SqlParserPos.ZERO), @@ -177,26 +205,25 @@ private static SqlUserDefinedFunction createUdfOperator( Class methodClass, String methodName, SqlReturnTypeInference returnTypeInference, - List paramTypes) { + List paramTypes, + String funGroup) { return new SqlUserDefinedFunction( new SqlIdentifier(name, SqlParserPos.ZERO), returnTypeInference, null, null, paramTypes, - ScalarFunctionImpl.create(methodClass, methodName)); + ZetaSqlScalarFunctionImpl.create(methodClass, methodName, funGroup)); } - // Helper function to create SqlUserDefinedFunction based on a function name and a method. - // SqlUserDefinedFunction will be able to pass through Calcite codegen and get proper function - // called. - private static SqlUserDefinedFunction createUdfOperator(String name, Method method) { - return createUdfOperator(name, method, SqlSyntax.FUNCTION); + public static SqlUserDefinedFunction createUdfOperator( + String name, Method method, String funGroup) { + return createUdfOperator(name, method, SqlSyntax.FUNCTION, funGroup); } private static SqlUserDefinedFunction createUdfOperator( - String name, Method method, final SqlSyntax syntax) { - Function function = ScalarFunctionImpl.create(method); + String name, Method method, final SqlSyntax syntax, String funGroup) { + Function function = ZetaSqlScalarFunctionImpl.create(method, funGroup); final RelDataTypeFactory typeFactory = createTypeFactory(); List argTypes = new ArrayList<>(); diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ZetaSqlScalarFunctionImpl.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ZetaSqlScalarFunctionImpl.java new file mode 100644 index 000000000000..5255eec75e59 --- /dev/null +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/ZetaSqlScalarFunctionImpl.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.beam.sdk.extensions.sql.zetasql.translation; + +import java.lang.reflect.Method; +import org.apache.beam.sdk.extensions.sql.impl.ScalarFunctionImpl; +import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.adapter.enumerable.CallImplementor; +import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.Function; +import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.ScalarFunction; + +/** ZetaSQL-specific extension to {@link ScalarFunctionImpl}. */ +public class ZetaSqlScalarFunctionImpl extends ScalarFunctionImpl { + /** + * ZetaSQL function group identifier. Different function groups may have divergent translation + * paths. + */ + public final String functionGroup; + + private ZetaSqlScalarFunctionImpl( + Method method, CallImplementor implementor, String functionGroup) { + super(method, implementor); + this.functionGroup = functionGroup; + } + + /** + * Creates {@link org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.Function} from + * given class. + * + *

    If a method of the given name is not found or it does not suit, returns {@code null}. + * + * @param clazz class that is used to implement the function + * @param methodName Method name (typically "eval") + * @param functionGroup ZetaSQL function group identifier. Different function groups may have + * divergent translation paths. + * @return created {@link ScalarFunction} or null + */ + public static Function create(Class clazz, String methodName, String functionGroup) { + return create(findMethod(clazz, methodName)); + } + + /** + * Creates {@link org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.Function} from + * given method. When {@code eval} method does not suit, {@code null} is returned. + * + * @param method method that is used to implement the function + * @param functionGroup ZetaSQL function group identifier. Different function groups may have + * divergent translation paths. + * @return created {@link Function} or null + */ + public static Function create(Method method, String functionGroup) { + validateMethod(method); + CallImplementor implementor = createImplementor(method); + return new ZetaSqlScalarFunctionImpl(method, implementor, functionGroup); + } + + /* + * Finds a method in a given class by name. + * @param clazz class to search method in + * @param name name of the method to find + * @return the first method with matching name or null when no method found + */ + private static Method findMethod(Class clazz, String name) { + for (Method method : clazz.getMethods()) { + if (method.getName().equals(name) && !method.isBridge()) { + return method; + } + } + throw new NoSuchMethodError( + String.format("Method %s not found in class %s.", name, clazz.getName())); + } +} From 0e8725ce3e59f08472a20093edb8f93f969a5344 Mon Sep 17 00:00:00 2001 From: Masato Nakamura Date: Wed, 25 Nov 2020 00:57:35 +0900 Subject: [PATCH 0016/2127] Bump Gradle to 6.7.1 update to latest version https://docs.gradle.org/6.7.1/userguide/userguide.html --- gradle/wrapper/gradle-wrapper.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index be52383ef49c..4d9ca1649142 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 03535541ad174c14e6beba4bebac5fad77a90e92 Mon Sep 17 00:00:00 2001 From: Costi Ciudatu Date: Tue, 24 Nov 2020 19:01:46 +0200 Subject: [PATCH 0017/2127] [BEAM-11337] Make the ThriftCoder class visible from outside its package --- .../main/java/org/apache/beam/sdk/io/thrift/ThriftCoder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftCoder.java b/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftCoder.java index 242d58137230..56d9d4d6b18c 100644 --- a/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftCoder.java +++ b/sdks/java/io/thrift/src/main/java/org/apache/beam/sdk/io/thrift/ThriftCoder.java @@ -37,7 +37,7 @@ @SuppressWarnings({ "nullness" // TODO(https://issues.apache.org/jira/browse/BEAM-10402) }) -class ThriftCoder extends CustomCoder { +public class ThriftCoder extends CustomCoder { private final Class type; private final TProtocolFactory protocolFactory; @@ -57,7 +57,7 @@ protected ThriftCoder(Class type, TProtocolFactory protocolFactory) { * @return ThriftCoder initialize with class to be encoded/decoded and {@link TProtocolFactory} * used to encode/decode. */ - static ThriftCoder of(Class clazz, TProtocolFactory protocolFactory) { + public static ThriftCoder of(Class clazz, TProtocolFactory protocolFactory) { return new ThriftCoder<>(clazz, protocolFactory); } From 1f790fcd92e66a808cf2099aa86adeaa2bf1c5ab Mon Sep 17 00:00:00 2001 From: Nam Bui Date: Wed, 25 Nov 2020 07:27:51 +0700 Subject: [PATCH 0018/2127] [BEAM-11182][Website revamp] Implemented Stay up to date with Beam and Changed Works with components (#13406) * Implemented About social media component * Implemented About social media component * Implemented Stay up to date component * Updated supported integrations component * Review: Added comments --- .../www/site/assets/icons/calendar-icon.svg | 23 ++ website/www/site/assets/scss/_calendar.scss | 238 ++++++++++++++++++ .../assets/scss/{_logos.sass => _logos.scss} | 73 ++++-- website/www/site/assets/scss/_typography.scss | 48 +++- website/www/site/assets/scss/_vars.sass | 3 + website/www/site/assets/scss/main.scss | 3 +- website/www/site/data/en/calendar_events.yaml | 27 ++ website/www/site/i18n/home/calendar/en.yaml | 44 ++++ website/www/site/i18n/home/en.yaml | 2 - website/www/site/i18n/home/logos/en.yaml | 14 ++ website/www/site/layouts/_default/baseof.html | 1 + website/www/site/layouts/index.html | 73 +++++- .../partials/calendar/calendar-events.html | 33 +++ .../partials/pillars/pillars-social.html | 2 +- 14 files changed, 555 insertions(+), 29 deletions(-) create mode 100644 website/www/site/assets/icons/calendar-icon.svg create mode 100644 website/www/site/assets/scss/_calendar.scss rename website/www/site/assets/scss/{_logos.sass => _logos.scss} (50%) create mode 100644 website/www/site/data/en/calendar_events.yaml create mode 100644 website/www/site/i18n/home/calendar/en.yaml create mode 100644 website/www/site/i18n/home/logos/en.yaml create mode 100644 website/www/site/layouts/partials/calendar/calendar-events.html diff --git a/website/www/site/assets/icons/calendar-icon.svg b/website/www/site/assets/icons/calendar-icon.svg new file mode 100644 index 000000000000..1aa91dd4d3f8 --- /dev/null +++ b/website/www/site/assets/icons/calendar-icon.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/scss/_calendar.scss b/website/www/site/assets/scss/_calendar.scss new file mode 100644 index 000000000000..69e2bf81a291 --- /dev/null +++ b/website/www/site/assets/scss/_calendar.scss @@ -0,0 +1,238 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.calendar { + padding: $pad-l $pad; + + .calendar-title { + @extend .component-title; + + text-align: center; + } + + .calendar-content { + display: flex; + justify-content: center; + align-items: flex-start; + margin-top: 84px; + + a { + text-decoration: none; + } + + // Left card + .calendar-card-big { + width: 100%; + max-width: 381px; + height: 468px; + border-radius: 16px; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + background-color: $color-white; + padding: 32px 24px; + transition: 0.2s; + + .calendar-card-big-title { + @extend .component-large-header; + + margin-top: 48px; + margin-bottom: 24px; + } + } + + .calendar-card-big-left:hover { + box-shadow: 0 4px 20px 0 rgba(0, 0, 0, 0.24), + 0 4px 6px 0 rgba(0, 0, 0, 0.24); + } + + // Middle cards + .calendar-card-box { + margin: 0 37px; + min-height: 468px; + display: flex; + flex-direction: column; + justify-content: space-between; + + .calendar-card-small { + width: 100%; + max-width: 381px; + height: 216px; + border-radius: 16px; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + background-color: $color-white; + padding: 32px 24px; + transition: 0.2s; + + .calendar-card-small-title { + @extend .component-small-header; + + margin-top: 40px; + margin-bottom: 12px; + } + } + + .calendar-card-small:hover { + box-shadow: 0 4px 20px 0 rgba(0, 0, 0, 0.24), + 0 4px 6px 0 rgba(0, 0, 0, 0.24); + } + } + + // Right card + .calendar-card-big-right { + .calendar-card-event-title { + @extend .component-header; + } + + .calendar-card-events { + margin-top: 30px; + margin-bottom: 70px; + + .calendar-event { + display: flex; + padding: 14px; + + .calendar-event-icon { + margin-right: 12px; + } + + .calendar-event-description { + .calendar-event-title { + @extend .component-label; + } + + .calendar-event-place { + @extend .component-tag; + } + + .calendar-event-time { + @extend .component-tag; + } + } + } + + .calendar-event:hover { + background-color: rgba(196, 196, 196, 0.16); + border-radius: 16px; + } + + :last-child { + margin-bottom: 0; + } + } + + .calendar-card-events-button { + width: 115px; + height: 36px; + border-radius: 100px; + background-color: $color-sun; + border: none; + outline: none; + float: right; + + span { + @extend .component-label; + + font-size: 14px; + color: $color-white; + } + } + + button:hover { + opacity: 0.84; + } + } + } +} + +// Category for left and middle cards +.calendar-category { + display: flex; + justify-content: space-between; + + .calendar-category-tag { + @extend .component-tag; + + font-size: 14px; + font-weight: 500; + text-transform: uppercase; + } + + .calendar-category-date { + @extend .component-tag; + } +} + +// Author for left and middle cards +.calendar-card-author { + @extend .component-tag; +} + +@media (max-width: $tablet) { + .calendar { + padding: $pad-md $pad-s; + + .calendar-content { + flex-direction: column; + align-items: center; + margin-top: 64px; + + .calendar-card-big { + max-width: 327px; + height: 356px; + padding: 32px 20px; + + .calendar-card-big-title { + margin-top: 35px; + margin-bottom: 16px; + } + } + + .calendar-card-box { + margin: 24px 0px; + min-height: 456px; + + .calendar-card-small { + max-width: 327px; + height: 216px; + padding: 24px 20px; + + .calendar-card-small-title { + margin-top: 30px; + margin-bottom: 10px; + width: 280px; + } + } + } + + .calendar-card-big-right { + height: 404px; + + .calendar-card-events { + margin-top: 20px; + margin-bottom: 15px; + + .calendar-event { + padding: 14px 5px; + } + } + } + } + } +} diff --git a/website/www/site/assets/scss/_logos.sass b/website/www/site/assets/scss/_logos.scss similarity index 50% rename from website/www/site/assets/scss/_logos.sass rename to website/www/site/assets/scss/_logos.scss index fd0f6f30058c..33cd632d1fb8 100644 --- a/website/www/site/assets/scss/_logos.sass +++ b/website/www/site/assets/scss/_logos.scss @@ -15,22 +15,57 @@ * limitations under the License. */ -.logos - text-align: center - margin: $pad-xl 0 - - .logos__title - +type-h2 - margin-bottom: $pad-md - - .logos__logos - display: flex - justify-content: center - - .logos__logos__logo - line-height: 0 - margin: 0 $pad - +md - margin: 0 $pad-md - img - max-height: 70px +@import "media"; + +.logos { + padding: $pad-l $pad; + + .logos-title { + @extend .component-title; + + text-align: center; + } + + .logos-logos { + display: flex; + justify-content: space-between; + width: 100%; + max-width: 1111px; + margin: 70px auto 60px; + + .logos-logo { + line-height: 0; + + img { + max-height: 70px; + } + } + } +} + +@media (max-width: $tablet) { + .logos { + padding: $pad-md $pad-s; + + .logos-logos { + max-width: 360px; + flex-wrap: wrap; + justify-content: center; + margin: 50px auto 20px; + + .logos-logo { + margin-right: 60px; + margin-bottom: 50px; + + img { + max-height: 45px; + } + } + + :nth-child(3), + :last-child { + margin-right: 0; + } + } + } +} diff --git a/website/www/site/assets/scss/_typography.scss b/website/www/site/assets/scss/_typography.scss index d093140ace72..99eacfe4ca04 100644 --- a/website/www/site/assets/scss/_typography.scss +++ b/website/www/site/assets/scss/_typography.scss @@ -23,6 +23,17 @@ font-style: normal; line-height: 1.1; letter-spacing: normal; + margin: 0; + color: $color-gray; +} + +.component-large-header { + font-size: 30px; + font-weight: 500; + font-style: normal; + line-height: 1.4; + letter-spacing: normal; + margin: 0; color: $color-gray; } @@ -32,6 +43,17 @@ font-style: normal; line-height: 1.45; letter-spacing: 0.43px; + margin: 0; + color: $color-gray; +} + +.component-small-header { + font-size: 18px; + font-weight: 500; + font-style: normal; + line-height: 1.56; + letter-spacing: 0.43px; + margin: 0; color: $color-gray; } @@ -41,14 +63,36 @@ font-style: normal; line-height: 1.63; letter-spacing: 0.43px; + margin: 0; color: $color-gray; } +.component-label { + font-size: 16px; + font-weight: bold; + font-stretch: normal; + font-style: normal; + line-height: 1.5; + letter-spacing: 0.43px; + margin: 0; + color: $color-gray; +} + +.component-tag { + font-size: 16px; + font-weight: normal; + font-style: normal; + line-height: 1.63; + letter-spacing: 0.43px; + margin: 0; + color: $color-smoke; +} + @media (max-width: $tablet) { .component-title { font-size: 28px; } - .component-header { - font-weight: normal; + .component-large-header { + font-size: 24px; } } \ No newline at end of file diff --git a/website/www/site/assets/scss/_vars.sass b/website/www/site/assets/scss/_vars.sass index 6a98df42306a..df4276a042b1 100644 --- a/website/www/site/assets/scss/_vars.sass +++ b/website/www/site/assets/scss/_vars.sass @@ -21,6 +21,9 @@ $color-white: #FFF $color-light-gray: #F7F7F7 $color-dark-gray: #555 $color-gray: #333333 +$color-smoke: #8C8B8E +$color-sun: #F26628 +$color-silver: #C4C4C4 $pad-sm: 15px $pad-s: 24px diff --git a/website/www/site/assets/scss/main.scss b/website/www/site/assets/scss/main.scss index 8007760cbbbd..e4075a1fe51a 100644 --- a/website/www/site/assets/scss/main.scss +++ b/website/www/site/assets/scss/main.scss @@ -37,8 +37,9 @@ @import "_graphic.sass"; @import "_header.sass"; @import "_hero.sass"; -@import "_logos.sass"; +@import "_logos.scss"; @import "_pillars.scss"; @import "_section-nav.sass"; @import "_page-nav.sass"; @import "_table-wrapper.sass"; +@import "_calendar.scss"; \ No newline at end of file diff --git a/website/www/site/data/en/calendar_events.yaml b/website/www/site/data/en/calendar_events.yaml new file mode 100644 index 000000000000..865542f9c637 --- /dev/null +++ b/website/www/site/data/en/calendar_events.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- title: This is the title + place: Data Con LA + time: 2020/02/21 + icon: icons/calendar-icon.svg + url: # +- title: This is the title + place: Data Con LA + time: 2020/02/21 + icon: icons/calendar-icon.svg + url: # +- title: This is the title + place: Data Con LA + time: 2020/02/21 + icon: icons/calendar-icon.svg + url: # \ No newline at end of file diff --git a/website/www/site/i18n/home/calendar/en.yaml b/website/www/site/i18n/home/calendar/en.yaml new file mode 100644 index 000000000000..948854e1a09d --- /dev/null +++ b/website/www/site/i18n/home/calendar/en.yaml @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: +# Placeholder texts should be updated later +- id: home-calendar-title + translation: "Stay up to date with Beam" +- id: home-calendar-category-tag-first + translation: "Category" +- id: home-calendar-category-date-first + translation: "2020/02/21" +- id: home-calendar-card-title-first + translation: "Performance-Driven Runtime Type Checking for the Python SDK" +- id: home-calendar-card-author-first + translation: "Saavan Nanavati" +- id: home-calendar-category-tag-second + translation: "Category" +- id: home-calendar-category-date-second + translation: "2020/02/21" +- id: home-calendar-card-title-second + translation: "Performance-Driven Runtime Type Checking for the Python SDK" +- id: home-calendar-card-author-second + translation: "Saavan Nanavati" +- id: home-calendar-category-tag-third + translation: "Category" +- id: home-calendar-category-date-third + translation: "2020/02/21" +- id: home-calendar-card-title-third + translation: "Performance-Driven Runtime Type Checking for the Python SDK" +- id: home-calendar-card-author-third + translation: "Saavan Nanavati" +- id: home-calendar-card-events-title + translation: "Upcoming events" +- id: home-calendar-card-events-button + translation: "SEE MORE" diff --git a/website/www/site/i18n/home/en.yaml b/website/www/site/i18n/home/en.yaml index 43a3fea9bbc8..8a1ac53eecf8 100644 --- a/website/www/site/i18n/home/en.yaml +++ b/website/www/site/i18n/home/en.yaml @@ -28,8 +28,6 @@ translation: "Go Quickstart" - id: home-hero-blog-title translation: "The latest from the blog" -- id: home-logos-title - translation: "Works with" - id: home-cards-title translation: "Testimonials" - id: home-cards-body diff --git a/website/www/site/i18n/home/logos/en.yaml b/website/www/site/i18n/home/logos/en.yaml new file mode 100644 index 000000000000..7ed599a42ab5 --- /dev/null +++ b/website/www/site/i18n/home/logos/en.yaml @@ -0,0 +1,14 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- id: home-logos-title + translation: "Supported runners" \ No newline at end of file diff --git a/website/www/site/layouts/_default/baseof.html b/website/www/site/layouts/_default/baseof.html index f8fbfb6c04b1..6c245c51ac9f 100644 --- a/website/www/site/layouts/_default/baseof.html +++ b/website/www/site/layouts/_default/baseof.html @@ -21,6 +21,7 @@ {{ block "hero-section" . }}{{ end }} {{ block "pillars-section" . }}{{ end }} {{ block "graphic-section" . }}{{ end }} + {{ block "calendar-section" . }}{{ end }} {{ block "logos-section" . }}{{ end }} {{ block "cards-section" . }}{{ end }} {{ block "ctas-section" . }}{{ end }} diff --git a/website/www/site/layouts/index.html b/website/www/site/layouts/index.html index f173e4103a71..bbfdf71b9132 100644 --- a/website/www/site/layouts/index.html +++ b/website/www/site/layouts/index.html @@ -89,14 +89,79 @@

    {{ end }} +{{/* + TODO: + This is the implementation of the design. + The event sync functionality will be implemented when we have the event page. +*/}} +{{ define "calendar-section" }} +
    +

    + {{ T "home-calendar-title" }} +

    +
    + +
    +
    +

    {{ T "home-calendar-category-tag-first" }}

    +

    {{ T "home-calendar-category-date-first" }}

    +
    +

    + {{ T "home-calendar-card-title-first" }} +

    +

    {{ T "home-calendar-card-author-first" }}

    +
    +     +
    + +
    +
    +

    {{ T "home-calendar-category-tag-second" }}

    +

    {{ T "home-calendar-category-date-second" }}

    +
    +

    + {{ T "home-calendar-card-title-second" }} +

    +

    {{ T "home-calendar-card-author-second" }}

    +
    +     + +
    +
    +

    {{ T "home-calendar-category-tag-third" }}

    +

    {{ T "home-calendar-category-date-third" }}

    +
    +

    + {{ T "home-calendar-card-title-third" }} +

    +

    {{ T "home-calendar-card-author-third" }}

    +
    +     +
    +
    +

    {{ T "home-calendar-card-events-title" }}

    +
    + {{ $data := index $.Site.Data .Site.Language.Lang }} + {{ range $event := $data.calendar_events }} + {{ partial "calendar/calendar-events" (dict "icon" $event.icon "title" $event.title "place" $event.place "time" $event.time "url" $event.url) }} + {{ end }} +
    + + + +
    +
    +
    +{{ end }} + {{ define "logos-section" }} -
    -
    +
    +
    {{ T "home-logos-title" }}
    -
    +
    {{ range $logo := $.Site.Data.works_with }} - \ No newline at end of file +
    From 542e8ceb40a690ffc5a297c465f559840c9e965f Mon Sep 17 00:00:00 2001 From: Kyle Weaver Date: Thu, 19 Nov 2020 18:50:37 -0800 Subject: [PATCH 0019/2127] Make createUd(a)fOperator methods package-private. --- .../sdk/extensions/sql/zetasql/translation/SqlOperators.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java index 4564c1139f1a..592c8d25fa66 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java @@ -186,7 +186,7 @@ public static SqlFunction createZetaSqlFunction(String name, SqlTypeName returnT SqlFunctionCategory.USER_DEFINED_FUNCTION); } - public static SqlUserDefinedAggFunction createUdafOperator( + static SqlUserDefinedAggFunction createUdafOperator( String name, SqlReturnTypeInference returnTypeInference, AggregateFunction function) { return new SqlUserDefinedAggFunction( new SqlIdentifier(name, SqlParserPos.ZERO), @@ -216,8 +216,7 @@ private static SqlUserDefinedFunction createUdfOperator( ZetaSqlScalarFunctionImpl.create(methodClass, methodName, funGroup)); } - public static SqlUserDefinedFunction createUdfOperator( - String name, Method method, String funGroup) { + static SqlUserDefinedFunction createUdfOperator(String name, Method method, String funGroup) { return createUdfOperator(name, method, SqlSyntax.FUNCTION, funGroup); } From 7d88858e9fbce14befcc8f92dcbc932a6fe69b27 Mon Sep 17 00:00:00 2001 From: Sam Whittle Date: Fri, 30 Oct 2020 06:44:43 -0700 Subject: [PATCH 0020/2127] [BEAM-11300] Improve Nexmark performance: Change Query3 to use stateful processing and timers without CoGroupByKey which introduces additional windowing and shuffling. Change Query5 to avoid unnecessary lists when combining. Change Query10 to avoid excessive logging. --- .../beam/sdk/nexmark/queries/Query10.java | 8 +- .../beam/sdk/nexmark/queries/Query3.java | 191 +++++++++--------- .../beam/sdk/nexmark/queries/Query5.java | 133 ++++++++---- 3 files changed, 183 insertions(+), 149 deletions(-) diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java index c8b58f9a5d6b..0c5e8b901bf6 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query10.java @@ -188,7 +188,7 @@ public PCollection expand(PCollection events) { public void processElement(ProcessContext c) { if (c.element().hasAnnotation("LATE")) { lateCounter.inc(); - LOG.info("Observed late: {}", c.element()); + LOG.debug("Observed late: {}", c.element()); } else { onTimeCounter.inc(); } @@ -240,7 +240,7 @@ public void processElement(ProcessContext c, BoundedWindow window) { } } String shard = c.element().getKey(); - LOG.info( + LOG.debug( String.format( "%s with timestamp %s has %d actually late and %d on-time " + "elements in pane %s for window %s", @@ -289,7 +289,7 @@ public void processElement(ProcessContext c, BoundedWindow window) String shard = c.element().getKey(); GcsOptions options = c.getPipelineOptions().as(GcsOptions.class); OutputFile outputFile = outputFileFor(window, shard, c.pane()); - LOG.info( + LOG.debug( String.format( "Writing %s with record timestamp %s, window timestamp %s, pane %s", shard, c.timestamp(), window.maxTimestamp(), c.pane())); @@ -350,7 +350,7 @@ public void processElement(ProcessContext c, BoundedWindow window) LOG.error("ERROR! Unexpected ON_TIME pane index: {}", c.pane()); } else { GcsOptions options = c.getPipelineOptions().as(GcsOptions.class); - LOG.info( + LOG.debug( "Index with record timestamp {}, window timestamp {}, pane {}", c.timestamp(), window.maxTimestamp(), diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3.java index 0d23d5fa422d..11ca3d7aa792 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query3.java @@ -17,9 +17,6 @@ */ package org.apache.beam.sdk.nexmark.queries; -import java.util.ArrayList; -import java.util.List; -import org.apache.beam.sdk.coders.ListCoder; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.nexmark.NexmarkConfiguration; @@ -27,6 +24,7 @@ import org.apache.beam.sdk.nexmark.model.Event; import org.apache.beam.sdk.nexmark.model.NameCityStateId; import org.apache.beam.sdk.nexmark.model.Person; +import org.apache.beam.sdk.state.BagState; import org.apache.beam.sdk.state.StateSpec; import org.apache.beam.sdk.state.StateSpecs; import org.apache.beam.sdk.state.TimeDomain; @@ -36,16 +34,11 @@ import org.apache.beam.sdk.state.ValueState; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.Filter; +import org.apache.beam.sdk.transforms.Flatten; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.transforms.join.CoGbkResult; -import org.apache.beam.sdk.transforms.join.CoGroupByKey; -import org.apache.beam.sdk.transforms.join.KeyedPCollectionTuple; -import org.apache.beam.sdk.transforms.windowing.AfterPane; -import org.apache.beam.sdk.transforms.windowing.GlobalWindows; -import org.apache.beam.sdk.transforms.windowing.Repeatedly; -import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PCollectionList; import org.joda.time.Duration; import org.joda.time.Instant; import org.slf4j.Logger; @@ -83,27 +76,29 @@ public Query3(NexmarkConfiguration configuration) { @Override public PCollection expand(PCollection events) { - int numEventsInPane = 30; - - PCollection eventsWindowed = - events.apply( - Window.into(new GlobalWindows()) - .triggering(Repeatedly.forever(AfterPane.elementCountAtLeast(numEventsInPane))) - .discardingFiredPanes() - .withAllowedLateness(Duration.ZERO)); - PCollection> auctionsBySellerId = - eventsWindowed + PCollection> auctionsBySellerId = + events // Only want the new auction events. .apply(NexmarkQueryUtil.JUST_NEW_AUCTIONS) // We only want auctions in category 10. .apply(name + ".InCategory", Filter.by(auction -> auction.category == 10)) - // Key auctions by their seller id. - .apply("AuctionBySeller", NexmarkQueryUtil.AUCTION_BY_SELLER); + // Key auctions by their seller id and move to union Event type. + .apply( + "EventByAuctionSeller", + ParDo.of( + new DoFn>() { + @ProcessElement + public void processElement(ProcessContext c) { + Event e = new Event(); + e.newAuction = c.element(); + c.output(KV.of(c.element().seller, e)); + } + })); - PCollection> personsById = - eventsWindowed + PCollection> personsById = + events // Only want the new people events. .apply(NexmarkQueryUtil.JUST_NEW_PERSONS) @@ -116,18 +111,24 @@ public PCollection expand(PCollection events) { || "ID".equals(person.state) || "CA".equals(person.state))) - // Key people by their id. - .apply("PersonById", NexmarkQueryUtil.PERSON_BY_ID); + // Key persons by their id and move to the union event type. + .apply( + "EventByPersonId", + ParDo.of( + new DoFn>() { + @ProcessElement + public void processElement(ProcessContext c) { + Event e = new Event(); + e.newPerson = c.element(); + c.output(KV.of(c.element().id, e)); + } + })); - return // Join auctions and people. - // concatenate KeyedPCollections - KeyedPCollectionTuple.of(NexmarkQueryUtil.AUCTION_TAG, auctionsBySellerId) - .and(NexmarkQueryUtil.PERSON_TAG, personsById) - // group auctions and persons by personId - .apply(CoGroupByKey.create()) + return PCollectionList.of(auctionsBySellerId) + .and(personsById) + .apply(Flatten.pCollections()) .apply(name + ".Join", ParDo.of(joinDoFn)) - // Project what we want. .apply( name + ".Project", @@ -154,8 +155,11 @@ public void processElement(ProcessContext c) { *

    However we know that each auction is associated with at most one person, so only need to * store auction records in persistent state until we have seen the corresponding person record. * And of course may have already seen that record. + * + *

    To prevent state from accumulating over time, we cleanup buffered people or auctions after a + * max waiting time. */ - private static class JoinDoFn extends DoFn, KV> { + private static class JoinDoFn extends DoFn, KV> { private final int maxAuctionsWaitingTime; private static final String AUCTIONS = "auctions"; @@ -164,13 +168,12 @@ private static class JoinDoFn extends DoFn, KV> personSpec = StateSpecs.value(Person.CODER); - private static final String PERSON_STATE_EXPIRING = "personStateExpiring"; + private static final String STATE_EXPIRING = "stateExpiring"; @StateId(AUCTIONS) - private final StateSpec>> auctionsSpec = - StateSpecs.value(ListCoder.of(Auction.CODER)); + private final StateSpec> auctionsSpec = StateSpecs.bag(Auction.CODER); - @TimerId(PERSON_STATE_EXPIRING) + @TimerId(STATE_EXPIRING) private final TimerSpec timerSpec = TimerSpecs.timer(TimeDomain.EVENT_TIME); // Used to refer the metrics namespace @@ -178,7 +181,6 @@ private static class JoinDoFn extends DoFn, KV personState, - @StateId(AUCTIONS) ValueState> auctionsState) { + @Element KV element, + OutputReceiver> output, + @TimerId(STATE_EXPIRING) Timer timer, + @StateId(PERSON) @AlwaysFetched ValueState personState, + @StateId(AUCTIONS) BagState auctionsState) { // We would *almost* implement this by rewindowing into the global window and // running a combiner over the result. The combiner's accumulator would be the // state we use below. However, combiners cannot emit intermediate results, thus - // we need to wait for the pending ReduceFn API. + // we need to wait for the pending ReduceFn API Person existingPerson = personState.read(); - if (existingPerson != null) { - // We've already seen the new person event for this person id. - // We can join with any new auctions on-the-fly without needing any - // additional persistent state. - for (Auction newAuction : c.element().getValue().getAll(NexmarkQueryUtil.AUCTION_TAG)) { - newAuctionCounter.inc(); - newOldOutputCounter.inc(); - c.output(KV.of(newAuction, existingPerson)); - } - return; - } - - Person theNewPerson = null; - for (Person newPerson : c.element().getValue().getAll(NexmarkQueryUtil.PERSON_TAG)) { - if (theNewPerson == null) { - theNewPerson = newPerson; + Event event = element.getValue(); + Instant eventTime = null; + // Event is a union object, handle a new person or auction. + if (event.newPerson != null) { + Person person = event.newPerson; + eventTime = person.dateTime; + if (existingPerson == null) { + newPersonCounter.inc(); + personState.write(person); + // We've now seen the person for this person id so can flush any + // pending auctions for the same seller id (an auction is done by only one seller). + Iterable pendingAuctions = auctionsState.read(); + if (pendingAuctions != null) { + for (Auction pendingAuction : pendingAuctions) { + oldNewOutputCounter.inc(); + output.output(KV.of(pendingAuction, person)); + } + auctionsState.clear(); + } } else { - if (theNewPerson.equals(newPerson)) { - LOG.error("Duplicate person {}", theNewPerson); + if (person.equals(existingPerson)) { + LOG.error("Duplicate person {}", person); } else { - LOG.error("Conflicting persons {} and {}", theNewPerson, newPerson); + LOG.error("Conflicting persons {} and {}", existingPerson, person); } fatalCounter.inc(); - continue; } - newPersonCounter.inc(); - // We've now seen the person for this person id so can flush any - // pending auctions for the same seller id (an auction is done by only one seller). - List pendingAuctions = auctionsState.read(); - if (pendingAuctions != null) { - for (Auction pendingAuction : pendingAuctions) { - oldNewOutputCounter.inc(); - c.output(KV.of(pendingAuction, newPerson)); - } - auctionsState.clear(); - } - // Also deal with any new auctions. - for (Auction newAuction : c.element().getValue().getAll(NexmarkQueryUtil.AUCTION_TAG)) { + } else if (event.newAuction != null) { + Auction auction = event.newAuction; + eventTime = auction.dateTime; + newAuctionCounter.inc(); + if (existingPerson == null) { + auctionsState.add(auction); + } else { newAuctionCounter.inc(); - newNewOutputCounter.inc(); - c.output(KV.of(newAuction, newPerson)); + newOldOutputCounter.inc(); + output.output(KV.of(auction, existingPerson)); } - // Remember this person for any future auctions. - personState.write(newPerson); - // set a time out to clear this state + } else { + LOG.error("Only expecting people or auctions but received {}", event); + fatalCounter.inc(); + } + if (eventTime != null) { + // Set or reset the cleanup timer to clear the state. Instant firingTime = - new Instant(newPerson.dateTime).plus(Duration.standardSeconds(maxAuctionsWaitingTime)); + new Instant(eventTime).plus(Duration.standardSeconds(maxAuctionsWaitingTime)); timer.set(firingTime); } - if (theNewPerson != null) { - return; - } - - // We'll need to remember the auctions until we see the corresponding - // new person event. - List pendingAuctions = auctionsState.read(); - if (pendingAuctions == null) { - pendingAuctions = new ArrayList<>(); - } - for (Auction newAuction : c.element().getValue().getAll(NexmarkQueryUtil.AUCTION_TAG)) { - newAuctionCounter.inc(); - pendingAuctions.add(newAuction); - } - auctionsState.write(pendingAuctions); } - @OnTimer(PERSON_STATE_EXPIRING) + @OnTimer(STATE_EXPIRING) public void onTimerCallback( - OnTimerContext context, @StateId(PERSON) ValueState personState) { + OnTimerContext context, + @StateId(PERSON) ValueState personState, + @StateId(AUCTIONS) BagState auctionState) { personState.clear(); + auctionState.clear(); } } } diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java index 76d9e64d3245..77d64ab0b07b 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/queries/Query5.java @@ -18,12 +18,18 @@ package org.apache.beam.sdk.nexmark.queries; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.Objects; +import org.apache.beam.sdk.coders.Coder; +import org.apache.beam.sdk.coders.CoderRegistry; import org.apache.beam.sdk.nexmark.NexmarkConfiguration; import org.apache.beam.sdk.nexmark.model.AuctionCount; import org.apache.beam.sdk.nexmark.model.Event; +import org.apache.beam.sdk.nexmark.queries.Query5.TopCombineFn.Accum; +import org.apache.beam.sdk.schemas.JavaFieldSchema; +import org.apache.beam.sdk.schemas.SchemaCoder; import org.apache.beam.sdk.transforms.Combine; +import org.apache.beam.sdk.transforms.Combine.AccumulatingCombineFn; import org.apache.beam.sdk.transforms.Count; import org.apache.beam.sdk.transforms.DoFn; import org.apache.beam.sdk.transforms.ParDo; @@ -31,6 +37,10 @@ import org.apache.beam.sdk.transforms.windowing.Window; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.TypeDescriptor; +import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables; +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; /** @@ -53,6 +63,80 @@ public class Query5 extends NexmarkQueryTransform { private final NexmarkConfiguration configuration; + /** CombineFn that takes bidders with counts and keeps all bidders with the top count. */ + public static class TopCombineFn + extends AccumulatingCombineFn, Accum, KV>> { + @Override + public Accum createAccumulator() { + return new Accum(); + } + + @Override + public Coder getAccumulatorCoder( + @NonNull CoderRegistry registry, @NonNull Coder> inputCoder) { + JavaFieldSchema provider = new JavaFieldSchema(); + TypeDescriptor typeDescriptor = new TypeDescriptor() {}; + return SchemaCoder.of( + provider.schemaFor(typeDescriptor), + typeDescriptor, + provider.toRowFunction(typeDescriptor), + provider.fromRowFunction(typeDescriptor)); + } + + /** Accumulator that takes bidders with counts and keeps all bidders with the top count. */ + public static class Accum + implements AccumulatingCombineFn.Accumulator, Accum, KV>> { + + public ArrayList auctions = new ArrayList<>(); + public long count = 0; + + @Override + public void addInput(KV input) { + if (input.getValue() > count) { + count = input.getValue(); + auctions.clear(); + auctions.add(input.getKey()); + } else if (input.getValue() == count) { + auctions.add(input.getKey()); + } + } + + @Override + public void mergeAccumulator(Accum other) { + if (other.count > this.count) { + this.count = other.count; + this.auctions.clear(); + this.auctions.addAll(other.auctions); + } else if (other.count == this.count) { + this.auctions.addAll(other.auctions); + } + } + + @Override + public KV> extractOutput() { + return KV.of(count, auctions); + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Accum other = (Accum) o; + return this.count == other.count && Iterables.elementsEqual(this.auctions, other.auctions); + } + + @Override + public int hashCode() { + return Objects.hash(count, auctions); + } + } + } + public Query5(NexmarkConfiguration configuration) { super("Query5"); this.configuration = configuration; @@ -74,58 +158,19 @@ public PCollection expand(PCollection events) { // Count the number of bids per auction id. .apply(Count.perElement()) - // We'll want to keep all auctions with the maximal number of bids. - // Start by lifting each into a singleton list. - // need to do so because bellow combine returns a list of auctions in the key in case of - // equal number of bids. Combine needs to have same input type and return type. - .apply( - name + ".ToSingletons", - ParDo.of( - new DoFn, KV, Long>>() { - @ProcessElement - public void processElement(ProcessContext c) { - c.output( - KV.of( - Collections.singletonList(c.element().getKey()), - c.element().getValue())); - } - })) - // Keep only the auction ids with the most bids. .apply( - Combine.globally( - new Combine.BinaryCombineFn, Long>>() { - @Override - public KV, Long> apply( - KV, Long> left, KV, Long> right) { - List leftBestAuctions = left.getKey(); - long leftCount = left.getValue(); - List rightBestAuctions = right.getKey(); - long rightCount = right.getValue(); - if (leftCount > rightCount) { - return left; - } else if (leftCount < rightCount) { - return right; - } else { - List newBestAuctions = new ArrayList<>(); - newBestAuctions.addAll(leftBestAuctions); - newBestAuctions.addAll(rightBestAuctions); - return KV.of(newBestAuctions, leftCount); - } - } - }) - .withoutDefaults() - .withFanout(configuration.fanout)) + Combine.globally(new TopCombineFn()).withoutDefaults().withFanout(configuration.fanout)) // Project into result. .apply( name + ".Select", ParDo.of( - new DoFn, Long>, AuctionCount>() { + new DoFn>, AuctionCount>() { @ProcessElement public void processElement(ProcessContext c) { - long count = c.element().getValue(); - for (long auction : c.element().getKey()) { + long count = c.element().getKey(); + for (long auction : c.element().getValue()) { c.output(new AuctionCount(auction, count)); } } From 243e5df54bd7704d090a429bef442d44409a34f1 Mon Sep 17 00:00:00 2001 From: Sam Whittle Date: Fri, 20 Nov 2020 01:51:34 -0800 Subject: [PATCH 0021/2127] [BEAM-11391] Improve Nexmark Kafka support to track watermarks using commit time, allow specifying the number of partitions instead of requiring launcher to have access to kafka cluster, and removing unneeded deserialization of keys. --- .../beam/sdk/nexmark/NexmarkLauncher.java | 38 ++++++++++++++----- .../beam/sdk/nexmark/NexmarkOptions.java | 14 +++++++ 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java index fd2044f652e9..52b96a4ef39e 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkLauncher.java @@ -104,9 +104,9 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps; +import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.serialization.ByteArrayDeserializer; import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.LongDeserializer; import org.apache.kafka.common.serialization.StringSerializer; import org.checkerframework.checker.nullness.qual.Nullable; import org.joda.time.Duration; @@ -717,11 +717,12 @@ private void sinkEventsToKafka(PCollection events) { .withBootstrapServers(options.getBootstrapServers()) .withTopic(options.getKafkaTopic()) .withValueSerializer(ByteArraySerializer.class) + .withInputTimestamp() .values()); } - static final DoFn, Event> BYTEARRAY_TO_EVENT = - new DoFn, Event>() { + static final DoFn, Event> BYTEARRAY_TO_EVENT = + new DoFn, Event>() { @ProcessElement public void processElement(ProcessContext c) throws IOException { byte[] encodedEvent = c.element().getValue(); @@ -731,20 +732,35 @@ public void processElement(ProcessContext c) throws IOException { }; /** Return source of events from Kafka. */ - private PCollection sourceEventsFromKafka(Pipeline p, final Instant now) { + private PCollection sourceEventsFromKafka(Pipeline p, final Instant start) { checkArgument((options.getBootstrapServers() != null), "Missing --bootstrapServers"); NexmarkUtils.console("Reading events from Kafka Topic %s", options.getKafkaTopic()); - KafkaIO.Read read = - KafkaIO.read() + KafkaIO.Read read = + KafkaIO.read() .withBootstrapServers(options.getBootstrapServers()) - .withTopic(options.getKafkaTopic()) - .withKeyDeserializer(LongDeserializer.class) + .withKeyDeserializer(ByteArrayDeserializer.class) .withValueDeserializer(ByteArrayDeserializer.class) - .withStartReadTime(now) + .withStartReadTime(start) .withMaxNumRecords( options.getNumEvents() != null ? options.getNumEvents() : Long.MAX_VALUE); + if (options.getKafkaTopicCreateTimeMaxDelaySec() >= 0) { + read = + read.withCreateTime( + Duration.standardSeconds(options.getKafkaTopicCreateTimeMaxDelaySec())); + } + + if (options.getNumKafkaTopicPartitions() > 0) { + ArrayList partitionArrayList = new ArrayList<>(); + for (int i = 0; i < options.getNumKafkaTopicPartitions(); ++i) { + partitionArrayList.add(new TopicPartition(options.getKafkaTopic(), i)); + } + read = read.withTopicPartitions(partitionArrayList); + } else { + read = read.withTopic(options.getKafkaTopic()); + } + return p.apply(queryName + ".ReadKafkaEvents", read.withoutMetadata()) .apply(queryName + ".KafkaToEvents", ParDo.of(BYTEARRAY_TO_EVENT)); } @@ -802,6 +818,7 @@ private void sinkResultsToKafka(PCollection formattedResults) { .withBootstrapServers(options.getBootstrapServers()) .withTopic(options.getKafkaResultsTopic()) .withValueSerializer(StringSerializer.class) + .withInputTimestamp() .values()); } @@ -1012,7 +1029,8 @@ private PCollection createSource(Pipeline p, final Instant now) throws IO // finished. In other case. when pubSubMode=SUBSCRIBE_ONLY, now should be null and // it will be ignored. source = - sourceEventsFromKafka(p, configuration.pubSubMode == COMBINED ? now : null); + sourceEventsFromKafka( + p, configuration.pubSubMode == COMBINED ? now : Instant.EPOCH); } else { source = sourceEventsFromPubsub(p); } diff --git a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java index 31a459edf736..7fa560759c1a 100644 --- a/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java +++ b/sdks/java/testing/nexmark/src/main/java/org/apache/beam/sdk/nexmark/NexmarkOptions.java @@ -429,6 +429,20 @@ void setPubsubMessageSerializationMethod( void setKafkaTopic(String value); + @Description( + "Number of partitions for Kafka topic in streaming mode. If unspecified, the broker will be queried for all partitions.") + int getNumKafkaTopicPartitions(); + + void setNumKafkaTopicPartitions(int value); + + @Description( + "If non-negative, events from the Kafka topic will get their timestamps from the Kafka createtime, with the maximum delay for" + + "disorder as specified.") + @Default.Integer(60) + int getKafkaTopicCreateTimeMaxDelaySec(); + + void setKafkaTopicCreateTimeMaxDelaySec(int value); + @Description("Base name of Kafka results topic in streaming mode.") @Default.String("nexmark-results") @Nullable From dad20e96472d16b19000ec61d679c9ba23038e6f Mon Sep 17 00:00:00 2001 From: Sruthi Sree Kumar Date: Thu, 3 Dec 2020 16:33:03 +0100 Subject: [PATCH 0022/2127] added not run tests --- .test-infra/validate-runner/build.gradle | 12 +- .test-infra/validate-runner/gradlew | 172 ------------------ .test-infra/validate-runner/gradlew.bat | 84 --------- .test-infra/validate-runner/settings.gradle | 2 - .../org/apache/beam/validate/runner/Main.java | 8 +- .../validate/runner/model/CaseResult.java | 17 ++ .../runner/service/BatchTestService.java | 16 +- .../runner/service/StreamTestService.java | 19 +- .../validate/runner/service/TestService.java | 51 ++++-- 9 files changed, 85 insertions(+), 296 deletions(-) delete mode 100755 .test-infra/validate-runner/gradlew delete mode 100644 .test-infra/validate-runner/gradlew.bat delete mode 100644 .test-infra/validate-runner/settings.gradle diff --git a/.test-infra/validate-runner/build.gradle b/.test-infra/validate-runner/build.gradle index 530a40b8fdc5..d5a29ccd6e4e 100644 --- a/.test-infra/validate-runner/build.gradle +++ b/.test-infra/validate-runner/build.gradle @@ -17,9 +17,9 @@ */ //plugins { id 'org.apache.beam.module' } -plugins { - id 'java' -} +apply plugin : "application" +mainClassName = 'org.apache.beam.validate.runner.Main' + group 'org.apache.beam' //version '1.0-SNAPSHOT' description = "Apache Beam :: Validate :: Runner" @@ -43,9 +43,3 @@ dependencies { compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.11.3' compile 'org.slf4j:slf4j-simple:1.7.9' } - - - -//wrapper { -// gradleVersion = '5.0' -//} \ No newline at end of file diff --git a/.test-infra/validate-runner/gradlew b/.test-infra/validate-runner/gradlew deleted file mode 100755 index af6708ff229f..000000000000 --- a/.test-infra/validate-runner/gradlew +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env sh - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin, switch paths to Windows format before running java -if $cygwin ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/.test-infra/validate-runner/gradlew.bat b/.test-infra/validate-runner/gradlew.bat deleted file mode 100644 index 6d57edc706c9..000000000000 --- a/.test-infra/validate-runner/gradlew.bat +++ /dev/null @@ -1,84 +0,0 @@ -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/.test-infra/validate-runner/settings.gradle b/.test-infra/validate-runner/settings.gradle deleted file mode 100644 index ed87891f81d3..000000000000 --- a/.test-infra/validate-runner/settings.gradle +++ /dev/null @@ -1,2 +0,0 @@ -rootProject.name = 'validare-runner' - diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java index 016921cd2809..fe83c5867bd6 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java @@ -1,6 +1,6 @@ package org.apache.beam.validate.runner; -import net.sf.json.JSONObject; +import net.sf.json.JSONArray; import org.apache.beam.validate.runner.service.BatchTestService; import org.apache.beam.validate.runner.service.StreamTestService; import org.slf4j.Logger; @@ -15,15 +15,15 @@ public static void main(String args[]) { try { final Logger logger = LoggerFactory.getLogger(Main.class); - JSONObject outputDetails = new JSONObject(); + JSONArray outputDetails = new JSONArray(); logger.info("Processing Batch Jobs:"); BatchTestService batchTestService = new BatchTestService(); - outputDetails.put("batch", batchTestService.getBatchTests()); + outputDetails.add(batchTestService.getBatchTests()); logger.info("Processing Stream Jobs:"); StreamTestService streamTestService = new StreamTestService(); - outputDetails.put("stream", streamTestService.getStreamTests()); + outputDetails.add(streamTestService.getStreamTests()); try (FileWriter file = new FileWriter("out.json")) { file.write(outputDetails.toString(3)); diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/model/CaseResult.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/model/CaseResult.java index 43966429d626..885a5fe9a470 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/model/CaseResult.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/model/CaseResult.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import java.util.Objects; + @JsonIgnoreProperties(ignoreUnknown = true) public class CaseResult { private String className; @@ -41,4 +43,19 @@ public String getStatus() { public void setStatus(String status) { this.status = status; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CaseResult that = (CaseResult) o; + return className.equals(that.className) && + status.equals(that.status) && + name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(className, status, name); + } } diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/BatchTestService.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/BatchTestService.java index 38eff7574627..90b527102746 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/BatchTestService.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/BatchTestService.java @@ -1,24 +1,30 @@ package org.apache.beam.validate.runner.service; import com.fasterxml.jackson.databind.ObjectMapper; -import net.sf.json.JSONArray; import net.sf.json.JSONObject; +import org.apache.beam.validate.runner.model.CaseResult; import org.apache.beam.validate.runner.model.Configuration; import org.apache.beam.validate.runner.model.TestResult; import org.apache.beam.validate.runner.util.FileReaderUtil; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; public class BatchTestService implements TestService { - + private static Set> batchTests = new HashSet<>(); + private HashMap> map = new HashMap<>(); public JSONObject getBatchTests() { - JSONArray batchObject = new JSONArray(); try { Configuration configuration = FileReaderUtil.readConfiguration(); for(Map job : configuration.getBatch()) { try { TestResult testResult = new ObjectMapper().readValue(getUrl(job,configuration), TestResult.class); - batchObject.add(getBatchObject(job,testResult)); + batchTests.addAll(getTestNames(testResult)); + map.put((String) job.keySet().toArray()[0], getAllTests(testResult)); } catch (Exception ex) { ex.printStackTrace(); } @@ -27,7 +33,7 @@ public JSONObject getBatchTests() { ex.printStackTrace(); } JSONObject outputDetails = new JSONObject(); - outputDetails.put("batch", batchObject); + outputDetails.put("batch", process(batchTests, map)); return outputDetails; } } diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/StreamTestService.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/StreamTestService.java index 1a8f42f5b55e..05cd0555af95 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/StreamTestService.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/StreamTestService.java @@ -1,25 +1,30 @@ package org.apache.beam.validate.runner.service; import com.fasterxml.jackson.databind.ObjectMapper; -import net.sf.json.JSONArray; import net.sf.json.JSONObject; +import org.apache.beam.validate.runner.model.CaseResult; import org.apache.beam.validate.runner.model.Configuration; import org.apache.beam.validate.runner.model.TestResult; import org.apache.beam.validate.runner.util.FileReaderUtil; +import org.apache.commons.lang3.tuple.Pair; -import java.net.URISyntaxException; +import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; -public class StreamTestService implements TestService { - public JSONObject getStreamTests() throws URISyntaxException { - JSONArray streamObject = new JSONArray(); +public class StreamTestService implements TestService { + private static Set> streamTests = new HashSet<>(); + private HashMap> map = new HashMap<>(); + public JSONObject getStreamTests() { try { Configuration configuration = FileReaderUtil.readConfiguration(); for (Map job : configuration.getStream()) { try { TestResult result = new ObjectMapper().readValue(getUrl(job, configuration), TestResult.class); - streamObject.add(getBatchObject(job, result)); + streamTests.addAll(getTestNames(result)); + map.put((String) job.keySet().toArray()[0], getAllTests(result)); } catch (Exception ex) { ex.printStackTrace(); } @@ -28,7 +33,7 @@ public JSONObject getStreamTests() throws URISyntaxException { ex.printStackTrace(); } JSONObject outputDetails = new JSONObject(); - outputDetails.put("stream", streamObject); + outputDetails.put("stream", process(streamTests, map)); return outputDetails; } } diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java index b94bb5c761a5..62ffcbd652d3 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java @@ -8,20 +8,19 @@ import org.apache.beam.validate.runner.model.CaseResult; import org.apache.beam.validate.runner.model.Configuration; import org.apache.beam.validate.runner.model.TestResult; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; public interface TestService { - default List getAllTests(TestResult testResult) { - List caseResults = new ArrayList<>(); + default Set getAllTests(TestResult testResult) { + Set caseResults = new HashSet<>(); Optional.ofNullable(testResult.getSuites()).ifPresent(suites -> suites.forEach(item -> caseResults.addAll(item.getCases()))); // List tests = new ArrayList<>(); // Optional.ofNullable(caseResults).ifPresent(cases -> cases.forEach(item -> tests.add(item.getClassName() + "." + item.getName() + " : " + item.getStatus()))); @@ -34,15 +33,41 @@ default URL getUrl(Map job, Configuration configuration) throws return new URL(jobWithDetails.getLastSuccessfulBuild().getUrl() + configuration.getJsonapi()); } - default JSONObject getBatchObject(Map job, TestResult result) { - JSONArray tests = new JSONArray(); - tests.addAll(getAllTests(result)); + default Set> getTestNames(TestResult testResult) { + Set> caseResults = new HashSet<>(); + Optional.ofNullable(testResult.getSuites()).ifPresent(suites -> suites.forEach(item -> item.getCases().forEach(caseResult -> caseResults.add(new ImmutablePair<>(caseResult.getName(), caseResult.getClassName()))))); +// List tests = new ArrayList<>(); +// Optional.ofNullable(caseResults).ifPresent(cases -> cases.forEach(item -> tests.add(item.getClassName() + "." + item.getName() + " : " + item.getStatus()))); + return caseResults; + } - JSONObject jsonOut = new JSONObject(); - jsonOut.put("testCases", tests); - JSONObject jsonMain = new JSONObject(); - jsonMain.put(job.keySet().toArray()[0], jsonOut); + default JSONObject process(Set> testnames, HashMap> map) { + map.forEach((k, v) -> { + Set caseResults = v; + for(Pair pair : testnames) { + boolean found = false; + for(CaseResult caseResult : caseResults) { + if (caseResult.getName().equals(pair.getKey())) { + found = true; + break; + } + } + if(found) { + found = false; + } else { + caseResults.add(new CaseResult(pair.getValue(), "NOT RUN", pair.getKey())); + } + } + }); + JSONObject jsonMain = new JSONObject(); + map.forEach((k, v) -> { + JSONArray tests = new JSONArray(); + tests.addAll(v); + JSONObject jsonOut = new JSONObject(); + jsonOut.put("testCases", tests); + jsonMain.put(k, jsonOut); + }); return jsonMain; } } From 102c317ec265973d52dee25883d5d64ee66c563a Mon Sep 17 00:00:00 2001 From: Sruthi Sree Kumar Date: Thu, 3 Dec 2020 16:34:36 +0100 Subject: [PATCH 0023/2127] added readme --- .test-infra/validate-runner/readme.md | 2 ++ settings.gradle | 2 ++ 2 files changed, 4 insertions(+) create mode 100644 .test-infra/validate-runner/readme.md diff --git a/.test-infra/validate-runner/readme.md b/.test-infra/validate-runner/readme.md new file mode 100644 index 000000000000..a72040c3a034 --- /dev/null +++ b/.test-infra/validate-runner/readme.md @@ -0,0 +1,2 @@ +Run the project +./gradlew beam-validate-runner:run \ No newline at end of file diff --git a/settings.gradle b/settings.gradle index 3d4bd3be7735..0385aecbfa28 100644 --- a/settings.gradle +++ b/settings.gradle @@ -191,3 +191,5 @@ include "beam-test-tools" project(":beam-test-tools").dir = file(".test-infra/tools") include "beam-test-jenkins" project(":beam-test-jenkins").dir = file(".test-infra/jenkins") +include "beam-validate-runner" +project(":beam-validate-runner").dir = file(".test-infra/validate-runner") From 70f94788f0ca1f48d9cd52caa70c6ed9dd797461 Mon Sep 17 00:00:00 2001 From: Sruthi Sree Kumar Date: Thu, 3 Dec 2020 16:37:29 +0100 Subject: [PATCH 0024/2127] deleted gradle jar --- .../gradle/wrapper/gradle-wrapper.jar | Bin 55190 -> 0 bytes .../gradle/wrapper/gradle-wrapper.properties | 5 ----- 2 files changed, 5 deletions(-) delete mode 100644 .test-infra/validate-runner/gradle/wrapper/gradle-wrapper.jar delete mode 100644 .test-infra/validate-runner/gradle/wrapper/gradle-wrapper.properties diff --git a/.test-infra/validate-runner/gradle/wrapper/gradle-wrapper.jar b/.test-infra/validate-runner/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index 87b738cbd051603d91cc39de6cb000dd98fe6b02..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 55190 zcmafaW0WS*vSoFbZQHhO+s0S6%`V%vZQJa!ZQHKus_B{g-pt%P_q|ywBQt-*Stldc z$+IJ3?^KWm27v+sf`9-50uuadKtMnL*BJ;1^6ynvR7H?hQcjE>7)art9Bu0Pcm@7C z@c%WG|JzYkP)<@zR9S^iR_sA`azaL$mTnGKnwDyMa;8yL_0^>Ba^)phg0L5rOPTbm7g*YIRLg-2^{qe^`rb!2KqS zk~5wEJtTdD?)3+}=eby3x6%i)sb+m??NHC^u=tcG8p$TzB<;FL(WrZGV&cDQb?O0GMe6PBV=V z?tTO*5_HTW$xea!nkc~Cnx#cL_rrUGWPRa6l+A{aiMY=<0@8y5OC#UcGeE#I>nWh}`#M#kIn-$A;q@u-p71b#hcSItS!IPw?>8 zvzb|?@Ahb22L(O4#2Sre&l9H(@TGT>#Py)D&eW-LNb!=S;I`ZQ{w;MaHW z#to!~TVLgho_Pm%zq@o{K3Xq?I|MVuVSl^QHnT~sHlrVxgsqD-+YD?Nz9@HA<;x2AQjxP)r6Femg+LJ-*)k%EZ}TTRw->5xOY z9#zKJqjZgC47@AFdk1$W+KhTQJKn7e>A&?@-YOy!v_(}GyV@9G#I?bsuto4JEp;5|N{orxi_?vTI4UF0HYcA( zKyGZ4<7Fk?&LZMQb6k10N%E*$gr#T&HsY4SPQ?yerqRz5c?5P$@6dlD6UQwZJ*Je9 z7n-@7!(OVdU-mg@5$D+R%gt82Lt%&n6Yr4=|q>XT%&^z_D*f*ug8N6w$`woqeS-+#RAOfSY&Rz z?1qYa5xi(7eTCrzCFJfCxc%j{J}6#)3^*VRKF;w+`|1n;Xaojr2DI{!<3CaP`#tXs z*`pBQ5k@JLKuCmovFDqh_`Q;+^@t_;SDm29 zCNSdWXbV?9;D4VcoV`FZ9Ggrr$i<&#Dx3W=8>bSQIU_%vf)#(M2Kd3=rN@^d=QAtC zI-iQ;;GMk|&A++W5#hK28W(YqN%?!yuW8(|Cf`@FOW5QbX|`97fxmV;uXvPCqxBD zJ9iI37iV)5TW1R+fV16y;6}2tt~|0J3U4E=wQh@sx{c_eu)t=4Yoz|%Vp<#)Qlh1V z0@C2ZtlT>5gdB6W)_bhXtcZS)`9A!uIOa`K04$5>3&8An+i9BD&GvZZ=7#^r=BN=k za+=Go;qr(M)B~KYAz|<^O3LJON}$Q6Yuqn8qu~+UkUKK~&iM%pB!BO49L+?AL7N7o z(OpM(C-EY753=G=WwJHE`h*lNLMNP^c^bBk@5MyP5{v7x>GNWH>QSgTe5 z!*GPkQ(lcbEs~)4ovCu!Zt&$${9$u(<4@9%@{U<-ksAqB?6F`bQ;o-mvjr)Jn7F&j$@`il1Mf+-HdBs<-`1FahTxmPMMI)@OtI&^mtijW6zGZ67O$UOv1Jj z;a3gmw~t|LjPkW3!EZ=)lLUhFzvO;Yvj9g`8hm%6u`;cuek_b-c$wS_0M4-N<@3l|88 z@V{Sd|M;4+H6guqMm4|v=C6B7mlpP(+It%0E;W`dxMOf9!jYwWj3*MRk`KpS_jx4c z=hrKBkFK;gq@;wUV2eqE3R$M+iUc+UD0iEl#-rECK+XmH9hLKrC={j@uF=f3UiceB zU5l$FF7#RKjx+6!JHMG5-!@zI-eG=a-!Bs^AFKqN_M26%cIIcSs61R$yuq@5a3c3& z4%zLs!g}+C5%`ja?F`?5-og0lv-;(^e<`r~p$x%&*89_Aye1N)9LNVk?9BwY$Y$$F^!JQAjBJvywXAesj7lTZ)rXuxv(FFNZVknJha99lN=^h`J2> zl5=~(tKwvHHvh|9-41@OV`c;Ws--PE%{7d2sLNbDp;A6_Ka6epzOSFdqb zBa0m3j~bT*q1lslHsHqaHIP%DF&-XMpCRL(v;MV#*>mB^&)a=HfLI7efblG z(@hzN`|n+oH9;qBklb=d^S0joHCsArnR1-h{*dIUThik>ot^!6YCNjg;J_i3h6Rl0ji)* zo(tQ~>xB!rUJ(nZjCA^%X;)H{@>uhR5|xBDA=d21p@iJ!cH?+%U|VSh2S4@gv`^)^ zNKD6YlVo$%b4W^}Rw>P1YJ|fTb$_(7C;hH+ z1XAMPb6*p^h8)e5nNPKfeAO}Ik+ZN_`NrADeeJOq4Ak;sD~ zTe77no{Ztdox56Xi4UE6S7wRVxJzWxKj;B%v7|FZ3cV9MdfFp7lWCi+W{}UqekdpH zdO#eoOuB3Fu!DU`ErfeoZWJbWtRXUeBzi zBTF-AI7yMC^ntG+8%mn(I6Dw}3xK8v#Ly{3w3_E?J4(Q5JBq~I>u3!CNp~Ekk&YH` z#383VO4O42NNtcGkr*K<+wYZ>@|sP?`AQcs5oqX@-EIqgK@Pmp5~p6O6qy4ml~N{D z{=jQ7k(9!CM3N3Vt|u@%ssTw~r~Z(}QvlROAkQQ?r8OQ3F0D$aGLh zny+uGnH5muJ<67Z=8uilKvGuANrg@s3Vu_lU2ajb?rIhuOd^E@l!Kl0hYIxOP1B~Q zggUmXbh$bKL~YQ#!4fos9UUVG#}HN$lIkM<1OkU@r>$7DYYe37cXYwfK@vrHwm;pg zbh(hEU|8{*d$q7LUm+x&`S@VbW*&p-sWrplWnRM|I{P;I;%U`WmYUCeJhYc|>5?&& zj}@n}w~Oo=l}iwvi7K6)osqa;M8>fRe}>^;bLBrgA;r^ZGgY@IC^ioRmnE&H4)UV5 zO{7egQ7sBAdoqGsso5q4R(4$4Tjm&&C|7Huz&5B0wXoJzZzNc5Bt)=SOI|H}+fbit z-PiF5(NHSy>4HPMrNc@SuEMDuKYMQ--G+qeUPqO_9mOsg%1EHpqoX^yNd~~kbo`cH zlV0iAkBFTn;rVb>EK^V6?T~t~3vm;csx+lUh_%ROFPy0(omy7+_wYjN!VRDtwDu^h4n|xpAMsLepm% zggvs;v8+isCW`>BckRz1MQ=l>K6k^DdT`~sDXTWQ<~+JtY;I~I>8XsAq3yXgxe>`O zZdF*{9@Z|YtS$QrVaB!8&`&^W->_O&-JXn1n&~}o3Z7FL1QE5R*W2W@=u|w~7%EeC1aRfGtJWxImfY-D3t!!nBkWM> zafu>^Lz-ONgT6ExjV4WhN!v~u{lt2-QBN&UxwnvdH|I%LS|J-D;o>@@sA62@&yew0 z)58~JSZP!(lX;da!3`d)D1+;K9!lyNlkF|n(UduR-%g>#{`pvrD^ClddhJyfL7C-(x+J+9&7EsC~^O`&}V%)Ut8^O_7YAXPDpzv8ir4 zl`d)(;imc6r16k_d^)PJZ+QPxxVJS5e^4wX9D=V2zH&wW0-p&OJe=}rX`*->XT=;_qI&)=WHkYnZx6bLoUh_)n-A}SF_ z9z7agNTM5W6}}ui=&Qs@pO5$zHsOWIbd_&%j^Ok5PJ3yUWQw*i4*iKO)_er2CDUME ztt+{Egod~W-fn^aLe)aBz)MOc_?i-stTj}~iFk7u^-gGSbU;Iem06SDP=AEw9SzuF zeZ|hKCG3MV(z_PJg0(JbqTRf4T{NUt%kz&}4S`)0I%}ZrG!jgW2GwP=WTtkWS?DOs znI9LY!dK+1_H0h+i-_~URb^M;4&AMrEO_UlDV8o?E>^3x%ZJyh$JuDMrtYL8|G3If zPf2_Qb_W+V?$#O; zydKFv*%O;Y@o_T_UAYuaqx1isMKZ^32JtgeceA$0Z@Ck0;lHbS%N5)zzAW9iz; z8tTKeK7&qw!8XVz-+pz>z-BeIzr*#r0nB^cntjQ9@Y-N0=e&ZK72vlzX>f3RT@i7@ z=z`m7jNk!9%^xD0ug%ptZnM>F;Qu$rlwo}vRGBIymPL)L|x}nan3uFUw(&N z24gdkcb7!Q56{0<+zu zEtc5WzG2xf%1<@vo$ZsuOK{v9gx^0`gw>@h>ZMLy*h+6ueoie{D#}}` zK2@6Xxq(uZaLFC%M!2}FX}ab%GQ8A0QJ?&!vaI8Gv=vMhd);6kGguDmtuOElru()) zuRk&Z{?Vp!G~F<1#s&6io1`poBqpRHyM^p;7!+L??_DzJ8s9mYFMQ0^%_3ft7g{PD zZd}8E4EV}D!>F?bzcX=2hHR_P`Xy6?FOK)mCj)Ym4s2hh z0OlOdQa@I;^-3bhB6mpw*X5=0kJv8?#XP~9){G-+0ST@1Roz1qi8PhIXp1D$XNqVG zMl>WxwT+K`SdO1RCt4FWTNy3!i?N>*-lbnn#OxFJrswgD7HjuKpWh*o@QvgF&j+CT z{55~ZsUeR1aB}lv#s_7~+9dCix!5(KR#c?K?e2B%P$fvrsZxy@GP#R#jwL{y#Ld$} z7sF>QT6m|}?V;msb?Nlohj7a5W_D$y+4O6eI;Zt$jVGymlzLKscqer9#+p2$0It&u zWY!dCeM6^B^Z;ddEmhi?8`scl=Lhi7W%2|pT6X6^%-=q90DS(hQ-%c+E*ywPvmoF(KqDoW4!*gmQIklm zk#!GLqv|cs(JRF3G?=AYY19{w@~`G3pa z@xR9S-Hquh*&5Yas*VI};(%9%PADn`kzm zeWMJVW=>>wap*9|R7n#!&&J>gq04>DTCMtj{P^d12|2wXTEKvSf?$AvnE!peqV7i4 zE>0G%CSn%WCW1yre?yi9*aFP{GvZ|R4JT}M%x_%Hztz2qw?&28l&qW<6?c6ym{f$d z5YCF+k#yEbjCN|AGi~-NcCG8MCF1!MXBFL{#7q z)HO+WW173?kuI}^Xat;Q^gb4Hi0RGyB}%|~j8>`6X4CPo+|okMbKy9PHkr58V4bX6<&ERU)QlF8%%huUz&f+dwTN|tk+C&&o@Q1RtG`}6&6;ncQuAcfHoxd5AgD7`s zXynq41Y`zRSiOY@*;&1%1z>oNcWTV|)sjLg1X8ijg1Y zbIGL0X*Sd}EXSQ2BXCKbJmlckY(@EWn~Ut2lYeuw1wg?hhj@K?XB@V_ZP`fyL~Yd3n3SyHU-RwMBr6t-QWE5TinN9VD4XVPU; zonIIR!&pGqrLQK)=#kj40Im%V@ij0&Dh0*s!lnTw+D`Dt-xmk-jmpJv$1-E-vfYL4 zqKr#}Gm}~GPE+&$PI@4ag@=M}NYi7Y&HW82Q`@Y=W&PE31D110@yy(1vddLt`P%N^ z>Yz195A%tnt~tvsSR2{m!~7HUc@x<&`lGX1nYeQUE(%sphTi>JsVqSw8xql*Ys@9B z>RIOH*rFi*C`ohwXjyeRBDt8p)-u{O+KWP;$4gg||%*u{$~yEj+Al zE(hAQRQ1k7MkCq9s4^N3ep*$h^L%2Vq?f?{+cicpS8lo)$Cb69b98au+m2J_e7nYwID0@`M9XIo1H~|eZFc8Hl!qly612ADCVpU zY8^*RTMX(CgehD{9v|^9vZ6Rab`VeZ2m*gOR)Mw~73QEBiktViBhR!_&3l$|be|d6 zupC`{g89Y|V3uxl2!6CM(RNpdtynaiJ~*DqSTq9Mh`ohZnb%^3G{k;6%n18$4nAqR zjPOrP#-^Y9;iw{J@XH9=g5J+yEVh|e=4UeY<^65`%gWtdQ=-aqSgtywM(1nKXh`R4 zzPP&7r)kv_uC7X9n=h=!Zrf<>X=B5f<9~Q>h#jYRD#CT7D~@6@RGNyO-#0iq0uHV1 zPJr2O4d_xLmg2^TmG7|dpfJ?GGa`0|YE+`2Rata9!?$j#e9KfGYuLL(*^z z!SxFA`$qm)q-YKh)WRJZ@S+-sD_1E$V?;(?^+F3tVcK6 z2fE=8hV*2mgiAbefU^uvcM?&+Y&E}vG=Iz!%jBF7iv){lyC`)*yyS~D8k+Mx|N3bm zI~L~Z$=W9&`x)JnO;8c>3LSDw!fzN#X3qi|0`sXY4?cz{*#xz!kvZ9bO=K3XbN z5KrgN=&(JbXH{Wsu9EdmQ-W`i!JWEmfI;yVTT^a-8Ch#D8xf2dtyi?7p z%#)W3n*a#ndFpd{qN|+9Jz++AJQO#-Y7Z6%*%oyEP5zs}d&kKIr`FVEY z;S}@d?UU=tCdw~EJ{b}=9x}S2iv!!8<$?d7VKDA8h{oeD#S-$DV)-vPdGY@x08n)@ zag?yLF_E#evvRTj4^CcrLvBL=fft&@HOhZ6Ng4`8ijt&h2y}fOTC~7GfJi4vpomA5 zOcOM)o_I9BKz}I`q)fu+Qnfy*W`|mY%LO>eF^a z;$)?T4F-(X#Q-m}!-k8L_rNPf`Mr<9IWu)f&dvt=EL+ESYmCvErd@8B9hd)afc(ZL94S z?rp#h&{7Ah5IJftK4VjATklo7@hm?8BX*~oBiz)jyc9FuRw!-V;Uo>p!CWpLaIQyt zAs5WN)1CCeux-qiGdmbIk8LR`gM+Qg=&Ve}w?zA6+sTL)abU=-cvU`3E?p5$Hpkxw znu0N659qR=IKnde*AEz_7z2pdi_Bh-sb3b=PdGO1Pdf_q2;+*Cx9YN7p_>rl``knY zRn%aVkcv1(W;`Mtp_DNOIECtgq%ufk-mu_<+Fu3Q17Tq4Rr(oeq)Yqk_CHA7LR@7@ zIZIDxxhS&=F2IQfusQ+Nsr%*zFK7S4g!U0y@3H^Yln|i;0a5+?RPG;ZSp6Tul>ezM z`40+516&719qT)mW|ArDSENle5hE2e8qY+zfeZoy12u&xoMgcP)4=&P-1Ib*-bAy` zlT?>w&B|ei-rCXO;sxo7*G;!)_p#%PAM-?m$JP(R%x1Hfas@KeaG%LO?R=lmkXc_MKZW}3f%KZ*rAN?HYvbu2L$ zRt_uv7~-IejlD1x;_AhwGXjB94Q=%+PbxuYzta*jw?S&%|qb=(JfJ?&6P=R7X zV%HP_!@-zO*zS}46g=J}#AMJ}rtWBr21e6hOn&tEmaM%hALH7nlm2@LP4rZ>2 zebe5aH@k!e?ij4Zwak#30|}>;`bquDQK*xmR=zc6vj0yuyC6+U=LusGnO3ZKFRpen z#pwzh!<+WBVp-!$MAc<0i~I%fW=8IO6K}bJ<-Scq>e+)951R~HKB?Mx2H}pxPHE@} zvqpq5j81_jtb_WneAvp<5kgdPKm|u2BdQx9%EzcCN&U{l+kbkhmV<1}yCTDv%&K^> zg;KCjwh*R1f_`6`si$h6`jyIKT7rTv5#k~x$mUyIw)_>Vr)D4fwIs@}{FSX|5GB1l z4vv;@oS@>Bu7~{KgUa_8eg#Lk6IDT2IY$41$*06{>>V;Bwa(-@N;ex4;D`(QK*b}{ z{#4$Hmt)FLqERgKz=3zXiV<{YX6V)lvYBr3V>N6ajeI~~hGR5Oe>W9r@sg)Na(a4- zxm%|1OKPN6^%JaD^^O~HbLSu=f`1px>RawOxLr+1b2^28U*2#h*W^=lSpSY4(@*^l z{!@9RSLG8Me&RJYLi|?$c!B0fP=4xAM4rerxX{xy{&i6=AqXueQAIBqO+pmuxy8Ib z4X^}r!NN3-upC6B#lt7&x0J;)nb9O~xjJMemm$_fHuP{DgtlU3xiW0UesTzS30L+U zQzDI3p&3dpONhd5I8-fGk^}@unluzu%nJ$9pzoO~Kk!>dLxw@M)M9?pNH1CQhvA`z zV;uacUtnBTdvT`M$1cm9`JrT3BMW!MNVBy%?@ZX%;(%(vqQAz<7I!hlDe|J3cn9=} zF7B;V4xE{Ss76s$W~%*$JviK?w8^vqCp#_G^jN0j>~Xq#Zru26e#l3H^{GCLEXI#n z?n~F-Lv#hU(bZS`EI9(xGV*jT=8R?CaK)t8oHc9XJ;UPY0Hz$XWt#QyLBaaz5+}xM zXk(!L_*PTt7gwWH*HLWC$h3Ho!SQ-(I||nn_iEC{WT3S{3V{8IN6tZ1C+DiFM{xlI zeMMk{o5;I6UvaC)@WKp9D+o?2Vd@4)Ue-nYci()hCCsKR`VD;hr9=vA!cgGL%3k^b(jADGyPi2TKr(JNh8mzlIR>n(F_hgiV(3@Ds(tjbNM7GoZ;T|3 zWzs8S`5PrA!9){jBJuX4y`f<4;>9*&NY=2Sq2Bp`M2(fox7ZhIDe!BaQUb@P(ub9D zlP8!p(AN&CwW!V&>H?yPFMJ)d5x#HKfwx;nS{Rr@oHqpktOg)%F+%1#tsPtq7zI$r zBo-Kflhq-=7_eW9B2OQv=@?|y0CKN77)N;z@tcg;heyW{wlpJ1t`Ap!O0`Xz{YHqO zI1${8Hag^r!kA<2_~bYtM=<1YzQ#GGP+q?3T7zYbIjN6Ee^V^b&9en$8FI*NIFg9G zPG$OXjT0Ku?%L7fat8Mqbl1`azf1ltmKTa(HH$Dqlav|rU{zP;Tbnk-XkGFQ6d+gi z-PXh?_kEJl+K98&OrmzgPIijB4!Pozbxd0H1;Usy!;V>Yn6&pu*zW8aYx`SC!$*ti zSn+G9p=~w6V(fZZHc>m|PPfjK6IN4(o=IFu?pC?+`UZAUTw!e`052{P=8vqT^(VeG z=psASIhCv28Y(;7;TuYAe>}BPk5Qg=8$?wZj9lj>h2kwEfF_CpK=+O6Rq9pLn4W)# zeXCKCpi~jsfqw7Taa0;!B5_C;B}e56W1s8@p*)SPzA;Fd$Slsn^=!_&!mRHV*Lmt| zBGIDPuR>CgS4%cQ4wKdEyO&Z>2aHmja;Pz+n|7(#l%^2ZLCix%>@_mbnyPEbyrHaz z>j^4SIv;ZXF-Ftzz>*t4wyq)ng8%0d;(Z_ExZ-cxwei=8{(br-`JYO(f23Wae_MqE z3@{Mlf^%M5G1SIN&en1*| zH~ANY1h3&WNsBy$G9{T=`kcxI#-X|>zLX2r*^-FUF+m0{k)n#GTG_mhG&fJfLj~K& zU~~6othMlvMm9<*SUD2?RD+R17|Z4mgR$L*R3;nBbo&Vm@39&3xIg;^aSxHS>}gwR zmzs?h8oPnNVgET&dx5^7APYx6Vv6eou07Zveyd+^V6_LzI$>ic+pxD_8s~ zC<}ucul>UH<@$KM zT4oI=62M%7qQO{}re-jTFqo9Z;rJKD5!X5$iwUsh*+kcHVhID08MB5cQD4TBWB(rI zuWc%CA}}v|iH=9gQ?D$1#Gu!y3o~p7416n54&Hif`U-cV?VrUMJyEqo_NC4#{puzU zzXEE@UppeeRlS9W*^N$zS`SBBi<@tT+<%3l@KhOy^%MWB9(A#*J~DQ;+MK*$rxo6f zcx3$3mcx{tly!q(p2DQrxcih|)0do_ZY77pyHGE#Q(0k*t!HUmmMcYFq%l$-o6%lS zDb49W-E?rQ#Hl``C3YTEdGZjFi3R<>t)+NAda(r~f1cT5jY}s7-2^&Kvo&2DLTPYP zhVVo-HLwo*vl83mtQ9)PR#VBg)FN}+*8c-p8j`LnNUU*Olm1O1Qqe62D#$CF#?HrM zy(zkX|1oF}Z=T#3XMLWDrm(|m+{1&BMxHY7X@hM_+cV$5-t!8HT(dJi6m9{ja53Yw z3f^`yb6Q;(e|#JQIz~B*=!-GbQ4nNL-NL z@^NWF_#w-Cox@h62;r^;Y`NX8cs?l^LU;5IWE~yvU8TqIHij!X8ydbLlT0gwmzS9} z@5BccG?vO;rvCs$mse1*ANi-cYE6Iauz$Fbn3#|ToAt5v7IlYnt6RMQEYLldva{~s zvr>1L##zmeoYgvIXJ#>bbuCVuEv2ZvZ8I~PQUN3wjP0UC)!U+wn|&`V*8?)` zMSCuvnuGec>QL+i1nCPGDAm@XSMIo?A9~C?g2&G8aNKjWd2pDX{qZ?04+2 zeyLw}iEd4vkCAWwa$ zbrHlEf3hfN7^1g~aW^XwldSmx1v~1z(s=1az4-wl} z`mM+G95*N*&1EP#u3}*KwNrPIgw8Kpp((rdEOO;bT1;6ea~>>sK+?!;{hpJ3rR<6UJb`O8P4@{XGgV%63_fs%cG8L zk9Fszbdo4tS$g0IWP1>t@0)E%-&9yj%Q!fiL2vcuL;90fPm}M==<>}Q)&sp@STFCY z^p!RzmN+uXGdtPJj1Y-khNyCb6Y$Vs>eZyW zPaOV=HY_T@FwAlleZCFYl@5X<<7%5DoO(7S%Lbl55?{2vIr_;SXBCbPZ(up;pC6Wx={AZL?shYOuFxLx1*>62;2rP}g`UT5+BHg(ju z&7n5QSvSyXbioB9CJTB#x;pexicV|9oaOpiJ9VK6EvKhl4^Vsa(p6cIi$*Zr0UxQ z;$MPOZnNae2Duuce~7|2MCfhNg*hZ9{+8H3?ts9C8#xGaM&sN;2lriYkn9W>&Gry! z3b(Xx1x*FhQkD-~V+s~KBfr4M_#0{`=Yrh90yj}Ph~)Nx;1Y^8<418tu!$1<3?T*~ z7Dl0P3Uok-7w0MPFQexNG1P5;y~E8zEvE49>$(f|XWtkW2Mj`udPn)pb%} zrA%wRFp*xvDgC767w!9`0vx1=q!)w!G+9(-w&p*a@WXg{?T&%;qaVcHo>7ca%KX$B z^7|KBPo<2;kM{2mRnF8vKm`9qGV%|I{y!pKm8B(q^2V;;x2r!1VJ^Zz8bWa)!-7a8 zSRf@dqEPlsj!7}oNvFFAA)75})vTJUwQ03hD$I*j6_5xbtd_JkE2`IJD_fQ;a$EkO z{fQ{~e%PKgPJsD&PyEvDmg+Qf&p*-qu!#;1k2r_(H72{^(Z)htgh@F?VIgK#_&eS- z$~(qInec>)XIkv@+{o6^DJLpAb>!d}l1DK^(l%#OdD9tKK6#|_R?-%0V!`<9Hj z3w3chDwG*SFte@>Iqwq`J4M&{aHXzyigT620+Vf$X?3RFfeTcvx_e+(&Q*z)t>c0e zpZH$1Z3X%{^_vylHVOWT6tno=l&$3 z9^eQ@TwU#%WMQaFvaYp_we%_2-9=o{+ck zF{cKJCOjpW&qKQquyp2BXCAP920dcrZ}T1@piukx_NY;%2W>@Wca%=Ch~x5Oj58Hv z;D-_ALOZBF(Mqbcqjd}P3iDbek#Dwzu`WRs`;hRIr*n0PV7vT+%Io(t}8KZ zpp?uc2eW!v28ipep0XNDPZt7H2HJ6oey|J3z!ng#1H~x_k%35P+Cp%mqXJ~cV0xdd z^4m5^K_dQ^Sg?$P`))ccV=O>C{Ds(C2WxX$LMC5vy=*44pP&)X5DOPYfqE${)hDg< z3hcG%U%HZ39=`#Ko4Uctg&@PQLf>?0^D|4J(_1*TFMOMB!Vv1_mnOq$BzXQdOGqgy zOp#LBZ!c>bPjY1NTXksZmbAl0A^Y&(%a3W-k>bE&>K?px5Cm%AT2E<&)Y?O*?d80d zgI5l~&Mve;iXm88Q+Fw7{+`PtN4G7~mJWR^z7XmYQ>uoiV!{tL)hp|= zS(M)813PM`d<501>{NqaPo6BZ^T{KBaqEVH(2^Vjeq zgeMeMpd*1tE@@);hGjuoVzF>Cj;5dNNwh40CnU+0DSKb~GEMb_# zT8Z&gz%SkHq6!;_6dQFYE`+b`v4NT7&@P>cA1Z1xmXy<2htaDhm@XXMp!g($ zw(7iFoH2}WR`UjqjaqOQ$ecNt@c|K1H1kyBArTTjLp%-M`4nzOhkfE#}dOpcd;b#suq8cPJ&bf5`6Tq>ND(l zib{VrPZ>{KuaIg}Y$W>A+nrvMg+l4)-@2jpAQ5h(Tii%Ni^-UPVg{<1KGU2EIUNGaXcEkOedJOusFT9X3%Pz$R+-+W+LlRaY-a$5r?4V zbPzgQl22IPG+N*iBRDH%l{Zh$fv9$RN1sU@Hp3m=M}{rX%y#;4(x1KR2yCO7Pzo>rw(67E{^{yUR`91nX^&MxY@FwmJJbyPAoWZ9Z zcBS$r)&ogYBn{DOtD~tIVJUiq|1foX^*F~O4hlLp-g;Y2wKLLM=?(r3GDqsPmUo*? zwKMEi*%f)C_@?(&&hk>;m07F$X7&i?DEK|jdRK=CaaNu-)pX>n3}@%byPKVkpLzBq z{+Py&!`MZ^4@-;iY`I4#6G@aWMv{^2VTH7|WF^u?3vsB|jU3LgdX$}=v7#EHRN(im zI(3q-eU$s~r=S#EWqa_2!G?b~ z<&brq1vvUTJH380=gcNntZw%7UT8tLAr-W49;9y^=>TDaTC|cKA<(gah#2M|l~j)w zY8goo28gj$n&zcNgqX1Qn6=<8?R0`FVO)g4&QtJAbW3G#D)uNeac-7cH5W#6i!%BH z=}9}-f+FrtEkkrQ?nkoMQ1o-9_b+&=&C2^h!&mWFga#MCrm85hW;)1pDt;-uvQG^D zntSB?XA*0%TIhtWDS!KcI}kp3LT>!(Nlc(lQN?k^bS8Q^GGMfo}^|%7s;#r+pybl@?KA++|FJ zr%se9(B|g*ERQU96az%@4gYrxRRxaM2*b}jNsG|0dQi;Rw{0WM0E>rko!{QYAJJKY z)|sX0N$!8d9E|kND~v|f>3YE|uiAnqbkMn)hu$if4kUkzKqoNoh8v|S>VY1EKmgO} zR$0UU2o)4i4yc1inx3}brso+sio{)gfbLaEgLahj8(_Z#4R-v) zglqwI%`dsY+589a8$Mu7#7_%kN*ekHupQ#48DIN^uhDxblDg3R1yXMr^NmkR z7J_NWCY~fhg}h!_aXJ#?wsZF$q`JH>JWQ9`jbZzOBpS`}-A$Vgkq7+|=lPx9H7QZG z8i8guMN+yc4*H*ANr$Q-3I{FQ-^;8ezWS2b8rERp9TMOLBxiG9J*g5=?h)mIm3#CGi4JSq1ohFrcrxx@`**K5%T}qbaCGldV!t zVeM)!U3vbf5FOy;(h08JnhSGxm)8Kqxr9PsMeWi=b8b|m_&^@#A3lL;bVKTBx+0v8 zLZeWAxJ~N27lsOT2b|qyp$(CqzqgW@tyy?CgwOe~^i;ZH zlL``i4r!>i#EGBNxV_P@KpYFQLz4Bdq{#zA&sc)*@7Mxsh9u%e6Ke`?5Yz1jkTdND zR8!u_yw_$weBOU}24(&^Bm|(dSJ(v(cBct}87a^X(v>nVLIr%%D8r|&)mi+iBc;B;x;rKq zd8*X`r?SZsTNCPQqoFOrUz8nZO?225Z#z(B!4mEp#ZJBzwd7jW1!`sg*?hPMJ$o`T zR?KrN6OZA1H{9pA;p0cSSu;@6->8aJm1rrO-yDJ7)lxuk#npUk7WNER1Wwnpy%u zF=t6iHzWU(L&=vVSSc^&D_eYP3TM?HN!Tgq$SYC;pSIPWW;zeNm7Pgub#yZ@7WPw#f#Kl)W4%B>)+8%gpfoH1qZ;kZ*RqfXYeGXJ_ zk>2otbp+1By`x^1V!>6k5v8NAK@T;89$`hE0{Pc@Q$KhG0jOoKk--Qx!vS~lAiypV zCIJ&6B@24`!TxhJ4_QS*S5;;Pk#!f(qIR7*(c3dN*POKtQe)QvR{O2@QsM%ujEAWEm) z+PM=G9hSR>gQ`Bv2(k}RAv2+$7qq(mU`fQ+&}*i%-RtSUAha>70?G!>?w%F(b4k!$ zvm;E!)2`I?etmSUFW7WflJ@8Nx`m_vE2HF#)_BiD#FaNT|IY@!uUbd4v$wTglIbIX zblRy5=wp)VQzsn0_;KdM%g<8@>#;E?vypTf=F?3f@SSdZ;XpX~J@l1;p#}_veWHp>@Iq_T z@^7|h;EivPYv1&u0~l9(a~>dV9Uw10QqB6Dzu1G~-l{*7IktljpK<_L8m0|7VV_!S zRiE{u97(%R-<8oYJ{molUd>vlGaE-C|^<`hppdDz<7OS13$#J zZ+)(*rZIDSt^Q$}CRk0?pqT5PN5TT`Ya{q(BUg#&nAsg6apPMhLTno!SRq1e60fl6GvpnwDD4N> z9B=RrufY8+g3_`@PRg+(+gs2(bd;5#{uTZk96CWz#{=&h9+!{_m60xJxC%r&gd_N! z>h5UzVX%_7@CUeAA1XFg_AF%(uS&^1WD*VPS^jcC!M2v@RHZML;e(H-=(4(3O&bX- zI6>usJOS+?W&^S&DL{l|>51ZvCXUKlH2XKJPXnHjs*oMkNM#ZDLx!oaM5(%^)5XaP zk6&+P16sA>vyFe9v`Cp5qnbE#r#ltR5E+O3!WnKn`56Grs2;sqr3r# zp@Zp<^q`5iq8OqOlJ`pIuyK@3zPz&iJ0Jcc`hDQ1bqos2;}O|$i#}e@ua*x5VCSx zJAp}+?Hz++tm9dh3Fvm_bO6mQo38al#>^O0g)Lh^&l82+&x)*<n7^Sw-AJo9tEzZDwyJ7L^i7|BGqHu+ea6(&7jKpBq>~V z8CJxurD)WZ{5D0?s|KMi=e7A^JVNM6sdwg@1Eg_+Bw=9j&=+KO1PG|y(mP1@5~x>d z=@c{EWU_jTSjiJl)d(>`qEJ;@iOBm}alq8;OK;p(1AdH$)I9qHNmxxUArdzBW0t+Qeyl)m3?D09770g z)hzXEOy>2_{?o%2B%k%z4d23!pZcoxyW1Ik{|m7Q1>fm4`wsRrl)~h z_=Z*zYL+EG@DV1{6@5@(Ndu!Q$l_6Qlfoz@79q)Kmsf~J7t1)tl#`MD<;1&CAA zH8;i+oBm89dTTDl{aH`cmTPTt@^K-%*sV+t4X9q0Z{A~vEEa!&rRRr=0Rbz4NFCJr zLg2u=0QK@w9XGE=6(-JgeP}G#WG|R&tfHRA3a9*zh5wNTBAD;@YYGx%#E4{C#Wlfo z%-JuW9=FA_T6mR2-Vugk1uGZvJbFvVVWT@QOWz$;?u6+CbyQsbK$>O1APk|xgnh_8 zc)s@Mw7#0^wP6qTtyNq2G#s?5j~REyoU6^lT7dpX{T-rhZWHD%dik*=EA7bIJgOVf_Ga!yC8V^tkTOEHe+JK@Fh|$kfNxO^= z#lpV^(ZQ-3!^_BhV>aXY~GC9{8%1lOJ}6vzXDvPhC>JrtXwFBC+!3a*Z-%#9}i z#<5&0LLIa{q!rEIFSFc9)>{-_2^qbOg5;_A9 ztQ))C6#hxSA{f9R3Eh^`_f${pBJNe~pIQ`tZVR^wyp}=gLK}e5_vG@w+-mp#Fu>e| z*?qBp5CQ5zu+Fi}xAs)YY1;bKG!htqR~)DB$ILN6GaChoiy%Bq@i+1ZnANC0U&D z_4k$=YP47ng+0NhuEt}6C;9-JDd8i5S>`Ml==9wHDQFOsAlmtrVwurYDw_)Ihfk35 zJDBbe!*LUpg%4n>BExWz>KIQ9vexUu^d!7rc_kg#Bf= z7TLz|l*y*3d2vi@c|pX*@ybf!+Xk|2*z$@F4K#MT8Dt4zM_EcFmNp31#7qT6(@GG? zdd;sSY9HHuDb=w&|K%sm`bYX#%UHKY%R`3aLMO?{T#EI@FNNFNO>p@?W*i0z(g2dt z{=9Ofh80Oxv&)i35AQN>TPMjR^UID-T7H5A?GI{MD_VeXZ%;uo41dVm=uT&ne2h0i zv*xI%9vPtdEK@~1&V%p1sFc2AA`9?H)gPnRdlO~URx!fiSV)j?Tf5=5F>hnO=$d$x zzaIfr*wiIc!U1K*$JO@)gP4%xp!<*DvJSv7p}(uTLUb=MSb@7_yO+IsCj^`PsxEl& zIxsi}s3L?t+p+3FXYqujGhGwTx^WXgJ1}a@Yq5mwP0PvGEr*qu7@R$9j>@-q1rz5T zriz;B^(ex?=3Th6h;7U`8u2sDlfS{0YyydK=*>-(NOm9>S_{U|eg(J~C7O zIe{|LK=Y`hXiF_%jOM8Haw3UtaE{hWdzo3BbD6ud7br4cODBtN(~Hl+odP0SSWPw;I&^m)yLw+nd#}3#z}?UIcX3=SssI}`QwY=% zAEXTODk|MqTx}2DVG<|~(CxgLyi*A{m>M@1h^wiC)4Hy>1K7@|Z&_VPJsaQoS8=ex zDL&+AZdQa>ylxhT_Q$q=60D5&%pi6+qlY3$3c(~rsITX?>b;({FhU!7HOOhSP7>bmTkC8KM%!LRGI^~y3Ug+gh!QM=+NZXznM)?L3G=4=IMvFgX3BAlyJ z`~jjA;2z+65D$j5xbv9=IWQ^&-K3Yh`vC(1Qz2h2`o$>Cej@XRGff!it$n{@WEJ^N z41qk%Wm=}mA*iwCqU_6}Id!SQd13aFER3unXaJJXIsSnxvG2(hSCP{i&QH$tL&TPx zDYJsuk+%laN&OvKb-FHK$R4dy%M7hSB*yj#-nJy?S9tVoxAuDei{s}@+pNT!vLOIC z8g`-QQW8FKp3cPsX%{)0B+x+OhZ1=L7F-jizt|{+f1Ga7%+!BXqjCjH&x|3%?UbN# zh?$I1^YokvG$qFz5ySK+Ja5=mkR&p{F}ev**rWdKMko+Gj^?Or=UH?SCg#0F(&a_y zXOh}dPv0D9l0RVedq1~jCNV=8?vZfU-Xi|nkeE->;ohG3U7z+^0+HV17~-_Mv#mV` zzvwUJJ15v5wwKPv-)i@dsEo@#WEO9zie7mdRAbgL2kjbW4&lk$vxkbq=w5mGKZK6@ zjXWctDkCRx58NJD_Q7e}HX`SiV)TZMJ}~zY6P1(LWo`;yDynY_5_L?N-P`>ALfmyl z8C$a~FDkcwtzK9m$tof>(`Vu3#6r#+v8RGy#1D2)F;vnsiL&P-c^PO)^B-4VeJteLlT@25sPa z%W~q5>YMjj!mhN})p$47VA^v$Jo6_s{!y?}`+h+VM_SN`!11`|;C;B};B&Z<@%FOG z_YQVN+zFF|q5zKab&e4GH|B;sBbKimHt;K@tCH+S{7Ry~88`si7}S)1E{21nldiu5 z_4>;XTJa~Yd$m4A9{Qbd)KUAm7XNbZ4xHbg3a8-+1uf*$1PegabbmCzgC~1WB2F(W zYj5XhVos!X!QHuZXCatkRsdEsSCc+D2?*S7a+(v%toqyxhjz|`zdrUvsxQS{J>?c& zvx*rHw^8b|v^7wq8KWVofj&VUitbm*a&RU_ln#ZFA^3AKEf<#T%8I!Lg3XEsdH(A5 zlgh&M_XEoal)i#0tcq8c%Gs6`xu;vvP2u)D9p!&XNt z!TdF_H~;`g@fNXkO-*t<9~;iEv?)Nee%hVe!aW`N%$cFJ(Dy9+Xk*odyFj72T!(b%Vo5zvCGZ%3tkt$@Wcx8BWEkefI1-~C_3y*LjlQ5%WEz9WD8i^ z2MV$BHD$gdPJV4IaV)G9CIFwiV=ca0cfXdTdK7oRf@lgyPx;_7*RRFk=?@EOb9Gcz zg~VZrzo*Snp&EE{$CWr)JZW)Gr;{B2ka6B!&?aknM-FENcl%45#y?oq9QY z3^1Y5yn&^D67Da4lI}ljDcphaEZw2;tlYuzq?uB4b9Mt6!KTW&ptxd^vF;NbX=00T z@nE1lIBGgjqs?ES#P{ZfRb6f!At51vk%<0X%d_~NL5b8UyfQMPDtfU@>ijA0NP3UU zh{lCf`Wu7cX!go`kUG`1K=7NN@SRGjUKuo<^;@GS!%iDXbJs`o6e`v3O8-+7vRkFm z)nEa$sD#-v)*Jb>&Me+YIW3PsR1)h=-Su)))>-`aRcFJG-8icomO4J@60 zw10l}BYxi{eL+Uu0xJYk-Vc~BcR49Qyyq!7)PR27D`cqGrik=?k1Of>gY7q@&d&Ds zt7&WixP`9~jjHO`Cog~RA4Q%uMg+$z^Gt&vn+d3&>Ux{_c zm|bc;k|GKbhZLr-%p_f%dq$eiZ;n^NxoS-Nu*^Nx5vm46)*)=-Bf<;X#?`YC4tLK; z?;u?shFbXeks+dJ?^o$l#tg*1NA?(1iFff@I&j^<74S!o;SWR^Xi);DM%8XiWpLi0 zQE2dL9^a36|L5qC5+&Pf0%>l&qQ&)OU4vjd)%I6{|H+pw<0(a``9w(gKD&+o$8hOC zNAiShtc}e~ob2`gyVZx59y<6Fpl*$J41VJ-H*e-yECWaDMmPQi-N8XI3 z%iI@ljc+d}_okL1CGWffeaejlxWFVDWu%e=>H)XeZ|4{HlbgC-Uvof4ISYQzZ0Um> z#Ov{k1c*VoN^f(gfiueuag)`TbjL$XVq$)aCUBL_M`5>0>6Ska^*Knk__pw{0I>jA zzh}Kzg{@PNi)fcAk7jMAdi-_RO%x#LQszDMS@_>iFoB+zJ0Q#CQJzFGa8;pHFdi`^ zxnTC`G$7Rctm3G8t8!SY`GwFi4gF|+dAk7rh^rA{NXzc%39+xSYM~($L(pJ(8Zjs* zYdN_R^%~LiGHm9|ElV4kVZGA*T$o@YY4qpJOxGHlUi*S*A(MrgQ{&xoZQo+#PuYRs zv3a$*qoe9gBqbN|y|eaH=w^LE{>kpL!;$wRahY(hhzRY;d33W)m*dfem@)>pR54Qy z ze;^F?mwdU?K+=fBabokSls^6_6At#1Sh7W*y?r6Ss*dmZP{n;VB^LDxM1QWh;@H0J z!4S*_5j_;+@-NpO1KfQd&;C7T`9ak;X8DTRz$hDNcjG}xAfg%gwZSb^zhE~O);NMO zn2$fl7Evn%=Lk!*xsM#(y$mjukN?A&mzEw3W5>_o+6oh62kq=4-`e3B^$rG=XG}Kd zK$blh(%!9;@d@3& zGFO60j1Vf54S}+XD?%*uk7wW$f`4U3F*p7@I4Jg7f`Il}2H<{j5h?$DDe%wG7jZQL zI{mj?t?Hu>$|2UrPr5&QyK2l3mas?zzOk0DV30HgOQ|~xLXDQ8M3o#;CNKO8RK+M; zsOi%)js-MU>9H4%Q)#K_me}8OQC1u;f4!LO%|5toa1|u5Q@#mYy8nE9IXmR}b#sZK z3sD395q}*TDJJA9Er7N`y=w*S&tA;mv-)Sx4(k$fJBxXva0_;$G6!9bGBw13c_Uws zXks4u(8JA@0O9g5f?#V~qR5*u5aIe2HQO^)RW9TTcJk28l`Syl>Q#ZveEE4Em+{?%iz6=V3b>rCm9F zPQQm@-(hfNdo2%n?B)u_&Qh7^^@U>0qMBngH8}H|v+Ejg*Dd(Y#|jgJ-A zQ_bQscil%eY}8oN7ZL+2r|qv+iJY?*l)&3W_55T3GU;?@Om*(M`u0DXAsQ7HSl56> z4P!*(%&wRCb?a4HH&n;lAmr4rS=kMZb74Akha2U~Ktni>>cD$6jpugjULq)D?ea%b zk;UW0pAI~TH59P+o}*c5Ei5L-9OE;OIBt>^(;xw`>cN2`({Rzg71qrNaE=cAH^$wP zNrK9Glp^3a%m+ilQj0SnGq`okjzmE7<3I{JLD6Jn^+oas=h*4>Wvy=KXqVBa;K&ri z4(SVmMXPG}0-UTwa2-MJ=MTfM3K)b~DzSVq8+v-a0&Dsv>4B65{dBhD;(d44CaHSM zb!0ne(*<^Q%|nuaL`Gb3D4AvyO8wyygm=1;9#u5x*k0$UOwx?QxR*6Od8>+ujfyo0 zJ}>2FgW_iv(dBK2OWC-Y=Tw!UwIeOAOUUC;h95&S1hn$G#if+d;*dWL#j#YWswrz_ zMlV=z+zjZJ%SlDhxf)vv@`%~$Afd)T+MS1>ZE7V$Rj#;J*<9Ld=PrK0?qrazRJWx) z(BTLF@Wk279nh|G%ZY7_lK7=&j;x`bMND=zgh_>>-o@6%8_#Bz!FnF*onB@_k|YCF z?vu!s6#h9bL3@tPn$1;#k5=7#s*L;FLK#=M89K^|$3LICYWIbd^qguQp02w5>8p-H z+@J&+pP_^iF4Xu>`D>DcCnl8BUwwOlq6`XkjHNpi@B?OOd`4{dL?kH%lt78(-L}eah8?36zw9d-dI6D{$s{f=M7)1 zRH1M*-82}DoFF^Mi$r}bTB5r6y9>8hjL54%KfyHxn$LkW=AZ(WkHWR;tIWWr@+;^^ zVomjAWT)$+rn%g`LHB6ZSO@M3KBA? z+W7ThSBgpk`jZHZUrp`F;*%6M5kLWy6AW#T{jFHTiKXP9ITrMlEdti7@&AT_a-BA!jc(Kt zWk>IdY-2Zbz?U1)tk#n_Lsl?W;0q`;z|t9*g-xE!(}#$fScX2VkjSiboKWE~afu5d z2B@9mvT=o2fB_>Mnie=TDJB+l`GMKCy%2+NcFsbpv<9jS@$X37K_-Y!cvF5NEY`#p z3sWEc<7$E*X*fp+MqsOyMXO=<2>o8)E(T?#4KVQgt=qa%5FfUG_LE`n)PihCz2=iNUt7im)s@;mOc9SR&{`4s9Q6)U31mn?}Y?$k3kU z#h??JEgH-HGt`~%)1ZBhT9~uRi8br&;a5Y3K_Bl1G)-y(ytx?ok9S*Tz#5Vb=P~xH z^5*t_R2It95=!XDE6X{MjLYn4Eszj9Y91T2SFz@eYlx9Z9*hWaS$^5r7=W5|>sY8}mS(>e9Ez2qI1~wtlA$yv2e-Hjn&K*P z2zWSrC~_8Wrxxf#%QAL&f8iH2%R)E~IrQLgWFg8>`Vnyo?E=uiALoRP&qT{V2{$79 z%9R?*kW-7b#|}*~P#cA@q=V|+RC9=I;aK7Pju$K-n`EoGV^-8Mk=-?@$?O37evGKn z3NEgpo_4{s>=FB}sqx21d3*=gKq-Zk)U+bM%Q_}0`XGkYh*+jRaP+aDnRv#Zz*n$pGp zEU9omuYVXH{AEx>=kk}h2iKt!yqX=EHN)LF}z1j zJx((`CesN1HxTFZ7yrvA2jTPmKYVij>45{ZH2YtsHuGzIRotIFj?(8T@ZWUv{_%AI zgMZlB03C&FtgJqv9%(acqt9N)`4jy4PtYgnhqev!r$GTIOvLF5aZ{tW5MN@9BDGu* zBJzwW3sEJ~Oy8is`l6Ly3an7RPtRr^1Iu(D!B!0O241Xua>Jee;Rc7tWvj!%#yX#m z&pU*?=rTVD7pF6va1D@u@b#V@bShFr3 zMyMbNCZwT)E-%L-{%$3?n}>EN>ai7b$zR_>=l59mW;tfKj^oG)>_TGCJ#HbLBsNy$ zqAqPagZ3uQ(Gsv_-VrZmG&hHaOD#RB#6J8&sL=^iMFB=gH5AIJ+w@sTf7xa&Cnl}@ zxrtzoNq>t?=(+8bS)s2p3>jW}tye0z2aY_Dh@(18-vdfvn;D?sv<>UgL{Ti08$1Q+ zZI3q}yMA^LK=d?YVg({|v?d1|R?5 zL0S3fw)BZazRNNX|7P4rh7!+3tCG~O8l+m?H} z(CB>8(9LtKYIu3ohJ-9ecgk+L&!FX~Wuim&;v$>M4 zUfvn<=Eok(63Ubc>mZrd8d7(>8bG>J?PtOHih_xRYFu1Hg{t;%+hXu2#x%a%qzcab zv$X!ccoj)exoOnaco_jbGw7KryOtuf(SaR-VJ0nAe(1*AA}#QV1lMhGtzD>RoUZ;WA?~!K{8%chYn?ttlz17UpDLlhTkGcVfHY6R<2r4E{mU zq-}D?+*2gAkQYAKrk*rB%4WFC-B!eZZLg4(tR#@kUQHIzEqV48$9=Q(~J_0 zy1%LSCbkoOhRO!J+Oh#;bGuXe;~(bIE*!J@i<%_IcB7wjhB5iF#jBn5+u~fEECN2* z!QFh!m<(>%49H12Y33+?$JxKV3xW{xSs=gxkxW-@Xds^|O1`AmorDKrE8N2-@ospk z=Au%h=f!`_X|G^A;XWL}-_L@D6A~*4Yf!5RTTm$!t8y&fp5_oqvBjW{FufS`!)5m% z2g(=9Ap6Y2y(9OYOWuUVGp-K=6kqQ)kM0P^TQT{X{V$*sN$wbFb-DaUuJF*!?EJPl zJev!UsOB^UHZ2KppYTELh+kqDw+5dPFv&&;;C~=u$Mt+Ywga!8YkL2~@g67}3wAQP zrx^RaXb1(c7vwU8a2se75X(cX^$M{FH4AHS7d2}heqqg4F0!1|Na>UtAdT%3JnS!B)&zelTEj$^b0>Oyfw=P-y-Wd^#dEFRUN*C{!`aJIHi<_YA2?piC%^ zj!p}+ZnBrM?ErAM+D97B*7L8U$K zo(IR-&LF(85p+fuct9~VTSdRjs`d-m|6G;&PoWvC&s8z`TotPSoksp;RsL4VL@CHf z_3|Tn%`ObgRhLmr60<;ya-5wbh&t z#ycN_)3P_KZN5CRyG%LRO4`Ot)3vY#dNX9!f!`_>1%4Q`81E*2BRg~A-VcN7pcX#j zrbl@7`V%n z6J53(m?KRzKb)v?iCuYWbH*l6M77dY4keS!%>}*8n!@ROE4!|7mQ+YS4dff1JJC(t z6Fnuf^=dajqHpH1=|pb(po9Fr8it^;2dEk|Ro=$fxqK$^Yix{G($0m-{RCFQJ~LqUnO7jJcjr zl*N*!6WU;wtF=dLCWzD6kW;y)LEo=4wSXQDIcq5WttgE#%@*m><@H;~Q&GniA-$in z`sjWFLgychS1kIJmPtd-w6%iKkj&dGhtB%0)pyy0M<4HZ@ZY0PWLAd7FCrj&i|NRh?>hZj*&FYnyu%Ur`JdiTu&+n z78d3n)Rl6q&NwVj_jcr#s5G^d?VtV8bkkYco5lV0LiT+t8}98LW>d)|v|V3++zLbHC(NC@X#Hx?21J0M*gP2V`Yd^DYvVIr{C zSc4V)hZKf|OMSm%FVqSRC!phWSyuUAu%0fredf#TDR$|hMZihJ__F!)Nkh6z)d=NC z3q4V*K3JTetxCPgB2_)rhOSWhuXzu+%&>}*ARxUaDeRy{$xK(AC0I=9%X7dmc6?lZNqe-iM(`?Xn3x2Ov>sej6YVQJ9Q42>?4lil?X zew-S>tm{=@QC-zLtg*nh5mQojYnvVzf3!4TpXPuobW_*xYJs;9AokrXcs!Ay z;HK>#;G$*TPN2M!WxdH>oDY6k4A6S>BM0Nimf#LfboKxJXVBC=RBuO&g-=+@O-#0m zh*aPG16zY^tzQLNAF7L(IpGPa+mDsCeAK3k=IL6^LcE8l0o&)k@?dz!79yxUquQIe($zm5DG z5RdXTv)AjHaOPv6z%99mPsa#8OD@9=URvHoJ1hYnV2bG*2XYBgB!-GEoP&8fLmWGg z9NG^xl5D&3L^io&3iYweV*qhc=m+r7C#Jppo$Ygg;jO2yaFU8+F*RmPL` zYxfGKla_--I}YUT353k}nF1zt2NO?+kofR8Efl$Bb^&llgq+HV_UYJUH7M5IoN0sT z4;wDA0gs55ZI|FmJ0}^Pc}{Ji-|#jdR$`!s)Di4^g3b_Qr<*Qu2rz}R6!B^;`Lj3sKWzjMYjexX)-;f5Y+HfkctE{PstO-BZan0zdXPQ=V8 zS8cBhnQyy4oN?J~oK0zl!#S|v6h-nx5to7WkdEk0HKBm;?kcNO*A+u=%f~l&aY*+J z>%^Dz`EQ6!+SEX$>?d(~|MNWU-}JTrk}&`IR|Ske(G^iMdk04)Cxd@}{1=P0U*%L5 zMFH_$R+HUGGv|ju2Z>5x(-aIbVJLcH1S+(E#MNe9g;VZX{5f%_|Kv7|UY-CM(>vf= z!4m?QS+AL+rUyfGJ;~uJGp4{WhOOc%2ybVP68@QTwI(8kDuYf?#^xv zBmOHCZU8O(x)=GVFn%tg@TVW1)qJJ_bU}4e7i>&V?r zh-03>d3DFj&@}6t1y3*yOzllYQ++BO-q!)zsk`D(z||)y&}o%sZ-tUF>0KsiYKFg6 zTONq)P+uL5Vm0w{D5Gms^>H1qa&Z##*X31=58*r%Z@Ko=IMXX{;aiMUp-!$As3{sq z0EEk02MOsgGm7$}E%H1ys2$yftNbB%1rdo@?6~0!a8Ym*1f;jIgfcYEF(I_^+;Xdr z2a>&oc^dF3pm(UNpazXgVzuF<2|zdPGjrNUKpdb$HOgNp*V56XqH`~$c~oSiqx;8_ zEz3fHoU*aJUbFJ&?W)sZB3qOSS;OIZ=n-*#q{?PCXi?Mq4aY@=XvlNQdA;yVC0Vy+ z{Zk6OO!lMYWd`T#bS8FV(`%flEA9El;~WjZKU1YmZpG#49`ku`oV{Bdtvzyz3{k&7 zlG>ik>eL1P93F zd&!aXluU_qV1~sBQf$F%sM4kTfGx5MxO0zJy<#5Z&qzNfull=k1_CZivd-WAuIQf> zBT3&WR|VD|=nKelnp3Q@A~^d_jN3@$x2$f@E~e<$dk$L@06Paw$);l*ewndzL~LuU zq`>vfKb*+=uw`}NsM}~oY}gW%XFwy&A>bi{7s>@(cu4NM;!%ieP$8r6&6jfoq756W z$Y<`J*d7nK4`6t`sZ;l%Oen|+pk|Ry2`p9lri5VD!Gq`U#Ms}pgX3ylAFr8(?1#&dxrtJgB>VqrlWZf61(r`&zMXsV~l{UGjI7R@*NiMJLUoK*kY&gY9kC@^}Fj* zd^l6_t}%Ku<0PY71%zQL`@}L}48M!@=r)Q^Ie5AWhv%#l+Rhu6fRpvv$28TH;N7Cl z%I^4ffBqx@Pxpq|rTJV)$CnxUPOIn`u278s9#ukn>PL25VMv2mff)-RXV&r`Dwid7}TEZxXX1q(h{R6v6X z&x{S_tW%f)BHc!jHNbnrDRjGB@cam{i#zZK*_*xlW@-R3VDmp)<$}S%t*@VmYX;1h zFWmpXt@1xJlc15Yjs2&e%)d`fimRfi?+fS^BoTcrsew%e@T^}wyVv6NGDyMGHSKIQ zC>qFr4GY?#S#pq!%IM_AOf`#}tPoMn7JP8dHXm(v3UTq!aOfEXNRtEJ^4ED@jx%le zvUoUs-d|2(zBsrN0wE(Pj^g5wx{1YPg9FL1)V1JupsVaXNzq4fX+R!oVX+q3tG?L= z>=s38J_!$eSzy0m?om6Wv|ZCbYVHDH*J1_Ndajoh&?L7h&(CVii&rmLu+FcI;1qd_ zHDb3Vk=(`WV?Uq;<0NccEh0s`mBXcEtmwt6oN99RQt7MNER3`{snV$qBTp={Hn!zz z1gkYi#^;P8s!tQl(Y>|lvz{5$uiXsitTD^1YgCp+1%IMIRLiSP`sJru0oY-p!FPbI)!6{XM%)(_Dolh1;$HlghB-&e><;zU&pc=ujpa-(+S&Jj zX1n4T#DJDuG7NP;F5TkoG#qjjZ8NdXxF0l58RK?XO7?faM5*Z17stidTP|a%_N z^e$D?@~q#Pf+708cLSWCK|toT1YSHfXVIs9Dnh5R(}(I;7KhKB7RD>f%;H2X?Z9eR z{lUMuO~ffT!^ew= z7u13>STI4tZpCQ?yb9;tSM-(EGb?iW$a1eBy4-PVejgMXFIV_Ha^XB|F}zK_gzdhM z!)($XfrFHPf&uyFQf$EpcAfk83}91Y`JFJOiQ;v5ca?)a!IxOi36tGkPk4S6EW~eq z>WiK`Vu3D1DaZ}515nl6>;3#xo{GQp1(=uTXl1~ z4gdWxr-8a$L*_G^UVd&bqW_nzMM&SlNW$8|$lAfo@zb+P>2q?=+T^qNwblP*RsN?N zdZE%^Zs;yAwero1qaoqMp~|KL=&npffh981>2om!fseU(CtJ=bW7c6l{U5(07*e0~ zJRbid6?&psp)ilmYYR3ZIg;t;6?*>hoZ3uq7dvyyq-yq$zH$yyImjfhpQb@WKENSP zl;KPCE+KXzU5!)mu12~;2trrLfs&nlEVOndh9&!SAOdeYd}ugwpE-9OF|yQs(w@C9 zoXVX`LP~V>%$<(%~tE*bsq(EFm zU5z{H@Fs^>nm%m%wZs*hRl=KD%4W3|(@j!nJr{Mmkl`e_uR9fZ-E{JY7#s6i()WXB0g-b`R{2r@K{2h3T+a>82>722+$RM*?W5;Bmo6$X3+Ieg9&^TU(*F$Q3 zT572!;vJeBr-)x?cP;^w1zoAM`nWYVz^<6N>SkgG3s4MrNtzQO|A?odKurb6DGZffo>DP_)S0$#gGQ_vw@a9JDXs2}hV&c>$ zUT0;1@cY5kozKOcbN6)n5v)l#>nLFL_x?2NQgurQH(KH@gGe>F|$&@ zq@2A!EXcIsDdzf@cWqElI5~t z4cL9gg7{%~4@`ANXnVAi=JvSsj95-7V& zME3o-%9~2?cvlH#twW~99=-$C=+b5^Yv}Zh4;Mg-!LS zw>gqc=}CzS9>v5C?#re>JsRY!w|Mtv#%O3%Ydn=S9cQarqkZwaM4z(gL~1&oJZ;t; zA5+g3O6itCsu93!G1J_J%Icku>b3O6qBW$1Ej_oUWc@MI)| zQ~eyS-EAAnVZp}CQnvG0N>Kc$h^1DRJkE7xZqJ0>p<>9*apXgBMI-v87E0+PeJ-K& z#(8>P_W^h_kBkI;&e_{~!M+TXt@z8Po*!L^8XBn{of)knd-xp{heZh~@EunB2W)gd zAVTw6ZZasTi>((qpBFh(r4)k zz&@Mc@ZcI-4d639AfcOgHOU+YtpZ)rC%Bc5gw5o~+E-i+bMm(A6!uE>=>1M;V!Wl4 z<#~muol$FsY_qQC{JDc8b=$l6Y_@_!$av^08`czSm!Xan{l$@GO-zPq1s>WF)G=wv zDD8j~Ht1pFj)*-b7h>W)@O&m&VyYci&}K|0_Z*w`L>1jnGfCf@6p}Ef*?wdficVe_ zmPRUZ(C+YJU+hIj@_#IiM7+$4kH#VS5tM!Ksz01siPc-WUe9Y3|pb4u2qnn zRavJiRpa zq?tr&YV?yKt<@-kAFl3s&Kq#jag$hN+Y%%kX_ytvpCsElgFoN3SsZLC>0f|m#&Jhu zp7c1dV$55$+k78FI2q!FT}r|}cIV;zp~#6X2&}22$t6cHx_95FL~T~1XW21VFuatb zpM@6w>c^SJ>Pq6{L&f9()uy)TAWf;6LyHH3BUiJ8A4}od)9sriz~e7}l7Vr0e%(=>KG1Jay zW0azuWC`(|B?<6;R)2}aU`r@mt_#W2VrO{LcX$Hg9f4H#XpOsAOX02x^w9+xnLVAt z^~hv2guE-DElBG+`+`>PwXn5kuP_ZiOO3QuwoEr)ky;o$n7hFoh}Aq0@Ar<8`H!n} zspCC^EB=6>$q*gf&M2wj@zzfBl(w_@0;h^*fC#PW9!-kT-dt*e7^)OIU{Uw%U4d#g zL&o>6`hKQUps|G4F_5AuFU4wI)(%9(av7-u40(IaI|%ir@~w9-rLs&efOR@oQy)}{ z&T#Qf`!|52W0d+>G!h~5A}7VJky`C3^fkJzt3|M&xW~x-8rSi-uz=qBsgODqbl(W#f{Ew#ui(K)(Hr&xqZs` zfrK^2)tF#|U=K|_U@|r=M_Hb;qj1GJG=O=d`~#AFAccecIaq3U`(Ds1*f*TIs=IGL zp_vlaRUtFNK8(k;JEu&|i_m39c(HblQkF8g#l|?hPaUzH2kAAF1>>Yykva0;U@&oRV8w?5yEK??A0SBgh?@Pd zJg{O~4xURt7!a;$rz9%IMHQeEZHR8KgFQixarg+MfmM_OeX#~#&?mx44qe!wt`~dd zqyt^~ML>V>2Do$huU<7}EF2wy9^kJJSm6HoAD*sRz%a|aJWz_n6?bz99h)jNMp}3k ztPVbos1$lC1nX_OK0~h>=F&v^IfgBF{#BIi&HTL}O7H-t4+wwa)kf3AE2-Dx@#mTA z!0f`>vz+d3AF$NH_-JqkuK1C+5>yns0G;r5ApsU|a-w9^j4c+FS{#+7- zH%skr+TJ~W_8CK_j$T1b;$ql_+;q6W|D^BNK*A+W5XQBbJy|)(IDA=L9d>t1`KX2b zOX(Ffv*m?e>! zS3lc>XC@IqPf1g-%^4XyGl*1v0NWnwZTW?z4Y6sncXkaA{?NYna3(n@(+n+#sYm}A zGQS;*Li$4R(Ff{obl3#6pUsA0fKuWurQo$mWXMNPV5K66V!XYOyc})^>889Hg3I<{V^Lj9($B4Zu$xRr=89-lDz9x`+I8q(vEAimx1K{sTbs|5x7S zZ+7o$;9&9>@3K;5-DVzGw=kp7ez%1*kxhGytdLS>Q)=xUWv3k_x(IsS8we39Tijvr z`GKk>gkZTHSht;5q%fh9z?vk%sWO}KR04G9^jleJ^@ovWrob7{1xy7V=;S~dDVt%S za$Q#Th%6g1(hiP>hDe}7lcuI94K-2~Q0R3A1nsb7Y*Z!DtQ(Ic<0;TDKvc6%1kBdJ z$hF!{uALB0pa?B^TC}#N5gZ|CKjy|BnT$7eaKj;f>Alqdb_FA3yjZ4CCvm)D&ibL) zZRi91HC!TIAUl<|`rK_6avGh`!)TKk=j|8*W|!vb9>HLv^E%t$`@r@piI(6V8pqDG zBON7~=cf1ZWF6jc{qkKm;oYBtUpIdau6s+<-o^5qNi-p%L%xAtn9OktFd{@EjVAT% z#?-MJ5}Q9QiK_jYYWs+;I4&!N^(mb!%4zx7qO6oCEDn=8oL6#*9XIJ&iJ30O`0vsFy|fEVkw}*jd&B6!IYi+~Y)qv6QlM&V9g0 zh)@^BVDB|P&#X{31>G*nAT}Mz-j~zd>L{v{9AxrxKFw8j;ccQ$NE0PZCc(7fEt1xd z`(oR2!gX6}R+Z77VkDz^{I)@%&HQT5q+1xlf*3R^U8q%;IT8-B53&}dNA7GW`Ki&= z$lrdH zDCu;j$GxW<&v_4Te7=AE2J0u1NM_7Hl9$u{z(8#%8vvrx2P#R7AwnY|?#LbWmROa; zOJzU_*^+n(+k;Jd{e~So9>OF>fPx$Hb$?~K1ul2xr>>o@**n^6IMu8+o3rDp(X$cC z`wQt9qIS>yjA$K~bg{M%kJ00A)U4L+#*@$8UlS#lN3YA{R{7{-zu#n1>0@(#^eb_% zY|q}2)jOEM8t~9p$X5fpT7BZQ1bND#^Uyaa{mNcFWL|MoYb@>y`d{VwmsF&haoJuS2W7azZU0{tu#Jj_-^QRc35tjW~ae&zhKk!wD}#xR1WHu z_7Fys#bp&R?VXy$WYa$~!dMxt2@*(>@xS}5f-@6eoT%rwH zv_6}M?+piNE;BqaKzm1kK@?fTy$4k5cqYdN8x-<(o6KelwvkTqC3VW5HEnr+WGQlF zs`lcYEm=HPpmM4;Ich7A3a5Mb3YyQs7(Tuz-k4O0*-YGvl+2&V(B&L1F8qfR0@vQM-rF<2h-l9T12eL}3LnNAVyY_z51xVr$%@VQ-lS~wf3mnHc zoM({3Z<3+PpTFCRn_Y6cbxu9v>_>eTN0>hHPl_NQQuaK^Mhrv zX{q#80ot;ptt3#js3>kD&uNs{G0mQp>jyc0GG?=9wb33hm z`y2jL=J)T1JD7eX3xa4h$bG}2ev=?7f>-JmCj6){Upo&$k{2WA=%f;KB;X5e;JF3IjQBa4e-Gp~xv- z|In&Rad7LjJVz*q*+splCj|{7=kvQLw0F@$vPuw4m^z=B^7=A4asK_`%lEf_oIJ-O z{L)zi4bd#&g0w{p1$#I&@bz3QXu%Y)j46HAJKWVfRRB*oXo4lIy7BcVl4hRs<%&iQ zr|)Z^LUJ>qn>{6y`JdabfNNFPX7#3`x|uw+z@h<`x{J4&NlDjnknMf(VW_nKWT!Jh zo1iWBqT6^BR-{T=4Ybe+?6zxP_;A5Uo{}Xel%*=|zRGm1)pR43K39SZ=%{MDCS2d$~}PE-xPw4ZK6)H;Zc&0D5p!vjCn0wCe&rVIhchR9ql!p2`g0b@JsC^J#n_r*4lZ~u0UHKwo(HaHUJDHf^gdJhTdTW z3i7Zp_`xyKC&AI^#~JMVZj^9WsW}UR#nc#o+ifY<4`M+?Y9NTBT~p`ONtAFf8(ltr*ER-Ig!yRs2xke#NN zkyFcaQKYv>L8mQdrL+#rjgVY>Z2_$bIUz(kaqL}cYENh-2S6BQK-a(VNDa_UewSW` zMgHi<3`f!eHsyL6*^e^W7#l?V|42CfAjsgyiJsA`yNfAMB*lAsJj^K3EcCzm1KT zDU2+A5~X%ax-JJ@&7>m`T;;}(-e%gcYQtj}?ic<*gkv)X2-QJI5I0tA2`*zZRX(;6 zJ0dYfMbQ+{9Rn3T@Iu4+imx3Y%bcf2{uT4j-msZ~eO)5Z_T7NC|Nr3)|NWjomhv=E zXaVin)MY)`1QtDyO7mUCjG{5+o1jD_anyKn73uflH*ASA8rm+S=gIfgJ);>Zx*hNG z!)8DDCNOrbR#9M7Ud_1kf6BP)x^p(|_VWCJ+(WGDbYmnMLWc?O4zz#eiP3{NfP1UV z(n3vc-axE&vko^f+4nkF=XK-mnHHQ7>w05$Q}iv(kJc4O3TEvuIDM<=U9@`~WdKN* zp4e4R1ncR_kghW}>aE$@OOc~*aH5OOwB5U*Z)%{LRlhtHuigxH8KuDwvq5{3Zg{Vr zrd@)KPwVKFP2{rXho(>MTZZfkr$*alm_lltPob4N4MmhEkv`J(9NZFzA>q0Ch;!Ut zi@jS_=0%HAlN+$-IZGPi_6$)ap>Z{XQGt&@ZaJ(es!Po5*3}>R4x66WZNsjE4BVgn z>}xm=V?F#tx#e+pimNPH?Md5hV7>0pAg$K!?mpt@pXg6UW9c?gvzlNe0 z3QtIWmw$0raJkjQcbv-7Ri&eX6Ks@@EZ&53N|g7HU<;V1pkc&$3D#8k!coJ=^{=vf z-pCP;vr2#A+i#6VA?!hs6A4P@mN62XYY$#W9;MwNia~89i`=1GoFESI+%Mbrmwg*0 zbBq4^bA^XT#1MAOum)L&ARDXJ6S#G>&*72f50M1r5JAnM1p7GFIv$Kf9eVR(u$KLt z9&hQ{t^i16zL1c(tRa~?qr?lbSN;1k;%;p*#gw_BwHJRjcYPTj6>y-rw*dFTnEs95 z`%-AoPL!P16{=#RI0 zUb6#`KR|v^?6uNnY`zglZ#Wd|{*rZ(x&Hk8N6ob6mpX~e^qu5kxvh$2TLJA$M=rx zc!#ot+sS+-!O<0KR6+Lx&~zgEhCsbFY{i_DQCihspM?e z-V}HemMAvFzXR#fV~a=Xf-;tJ1edd}Mry@^=9BxON;dYr8vDEK<<{ zW~rg(ZspxuC&aJo$GTM!9_sXu(EaQJNkV9AC(ob#uA=b4*!Uf}B*@TK=*dBvKKPAF z%14J$S)s-ws9~qKsf>DseEW(ssVQ9__YNg}r9GGx3AJiZR@w_QBlGP>yYh0lQCBtf zx+G;mP+cMAg&b^7J!`SiBwC81M_r0X9kAr2y$0(Lf1gZK#>i!cbww(hn$;fLIxRf? z!AtkSZc-h76KGSGz%48Oe`8ZBHkSXeVb!TJt_VC>$m<#}(Z}!(3h631ltKb3CDMw^fTRy%Ia!b&at`^g7Ew-%WLT9(#V0OP9CE?uj62s>`GI3NA z!`$U+i<`;IQyNBkou4|-7^9^ylac-Xu!M+V5p5l0Ve?J0wTSV+$gYtoc=+Ve*OJUJ z$+uIGALW?}+M!J9+M&#bT=Hz@{R2o>NtNGu1yS({pyteyb>*sg4N`KAD?`u3F#C1y z2K4FKOAPASGZTep54PqyCG(h3?kqQQAxDSW@>T2d!n;9C8NGS;3A8YMRcL>b=<<%M zMiWf$jY;`Ojq5S{kA!?28o)v$;)5bTL<4eM-_^h4)F#eeC2Dj*S`$jl^yn#NjJOYT zx%yC5Ww@eX*zsM)P(5#wRd=0+3~&3pdIH7CxF_2iZSw@>kCyd z%M}$1p((Bidw4XNtk&`BTkU{-PG)SXIZ)yQ!Iol6u8l*SQ1^%zC72FP zLvG>_Z0SReMvB%)1@+et0S{<3hV@^SY3V~5IY(KUtTR{*^xJ^2NN{sIMD9Mr9$~(C$GLNlSpzS=fsbw-DtHb_T|{s z9OR|sx!{?F``H!gVUltY7l~dx^a(2;OUV^)7 z%@hg`8+r&xIxmzZ;Q&v0X%9P)U0SE@r@(lKP%TO(>6I_iF{?PX(bez6v8Gp!W_nd5 z<8)`1jcT)ImNZp-9rr4_1MQ|!?#8sJQx{`~7)QZ75I=DPAFD9Mt{zqFrcrXCU9MG8 zEuGcy;nZ?J#M3!3DWW?Zqv~dnN6ijlIjPfJx(#S0cs;Z=jDjKY|$w2s4*Xa1Iz953sN2Lt!Vmk|%ZwOOqj`sA--5Hiaq8!C%LV zvWZ=bxeRV(&%BffMJ_F~~*FdcjhRVNUXu)MS(S#67rDe%Ler=GS+WysC1I2=Bmbh3s6wdS}o$0 zz%H08#SPFY9JPdL6blGD$D-AaYi;X!#zqib`(XX*i<*eh+2UEPzU4}V4RlC3{<>-~ zadGA8lSm>b7Z!q;D_f9DT4i)Q_}ByElGl*Cy~zX%IzHp)@g-itZB6xM70psn z;AY8II99e6P2drgtTG5>`^|7qg`9MTp%T~|1N3tBqV}2zgow3TFAH{XPor0%=HrkXnKyxyozHlJ6 zd3}OWkl?H$l#yZqOzZbMI+lDLoH48;s10!m1!K87g;t}^+A3f3e&w{EYhVPR0Km*- zh5-ku$Z|Ss{2?4pGm(Rz!0OQb^_*N`)rW{z)^Cw_`a(_L9j=&HEJl(!4rQy1IS)>- zeTIr>hOii`gc(fgYF(cs$R8l@q{mJzpoB5`5r>|sG zBpsY}RkY(g5`bj~D>(;F8v*DyjX(#nVLSs>)XneWI&%Wo>a0u#4A?N<1SK4D}&V1oN)76 z%S>a2n3n>G`YY1>0Hvn&AMtMuI_?`5?4y3w2Hnq4Qa2YH5 zxKdfM;k467djL31Y$0kd9FCPbU=pHBp@zaIi`Xkd80;%&66zvSqsq6%aY)jZacfvw ztkWE{ZV6V2WL9e}Dvz|!d96KqVkJU@5ryp#rReeWu>mSrOJxY^tWC9wd0)$+lZc%{ zY=c4#%OSyQJvQUuy^u}s8DN8|8T%TajOuaY^)R-&8s@r9D`(Ic4NmEu)fg1f!u`xUb;9t#rM z>}cY=648@d5(9A;J)d{a^*ORdVtJrZ77!g~^lZ9@)|-ojvW#>)Jhe8$7W3mhmQh@S zU=CSO+1gSsQ+Tv=x-BD}*py_Ox@;%#hPb&tqXqyUW9jV+fonnuCyVw=?HR>dAB~Fg z^vl*~y*4|)WUW*9RC%~O1gHW~*tJb^a-j;ae2LRNo|0S2`RX>MYqGKB^_ng7YRc@! zFxg1X!VsvXkNuv^3mI`F2=x6$(pZdw=jfYt1ja3FY7a41T07FPdCqFhU6%o|Yb6Z4 zpBGa=(ao3vvhUv#*S{li|EyujXQPUV;0sa5!0Ut)>tPWyC9e0_9(=v*z`TV5OUCcx zT=w=^8#5u~7<}8Mepqln4lDv*-~g^VoV{(+*4w(q{At6d^E-Usa2`JXty++Oh~on^ z;;WHkJsk2jvh#N|?(2PLl+g!M0#z_A;(#Uy=TzL&{Ei5G9#V{JbhKV$Qmkm%5tn!CMA? z@hM=b@2DZWTQ6>&F6WCq6;~~WALiS#@{|I+ucCmD6|tBf&e;$_)%JL8$oIQ%!|Xih1v4A$=7xNO zZVz$G8;G5)rxyD+M0$20L$4yukA_D+)xmK3DMTH3Q+$N&L%qB)XwYx&s1gkh=%qGCCPwnwhbT4p%*3R)I}S#w7HK3W^E%4w z2+7ctHPx3Q97MFYB48HfD!xKKb(U^K_4)Bz(5dvwyl*R?)k;uHEYVi|{^rvh)w7}t z`tnH{v9nlVHj2ign|1an_wz0vO)*`3RaJc#;(W-Q6!P&>+@#fptCgtUSn4!@b7tW0&pE2Qj@7}f#ugu4*C)8_}AMRuz^WG zc)XDcOPQjRaGptRD^57B83B-2NKRo!j6TBAJntJPHNQG;^Oz}zt5F^kId~miK3J@l ztc-IKp6qL!?u~q?qfGP0I~$5gvq#-0;R(oLU@sYayr*QH95fnrYA*E|n%&FP@Cz`a zSdJ~(c@O^>qaO`m9IQ8sd8!L<+)GPJDrL7{4{ko2gWOZel^3!($Gjt|B&$4dtfTmBmC>V`R&&6$wpgvdmns zxcmfS%9_ZoN>F~azvLFtA(9Q5HYT#A(byGkESnt{$Tu<73$W~reB4&KF^JBsoqJ6b zS?$D7DoUgzLO-?P`V?5_ub$nf1p0mF?I)StvPomT{uYjy!w&z$t~j&en=F~hw|O(1 zlV9$arQmKTc$L)Kupwz_zA~deT+-0WX6NzFPh&d+ly*3$%#?Ca9Z9lOJsGVoQ&1HNg+)tJ_sw)%oo*DK)iU~n zvL``LqTe=r=7SwZ@LB)9|3QB5`0(B9r(iR}0nUwJss-v=dXnwMRQFYSRK1blS#^g(3@z{`=8_CGDm!LESTWig zzm1{?AG&7`uYJ;PoFO$o8RWuYsV26V{>D-iYTnvq7igWx9@w$EC*FV^vpvDl@i9yp zPIqiX@hEZF4VqzI3Y)CHhR`xKN8poL&~ak|wgbE4zR%Dm(a@?bw%(7(!^>CM!^4@J z6Z)KhoQP;WBq_Z_&<@i2t2&xq>N>b;Np2rX?yK|-!14iE2T}E|jC+=wYe~`y38g3J z8QGZquvqBaG!vw&VtdXWX5*i5*% zJP~7h{?&E|<#l{klGPaun`IgAJ4;RlbRqgJz5rmHF>MtJHbfqyyZi53?Lhj=(Ku#& z__ubmZIxzSq3F90Xur!1)Vqe6b@!ueHA!93H~jdHmaS5Q^CULso}^poy)0Op6!{^9 zWyCyyIrdBP4fkliZ%*g+J-A!6VFSRF6Liu6G^^=W>cn81>4&7(c7(6vCGSAJ zQZ|S3mb|^Wf=yJ(h~rq`iiW~|n#$+KcblIR<@|lDtm!&NBzSG-1;7#YaU+-@=xIm4 zE}edTYd~e&_%+`dIqqgFntL-FxL3!m4yTNt<(^Vt9c6F(`?9`u>$oNxoKB29<}9FE zgf)VK!*F}nW?}l95%RRk8N4^Rf8)Xf;drT4<|lUDLPj^NPMrBPL;MX&0oGCsS za3}vWcF(IPx&W6{s%zwX{UxHX2&xLGfT{d9bWP!g;Lg#etpuno$}tHoG<4Kd*=kpU z;4%y(<^yj(UlG%l-7E9z_Kh2KoQ19qT3CR@Ghr>BAgr3Vniz3LmpC4g=g|A3968yD2KD$P7v$ zx9Q8`2&qH3&y-iv0#0+jur@}k`6C%7fKbCr|tHX2&O%r?rBpg`YNy~2m+ z*L7dP$RANzVUsG_Lb>=__``6vA*xpUecuGsL+AW?BeSwyoQfDlXe8R1*R1M{0#M?M zF+m19`3<`gM{+GpgW^=UmuK*yMh3}x)7P738wL8r@(Na6%ULPgbPVTa6gh5Q(SR0f znr6kdRpe^(LVM;6Rt(Z@Lsz3EX*ry6(WZ?w>#ZRelx)N%sE+MN>5G|Z8{%@b&D+Ov zPU{shc9}%;G7l;qbonIb_1m^Qc8ez}gTC-k02G8Rl?7={9zBz8uRX2{XJQ{vZhs67avlRn| zgRtWl0Lhjet&!YC47GIm%1gdq%T24_^@!W3pCywc89X4I5pnBCZDn(%!$lOGvS*`0!AoMtqxNPFgaMR zwoW$p;8l6v%a)vaNsesED3f}$%(>zICnoE|5JwP&+0XI}JxPccd+D^gx`g`=GsUc0 z9Uad|C+_@_0%JmcObGnS@3+J^0P!tg+fUZ_w#4rk#TlJYPXJiO>SBxzs9(J;XV9d{ zmTQE1(K8EYaz9p^XLbdWudyIPJlGPo0U*)fAh-jnbfm@SYD_2+?|DJ-^P+ojG{2{6 z>HJtedEjO@j_tqZ4;Zq1t5*5cWm~W?HGP!@_f6m#btM@46cEMhhK{(yI&jG)fwL1W z^n_?o@G8a-jYt!}$H*;{0#z8lANlo!9b@!c5K8<(#lPlpE!z86Yq#>WT&2} z;;G1$pD%iNoj#Z=&kij5&V1KHIhN-h<;{HC5wD)PvkF>CzlQOEx_0;-TJ*!#&{Wzt zKcvq^SZIdop}y~iouNqtU7K7+?eIz-v_rfNM>t#i+dD$s_`M;sjGubTdP)WI*uL@xPOLHt#~T<@Yz>xt50ZoTw;a(a}lNiDN-J${gOdE zx?8LOA|tv{Mb}=TTR=LcqMqbCJkKj+@;4Mu)Cu0{`~ohix6E$g&tff)aHeUAQQ%M? zIN4uSUTzC1iMEWL*W-in1y)C`E+R8j?4_?X4&2Zv5?QdkNMz(k} zw##^Ikx`#_s>i&CO_mu@vJJ*|3ePRDl5pq$9V^>D;g0R%l>lw;ttyM6Sy`NBF{)Lr zSk)V>mZr96+aHY%vTLLt%vO-+juw6^SO_ zYGJaGeWX6W(TOQx=5oTGXOFqMMU*uZyt>MR-Y`vxW#^&)H zk0!F8f*@v6NO@Z*@Qo)+hlX40EWcj~j9dGrLaq%1;DE_%#lffXCcJ;!ZyyyZTz74Q zb2WSly6sX{`gQeToQsi1-()5EJ1nJ*kXGD`xpXr~?F#V^sxE3qSOwRSaC9x9oa~jJ zTG9`E|q zC5Qs1xh}jzb5UPYF`3N9YuMnI7xsZ41P;?@c|%w zl=OxLr6sMGR+`LStLvh)g?fA5p|xbUD;yFAMQg&!PEDYxVYDfA>oTY;CFt`cg?Li1 z0b})!9Rvw&j#*&+D2))kXLL z0+j=?7?#~_}N-qdEIP>DQaZh#F(#e0WNLzwUAj@r694VJ8?Dr5_io2X49XYsG^ zREt0$HiNI~6VV!ycvao+0v7uT$_ilKCvsC+VDNg7yG1X+eNe^3D^S==F3ByiW0T^F zH6EsH^}Uj^VPIE&m)xlmOScYR(w750>hclqH~~dM2+;%GDXT`u4zG!p((*`Hwx41M z4KB+`hfT(YA%W)Ve(n+Gu9kuXWKzxg{1ff^xNQw>w%L-)RySTk9kAS92(X0Shg^Q? zx1YXg_TLC^?h6!4mBqZ9pKhXByu|u~gF%`%`vdoaGBN3^j4l!4x?Bw4Jd)Z4^di}! zXlG1;hFvc>H?bmmu1E7Vx=%vahd!P1#ZGJOJYNbaek^$DHt`EOE|Hlij+hX>ocQFSLVu|wz`|KVl@Oa;m2k6b*mNK2Vo{~l9>Qa3@B7G7#k?)aLx;w6U ze8bBq%vF?5v>#TspEoaII!N}sRT~>bh-VWJ7Q*1qsz%|G)CFmnttbq$Ogb{~YK_=! z{{0vhlW@g!$>|}$&4E3@k`KPElW6x#tSX&dfle>o!irek$NAbDzdd2pVeNzk4&qgJ zXvNF0$R96~g0x+R1igR=Xu&X_Hc5;!Ze&C)eUTB$9wW&?$&o8Yxhm5s(S`;?{> z*F?9Gr0|!OiKA>Rq-ae=_okB6&yMR?!JDer{@iQgIn=cGxs-u^!8Q$+N&pfg2WM&Z zulHu=Uh~U>fS{=Nm0x>ACvG*4R`Dx^kJ65&Vvfj`rSCV$5>c04N26Rt2S?*kh3JKq z9(3}5T?*x*AP(X2Ukftym0XOvg~r6Ms$2x&R&#}Sz23aMGU&7sU-cFvE3Eq`NBJe84VoftWF#v7PDAp`@V zRFCS24_k~;@~R*L)eCx@Q9EYmM)Sn}HLbVMyxx%{XnMBDc-YZ<(DXDBYUt8$u5Zh} zBK~=M9cG$?_m_M61YG+#|9Vef7LfbH>(C21&aC)x$^Lg}fa#SF){RX|?-xZjSOrn# z2ZAwUF)$VB<&S;R3FhNSQOV~8w%A`V9dWyLiy zgt7G=Z4t|zU3!dh5|s(@XyS|waBr$>@=^Dspmem8)@L`Ns{xl%rGdX!R(BiC5C7Vo zXetb$oC_iXS}2x_Hy}T(hUUNbO47Q@+^4Q`h>(R-;OxCyW#eoOeC51jzxnM1yxBrp zz6}z`(=cngs6X05e79o_B7@3K|Qpe3n38Py_~ zpi?^rj!`pq!7PHGliC$`-8A^Ib?2qgJJCW+(&TfOnFGJ+@-<<~`7BR0f4oSINBq&R z2CM`0%WLg_Duw^1SPwj-{?BUl2Y=M4e+7yL1{C&&f&zjF06#xf>VdLozgNye(BNgSD`=fFbBy0HIosLl@JwCQl^s;eTnc( z3!r8G=K>zb`|bLLI0N|eFJk%s)B>oJ^M@AQzqR;HUjLsOqW<0v>1ksT_#24*U@R3HJu*A^#1o#P3%3_jq>icD@<`tqU6ICEgZrME(xX#?i^Z z%Id$_uyQGlFD-CcaiRtRdGn|K`Lq5L-rx7`vYYGH7I=eLfHRozPiUtSe~Tt;IN2^gCXmf2#D~g2@9bhzK}3nphhG%d?V7+Zq{I2?Gt*!NSn_r~dd$ zqkUOg{U=MI?Ehx@`(X%rQB?LP=CjJ*V!rec{#0W2WshH$X#9zep!K)tzZoge*LYd5 z@g?-j5_mtMp>_WW`p*UNUZTFN{_+#m*bJzt{hvAdkF{W40{#L3w6gzPztnsA_4?&0 z(+>pv!zB16rR-(nm(^c>Z(its{ny677vT8sF564^mlZvJ!h65}OW%Hn|2OXbOQM%b z{6C54Z2v;^hyMQ;UH+HwFD2!F!VlQ}6Z{L0_9g5~CH0@Mqz?ZC`^QkhOU#$Lx<4`B zyZsa9uPF!rZDo8ZVfzzR#raQ>5|)k~_Ef*wDqG^76o)j!C4 zykvT*o$!-MBko@?{b~*Zf2*YMlImrK`cEp|#D7f%Twm<|C|dWD Date: Thu, 3 Dec 2020 17:19:40 +0100 Subject: [PATCH 0025/2127] read output file from commandline --- .test-infra/validate-runner/build.gradle | 15 ++++++++++++--- .test-infra/validate-runner/readme.md | 2 +- .../org/apache/beam/validate/runner/Main.java | 10 +++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.test-infra/validate-runner/build.gradle b/.test-infra/validate-runner/build.gradle index d5a29ccd6e4e..c4df92a3e254 100644 --- a/.test-infra/validate-runner/build.gradle +++ b/.test-infra/validate-runner/build.gradle @@ -16,12 +16,9 @@ * limitations under the License. */ -//plugins { id 'org.apache.beam.module' } apply plugin : "application" -mainClassName = 'org.apache.beam.validate.runner.Main' group 'org.apache.beam' -//version '1.0-SNAPSHOT' description = "Apache Beam :: Validate :: Runner" repositories { @@ -43,3 +40,15 @@ dependencies { compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.11.3' compile 'org.slf4j:slf4j-simple:1.7.9' } +ext.javaMainClass = "org.apache.beam.validate.runner.Main" +if (project.hasProperty("args")) { + ext.cmdargs = project.getProperty("args") +} else { + ext.cmdargs = "" +} + +task runner(type: JavaExec) { + classpath = sourceSets.main.runtimeClasspath + main = "org.apache.beam.validate.runner.Main" + args cmdargs.split() +} diff --git a/.test-infra/validate-runner/readme.md b/.test-infra/validate-runner/readme.md index a72040c3a034..2c7186f6ac6c 100644 --- a/.test-infra/validate-runner/readme.md +++ b/.test-infra/validate-runner/readme.md @@ -1,2 +1,2 @@ Run the project -./gradlew beam-validate-runner:run \ No newline at end of file +./gradlew beam-validate-runner:run -Pargs="filename" \ No newline at end of file diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java index fe83c5867bd6..8ef49bf8cfaf 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java @@ -15,6 +15,14 @@ public static void main(String args[]) { try { final Logger logger = LoggerFactory.getLogger(Main.class); + String outputFile; + if (args.length == 0) { + logger.info("Output file name missing. Output will be saved to output.json"); + outputFile = "output"; + } else { + outputFile = args[0]; + logger.info("Output will be saved to {} .json", outputFile); + } JSONArray outputDetails = new JSONArray(); logger.info("Processing Batch Jobs:"); @@ -25,7 +33,7 @@ public static void main(String args[]) { StreamTestService streamTestService = new StreamTestService(); outputDetails.add(streamTestService.getStreamTests()); - try (FileWriter file = new FileWriter("out.json")) { + try (FileWriter file = new FileWriter(outputFile + ".json")) { file.write(outputDetails.toString(3)); file.flush(); } catch (IOException e) { From 8fbb56f6b9272181f16df8477a494c8cfc226524 Mon Sep 17 00:00:00 2001 From: Sruthi Sree Kumar Date: Thu, 3 Dec 2020 17:47:58 +0100 Subject: [PATCH 0026/2127] added code comments --- .../validate/runner/service/TestService.java | 32 ++++++++++++++++--- .../validate/runner/util/FileReaderUtil.java | 4 +++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java index 62ffcbd652d3..592c45ba82b6 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/TestService.java @@ -19,28 +19,52 @@ public interface TestService { + /** + * Returns all the tests {@link CaseResult} from test result. + * @param testResult {@link TestResult} + * @return set of case result + */ default Set getAllTests(TestResult testResult) { Set caseResults = new HashSet<>(); Optional.ofNullable(testResult.getSuites()).ifPresent(suites -> suites.forEach(item -> caseResults.addAll(item.getCases()))); -// List tests = new ArrayList<>(); -// Optional.ofNullable(caseResults).ifPresent(cases -> cases.forEach(item -> tests.add(item.getClassName() + "." + item.getName() + " : " + item.getStatus()))); return caseResults; } + /** + * Returns the jenkins URL for the last successful build. + * + * @param job Map of runner an job name retrieved from configuration + * @param configuration The input configuration + * + * @return The URL of last successful job. + * @throws URISyntaxException + * @throws IOException + */ default URL getUrl(Map job, Configuration configuration) throws URISyntaxException, IOException { Map jobs = new JenkinsServer(new URI(configuration.getServer())).getJobs(); JobWithDetails jobWithDetails = jobs.get(job.get(job.keySet().toArray()[0])).details(); return new URL(jobWithDetails.getLastSuccessfulBuild().getUrl() + configuration.getJsonapi()); } + /** + * Fetches the test name and class name from a test result. + * @param testResult {@link TestResult} + * @return set of pair of all testname and test class name + */ default Set> getTestNames(TestResult testResult) { Set> caseResults = new HashSet<>(); Optional.ofNullable(testResult.getSuites()).ifPresent(suites -> suites.forEach(item -> item.getCases().forEach(caseResult -> caseResults.add(new ImmutablePair<>(caseResult.getName(), caseResult.getClassName()))))); -// List tests = new ArrayList<>(); -// Optional.ofNullable(caseResults).ifPresent(cases -> cases.forEach(item -> tests.add(item.getClassName() + "." + item.getName() + " : " + item.getStatus()))); return caseResults; } + /** + * Method find the tests which are not run for a particular runner and add them as the tests + * which are "NOT RUN" for a particular runner + * + * @param testnames Set of Pair of test name and Test ClassName of all the tests which are run across runners. + * @param map Map of runner and the Tests run for that particular runner + * @return The JsonObject with each runner and all the tests which are RUN/ NOT RUN. + */ default JSONObject process(Set> testnames, HashMap> map) { map.forEach((k, v) -> { Set caseResults = v; diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/util/FileReaderUtil.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/util/FileReaderUtil.java index 6727a6946f3b..14d8fb5abccd 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/util/FileReaderUtil.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/util/FileReaderUtil.java @@ -6,6 +6,10 @@ import java.io.InputStream; +/** + * Reads the input configurations from the configuration.yaml file + * and passes as a {@link Configuration} object for processing. + */ public class FileReaderUtil { private static final String FILE_PATH = "/configuration.yaml"; From 842245d1105c6cedb6141fe7a1cefe38011fd095 Mon Sep 17 00:00:00 2001 From: Sruthi Sree Kumar Date: Thu, 3 Dec 2020 17:52:39 +0100 Subject: [PATCH 0027/2127] code comment --- .../beam/validate/runner/service/BatchTestService.java | 5 +++++ .../beam/validate/runner/service/StreamTestService.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/BatchTestService.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/BatchTestService.java index 90b527102746..920008e0feb3 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/BatchTestService.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/BatchTestService.java @@ -15,8 +15,13 @@ import java.util.Set; public class BatchTestService implements TestService { + + // Stores all the tests which are run across runners in batch mode private static Set> batchTests = new HashSet<>(); + + //Stores the tests which are run for the particular runner. private HashMap> map = new HashMap<>(); + public JSONObject getBatchTests() { try { Configuration configuration = FileReaderUtil.readConfiguration(); diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/StreamTestService.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/StreamTestService.java index 05cd0555af95..866bfafaa5be 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/StreamTestService.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/service/StreamTestService.java @@ -15,8 +15,13 @@ public class StreamTestService implements TestService { + + // Stores all the tests which are run across runners in stream mode private static Set> streamTests = new HashSet<>(); + + //Stores the tests which are run for the particular runner. private HashMap> map = new HashMap<>(); + public JSONObject getStreamTests() { try { Configuration configuration = FileReaderUtil.readConfiguration(); From e2abf14a28f1d7a96eb5bcb3230f35daddd0a77f Mon Sep 17 00:00:00 2001 From: Sruthi Sree Kumar Date: Sat, 5 Dec 2020 14:41:26 +0100 Subject: [PATCH 0028/2127] added Readme --- .test-infra/validate-runner/README.md | 19 +++++++++++++++++++ .test-infra/validate-runner/readme.md | 2 -- .../org/apache/beam/validate/runner/Main.java | 6 +++--- 3 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 .test-infra/validate-runner/README.md delete mode 100644 .test-infra/validate-runner/readme.md diff --git a/.test-infra/validate-runner/README.md b/.test-infra/validate-runner/README.md new file mode 100644 index 000000000000..2b13d114d897 --- /dev/null +++ b/.test-infra/validate-runner/README.md @@ -0,0 +1,19 @@ +#Overview +Apache Beam provides a portable API layer for building sophisticated data-parallel processing pipelines that may be executed across a diversity of execution engines, or runners. The core concepts of this layer are based upon the Beam Model (formerly referred to as the Dataflow Model), and implemented to varying degrees in each Beam runner. +Apache Beam maintains a capability matrix to track which Beam features are supported by which set of language SDKs + Runners. + +This module consistis of the scripts to automatically update the capability matrix with each project release so that its uptodate up to date with minimum supervision or ownership. +The workflow is as follows: + +- The script will run periodically, and using the latest runs from relevant test suites. The script outputs a capability matrix file in JSON format. +- The capability matrix file is uploaded to a public folder in GCS +- The Beam website will fetch the capability matrix file every time a user loads the Capability Matrix pagefile, and build the matrix + +###Run the project +This module can be run using the below command. It accept a single argument which is the output JSON filename. If not passes, the output will be written to the file capability.json + +`./gradlew beam-validate-runner:runner -Pargs="filename"` + +####Run Configurations +The project includes a [configuration file](src/main/resources/configuration.yaml) which includes the different configurations to generate the capablities. +Inoreder to add a new runner, the runner name and the Jenkins job name needs to be added to the [configuration file](src/main/resources/configuration.yaml) in the respective mode(batch/stream). \ No newline at end of file diff --git a/.test-infra/validate-runner/readme.md b/.test-infra/validate-runner/readme.md deleted file mode 100644 index 2c7186f6ac6c..000000000000 --- a/.test-infra/validate-runner/readme.md +++ /dev/null @@ -1,2 +0,0 @@ -Run the project -./gradlew beam-validate-runner:run -Pargs="filename" \ No newline at end of file diff --git a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java index 8ef49bf8cfaf..3fa74384cb4c 100644 --- a/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java +++ b/.test-infra/validate-runner/src/main/java/org/apache/beam/validate/runner/Main.java @@ -17,11 +17,11 @@ public static void main(String args[]) { String outputFile; if (args.length == 0) { - logger.info("Output file name missing. Output will be saved to output.json"); - outputFile = "output"; + logger.info("Output file name missing. Output will be saved to capability.json"); + outputFile = "capability"; } else { outputFile = args[0]; - logger.info("Output will be saved to {} .json", outputFile); + logger.info("Output will be saved to {}.json", outputFile); } JSONArray outputDetails = new JSONArray(); From 8dd9b382170322acefaca123ae63aa814f924151 Mon Sep 17 00:00:00 2001 From: Sam Whittle Date: Wed, 2 Dec 2020 03:28:16 -0800 Subject: [PATCH 0029/2127] [BEAM-11401][BEAM-11366] Change ReaderCache invalidations to be asynchronous and make the timeout configurable. Closing readers can block indefinitely and without this change invalidations run inline on threads possibly accessing different keys in the ReaderCache. --- .../options/DataflowPipelineDebugOptions.java | 10 +++++ .../runners/dataflow/worker/ReaderCache.java | 38 +++++++++++-------- .../worker/StreamingDataflowWorker.java | 7 +++- .../dataflow/worker/ReaderCacheTest.java | 6 +-- .../StreamingModeExecutionContextTest.java | 3 +- .../worker/WorkerCustomSourcesTest.java | 2 +- 6 files changed, 44 insertions(+), 22 deletions(-) diff --git a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java index 4811ec008189..f0c16ff0aa43 100644 --- a/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java +++ b/runners/google-cloud-dataflow-java/src/main/java/org/apache/beam/runners/dataflow/options/DataflowPipelineDebugOptions.java @@ -220,6 +220,16 @@ public Dataflow create(PipelineOptions options) { void setWorkerCacheMb(Integer value); + /** + * The amount of time before UnboundedReaders are considered idle and closed during streaming + * execution. + */ + @Description("The amount of time before UnboundedReaders are uncached, in seconds.") + @Default.Integer(60) + Integer getReaderCacheTimeoutSec(); + + void setReaderCacheTimeoutSec(Integer value); + /** * CAUTION: This option implies dumpHeapOnOOM, and has similar caveats. Specifically, heap dumps * can of comparable size to the default boot disk. Consider increasing the boot disk size before diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderCache.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderCache.java index bf65cfac1d4a..58f3abfc66a2 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderCache.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/ReaderCache.java @@ -18,6 +18,7 @@ package org.apache.beam.runners.dataflow.worker; import java.io.IOException; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import javax.annotation.concurrent.ThreadSafe; import org.apache.beam.sdk.io.UnboundedSource; @@ -41,6 +42,7 @@ class ReaderCache { private static final Logger LOG = LoggerFactory.getLogger(ReaderCache.class); + private final Executor invalidationExecutor; // Note on thread safety. This class is thread safe because: // - Guava Cache is thread safe. @@ -64,33 +66,36 @@ private static class CacheEntry { private final Cache cache; - /** ReaderCache with default 1 minute expiration for readers. */ - ReaderCache() { - this(Duration.standardMinutes(1)); - } - - /** Cache reader for {@code cacheDuration}. */ - ReaderCache(Duration cacheDuration) { + /** Cache reader for {@code cacheDuration}. Readers will be closed on {@code executor}. */ + ReaderCache(Duration cacheDuration, Executor invalidationExecutor) { + this.invalidationExecutor = invalidationExecutor; this.cache = CacheBuilder.newBuilder() .expireAfterWrite(cacheDuration.getMillis(), TimeUnit.MILLISECONDS) .removalListener( (RemovalNotification notification) -> { if (notification.getCause() != RemovalCause.EXPLICIT) { - LOG.info("Closing idle reader for {}", notification.getKey()); - closeReader(notification.getKey(), notification.getValue()); + LOG.info( + "Asynchronously closing reader for {} as it has been idle for over {}", + notification.getKey(), + cacheDuration); + asyncCloseReader(notification.getKey(), notification.getValue()); } }) .build(); } /** Close the reader and log a warning if close fails. */ - private void closeReader(WindmillComputationKey key, CacheEntry entry) { - try { - entry.reader.close(); - } catch (IOException e) { - LOG.warn("Failed to close UnboundedReader for {}", key, e); - } + private void asyncCloseReader(WindmillComputationKey key, CacheEntry entry) { + invalidationExecutor.execute( + () -> { + try { + entry.reader.close(); + LOG.info("Finished closing reader for {}", key); + } catch (IOException e) { + LOG.warn("Failed to close UnboundedReader for {}", key, e); + } + }); } /** @@ -112,7 +117,8 @@ UnboundedSource.UnboundedReader acquireReader( } else { // new cacheToken invalidates old one or this is a retried or stale request, // close the reader. - closeReader(computationKey, entry); + LOG.info("Asynchronously closing reader for {} as it is no longer valid", computationKey); + asyncCloseReader(computationKey, entry); } } return null; diff --git a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java index 6c127c84f849..b10cebcd4542 100644 --- a/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java +++ b/runners/google-cloud-dataflow-java/worker/src/main/java/org/apache/beam/runners/dataflow/worker/StreamingDataflowWorker.java @@ -54,6 +54,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; import java.util.concurrent.ThreadFactory; @@ -467,7 +468,7 @@ public int getSize() { private final EvictingQueue pendingFailuresToReport = EvictingQueue.create(MAX_FAILURES_TO_REPORT_IN_UPDATE); - private final ReaderCache readerCache = new ReaderCache(); + private final ReaderCache readerCache; private final WorkUnitClient workUnitClient; private final CompletableFuture isDoneFuture; @@ -597,6 +598,10 @@ public static StreamingDataflowWorker fromDataflowWorkerHarnessOptions( HotKeyLogger hotKeyLogger) throws IOException { this.stateCache = new WindmillStateCache(options.getWorkerCacheMb()); + this.readerCache = + new ReaderCache( + Duration.standardSeconds(options.getReaderCacheTimeoutSec()), + Executors.newCachedThreadPool()); this.mapTaskExecutorFactory = mapTaskExecutorFactory; this.workUnitClient = workUnitClient; this.options = options; diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReaderCacheTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReaderCacheTest.java index f9ab12c41b0e..25e5ecf31902 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReaderCacheTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/ReaderCacheTest.java @@ -61,7 +61,7 @@ public class ReaderCacheTest { @Before public void setUp() { - readerCache = new ReaderCache(); + readerCache = new ReaderCache(Duration.standardMinutes(1), Runnable::run); MockitoAnnotations.initMocks(this); } @@ -152,7 +152,7 @@ public void testReaderCacheExpiration() throws IOException, InterruptedException Duration cacheDuration = Duration.millis(10); // Create a cache with short expiry period. - ReaderCache readerCache = new ReaderCache(cacheDuration); + ReaderCache readerCache = new ReaderCache(cacheDuration, Runnable::run); readerCache.cacheReader( WindmillComputationKey.create(C_ID, KEY_1, SHARDING_KEY), 1, 0, reader1); @@ -172,7 +172,7 @@ public void testReaderCacheExpiration() throws IOException, InterruptedException @Test public void testReaderCacheRetries() throws IOException, InterruptedException { - ReaderCache readerCache = new ReaderCache(); + ReaderCache readerCache = new ReaderCache(Duration.standardMinutes(1), Runnable::run); readerCache.cacheReader( WindmillComputationKey.create(C_ID, KEY_1, SHARDING_KEY), 1, 1, reader1); diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java index 89ac7580b738..8d86c1a33431 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/StreamingModeExecutionContextTest.java @@ -69,6 +69,7 @@ import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.ByteString; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; import org.hamcrest.Matchers; +import org.joda.time.Duration; import org.joda.time.Instant; import org.junit.Before; import org.junit.Test; @@ -104,7 +105,7 @@ public void setUp() { new StreamingModeExecutionContext( counterSet, "computationId", - new ReaderCache(), + new ReaderCache(Duration.standardMinutes(1), Executors.newCachedThreadPool()), stateNameMap, new WindmillStateCache(options.getWorkerCacheMb()).forComputation("comp"), StreamingStepMetricsContainer.createRegistry(), diff --git a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java index cf12da5230b2..2f749da35032 100644 --- a/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java +++ b/runners/google-cloud-dataflow-java/worker/src/test/java/org/apache/beam/runners/dataflow/worker/WorkerCustomSourcesTest.java @@ -502,7 +502,7 @@ public void testReadUnboundedReader() throws Exception { CounterSet counterSet = new CounterSet(); StreamingModeExecutionStateRegistry executionStateRegistry = new StreamingModeExecutionStateRegistry(null); - ReaderCache readerCache = new ReaderCache(); + ReaderCache readerCache = new ReaderCache(Duration.standardMinutes(1), Runnable::run); StreamingModeExecutionContext context = new StreamingModeExecutionContext( counterSet, From 997680a6888f34ace9ea36ed57e66bd8f7313748 Mon Sep 17 00:00:00 2001 From: Sonam Ramchand Date: Mon, 7 Dec 2020 17:06:58 +0500 Subject: [PATCH 0030/2127] Implemented BIT_XOR for Zetasql dialect --- .../transform/BeamBuiltinAggregations.java | 64 ++++++++++++++----- .../SupportedZetaSqlBuiltinFunctions.java | 5 +- .../translation/SqlOperatorMappingTable.java | 4 +- .../sql/zetasql/translation/SqlOperators.java | 27 ++++---- .../translation/impl/BeamBuiltinMethods.java | 6 +- .../translation/impl/BitwiseFunctions.java | 18 ++++++ .../sql/zetasql/ZetaSqlDialectSpecTest.java | 41 +++++++----- 7 files changed, 112 insertions(+), 53 deletions(-) create mode 100644 sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/impl/BitwiseFunctions.java diff --git a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java index 9f668505ab23..741d66c91826 100644 --- a/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java +++ b/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/transform/BeamBuiltinAggregations.java @@ -17,33 +17,25 @@ */ package org.apache.beam.sdk.extensions.sql.impl.transform; -import java.math.BigDecimal; -import java.math.MathContext; -import java.math.RoundingMode; -import java.util.Map; -import java.util.function.Function; -import org.apache.beam.sdk.coders.BigDecimalCoder; -import org.apache.beam.sdk.coders.BigEndianIntegerCoder; -import org.apache.beam.sdk.coders.Coder; -import org.apache.beam.sdk.coders.CoderRegistry; -import org.apache.beam.sdk.coders.KvCoder; +import org.apache.beam.sdk.coders.*; import org.apache.beam.sdk.extensions.sql.impl.transform.agg.CovarianceFn; import org.apache.beam.sdk.extensions.sql.impl.transform.agg.VarianceFn; import org.apache.beam.sdk.extensions.sql.impl.utils.CalciteUtils; import org.apache.beam.sdk.schemas.Schema; import org.apache.beam.sdk.schemas.Schema.FieldType; import org.apache.beam.sdk.schemas.Schema.TypeName; -import org.apache.beam.sdk.transforms.Combine; +import org.apache.beam.sdk.transforms.*; import org.apache.beam.sdk.transforms.Combine.CombineFn; -import org.apache.beam.sdk.transforms.Count; -import org.apache.beam.sdk.transforms.Max; -import org.apache.beam.sdk.transforms.Min; -import org.apache.beam.sdk.transforms.Sample; -import org.apache.beam.sdk.transforms.Sum; import org.apache.beam.sdk.values.KV; import org.apache.beam.vendor.calcite.v1_20_0.com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.util.Map; +import java.util.function.Function; + /** Built-in aggregations functions for COUNT/MAX/MIN/SUM/AVG/VAR_POP/VAR_SAMP. */ @SuppressWarnings({ "rawtypes", // TODO(https://issues.apache.org/jira/browse/BEAM-10556) @@ -62,6 +54,7 @@ public class BeamBuiltinAggregations { .put("$SUM0", BeamBuiltinAggregations::createSum) .put("AVG", BeamBuiltinAggregations::createAvg) .put("BIT_OR", BeamBuiltinAggregations::createBitOr) + .put("BIT_XOR", BeamBuiltinAggregations::createBitXOr) // JIRA link:https://issues.apache.org/jira/browse/BEAM-10379 .put("BIT_AND", BeamBuiltinAggregations::createBitAnd) .put("VAR_POP", t -> VarianceFn.newPopulation(t.getTypeName())) @@ -433,4 +426,43 @@ public Long extractOutput(Long accum) { return accum; } } + + public static CombineFn createBitXOr(Schema.FieldType fieldType) { + if (fieldType.getTypeName() == TypeName.INT64) { + return new BitXOr(); + } + throw new UnsupportedOperationException(String.format("[%s] is not supported in BIT_XOR", fieldType)); + } + + public static class BitXOr extends CombineFn { + + @Override + public Integer createAccumulator() { + return 0; + } + + @Override + public Integer addInput(Integer mutableAccumulator, Integer input) { + if (input != null) { + return mutableAccumulator ^ input; + }else { + return 0; + } + } + + @Override + public Integer mergeAccumulators(Iterable accumulators) { + Integer merged = createAccumulator(); + for (Integer accum : accumulators) { + merged = merged ^ accum; + } + return merged; + } + + @Override + public Integer extractOutput(Integer accumulator) { + return accumulator; + } + } + } diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java index 16471fecf58a..3af4edef4e5f 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/SupportedZetaSqlBuiltinFunctions.java @@ -18,9 +18,10 @@ package org.apache.beam.sdk.extensions.sql.zetasql; import com.google.zetasql.ZetaSQLFunction.FunctionSignatureId; -import java.util.List; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; +import java.util.List; + /** * List of ZetaSQL builtin functions supported by Beam ZetaSQL. Keep this list in sync with * https://github.com/google/zetasql/blob/master/zetasql/public/builtin_function.proto. Uncomment @@ -415,7 +416,7 @@ class SupportedZetaSqlBuiltinFunctions { // JIRA link: https://issues.apache.org/jira/browse/BEAM-10379 // FunctionSignatureId.FN_BIT_AND_INT64, // bit_and FunctionSignatureId.FN_BIT_OR_INT64, // bit_or - // FunctionSignatureId.FN_BIT_XOR_INT64, // bit_xor + FunctionSignatureId.FN_BIT_XOR_INT64, // bit_xor // FunctionSignatureId.FN_LOGICAL_AND, // logical_and // FunctionSignatureId.FN_LOGICAL_OR, // logical_or // Approximate aggregate functions. diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperatorMappingTable.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperatorMappingTable.java index 58e6f812bf3b..b3bb97441413 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperatorMappingTable.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperatorMappingTable.java @@ -17,11 +17,12 @@ */ package org.apache.beam.sdk.extensions.sql.zetasql.translation; -import java.util.Map; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.SqlOperator; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap; +import java.util.Map; + /** SqlOperatorMappingTable. */ class SqlOperatorMappingTable { @@ -76,6 +77,7 @@ class SqlOperatorMappingTable { // https://issues.apache.org/jira/browse/BEAM-10379 .put("string_agg", SqlOperators.STRING_AGG_STRING_FN) // NULL values not supported .put("bit_or", SqlStdOperatorTable.BIT_OR) + .put("bit_xor", SqlOperators.BIT_XOR) .put("ceil", SqlStdOperatorTable.CEIL) .put("floor", SqlStdOperatorTable.FLOOR) .put("mod", SqlStdOperatorTable.MOD) diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java index 1c0835c4d96d..acd6dcc8caae 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/SqlOperators.java @@ -17,13 +17,11 @@ */ package org.apache.beam.sdk.extensions.sql.zetasql.translation; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.List; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.sdk.extensions.sql.impl.ScalarFunctionImpl; import org.apache.beam.sdk.extensions.sql.impl.UdafImpl; import org.apache.beam.sdk.extensions.sql.impl.planner.BeamRelDataTypeSystem; +import org.apache.beam.sdk.extensions.sql.impl.transform.BeamBuiltinAggregations; import org.apache.beam.sdk.extensions.sql.impl.udaf.StringAgg; import org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils; import org.apache.beam.sdk.extensions.sql.zetasql.translation.impl.BeamBuiltinMethods; @@ -36,20 +34,9 @@ import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.Function; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.FunctionParameter; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.schema.ScalarFunction; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.SqlFunction; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.SqlFunctionCategory; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.SqlIdentifier; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.SqlKind; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.SqlOperator; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.SqlSyntax; +import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.*; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.parser.SqlParserPos; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.FamilyOperandTypeChecker; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.InferTypes; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.OperandTypes; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.SqlReturnTypeInference; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.SqlTypeFactoryImpl; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.SqlTypeFamily; -import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.SqlTypeName; +import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.type.*; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.sql.validate.SqlUserDefinedFunction; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.util.Optionality; @@ -57,6 +44,10 @@ import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList; import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; + /** * A separate SqlOperators table for those functions that do not exist or not compatible with * Calcite. Most of functions within this class is copied from Calcite. @@ -135,6 +126,10 @@ public class SqlOperators { public static final SqlOperator DATE_OP = createUdfOperator("DATE", BeamBuiltinMethods.DATE_METHOD); + public static final SqlOperator BIT_XOR = + createUdafOperator("BIT_XOR", x -> createTypeFactory().createSqlType(SqlTypeName.INTEGER), + new UdafImpl<>(new BeamBuiltinAggregations.BitXOr())); + public static final SqlUserDefinedFunction CAST_OP = new SqlUserDefinedFunction( new SqlIdentifier("CAST", SqlParserPos.ZERO), diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/impl/BeamBuiltinMethods.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/impl/BeamBuiltinMethods.java index 223469b9bd4e..59306efd6f5e 100644 --- a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/impl/BeamBuiltinMethods.java +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/impl/BeamBuiltinMethods.java @@ -17,10 +17,11 @@ */ package org.apache.beam.sdk.extensions.sql.zetasql.translation.impl; -import java.lang.reflect.Method; import org.apache.beam.sdk.annotations.Internal; import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.linq4j.tree.Types; +import java.lang.reflect.Method; + /** BeamBuiltinMethods. */ @Internal public class BeamBuiltinMethods { @@ -70,4 +71,7 @@ public class BeamBuiltinMethods { public static final Method DATE_METHOD = Types.lookupMethod(DateFunctions.class, "date", Integer.class, Integer.class, Integer.class); + + public static final Method BIT_XOR = + Types.lookupMethod(BitwiseFunctions.class, "bitXOr", Integer.class, Integer.class); } diff --git a/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/impl/BitwiseFunctions.java b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/impl/BitwiseFunctions.java new file mode 100644 index 000000000000..bd4926917ed5 --- /dev/null +++ b/sdks/java/extensions/sql/zetasql/src/main/java/org/apache/beam/sdk/extensions/sql/zetasql/translation/impl/BitwiseFunctions.java @@ -0,0 +1,18 @@ +package org.apache.beam.sdk.extensions.sql.zetasql.translation.impl; + +import org.apache.beam.vendor.calcite.v1_20_0.org.apache.calcite.linq4j.function.Strict; + +public class BitwiseFunctions { + + @Strict + public static Integer bitXOr(Integer int1, Integer int2) { + return int1 ^ int2; + } + + @Strict + public static Integer bitXOr(Integer int1, Integer int2, Integer int3) { + return int1 ^ int2 ^ int3; + } + +} + diff --git a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlDialectSpecTest.java b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlDialectSpecTest.java index d37a76aef7f8..a2bf179e198b 100644 --- a/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlDialectSpecTest.java +++ b/sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlDialectSpecTest.java @@ -17,24 +17,12 @@ */ package org.apache.beam.sdk.extensions.sql.zetasql; -import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTimestampWithUTCTimeZone; -import static org.apache.beam.sdk.schemas.Schema.FieldType.DATETIME; -import static org.junit.Assert.assertTrue; - import com.google.protobuf.ByteString; import com.google.zetasql.StructType; import com.google.zetasql.StructType.StructField; import com.google.zetasql.TypeFactory; import com.google.zetasql.Value; import com.google.zetasql.ZetaSQLType.TypeKind; -import java.nio.charset.StandardCharsets; -import java.time.LocalDate; -import java.time.LocalTime; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; import org.apache.beam.sdk.extensions.sql.SqlTransform; import org.apache.beam.sdk.extensions.sql.impl.BeamSqlPipelineOptions; import org.apache.beam.sdk.extensions.sql.impl.rel.BeamRelNode; @@ -52,15 +40,20 @@ import org.joda.time.DateTime; import org.joda.time.Duration; import org.joda.time.chrono.ISOChronology; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalTime; +import java.util.*; + +import static org.apache.beam.sdk.extensions.sql.zetasql.DateTimeUtils.parseTimestampWithUTCTimeZone; +import static org.apache.beam.sdk.schemas.Schema.FieldType.DATETIME; +import static org.junit.Assert.assertTrue; + /** Tests for various operations/functions defined by ZetaSQL dialect. */ @RunWith(JUnit4.class) @SuppressWarnings({ @@ -4067,4 +4060,18 @@ public void testSimpleTableName() { Row.withSchema(singleField).addValues(15L).build()); pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES)); } + + @Test + public void testBitXorAggregation() { + String sql ="SELECT BIT_XOR(x) AS bit_xor FROM UNNEST([5678, 1234]) AS x"; + ZetaSQLQueryPlanner zetaSQLQueryPlanner = new ZetaSQLQueryPlanner(config); + BeamRelNode beamRelNode = zetaSQLQueryPlanner.convertToBeamRel(sql); + PCollection stream = BeamSqlRelUtils.toPCollection(pipeline, beamRelNode); + + Schema schema = Schema.builder().addInt64Field("rslt_field").build(); + PAssert.that(stream) + .containsInAnyOrder(Row.withSchema(schema).addValue(4860L).build()); + + pipeline.run().waitUntilFinish(Duration.standardMinutes(PIPELINE_EXECUTION_WAITTIME_MINUTES)); + } } From 140a9139328ac7a87b9291aaa16bee5724009842 Mon Sep 17 00:00:00 2001 From: Nam Bui Date: Tue, 8 Dec 2020 09:08:25 +0700 Subject: [PATCH 0031/2127] [Website revamp][11179 - 11180 - 11183] Implemented Quotes component, Header and Navbar (#13439) * Implemented About social media component * Implemented Quotes component * Implemented Hero component * Changed Navbar component * Added license * Review changes * Review changes --- build.gradle | 3 + .../site/assets/icons/install-button-icon.svg | 22 +++ .../site/assets/icons/navbar-arrow-icon.svg | 23 +++ .../icons/navbar-documentation-icon.svg | 22 +++ website/www/site/assets/icons/quote-icon.svg | 22 +++ website/www/site/assets/scss/_calendar.scss | 2 +- website/www/site/assets/scss/_cards.sass | 69 ------- website/www/site/assets/scss/_global.sass | 7 +- website/www/site/assets/scss/_graphic.sass | 2 + .../www/site/assets/scss/_hero-mobile.scss | 62 +++++++ website/www/site/assets/scss/_hero.sass | 156 ---------------- website/www/site/assets/scss/_hero.scss | 128 +++++++++++++ .../www/site/assets/scss/_keen-slider.scss | 40 ++++ .../www/site/assets/scss/_navbar-desktop.scss | 174 ++++++++++++++++++ .../{_navbar.sass => _navbar-mobile.sass} | 17 +- website/www/site/assets/scss/_quotes.scss | 151 +++++++++++++++ .../www/site/assets/scss/_section-nav.sass | 4 +- website/www/site/assets/scss/_typography.scss | 47 ++++- website/www/site/assets/scss/_vars.sass | 4 + website/www/site/assets/scss/main.scss | 12 +- website/www/site/data/en/quotes.yaml | 23 +++ website/www/site/i18n/home/en.yaml | 4 - .../en/cards.yaml => i18n/home/hero/en.yaml} | 14 +- website/www/site/i18n/home/quotes/en.yaml | 14 ++ website/www/site/i18n/navbar/en.yaml | 4 +- website/www/site/layouts/_default/baseof.html | 4 +- website/www/site/layouts/index.html | 121 +++++------- website/www/site/layouts/partials/header.html | 58 +++++- .../site/layouts/partials/hooks/body-end.html | 17 ++ .../layouts/partials/quotes/quote-mobile.html | 21 +++ .../site/layouts/partials/quotes/quote.html | 21 +++ .../site/static/images/quote-paypal-logo.png | Bin 0 -> 16047 bytes .../www/site/static/js/hero/hero-desktop.js | 21 +++ .../www/site/static/js/hero/hero-mobile.js | 21 +++ .../site/static/js/hero/lottie-light.min.js | 18 ++ website/www/site/static/js/keen-slider.min.js | 15 ++ website/www/site/static/js/quotes-slider.js | 42 +++++ website/www/site/static/js/section-nav.js | 2 +- 38 files changed, 1050 insertions(+), 337 deletions(-) create mode 100644 website/www/site/assets/icons/install-button-icon.svg create mode 100644 website/www/site/assets/icons/navbar-arrow-icon.svg create mode 100644 website/www/site/assets/icons/navbar-documentation-icon.svg create mode 100644 website/www/site/assets/icons/quote-icon.svg delete mode 100644 website/www/site/assets/scss/_cards.sass create mode 100644 website/www/site/assets/scss/_hero-mobile.scss delete mode 100644 website/www/site/assets/scss/_hero.sass create mode 100644 website/www/site/assets/scss/_hero.scss create mode 100644 website/www/site/assets/scss/_keen-slider.scss create mode 100644 website/www/site/assets/scss/_navbar-desktop.scss rename website/www/site/assets/scss/{_navbar.sass => _navbar-mobile.sass} (87%) create mode 100644 website/www/site/assets/scss/_quotes.scss create mode 100644 website/www/site/data/en/quotes.yaml rename website/www/site/{data/en/cards.yaml => i18n/home/hero/en.yaml} (60%) create mode 100644 website/www/site/i18n/home/quotes/en.yaml create mode 100644 website/www/site/layouts/partials/hooks/body-end.html create mode 100644 website/www/site/layouts/partials/quotes/quote-mobile.html create mode 100644 website/www/site/layouts/partials/quotes/quote.html create mode 100644 website/www/site/static/images/quote-paypal-logo.png create mode 100644 website/www/site/static/js/hero/hero-desktop.js create mode 100644 website/www/site/static/js/hero/hero-mobile.js create mode 100644 website/www/site/static/js/hero/lottie-light.min.js create mode 100644 website/www/site/static/js/keen-slider.min.js create mode 100644 website/www/site/static/js/quotes-slider.js diff --git a/build.gradle b/build.gradle index 15d1dd0d22fb..c836cea30c43 100644 --- a/build.gradle +++ b/build.gradle @@ -82,6 +82,9 @@ rat { "website/www/site/themes", "website/www/yarn.lock", "website/www/package.json", + "website/www/site/static/js/hero/lottie-light.min.js", + "website/www/site/static/js/keen-slider.min.js", + "website/www/site/assets/scss/_keen-slider.scss", // Ignore ownership files "ownership/**/*", diff --git a/website/www/site/assets/icons/install-button-icon.svg b/website/www/site/assets/icons/install-button-icon.svg new file mode 100644 index 000000000000..39c3ffcfbd31 --- /dev/null +++ b/website/www/site/assets/icons/install-button-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/navbar-arrow-icon.svg b/website/www/site/assets/icons/navbar-arrow-icon.svg new file mode 100644 index 000000000000..78724e930331 --- /dev/null +++ b/website/www/site/assets/icons/navbar-arrow-icon.svg @@ -0,0 +1,23 @@ + + + + + + diff --git a/website/www/site/assets/icons/navbar-documentation-icon.svg b/website/www/site/assets/icons/navbar-documentation-icon.svg new file mode 100644 index 000000000000..11c165b0feba --- /dev/null +++ b/website/www/site/assets/icons/navbar-documentation-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/icons/quote-icon.svg b/website/www/site/assets/icons/quote-icon.svg new file mode 100644 index 000000000000..0a32eea254ad --- /dev/null +++ b/website/www/site/assets/icons/quote-icon.svg @@ -0,0 +1,22 @@ + + + + + diff --git a/website/www/site/assets/scss/_calendar.scss b/website/www/site/assets/scss/_calendar.scss index 69e2bf81a291..c78c16081571 100644 --- a/website/www/site/assets/scss/_calendar.scss +++ b/website/www/site/assets/scss/_calendar.scss @@ -209,7 +209,7 @@ min-height: 456px; .calendar-card-small { - max-width: 327px; + width: 327px; height: 216px; padding: 24px 20px; diff --git a/website/www/site/assets/scss/_cards.sass b/website/www/site/assets/scss/_cards.sass deleted file mode 100644 index a1562d0dde1d..000000000000 --- a/website/www/site/assets/scss/_cards.sass +++ /dev/null @@ -1,69 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -.cards - background-image: url(../images/cards_bg.svg) - background-size: cover - background-repeat: no-repeat - background-position: top - text-align: center - margin-bottom: $pad*2 - - .cards__title - +type-h2 - color: #fff - padding-top: $pad-md - margin-bottom: $pad - - .cards__body - max-width: 550px - +type-body - margin: 0 auto - - .cards__cards - margin-bottom: $pad*2 - +md - display: flex - justify-content: center - align-items: center - - .cards__cards__card - background: #fff - box-shadow: $box-shadow - max-width: 300px - margin: 0 auto $pad - padding: $pad*1.5 - +md - margin: 0 $pad/2 - - .cards__cards__card__body - margin-bottom: $pad - +type-h3 - - .cards__cards__card__user - display: flex - justify-content: center - align-items: center - - .cards__cards__card__user__icon - border-radius: 100% - background: #efefef - width: 40px - height: 40px - - .cards__cards__card__user__name - margin-left: $pad/2 diff --git a/website/www/site/assets/scss/_global.sass b/website/www/site/assets/scss/_global.sass index a4dd35517e26..2410e0d64693 100644 --- a/website/www/site/assets/scss/_global.sass +++ b/website/www/site/assets/scss/_global.sass @@ -26,7 +26,6 @@ body .body background: #fff margin: 0 auto - padding-top: 130px &:not(.body--index) .body__contained @@ -67,4 +66,8 @@ body .container-main-content padding: 0 20px - position: relative \ No newline at end of file + position: relative + margin-top: 85px + + @media (min-width: $tablet) + margin-top: 0 \ No newline at end of file diff --git a/website/www/site/assets/scss/_graphic.sass b/website/www/site/assets/scss/_graphic.sass index f01c72a12528..cd171f7db00b 100644 --- a/website/www/site/assets/scss/_graphic.sass +++ b/website/www/site/assets/scss/_graphic.sass @@ -16,6 +16,8 @@ */ .graphic + padding: $pad-l $pad + .graphic__image text-align: center line-height: 0 diff --git a/website/www/site/assets/scss/_hero-mobile.scss b/website/www/site/assets/scss/_hero-mobile.scss new file mode 100644 index 000000000000..343c279f4c90 --- /dev/null +++ b/website/www/site/assets/scss/_hero-mobile.scss @@ -0,0 +1,62 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.hero-mobile { + position: relative; + margin-bottom: 0; + display: none; + + .hero-content { + position: absolute; + z-index: 1; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + text-align: center; + width: 100%; + max-width: 506px; + + h3 { + @extend .hero-title; + + text-transform: uppercase; + margin: 0 auto 16px auto; + } + + h1 { + @extend .hero-heading; + + width: 300px; + margin: 0 auto 24px auto; + } + + h2 { + @extend .hero-subheading; + + width: 300px; + margin: 0 auto; + } + } +} + +@media (max-width: $mobile) { + .hero-mobile { + display: inherit; + } +} diff --git a/website/www/site/assets/scss/_hero.sass b/website/www/site/assets/scss/_hero.sass deleted file mode 100644 index 63c22b9282a5..000000000000 --- a/website/www/site/assets/scss/_hero.sass +++ /dev/null @@ -1,156 +0,0 @@ -/*! - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -.hero-bg - background-image: url(../images/hero_bg_flat.svg) - background-repeat: no-repeat - background-size: cover - background-position: top center - margin-top: -50px - +md - background-size: 100% - padding-bottom: $pad - -.hero - padding-top: $pad-xl - margin-bottom: $pad-md - position: relative - z-index: 1 - +md - padding-top: $pad-sm - margin-bottom: $pad-xl - - .hero__content - position: relative - z-index: 1 - - .hero__image - bottom: 0 - content: '' - left: 0 - line-height: 0 - position: absolute - right: 0 - top: 0 - z-index: 0 - img - position: absolute - bottom: 0 - width: 100% - - .hero__title - +type-h1 - color: #fff - max-width: 500px - margin: 0 auto $pad - text-align: center - +md - margin: 0 0 $pad - text-align: left - - .hero__ctas - text-align: center - margin-bottom: $pad-md - +md - margin-bottom: 0 - text-align: left - - &--first - margin-bottom: $pad - +md - margin-bottom: $pad-sm - - .hero__subtitle - +type-h3 - color: #fff - max-width: 540px - margin: 0 auto $pad - font-weight: $font-weight-semibold - text-align: center - +md - margin: 0 0 $pad-md - text-align: left - - .hero__blog - .hero__blog__title - +type-h4 - font-weight: $font-weight-bold - margin-bottom: $pad - text-align: center - +md - color: #fff - text-align: left - margin-bottom: $pad/2 - - .hero__blog__cards - +md - display: flex - margin: 0 -10px - - .hero__blog__cards__card - background-color: #fff - color: inherit - box-shadow: $box-shadow - padding: 20px - display: block - transition: transform 300ms ease, box-shadow 300ms ease - position: relative - max-width: 300px - margin: 0 auto $pad - +md - margin: 0 10px - - &:before - background-image: url(../images/card_border.svg) - background-position: top - background-repeat: no-repeat - background-size: cover - content: ' ' - display: block - height: 2px - position: absolute - width: 100% - left: 0 - top: 0 - - &:hover - text-decoration: none - transform: translateY(-8px) - box-shadow: $box-shadow-hover - - .hero__blog__cards__card__title - +type-body - margin-bottom: $pad - - .hero__blog__cards__card__date - +type-body-sm - font-weight: $font-weight-semibold - text-transform: uppercase - letter-spacing: 1px - - .hero__cols - +md - display: flex - min-height: 500px - - .hero__cols__col - width: 50% - display: flex - align-items: flex-end - - &:first-child - align-items: center diff --git a/website/www/site/assets/scss/_hero.scss b/website/www/site/assets/scss/_hero.scss new file mode 100644 index 000000000000..e204d1bf015d --- /dev/null +++ b/website/www/site/assets/scss/_hero.scss @@ -0,0 +1,128 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.hero-desktop { + position: relative; + margin-bottom: 0; + width: 100%; + height: 100%; + display: inherit; + margin-top: -30px; + + .hero-content { + position: absolute; + z-index: 1; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + text-align: center; + + h3 { + @extend .hero-title; + + text-transform: uppercase; + margin: 0 auto 24px auto; + } + + h1 { + @extend .hero-heading; + width: 506px; + + margin: 0 auto 36px auto; + } + + h2 { + @extend .hero-subheading; + + width: 344px; + margin: 0 auto 56px auto; + } + + a { + text-decoration: none; + } + + button { + width: 184px; + height: 46px; + padding: 15px 28px 15px 26px; + border-radius: 100px; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + background-color: $color-white; + display: flex; + align-items: center; + justify-content: center; + margin: 0 auto; + border: none; + outline: none; + + span { + text-transform: uppercase; + font-size: 14px; + font-weight: bold; + letter-spacing: 0.6px; + color: $color-sun; + } + } + + button:hover { + background-color: $color-white; + box-shadow: 0 4px 20px 0 rgba(0, 0, 0, 0.24), + 0 4px 6px 0 rgba(0, 0, 0, 0.24); + } + + button:focus { + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + } + } +} + +@media (max-width: $tablet) { + .hero-desktop { + margin-top: 70px; + + .hero-content { + h3 { + margin: 0 auto 16px auto; + } + + h1 { + width: 300px; + margin: 0 auto 24px auto; + } + + h2 { + width: 300px; + margin: 0 auto; + } + + button { + display: none; + } + } + } +} + +@media (max-width: $mobile) { + .hero-desktop { + display: none; + } +} diff --git a/website/www/site/assets/scss/_keen-slider.scss b/website/www/site/assets/scss/_keen-slider.scss new file mode 100644 index 000000000000..08a4326e0042 --- /dev/null +++ b/website/www/site/assets/scss/_keen-slider.scss @@ -0,0 +1,40 @@ +/** + * keen-slider 5.3.2 + * The HTML touch slider carousel with the most native feeling you will get. + * https://keen-slider.io + * Copyright 2020-2020 Eric Beyer + * License: MIT + * Released on: 2020-11-10 + */ + +/*# sourceMappingURL=keen-slider.min.css.map */ +// This is pulled from "https://cdn.jsdelivr.net/npm/keen-slider@5.3.2/keen-slider.min.css" to serve the consistency +.keen-slider { + display: flex; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + user-select: none; + -webkit-touch-callout: none; + -khtml-user-select: none; + touch-action: pan-y; + -webkit-tap-highlight-color: transparent; +} +.keen-slider, +.keen-slider__slide { + overflow: hidden; + position: relative; +} +.keen-slider__slide { + width: 100%; + min-height: 100%; +} +.keen-slider[data-keen-slider-v] { + flex-wrap: wrap; +} +.keen-slider[data-keen-slider-v] .keen-slider__slide { + width: 100%; +} +.keen-slider[data-keen-slider-moves] * { + pointer-events: none; +} diff --git a/website/www/site/assets/scss/_navbar-desktop.scss b/website/www/site/assets/scss/_navbar-desktop.scss new file mode 100644 index 000000000000..57eb63350808 --- /dev/null +++ b/website/www/site/assets/scss/_navbar-desktop.scss @@ -0,0 +1,174 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.navigation-bar-mobile { + display: none; +} + +.navigation-bar-desktop { + display: flex; + height: 96px; + width: 100%; + align-items: center; + justify-content: space-between; + margin-bottom: 30px; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.06); + background-color: $color-white; + z-index: 10000; // just to make sure that navbar always on top of other elements + + a { + @extend .component-text; + + color: $color-dark-gray; + letter-spacing: 0.2; + margin-right: 56px; + text-decoration: none; + cursor: pointer; + } + + .navbar-logo { + margin-left: 58px; + + img { + width: 88px; + } + } + + .navbar-links { + display: flex; + align-items: center; + justify-content: space-between; + z-index: 10000; + + :last-child { + margin-right: 0; + } + + .navbar-link { + display: inline-block; + position: relative; + margin-bottom: 2px; + } + + .navbar-link::before { + transition: 0.3; + content: ""; + position: absolute; + background-color: $color-sun; + height: 0%; + width: 100%; + bottom: 0px; + border-radius: 5px; + } + + .navbar-link:hover::before { + height: 2px; + } + + .navbar-dropdown-documentation { + list-style-type: none; + + .dropdown-toggle { + margin: 0; + } + + ul { + width: 209px; + left: -25%; + text-align: center; + border: none; + box-shadow: none; + padding-top: 34px; + padding-bottom: 0; + + a { + @extend .component-text; + } + } + } + } + + .navbar-dropdown-apache { + margin-right: 90px; + list-style-type: none; + + .dropdown-toggle { + margin: 0; + } + + ul { + width: 209px; + left: 70%; + transform: translateX(-50%); + text-align: center; + border: none; + box-shadow: none; + padding-top: 35px; + padding-bottom: 0; + + a { + @extend .component-text; + + margin-right: 0 !important; + } + } + + .arrow-icon { + position: absolute; + top: 3px; + right: -30px; + } + } + + .navbar-dropdown-apache:hover { + .arrow-icon { + opacity: 0.84; + } + } + + .navbar-dropdown { + .dropdown-menu > li > a { + &:hover, + &:focus { + text-decoration: none; + color: $color-dropdown-link-hover-text; + background-color: $color-dropdown-link-hover-bg; + } + } + } + + .dropdown:hover .dropdown-menu { + display: block; + margin-top: 0; + } +} + +@media (max-width: $tablet) { + .navigation-bar-desktop { + display: none; + } + + .navigation-bar-mobile { + display: block; + } + + .page-nav { + margin-top: 30px; + } +} \ No newline at end of file diff --git a/website/www/site/assets/scss/_navbar.sass b/website/www/site/assets/scss/_navbar-mobile.sass similarity index 87% rename from website/www/site/assets/scss/_navbar.sass rename to website/www/site/assets/scss/_navbar-mobile.sass index f4b4ea6bcae7..762b5e517b9a 100644 --- a/website/www/site/assets/scss/_navbar.sass +++ b/website/www/site/assets/scss/_navbar-mobile.sass @@ -23,6 +23,7 @@ .navbar-header margin-left: $pad + float: none .navbar-brand +md @@ -40,16 +41,17 @@ color: $color-dark-gray .navbar-toggle - float: left + margin-right: 24px .icon-bar - background-color: $color-dark-gray + background-color: $color-sun + height: 3px - @media (max-width: $ak-breakpoint-lg) + @media (max-width: $tablet) display: block .navbar-container - @media (max-width: $ak-breakpoint-lg) + @media (max-width: $tablet) background-color: $color-white bottom: 0 min-height: 100vh @@ -59,12 +61,13 @@ top: 0 transition: transform 100ms linear width: calc(100% - 32px) + right: 0 .navbar-nav > li width: 100% &.closed - transform: translateX(-100%) + transform: translateX(100%) &.open transform: translateX(0) @@ -78,7 +81,7 @@ top: 0 transition: opacity 200ms - @media (max-width: $ak-breakpoint-lg) + @media (max-width: $tablet) display: block &.closed @@ -89,6 +92,6 @@ opacity: 0.5 width: 100% - @media (max-width: $ak-breakpoint-lg) + @media (max-width: $tablet) .navbar-right margin-right: -15px diff --git a/website/www/site/assets/scss/_quotes.scss b/website/www/site/assets/scss/_quotes.scss new file mode 100644 index 000000000000..6d59798289f0 --- /dev/null +++ b/website/www/site/assets/scss/_quotes.scss @@ -0,0 +1,151 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@import "media"; + +.quotes { + padding: $pad-l $pad; + background-color: $color-medium-gray; + + .quotes-title { + @extend .component-title; + + text-align: center; + border: none; + } + + .quotes-desktop { + display: flex; + justify-content: center; + + .quote-card { + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + width: 100%; + max-width: 381px; + height: 474px; + margin: 86px 36px 0 0; + padding: 55px 20px 24px 20px; + border-radius: 16px; + background-color: $color-white; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + margin-right: 36px; + + .quote-text { + @extend .component-quote; + + margin: 108px 0 20px 0; + } + + img { + width: 172px; + } + } + + :last-child { + margin-right: 0; + } + } + + // Sliding feature is only displayed on mobile version + .keen-slider { + display: none; + } + + .dots { + display: none; + } + + .keen-slider { + width: 327px; + margin: 0 auto; + border-radius: 16px; + background-color: $color-white; + box-shadow: 0 4px 16px 0 rgba(0, 0, 0, 0.16), + 0 4px 4px 0 rgba(0, 0, 0, 0.06); + + .keen-slider__slide { + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + width: 100%; + max-width: 327px; + height: 468px; + padding: 55px 24px 24px 24px; + + .quote-text { + @extend .component-quote; + + margin: 108px 0 20px 0; + } + + img { + width: 172px; + } + } + } + + .dots { + display: none; + padding: 10px 0; + justify-content: center; + margin-top: 46px; + } + + .dot { + border: none; + width: 13px; + height: 13px; + background: $color-smoke; + border-radius: 50%; + margin: 0 5px; + padding: 4px; + cursor: pointer; + } + + .dot:focus { + outline: none; + } + + .dot--active { + background: $color-sun; + } +} + +@media (max-width: $tablet) { + .quotes { + .quotes-title { + margin-bottom: 64px; + } + + .quotes-desktop { + display: none; + } + + .keen-slider { + display: flex; + } + + .dots { + display: flex; + } + } +} diff --git a/website/www/site/assets/scss/_section-nav.sass b/website/www/site/assets/scss/_section-nav.sass index 61aff9f462ea..d8c4ed195b83 100644 --- a/website/www/site/assets/scss/_section-nav.sass +++ b/website/www/site/assets/scss/_section-nav.sass @@ -97,7 +97,7 @@ @media (max-width: $ak-breakpoint-lg) background-color: $color-light-gray bottom: 0 - left: 0 + right: 0 max-width: 256px position: fixed top: 0 @@ -110,7 +110,7 @@ overflow-y: auto &.closed - transform: translateX(-100%) + transform: translateX(100%) &.open transform: translateX(0) diff --git a/website/www/site/assets/scss/_typography.scss b/website/www/site/assets/scss/_typography.scss index 99eacfe4ca04..807619cadd05 100644 --- a/website/www/site/assets/scss/_typography.scss +++ b/website/www/site/assets/scss/_typography.scss @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - @import "media"; + +@import "media"; .component-title { font-size: 36px; @@ -88,6 +88,44 @@ color: $color-smoke; } +.component-quote { + font-size: 20px; + font-weight: normal; + font-stretch: normal; + font-style: italic; + line-height: 1.44; + letter-spacing: 0.43px; + text-align: center; + color: $color-gray; +} + +.hero-title { + font-size: 16px; + font-weight: normal; + font-style: normal; + line-height: 1.88; + letter-spacing: 0.8px; + color: $color-white; +} + +.hero-heading { + font-size: 46px; + font-weight: 500; + font-style: normal; + line-height: 1; + letter-spacing: normal; + color: $color-white; +} + +.hero-subheading { + font-size: 20px; + font-weight: normal; + font-style: normal; + line-height: 1.44; + letter-spacing: normal; + color: $color-white; +} + @media (max-width: $tablet) { .component-title { font-size: 28px; @@ -95,4 +133,7 @@ .component-large-header { font-size: 24px; } -} \ No newline at end of file + .hero-heading { + font-size: 32px; + } +} diff --git a/website/www/site/assets/scss/_vars.sass b/website/www/site/assets/scss/_vars.sass index df4276a042b1..626313d993b2 100644 --- a/website/www/site/assets/scss/_vars.sass +++ b/website/www/site/assets/scss/_vars.sass @@ -24,6 +24,10 @@ $color-gray: #333333 $color-smoke: #8C8B8E $color-sun: #F26628 $color-silver: #C4C4C4 +$color-medium-gray: #FBFBFB + +$color-dropdown-link-hover-text: #E65D21 +$color-dropdown-link-hover-bg: #FFEDE5 $pad-sm: 15px $pad-s: 24px diff --git a/website/www/site/assets/scss/main.scss b/website/www/site/assets/scss/main.scss index e4075a1fe51a..b40befd414cd 100644 --- a/website/www/site/assets/scss/main.scss +++ b/website/www/site/assets/scss/main.scss @@ -20,26 +20,28 @@ // Globals. @import "_vars.sass"; +@import "_media.scss"; @import "_breakpoints.sass"; @import "_type.sass"; @import "_global.sass"; -@import "_navbar.sass"; +@import "_navbar-mobile.sass"; @import "_typography.scss"; -@import "_media.scss"; // Components. @import "_button.sass"; // Modules. -@import "_cards.sass"; @import "_ctas.sass"; @import "_footer.sass"; @import "_graphic.sass"; @import "_header.sass"; -@import "_hero.sass"; +@import "_hero.scss"; +@import "_hero-mobile.scss"; @import "_logos.scss"; @import "_pillars.scss"; @import "_section-nav.sass"; @import "_page-nav.sass"; @import "_table-wrapper.sass"; -@import "_calendar.scss"; \ No newline at end of file +@import "_calendar.scss"; +@import "_quotes.scss"; +@import "navbar-desktop.scss"; \ No newline at end of file diff --git a/website/www/site/data/en/quotes.yaml b/website/www/site/data/en/quotes.yaml new file mode 100644 index 000000000000..537171a7d4c2 --- /dev/null +++ b/website/www/site/data/en/quotes.yaml @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: +# Placeholder texts should be updated later +- text: A framework that delivers the flexibility and advanced functionality our customers need. + icon: icons/quote-icon.svg + logoUrl: images/quote-paypal-logo.png +- text: A framework that delivers the flexibility and advanced functionality our customers need. + icon: icons/quote-icon.svg + logoUrl: images/quote-paypal-logo.png +- text: A framework that delivers the flexibility and advanced functionality our customers need. + icon: icons/quote-icon.svg + logoUrl: images/quote-paypal-logo.png \ No newline at end of file diff --git a/website/www/site/i18n/home/en.yaml b/website/www/site/i18n/home/en.yaml index 8a1ac53eecf8..6ae87c04f465 100644 --- a/website/www/site/i18n/home/en.yaml +++ b/website/www/site/i18n/home/en.yaml @@ -10,10 +10,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -- id: home-hero-title - translation: "Apache Beam: An advanced unified programming model" -- id: home-hero-subtitle - translation: "Implement batch and streaming data processing jobs that run on any execution engine." - id: home-learn-more translation: "Learn more" - id: home-try-beam diff --git a/website/www/site/data/en/cards.yaml b/website/www/site/i18n/home/hero/en.yaml similarity index 60% rename from website/www/site/data/en/cards.yaml rename to website/www/site/i18n/home/hero/en.yaml index a2981e7fba86..85f4728d86be 100644 --- a/website/www/site/data/en/cards.yaml +++ b/website/www/site/i18n/home/hero/en.yaml @@ -10,9 +10,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -- quote: "A framework that delivers the flexibility and advanced functionality our customers need." - name: –Talend -- quote: "Apache Beam has powerful semantics that solve real-world challenges of stream processing." - name: –PayPal -- quote: "Apache Beam represents a principled approach for analyzing data streams." - name: –data Artisans +- id: home-hero-title + translation: Introducing Apache Beam +- id: home-hero-heading + translation: An advanced unified programming model +- id: home-hero-subheading + translation: Implement batch and streaming data processing jobs that run on any execution engine. +- id: home-hero-button + translation: Install Beam diff --git a/website/www/site/i18n/home/quotes/en.yaml b/website/www/site/i18n/home/quotes/en.yaml new file mode 100644 index 000000000000..95c8e85b6453 --- /dev/null +++ b/website/www/site/i18n/home/quotes/en.yaml @@ -0,0 +1,14 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- id: home-quotes-title + translation: "They tried it out" \ No newline at end of file diff --git a/website/www/site/i18n/navbar/en.yaml b/website/www/site/i18n/navbar/en.yaml index fd22c3a73821..ae6255e50683 100644 --- a/website/www/site/i18n/navbar/en.yaml +++ b/website/www/site/i18n/navbar/en.yaml @@ -16,10 +16,12 @@ translation: "Get Started" - id: nav-documentation translation: "Documentation" +- id: nav-documentation-general + translation: "General" - id: nav-languages translation: "Languages" - id: nav-runners - translation: "RUNNERS" + translation: "Runners" - id: nav-roadmap translation: "Roadmap" - id: nav-contribute diff --git a/website/www/site/layouts/_default/baseof.html b/website/www/site/layouts/_default/baseof.html index 6c245c51ac9f..b8b6e26cdfd9 100644 --- a/website/www/site/layouts/_default/baseof.html +++ b/website/www/site/layouts/_default/baseof.html @@ -22,10 +22,12 @@ {{ block "pillars-section" . }}{{ end }} {{ block "graphic-section" . }}{{ end }} {{ block "calendar-section" . }}{{ end }} + {{ block "quotes-section" . }}{{ end }} + {{ block "quotes-mobile-section" . }}{{ end }} {{ block "logos-section" . }}{{ end }} - {{ block "cards-section" . }}{{ end }} {{ block "ctas-section" . }}{{ end }}

    {{ partial "footer.html" . }} + {{ partial "hooks/body-end.html"}} diff --git a/website/www/site/layouts/index.html b/website/www/site/layouts/index.html index bbfdf71b9132..4f7f0a536587 100644 --- a/website/www/site/layouts/index.html +++ b/website/www/site/layouts/index.html @@ -11,47 +11,28 @@ */}} {{ define "hero-section" }} -
    -
    -
    -
    -
    -
    - {{ T "home-hero-title" }} -
    -
    - {{ T "home-hero-subtitle" }} -
    -
    - {{ T "home-learn-more" }} - {{ T "home-try-beam" }} - {{ T "home-download-beam" . }} -
    -
    - {{ T "home-java-quickstart" }} - {{ T "home-python-quickstart" }} - {{ T "home-go-quickstart" }} -
    -
    -
    -
    -
    -
    - {{ T "home-hero-blog-title" }} -
    -
    - - {{ range ( where site.RegularPages "Section" "blog" | first 3 ) }} - -
    {{ .Title }}
    -
    {{ .Date.Format "Jan 2, 2006" }}
    -     - {{ end }} - -
    -
    -
    -
    +
    +
    +

    {{ T "home-hero-title" }}

    +

    {{ T "home-hero-heading" }}

    +

    {{ T "home-hero-subheading" }}

    + + + +
    +
    +
    +
    +

    {{ T "home-hero-title" }}

    +

    {{ T "home-hero-heading" }}

    +

    {{ T "home-hero-subheading" }}

    {{ end }} @@ -154,6 +135,32 @@

    {{ end }} +{{ define "quotes-section" }} +
    +
    + {{ T "home-quotes-title" }} +
    + +
    + {{ $data := index $.Site.Data .Site.Language.Lang }} + {{ range $quote := $data.quotes }} + {{ partial "quotes/quote.html" (dict "icon" $quote.icon "text" $quote.text "logoUrl" $quote.logoUrl) }} + {{ end }} +
    + + {{/* + The id "my-keen-slider" and "dots" should be named as defaults to make the external library (Keen Slider) works well + */}} +
    + {{ $data := index $.Site.Data .Site.Language.Lang }} + {{ range $quote := $data.quotes }} + {{ partial "quotes/quote-mobile.html" (dict "icon" $quote.icon "text" $quote.text "logoUrl" $quote.logoUrl) }} + {{ end }} +
    +
    +
    +{{ end }} + {{ define "logos-section" }}
    @@ -169,38 +176,6 @@

    {{ end }} -{{ define "cards-section" }} -
    -
    -
    - {{ T "home-cards-title" }} -
    -
    - {{ $data := index $.Site.Data .Site.Language.Lang }} - {{ range $card := $data.cards }} -
    -
    - {{ $card.quote }} -
    -
    - {{/* TODO: Implement icons (Original comment from Jekyll) -
    -
    - */}} -
    - {{ $card.name }} -
    -
    -
    - {{ end }} -
    -
    - {{ T "home-cards-body" }} {{ T "home-contribute" }} {{ T "home-section" }}. -
    -
    -
    -{{ end }} - {{ define "ctas-section" }}
    diff --git a/website/www/site/layouts/partials/header.html b/website/www/site/layouts/partials/header.html index 73c9867acffc..af8417087c20 100644 --- a/website/www/site/layouts/partials/header.html +++ b/website/www/site/layouts/partials/header.html @@ -10,18 +10,18 @@ limitations under the License. See accompanying LICENSE file. */}} -