[BEAM-13211][Playground] Add function to find all tagged files

playground/categories.yaml

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+categories:
+  - Side Input
+  - Multiple Outputs
+  - Testing
+  - Schemas
+  - Batch
+  - Streaming
+  - Combiners
+  - Dataframes
+  - Joins
+  - IO
+  - Metrics
+  - Options
+  - Coders
+  - Stateful Processing

playground/infrastructure/api/v1/api_pb2_grpc.py

@@ -53,19 +53,24 @@ def __init__(self, channel):
             request_serializer=api__pb2.GetCompileOutputRequest.SerializeToString,
             response_deserializer=api__pb2.GetCompileOutputResponse.FromString,
         )
-        self.GetListOfExamples = channel.unary_unary(
-            '/api.v1.PlaygroundService/GetListOfExamples',
-            request_serializer=api__pb2.GetListOfExamplesRequest.SerializeToString,
-            response_deserializer=api__pb2.GetListOfExamplesResponse.FromString,
+        self.Cancel = channel.unary_unary(
+            '/api.v1.PlaygroundService/Cancel',
+            request_serializer=api__pb2.CancelRequest.SerializeToString,
+            response_deserializer=api__pb2.CancelResponse.FromString,
         )
-        self.GetExample = channel.unary_unary(
-            '/api.v1.PlaygroundService/GetExample',
-            request_serializer=api__pb2.GetExampleRequest.SerializeToString,
-            response_deserializer=api__pb2.GetExampleResponse.FromString,
+        self.GetPrecompiledObjects = channel.unary_unary(
+            '/api.v1.PlaygroundService/GetPrecompiledObjects',
+            request_serializer=api__pb2.GetPrecompiledObjectsRequest.SerializeToString,
+            response_deserializer=api__pb2.GetPrecompiledObjectsResponse.FromString,
         )
-        self.GetExampleOutput = channel.unary_unary(
-            '/api.v1.PlaygroundService/GetExampleOutput',
-            request_serializer=api__pb2.GetExampleRequest.SerializeToString,
+        self.GetPrecompiledObjectCode = channel.unary_unary(
+            '/api.v1.PlaygroundService/GetPrecompiledObjectCode',
+            request_serializer=api__pb2.GetPrecompiledObjectRequest.SerializeToString,
+            response_deserializer=api__pb2.GetPrecompiledObjectCodeResponse.FromString,
+        )
+        self.GetPrecompiledObjectOutput = channel.unary_unary(
+            '/api.v1.PlaygroundService/GetPrecompiledObjectOutput',
+            request_serializer=api__pb2.GetPrecompiledObjectRequest.SerializeToString,
             response_deserializer=api__pb2.GetRunOutputResponse.FromString,
         )
 
@@ -108,22 +113,29 @@ def GetCompileOutput(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def GetListOfExamples(self, request, context):
-        """Get the list of precompiled examples.
+    def Cancel(self, request, context):
+        """Cancel code processing
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def GetPrecompiledObjects(self, request, context):
+        """Get all precompiled objects from the cloud storage.
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def GetExample(self, request, context):
-        """Get the code of an example.
+    def GetPrecompiledObjectCode(self, request, context):
+        """Get the code of an PrecompiledObject.
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def GetExampleOutput(self, request, context):
-        """Get the precompiled details of an example.
+    def GetPrecompiledObjectOutput(self, request, context):
+        """Get the precompiled details of an PrecompiledObject.
         """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
@@ -157,19 +169,24 @@ def add_PlaygroundServiceServicer_to_server(servicer, server):
             request_deserializer=api__pb2.GetCompileOutputRequest.FromString,
             response_serializer=api__pb2.GetCompileOutputResponse.SerializeToString,
         ),
-        'GetListOfExamples': grpc.unary_unary_rpc_method_handler(
-            servicer.GetListOfExamples,
-            request_deserializer=api__pb2.GetListOfExamplesRequest.FromString,
-            response_serializer=api__pb2.GetListOfExamplesResponse.SerializeToString,
+        'Cancel': grpc.unary_unary_rpc_method_handler(
+            servicer.Cancel,
+            request_deserializer=api__pb2.CancelRequest.FromString,
+            response_serializer=api__pb2.CancelResponse.SerializeToString,
+        ),
+        'GetPrecompiledObjects': grpc.unary_unary_rpc_method_handler(
+            servicer.GetPrecompiledObjects,
+            request_deserializer=api__pb2.GetPrecompiledObjectsRequest.FromString,
+            response_serializer=api__pb2.GetPrecompiledObjectsResponse.SerializeToString,
         ),
-        'GetExample': grpc.unary_unary_rpc_method_handler(
-            servicer.GetExample,
-            request_deserializer=api__pb2.GetExampleRequest.FromString,
-            response_serializer=api__pb2.GetExampleResponse.SerializeToString,
+        'GetPrecompiledObjectCode': grpc.unary_unary_rpc_method_handler(
+            servicer.GetPrecompiledObjectCode,
+            request_deserializer=api__pb2.GetPrecompiledObjectRequest.FromString,
+            response_serializer=api__pb2.GetPrecompiledObjectCodeResponse.SerializeToString,
         ),
-        'GetExampleOutput': grpc.unary_unary_rpc_method_handler(
-            servicer.GetExampleOutput,
-            request_deserializer=api__pb2.GetExampleRequest.FromString,
+        'GetPrecompiledObjectOutput': grpc.unary_unary_rpc_method_handler(
+            servicer.GetPrecompiledObjectOutput,
+            request_deserializer=api__pb2.GetPrecompiledObjectRequest.FromString,
             response_serializer=api__pb2.GetRunOutputResponse.SerializeToString,
         ),
     }
@@ -268,52 +285,69 @@ def GetCompileOutput(request,
                                              insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
-    def GetListOfExamples(request,
-                          target,
-                          options=(),
-                          channel_credentials=None,
-                          call_credentials=None,
-                          insecure=False,
-                          compression=None,
-                          wait_for_ready=None,
-                          timeout=None,
-                          metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetListOfExamples',
-                                             api__pb2.GetListOfExamplesRequest.SerializeToString,
-                                             api__pb2.GetListOfExamplesResponse.FromString,
+    def Cancel(request,
+               target,
+               options=(),
+               channel_credentials=None,
+               call_credentials=None,
+               insecure=False,
+               compression=None,
+               wait_for_ready=None,
+               timeout=None,
+               metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/Cancel',
+                                             api__pb2.CancelRequest.SerializeToString,
+                                             api__pb2.CancelResponse.FromString,
                                              options, channel_credentials,
                                              insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
-    def GetExample(request,
-                   target,
-                   options=(),
-                   channel_credentials=None,
-                   call_credentials=None,
-                   insecure=False,
-                   compression=None,
-                   wait_for_ready=None,
-                   timeout=None,
-                   metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetExample',
-                                             api__pb2.GetExampleRequest.SerializeToString,
-                                             api__pb2.GetExampleResponse.FromString,
+    def GetPrecompiledObjects(request,
+                              target,
+                              options=(),
+                              channel_credentials=None,
+                              call_credentials=None,
+                              insecure=False,
+                              compression=None,
+                              wait_for_ready=None,
+                              timeout=None,
+                              metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetPrecompiledObjects',
+                                             api__pb2.GetPrecompiledObjectsRequest.SerializeToString,
+                                             api__pb2.GetPrecompiledObjectsResponse.FromString,
                                              options, channel_credentials,
                                              insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
     @staticmethod
-    def GetExampleOutput(request,
-                         target,
-                         options=(),
-                         channel_credentials=None,
-                         call_credentials=None,
-                         insecure=False,
-                         compression=None,
-                         wait_for_ready=None,
-                         timeout=None,
-                         metadata=None):
-        return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetExampleOutput',
-                                             api__pb2.GetExampleRequest.SerializeToString,
+    def GetPrecompiledObjectCode(request,
+                                 target,
+                                 options=(),
+                                 channel_credentials=None,
+                                 call_credentials=None,
+                                 insecure=False,
+                                 compression=None,
+                                 wait_for_ready=None,
+                                 timeout=None,
+                                 metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetPrecompiledObjectCode',
+                                             api__pb2.GetPrecompiledObjectRequest.SerializeToString,
+                                             api__pb2.GetPrecompiledObjectCodeResponse.FromString,
+                                             options, channel_credentials,
+                                             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+    @staticmethod
+    def GetPrecompiledObjectOutput(request,
+                                   target,
+                                   options=(),
+                                   channel_credentials=None,
+                                   call_credentials=None,
+                                   insecure=False,
+                                   compression=None,
+                                   wait_for_ready=None,
+                                   timeout=None,
+                                   metadata=None):
+        return grpc.experimental.unary_unary(request, target, '/api.v1.PlaygroundService/GetPrecompiledObjectOutput',
+                                             api__pb2.GetPrecompiledObjectRequest.SerializeToString,
                                              api__pb2.GetRunOutputResponse.FromString,
                                              options, channel_credentials,
                                              insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

playground/infrastructure/ci_cd.py

@@ -17,7 +17,7 @@
 
 from cd_helper import CDHelper
 from ci_helper import CIHelper
-from helper import find_examples
+from helper import find_examples, get_supported_categories
 from logger import setup_logger
 
 
@@ -27,8 +27,10 @@ def ci_step():
     """
     setup_logger()
     root_dir = os.getenv("BEAM_ROOT_DIR")
+    categories_file = os.getenv("BEAM_EXAMPLE_CATEGORIES")
+    supported_categories = get_supported_categories(categories_file)
     ci_helper = CIHelper()
-    examples = find_examples(root_dir)
+    examples = find_examples(root_dir, supported_categories)
     asyncio.run(ci_helper.verify_examples(examples))
 
 
@@ -38,6 +40,8 @@ def cd_step():
     """
     setup_logger()
     root_dir = os.getenv("BEAM_ROOT_DIR")
+    categories_file = os.getenv("BEAM_EXAMPLE_CATEGORIES")
+    supported_categories = get_supported_categories(categories_file)
     cd_helper = CDHelper()
-    examples = find_examples(root_dir)
+    examples = find_examples(root_dir, supported_categories)
     cd_helper.store_examples(examples)

playground/infrastructure/config.py

@@ -14,16 +14,25 @@
 # limitations under the License.
 
 import os
+
 from dataclasses import dataclass
 from api.v1.api_pb2 import STATUS_VALIDATION_ERROR, STATUS_ERROR, STATUS_PREPARATION_ERROR, STATUS_COMPILE_ERROR, \
-    STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR
-
-from api.v1.api_pb2 import SDK_JAVA
+    STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR, SDK_JAVA, SDK_GO, SDK_PYTHON
 
 
 @dataclass(frozen=True)
 class Config:
     SERVER_ADDRESS = os.getenv("SERVER_ADDRESS", "localhost:8080")
     ERROR_STATUSES = [STATUS_VALIDATION_ERROR, STATUS_ERROR, STATUS_PREPARATION_ERROR, STATUS_COMPILE_ERROR,
                       STATUS_RUN_TIMEOUT, STATUS_RUN_ERROR]
-    SUPPORTED_SDK = {'java': SDK_JAVA}
+    SUPPORTED_SDK = {'java': SDK_JAVA, 'go': SDK_GO, 'py': SDK_PYTHON}
+    BEAM_PLAYGROUND_TITLE = "Beam-playground:\n"
+    BEAM_PLAYGROUND = "Beam-playground"
+
+
+@dataclass(frozen=True)
+class TagFields:
+    NAME: str = "name"
+    DESCRIPTION: str = "description"
+    MULTIFILE: str = "multifile"
+    CATEGORIES: str = "categories"

playground/infrastructure/grpc_client.py

@@ -12,8 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import uuid
 
+import uuid
 import grpc
 
 from api.v1 import api_pb2_grpc, api_pb2