diff --git a/bazel/envoy_build_system.bzl b/bazel/envoy_build_system.bzl
index c8c57941f795a..f144842b14649 100644
--- a/bazel/envoy_build_system.bzl
+++ b/bazel/envoy_build_system.bzl
@@ -207,6 +207,16 @@ def envoy_cc_test_library(name,
         linkstatic = 1,
     )
 
+# Envoy Python test binaries should be specified with this function.
+def envoy_py_test_binary(name,
+                         external_deps = [],
+                         **kargs):
+    # TODO(htuch): Add support for external_deps, e.g. jinja2 for config_test.
+    native.py_binary(
+        name = name,
+        **kargs
+    )
+
 # Envoy C++ mock targets should be specified with this function.
 def envoy_cc_mock(name, **kargs):
     envoy_cc_test_library(name = name, **kargs)
diff --git a/test/common/json/BUILD b/test/common/json/BUILD
index bc9952fda832f..4cf5c2b7d30cc 100644
--- a/test/common/json/BUILD
+++ b/test/common/json/BUILD
@@ -2,18 +2,11 @@ package(default_visibility = ["//visibility:public"])
 
 load("//bazel:envoy_build_system.bzl", "envoy_cc_test")
 
-filegroup(
-    name = "config_schema_test_generator",
-    srcs = glob([
-        "config_schemas_test_data/*.py",
-    ]),
-)
-
 envoy_cc_test(
     name = "config_schemas_test",
     srcs = ["config_schemas_test.cc"],
     data = [
-        "//test/common/json:config_schema_test_generator",
+        "//test/common/json/config_schemas_test_data:generate_test_data",
     ],
     deps = [
         "//source/common/json:config_schemas_lib",
diff --git a/test/common/json/config_schemas_test.cc b/test/common/json/config_schemas_test.cc
index 1c4c211bf0a6d..ddfb5c7d389ed 100644
--- a/test/common/json/config_schemas_test.cc
+++ b/test/common/json/config_schemas_test.cc
@@ -17,10 +17,12 @@ namespace Json {
 
 std::vector<std::string> generateTestInputs() {
   TestEnvironment::exec({TestEnvironment::runfilesPath(
-      "test/common/json/config_schemas_test_data/generate_test_data.py")});
+      "test/common/json/config_schemas_test_data/generate_test_data")});
 
   std::string test_path = TestEnvironment::temporaryDirectory() + "/config_schemas_test";
-  return TestUtility::listFiles(test_path, false);
+  auto file_list = TestUtility::listFiles(test_path, false);
+  EXPECT_EQ(17, file_list.size());
+  return file_list;
 }
 
 class ConfigSchemasTest : public ::testing::TestWithParam<std::string> {};
diff --git a/test/common/json/config_schemas_test_data/BUILD b/test/common/json/config_schemas_test_data/BUILD
new file mode 100644
index 0000000000000..bc850f5e1e888
--- /dev/null
+++ b/test/common/json/config_schemas_test_data/BUILD
@@ -0,0 +1,11 @@
+package(default_visibility = ["//visibility:public"])
+
+load("//bazel:envoy_build_system.bzl", "envoy_py_test_binary")
+
+envoy_py_test_binary(
+    name = "generate_test_data",
+    srcs = [
+        "generate_test_data.py",
+        "util.py",
+    ] + glob(["test_*.py"]),
+)
diff --git a/test/common/runtime/filesystem_setup.sh b/test/common/runtime/filesystem_setup.sh
index bc285c67f4792..a14db0e1ed2dd 100755
--- a/test/common/runtime/filesystem_setup.sh
+++ b/test/common/runtime/filesystem_setup.sh
@@ -4,5 +4,6 @@ set -e
 
 cd ${TEST_SRCDIR}/${TEST_WORKSPACE}
 cp -rfL --parents test/common/runtime/test_data ${TEST_TMPDIR}
+chmod -R u+rwX ${TEST_TMPDIR}/test/common/runtime/test_data
 ln -sf ${TEST_TMPDIR}/test/common/runtime/test_data/root ${TEST_TMPDIR}/test/common/runtime/test_data/current
 ln -sf ${TEST_TMPDIR}/test/common/runtime/test_data/root/envoy/subdir ${TEST_TMPDIR}/test/common/runtime/test_data/root/envoy/badlink