From 2993b50961220a80e792c29fd7996a599bf024ea Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:31:07 -0700 Subject: [PATCH 01/22] Add `.gitignore` --- .gitignore | 162 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..efa407c35f --- /dev/null +++ b/.gitignore @@ -0,0 +1,162 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file From 7d9afd89e35d9349d258af283af34444693bac18 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:33:31 -0700 Subject: [PATCH 02/22] Add github test workflow --- .github/workflows/ci-test.yml | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/ci-test.yml diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml new file mode 100644 index 0000000000..ed2fcd5317 --- /dev/null +++ b/.github/workflows/ci-test.yml @@ -0,0 +1,42 @@ +# Github action definitions for unit-tests with PRs. + +name: tfma-unit-tests +on: + pull_request: + branches: [ master ] + paths-ignore: + - '**.md' + - 'docs/**' + workflow_dispatch: + +jobs: + unit-tests: + if: github.actor != 'copybara-service[bot]' + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ['3.9', '3.10'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: | + setup.py + + - name: Install dependencies + run: | + sudo apt update + sudo apt install protobuf-compiler -y + pip install . + + - name: Run unit tests + shell: bash + run: | + python -m unittest discover -p "*_test.py" From 2543b3e10ff4b283ebe5682f00eb6ba06c36a018 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:34:28 -0700 Subject: [PATCH 03/22] Add push trigger --- .github/workflows/ci-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index ed2fcd5317..81fbc044e4 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -2,6 +2,7 @@ name: tfma-unit-tests on: + push: pull_request: branches: [ master ] paths-ignore: From a2e3e794376285fb0c05da68ef172569b7a7544e Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Tue, 22 Oct 2024 18:34:18 -0700 Subject: [PATCH 04/22] Add expected failures and skips --- tensorflow_model_analysis/api/model_eval_lib_test.py | 9 +++++++++ .../extractors/inference_base_test.py | 12 ++++++++++++ .../metrics/metric_specs_test.py | 3 +++ tensorflow_model_analysis/utils/model_util_test.py | 2 ++ 4 files changed, 26 insertions(+) diff --git a/tensorflow_model_analysis/api/model_eval_lib_test.py b/tensorflow_model_analysis/api/model_eval_lib_test.py index 6536230fb8..c54402bd32 100644 --- a/tensorflow_model_analysis/api/model_eval_lib_test.py +++ b/tensorflow_model_analysis/api/model_eval_lib_test.py @@ -16,6 +16,7 @@ import json import os import tempfile +import unittest from absl.testing import absltest from absl.testing import parameterized @@ -1122,6 +1123,8 @@ def testRunModelAnalysisWithQueryBasedMetrics(self): for k in expected_metrics[group]: self.assertIn(k, got_metrics[group]) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testRunModelAnalysisWithUncertainty(self): examples = [ self._makeExample(age=3.0, language='english', label=1.0), @@ -1391,6 +1394,8 @@ def testRunModelAnalysisWithSchema(self): self.assertEqual(1.0, got_buckets[1]['lowerThresholdInclusive']) self.assertEqual(2.0, got_buckets[-2]['upperThresholdExclusive']) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testLoadValidationResult(self): result = validation_result_pb2.ValidationResult(validation_ok=True) path = os.path.join(absltest.get_default_test_tmpdir(), 'results.tfrecord') @@ -1399,6 +1404,8 @@ def testLoadValidationResult(self): loaded_result = model_eval_lib.load_validation_result(path) self.assertTrue(loaded_result.validation_ok) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testLoadValidationResultDir(self): result = validation_result_pb2.ValidationResult(validation_ok=True) path = os.path.join( @@ -1409,6 +1416,8 @@ def testLoadValidationResultDir(self): loaded_result = model_eval_lib.load_validation_result(os.path.dirname(path)) self.assertTrue(loaded_result.validation_ok) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testLoadValidationResultEmptyFile(self): path = os.path.join( absltest.get_default_test_tmpdir(), constants.VALIDATIONS_KEY diff --git a/tensorflow_model_analysis/extractors/inference_base_test.py b/tensorflow_model_analysis/extractors/inference_base_test.py index f89d13f780..8c24e711b4 100644 --- a/tensorflow_model_analysis/extractors/inference_base_test.py +++ b/tensorflow_model_analysis/extractors/inference_base_test.py @@ -34,6 +34,8 @@ from tensorflow_serving.apis import logging_pb2 from tensorflow_serving.apis import prediction_log_pb2 +import unittest + class TfxBslPredictionsExtractorTest(testutil.TensorflowModelAnalysisTest): @@ -70,6 +72,8 @@ def _create_tfxio_and_feature_extractor( ) return tfx_io, feature_extractor + # PR 189: Remove the `skip` mark if the test passes + @unittest.skip("This test contains errors") def testIsValidConfigForBulkInferencePass(self): saved_model_proto = text_format.Parse( """ @@ -129,6 +133,8 @@ def testIsValidConfigForBulkInferencePass(self): ) ) + # PR 189: Remove the `skip` mark if the test passes + @unittest.skip("This test contains errors") def testIsValidConfigForBulkInferencePassDefaultSignatureLookUp(self): saved_model_proto = text_format.Parse( """ @@ -184,6 +190,8 @@ def testIsValidConfigForBulkInferencePassDefaultSignatureLookUp(self): ) ) + # PR 189: Remove the `skip` mark if the test passes + @unittest.skip("This test contains errors") def testIsValidConfigForBulkInferenceFailNoSignatureFound(self): saved_model_proto = text_format.Parse( """ @@ -239,6 +247,8 @@ def testIsValidConfigForBulkInferenceFailNoSignatureFound(self): ) ) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testIsValidConfigForBulkInferenceFailKerasModel(self): saved_model_proto = text_format.Parse( """ @@ -296,6 +306,8 @@ def testIsValidConfigForBulkInferenceFailKerasModel(self): ) ) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testIsValidConfigForBulkInferenceFailWrongInputType(self): saved_model_proto = text_format.Parse( """ diff --git a/tensorflow_model_analysis/metrics/metric_specs_test.py b/tensorflow_model_analysis/metrics/metric_specs_test.py index 9c307bc648..e50caa3ee6 100644 --- a/tensorflow_model_analysis/metrics/metric_specs_test.py +++ b/tensorflow_model_analysis/metrics/metric_specs_test.py @@ -14,6 +14,7 @@ """Tests for metric specs.""" import json +import unittest import tensorflow as tf from tensorflow_model_analysis.metrics import calibration @@ -37,6 +38,8 @@ def _maybe_add_fn_name(kv, name): class MetricSpecsTest(tf.test.TestCase): + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testSpecsFromMetrics(self): metrics_specs = metric_specs.specs_from_metrics( { diff --git a/tensorflow_model_analysis/utils/model_util_test.py b/tensorflow_model_analysis/utils/model_util_test.py index d10984f722..65253ddf9d 100644 --- a/tensorflow_model_analysis/utils/model_util_test.py +++ b/tensorflow_model_analysis/utils/model_util_test.py @@ -955,6 +955,8 @@ def testGetDefaultModelSignatureFromSavedModelProtoWithServingDefault(self): model_util.get_default_signature_name_from_saved_model_proto( saved_model_proto)) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testGetDefaultModelSignatureFromModelPath(self): saved_model_proto = text_format.Parse( """ From ec31f6f574424471da70aee900e2c85bacd1436d Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Wed, 23 Oct 2024 01:23:12 -0700 Subject: [PATCH 05/22] Also test on python version 3.11 --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 81fbc044e4..e43f0bc9f8 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - python-version: ['3.9', '3.10'] + python-version: ['3.9', '3.10', '3.11'] steps: - name: Checkout repository From 2b39506976a1ae1ae314afbd7dd75cbb5cc6fa73 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Wed, 23 Oct 2024 01:33:46 -0700 Subject: [PATCH 06/22] Remove python version 3.11 due to dependency error --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index e43f0bc9f8..81fbc044e4 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10'] steps: - name: Checkout repository From 93d3b12731818c758e76d231cbdc9bbb5b0e0401 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 24 Oct 2024 16:37:52 -0700 Subject: [PATCH 07/22] Add note on how to run tests --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 8aea43e90e..ece9a96782 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,16 @@ cd dist pip3 install tensorflow_model_analysis--py3-none-any.whl ``` +### Running tests + +To run tests, run + +``` +python -m unittest discover -p *_test.py +``` + +from the root project directory. + ### Jupyter Lab As of writing, because of https://github.com/pypa/pip/issues/9187, `pip install` From 609976af3a5c1e8a0c6596aac6943a11361a55d6 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Fri, 4 Jul 2025 14:49:06 -0700 Subject: [PATCH 08/22] Add `tf-keras` as a dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1b9a291a85..16b7b60090 100644 --- a/setup.py +++ b/setup.py @@ -342,6 +342,7 @@ def select_constraint(default, nightly=None, git_master=None): nightly='>=1.18.0.dev', git_master='@git+https://github.com/tensorflow/tfx-bsl@master', ), + 'tf-keras', ], 'extras_require': { 'all': [*_make_extra_packages_tfjs(), *_make_docs_packages()], From 3849ac98a7f9470631b3ea5c27503ff4ec682dbe Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Fri, 4 Jul 2025 15:59:05 -0700 Subject: [PATCH 09/22] Add expect failure for several tests and test classes all suffering from `AttributeError: module 'tensorflow_model_analysis' has no attribute 'EvalConfig'` These are to be addressed in a future PR --- tensorflow_model_analysis/metrics/bleu_test.py | 3 +++ tensorflow_model_analysis/metrics/example_count_test.py | 3 +++ .../object_detection_confusion_matrix_metrics_test.py | 5 ++++- .../metrics/object_detection_confusion_matrix_plot_test.py | 3 +++ .../metrics/object_detection_metrics_test.py | 4 ++++ tensorflow_model_analysis/metrics/rouge_test.py | 3 +++ .../metrics/set_match_confusion_matrix_metrics_test.py | 4 ++++ tensorflow_model_analysis/metrics/stats_test.py | 3 +++ tensorflow_model_analysis/utils/example_keras_model_test.py | 3 +++ 9 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tensorflow_model_analysis/metrics/bleu_test.py b/tensorflow_model_analysis/metrics/bleu_test.py index 8f25a23a42..0282d5fc0f 100644 --- a/tensorflow_model_analysis/metrics/bleu_test.py +++ b/tensorflow_model_analysis/metrics/bleu_test.py @@ -14,6 +14,7 @@ """Tests for BLEU metric.""" from absl.testing import parameterized +import unittest import apache_beam as beam from apache_beam.testing import util import numpy as np @@ -559,6 +560,8 @@ def test_bleu_merge_accumulators(self, accs_list, expected_merged_acc): class BleuEnd2EndTest(parameterized.TestCase): + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def test_bleu_end_2_end(self): # Same test as BleuTest.testBleuDefault with 'imperfect_score' eval_config = text_format.Parse( diff --git a/tensorflow_model_analysis/metrics/example_count_test.py b/tensorflow_model_analysis/metrics/example_count_test.py index 3526c36ecb..2ff130d2dc 100644 --- a/tensorflow_model_analysis/metrics/example_count_test.py +++ b/tensorflow_model_analysis/metrics/example_count_test.py @@ -14,6 +14,7 @@ """Tests for example count metric.""" from absl.testing import parameterized +import unittest import apache_beam as beam from apache_beam.testing import util import numpy as np @@ -94,6 +95,8 @@ def check_result(got): class ExampleCountEnd2EndTest(parameterized.TestCase): + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testExampleCountsWithoutLabelPredictions(self): eval_config = text_format.Parse( """ diff --git a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py index be25b2ce32..5ad90b5291 100644 --- a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py @@ -13,6 +13,7 @@ # limitations under the License. """Tests for object detection related confusion matrix metrics.""" from absl.testing import absltest +import unittest from absl.testing import parameterized import apache_beam as beam from apache_beam.testing import util @@ -21,7 +22,9 @@ from tensorflow_model_analysis.metrics import metric_types from google.protobuf import text_format - +# PR 189: Remove the `expectedFailure` mark if the test passes +# The test failures are `AttributeError: module 'tensorflow_model_analysis' has no attribute 'EvalConfig'` +@unittest.expectedFailure class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase): @parameterized.named_parameters(('_max_recall', diff --git a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py index 0cf544b2b8..63c48a99b0 100644 --- a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py @@ -14,6 +14,7 @@ """Tests for object detection confusion matrix plot.""" from absl.testing import absltest +import unittest import apache_beam as beam from apache_beam.testing import util import numpy as np @@ -28,6 +29,8 @@ class ObjectDetectionConfusionMatrixPlotTest( test_util.TensorflowModelAnalysisTest, absltest.TestCase ): + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testConfusionMatrixPlot(self): eval_config = text_format.Parse( """ diff --git a/tensorflow_model_analysis/metrics/object_detection_metrics_test.py b/tensorflow_model_analysis/metrics/object_detection_metrics_test.py index b5c46ba5aa..3d15b52db0 100644 --- a/tensorflow_model_analysis/metrics/object_detection_metrics_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_metrics_test.py @@ -13,6 +13,7 @@ # limitations under the License. """Tests for object detection related metrics.""" from absl.testing import absltest +import unittest from absl.testing import parameterized import apache_beam as beam from apache_beam.testing import util @@ -22,6 +23,9 @@ from google.protobuf import text_format +# PR 189: Remove the `expectedFailure` mark if the test passes +# The test failures are `AttributeError: module 'tensorflow_model_analysis' has no attribute 'EvalConfig'` +@unittest.expectedFailure class ObjectDetectionMetricsTest(parameterized.TestCase): """This tests the object detection metrics. diff --git a/tensorflow_model_analysis/metrics/rouge_test.py b/tensorflow_model_analysis/metrics/rouge_test.py index 1fc1afd6db..fd56d6f2a9 100644 --- a/tensorflow_model_analysis/metrics/rouge_test.py +++ b/tensorflow_model_analysis/metrics/rouge_test.py @@ -18,6 +18,7 @@ from absl.testing import parameterized import apache_beam as beam from apache_beam.testing import util +import unittest import numpy as np import tensorflow as tf import tensorflow_model_analysis as tfma @@ -632,6 +633,8 @@ def check_result(got): class RougeEnd2EndTest(parameterized.TestCase): + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testRougeEnd2End(self): # Same tests as RougeTest.testRougeMultipleExamplesWeighted eval_config = text_format.Parse( diff --git a/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py index 15d01ef7fe..ca0af82df2 100644 --- a/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py @@ -14,6 +14,7 @@ """Tests for set match related confusion matrix metrics.""" from absl.testing import absltest from absl.testing import parameterized +import unittest import apache_beam as beam from apache_beam.testing import util import numpy as np @@ -22,6 +23,9 @@ from google.protobuf import text_format +# PR 189: Remove the `expectedFailure` mark if the test passes +# The test failures are `AttributeError: module 'tensorflow_model_analysis' has no attribute 'EvalConfig'` +@unittest.expectedFailure class SetMatchConfusionMatrixMetricsTest(parameterized.TestCase): @parameterized.named_parameters( diff --git a/tensorflow_model_analysis/metrics/stats_test.py b/tensorflow_model_analysis/metrics/stats_test.py index bfc35a5af4..4f1abf3111 100644 --- a/tensorflow_model_analysis/metrics/stats_test.py +++ b/tensorflow_model_analysis/metrics/stats_test.py @@ -16,6 +16,7 @@ from absl.testing import parameterized import apache_beam as beam from apache_beam.testing import util +import unittest import numpy as np import tensorflow as tf import tensorflow_model_analysis as tfma @@ -290,6 +291,8 @@ def check_result(got): class MeanEnd2EndTest(parameterized.TestCase): + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testMeanEnd2End(self): extracts = [ { diff --git a/tensorflow_model_analysis/utils/example_keras_model_test.py b/tensorflow_model_analysis/utils/example_keras_model_test.py index b42471fd76..6ac3e7905a 100644 --- a/tensorflow_model_analysis/utils/example_keras_model_test.py +++ b/tensorflow_model_analysis/utils/example_keras_model_test.py @@ -21,6 +21,7 @@ import datetime import os import tempfile +import unittest import numpy as np import six @@ -91,6 +92,8 @@ def _write_tf_records(self, examples): writer.write(example.SerializeToString()) return data_location + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def test_example_keras_model(self): data = self._create_data() classifier = example_keras_model.get_example_classifier_model() From a9e43e630fe695e24ca30a411eea391c58aedbd7 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Fri, 4 Jul 2025 19:40:54 -0700 Subject: [PATCH 10/22] Remove import to nonexistant modules --- tensorflow_model_analysis/export_only/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow_model_analysis/export_only/__init__.py b/tensorflow_model_analysis/export_only/__init__.py index 5e02bf5c3f..e4ab459106 100644 --- a/tensorflow_model_analysis/export_only/__init__.py +++ b/tensorflow_model_analysis/export_only/__init__.py @@ -29,5 +29,3 @@ def eval_input_receiver_fn(): tfma_export.export.export_eval_saved_model(...) """ -from tensorflow_model_analysis.eval_saved_model import export -from tensorflow_model_analysis.eval_saved_model import exporter From a3812b9890d6bec6df19a137ec398d1e977cabe5 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Thu, 10 Jul 2025 22:35:00 -0700 Subject: [PATCH 11/22] Install `libprotobuf-c-dev` for unit tests in CI --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 81fbc044e4..a9487a9ea8 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -34,7 +34,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install protobuf-compiler -y + sudo apt install -y protobuf-compiler libprotobuf-c-dev pip install . - name: Run unit tests From da31bf3df6271682dfe78daeac9a760f73b50cda Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 19:53:51 -0700 Subject: [PATCH 12/22] Temporarily remove `expectedFailure`s --- .../extractors/inference_base_test.py | 10 ---------- tensorflow_model_analysis/metrics/bleu_test.py | 2 -- .../metrics/example_count_test.py | 2 -- tensorflow_model_analysis/metrics/metric_specs_test.py | 2 -- .../object_detection_confusion_matrix_metrics_test.py | 3 --- .../object_detection_confusion_matrix_plot_test.py | 2 -- .../metrics/object_detection_metrics_test.py | 3 --- tensorflow_model_analysis/metrics/rouge_test.py | 2 -- .../metrics/set_match_confusion_matrix_metrics_test.py | 3 --- tensorflow_model_analysis/metrics/stats_test.py | 2 -- .../utils/example_keras_model_test.py | 2 -- 11 files changed, 33 deletions(-) diff --git a/tensorflow_model_analysis/extractors/inference_base_test.py b/tensorflow_model_analysis/extractors/inference_base_test.py index 8c24e711b4..a56a4f3998 100644 --- a/tensorflow_model_analysis/extractors/inference_base_test.py +++ b/tensorflow_model_analysis/extractors/inference_base_test.py @@ -72,8 +72,6 @@ def _create_tfxio_and_feature_extractor( ) return tfx_io, feature_extractor - # PR 189: Remove the `skip` mark if the test passes - @unittest.skip("This test contains errors") def testIsValidConfigForBulkInferencePass(self): saved_model_proto = text_format.Parse( """ @@ -133,8 +131,6 @@ def testIsValidConfigForBulkInferencePass(self): ) ) - # PR 189: Remove the `skip` mark if the test passes - @unittest.skip("This test contains errors") def testIsValidConfigForBulkInferencePassDefaultSignatureLookUp(self): saved_model_proto = text_format.Parse( """ @@ -190,8 +186,6 @@ def testIsValidConfigForBulkInferencePassDefaultSignatureLookUp(self): ) ) - # PR 189: Remove the `skip` mark if the test passes - @unittest.skip("This test contains errors") def testIsValidConfigForBulkInferenceFailNoSignatureFound(self): saved_model_proto = text_format.Parse( """ @@ -247,8 +241,6 @@ def testIsValidConfigForBulkInferenceFailNoSignatureFound(self): ) ) - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def testIsValidConfigForBulkInferenceFailKerasModel(self): saved_model_proto = text_format.Parse( """ @@ -306,8 +298,6 @@ def testIsValidConfigForBulkInferenceFailKerasModel(self): ) ) - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def testIsValidConfigForBulkInferenceFailWrongInputType(self): saved_model_proto = text_format.Parse( """ diff --git a/tensorflow_model_analysis/metrics/bleu_test.py b/tensorflow_model_analysis/metrics/bleu_test.py index 0282d5fc0f..6762c4baca 100644 --- a/tensorflow_model_analysis/metrics/bleu_test.py +++ b/tensorflow_model_analysis/metrics/bleu_test.py @@ -560,8 +560,6 @@ def test_bleu_merge_accumulators(self, accs_list, expected_merged_acc): class BleuEnd2EndTest(parameterized.TestCase): - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def test_bleu_end_2_end(self): # Same test as BleuTest.testBleuDefault with 'imperfect_score' eval_config = text_format.Parse( diff --git a/tensorflow_model_analysis/metrics/example_count_test.py b/tensorflow_model_analysis/metrics/example_count_test.py index 2ff130d2dc..5def22e53d 100644 --- a/tensorflow_model_analysis/metrics/example_count_test.py +++ b/tensorflow_model_analysis/metrics/example_count_test.py @@ -95,8 +95,6 @@ def check_result(got): class ExampleCountEnd2EndTest(parameterized.TestCase): - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def testExampleCountsWithoutLabelPredictions(self): eval_config = text_format.Parse( """ diff --git a/tensorflow_model_analysis/metrics/metric_specs_test.py b/tensorflow_model_analysis/metrics/metric_specs_test.py index e50caa3ee6..0e18902977 100644 --- a/tensorflow_model_analysis/metrics/metric_specs_test.py +++ b/tensorflow_model_analysis/metrics/metric_specs_test.py @@ -38,8 +38,6 @@ def _maybe_add_fn_name(kv, name): class MetricSpecsTest(tf.test.TestCase): - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def testSpecsFromMetrics(self): metrics_specs = metric_specs.specs_from_metrics( { diff --git a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py index 5ad90b5291..d89cb81ddd 100644 --- a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py @@ -22,9 +22,6 @@ from tensorflow_model_analysis.metrics import metric_types from google.protobuf import text_format -# PR 189: Remove the `expectedFailure` mark if the test passes -# The test failures are `AttributeError: module 'tensorflow_model_analysis' has no attribute 'EvalConfig'` -@unittest.expectedFailure class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase): @parameterized.named_parameters(('_max_recall', diff --git a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py index 63c48a99b0..b07bb8b9c1 100644 --- a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py @@ -29,8 +29,6 @@ class ObjectDetectionConfusionMatrixPlotTest( test_util.TensorflowModelAnalysisTest, absltest.TestCase ): - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def testConfusionMatrixPlot(self): eval_config = text_format.Parse( """ diff --git a/tensorflow_model_analysis/metrics/object_detection_metrics_test.py b/tensorflow_model_analysis/metrics/object_detection_metrics_test.py index 3d15b52db0..8eef949e14 100644 --- a/tensorflow_model_analysis/metrics/object_detection_metrics_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_metrics_test.py @@ -23,9 +23,6 @@ from google.protobuf import text_format -# PR 189: Remove the `expectedFailure` mark if the test passes -# The test failures are `AttributeError: module 'tensorflow_model_analysis' has no attribute 'EvalConfig'` -@unittest.expectedFailure class ObjectDetectionMetricsTest(parameterized.TestCase): """This tests the object detection metrics. diff --git a/tensorflow_model_analysis/metrics/rouge_test.py b/tensorflow_model_analysis/metrics/rouge_test.py index fd56d6f2a9..c5f1f162aa 100644 --- a/tensorflow_model_analysis/metrics/rouge_test.py +++ b/tensorflow_model_analysis/metrics/rouge_test.py @@ -633,8 +633,6 @@ def check_result(got): class RougeEnd2EndTest(parameterized.TestCase): - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def testRougeEnd2End(self): # Same tests as RougeTest.testRougeMultipleExamplesWeighted eval_config = text_format.Parse( diff --git a/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py index ca0af82df2..5a9de1ae50 100644 --- a/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py @@ -23,9 +23,6 @@ from google.protobuf import text_format -# PR 189: Remove the `expectedFailure` mark if the test passes -# The test failures are `AttributeError: module 'tensorflow_model_analysis' has no attribute 'EvalConfig'` -@unittest.expectedFailure class SetMatchConfusionMatrixMetricsTest(parameterized.TestCase): @parameterized.named_parameters( diff --git a/tensorflow_model_analysis/metrics/stats_test.py b/tensorflow_model_analysis/metrics/stats_test.py index 4f1abf3111..2abb60fe80 100644 --- a/tensorflow_model_analysis/metrics/stats_test.py +++ b/tensorflow_model_analysis/metrics/stats_test.py @@ -291,8 +291,6 @@ def check_result(got): class MeanEnd2EndTest(parameterized.TestCase): - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def testMeanEnd2End(self): extracts = [ { diff --git a/tensorflow_model_analysis/utils/example_keras_model_test.py b/tensorflow_model_analysis/utils/example_keras_model_test.py index 6ac3e7905a..266f45ea71 100644 --- a/tensorflow_model_analysis/utils/example_keras_model_test.py +++ b/tensorflow_model_analysis/utils/example_keras_model_test.py @@ -92,8 +92,6 @@ def _write_tf_records(self, examples): writer.write(example.SerializeToString()) return data_location - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def test_example_keras_model(self): data = self._create_data() classifier = example_keras_model.get_example_classifier_model() From c0a63056250ba6198b214730aac9ffda0300b3ff Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 20:32:35 -0700 Subject: [PATCH 13/22] Fix `EvalConfig` imports --- .../metrics/bleu_test.py | 3 ++- .../metrics/example_count_test.py | 3 ++- ...detection_confusion_matrix_metrics_test.py | 11 +++++----- ...ct_detection_confusion_matrix_plot_test.py | 3 ++- .../metrics/object_detection_metrics_test.py | 21 ++++++++++--------- .../metrics/rouge_test.py | 3 ++- ...mentation_confusion_matrix_metrics_test.py | 9 ++++---- ...set_match_confusion_matrix_metrics_test.py | 15 ++++++------- .../metrics/stats_test.py | 5 +++-- .../utils/example_keras_model_test.py | 3 ++- 10 files changed, 43 insertions(+), 33 deletions(-) diff --git a/tensorflow_model_analysis/metrics/bleu_test.py b/tensorflow_model_analysis/metrics/bleu_test.py index 6762c4baca..04dee2e391 100644 --- a/tensorflow_model_analysis/metrics/bleu_test.py +++ b/tensorflow_model_analysis/metrics/bleu_test.py @@ -20,6 +20,7 @@ import numpy as np import tensorflow as tf import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis import constants from tensorflow_model_analysis.evaluators import metrics_plots_and_validations_evaluator from tensorflow_model_analysis.metrics import bleu @@ -574,7 +575,7 @@ def test_bleu_end_2_end(self): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ) example1 = { diff --git a/tensorflow_model_analysis/metrics/example_count_test.py b/tensorflow_model_analysis/metrics/example_count_test.py index 5def22e53d..1cc3930391 100644 --- a/tensorflow_model_analysis/metrics/example_count_test.py +++ b/tensorflow_model_analysis/metrics/example_count_test.py @@ -20,6 +20,7 @@ import numpy as np import tensorflow as tf import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis.metrics import example_count from tensorflow_model_analysis.metrics import metric_types from tensorflow_model_analysis.metrics import metric_util @@ -110,7 +111,7 @@ def testExampleCountsWithoutLabelPredictions(self): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ) name_list = ['example_count'] expected_results = [0.6] diff --git a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py index d89cb81ddd..288e8ba292 100644 --- a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py @@ -19,6 +19,7 @@ from apache_beam.testing import util import numpy as np import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis.metrics import metric_types from google.protobuf import text_format @@ -41,7 +42,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"maxrecall"' } } - """, tfma.EvalConfig()), ['maxrecall'], [2 / 3]), + """, config_pb2.EvalConfig()), ['maxrecall'], [2 / 3]), ('_precision_at_recall', text_format.Parse( """ @@ -59,7 +60,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"precisionatrecall"' } } - """, tfma.EvalConfig()), ['precisionatrecall'], [3 / 5]), + """, config_pb2.EvalConfig()), ['precisionatrecall'], [3 / 5]), ('_recall', text_format.Parse( """ @@ -77,7 +78,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"recall"' } } - """, tfma.EvalConfig()), ['recall'], [2 / 3]), ('_precision', + """, config_pb2.EvalConfig()), ['recall'], [2 / 3]), ('_precision', text_format.Parse( """ model_specs { @@ -94,7 +95,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"precision"' } } - """, tfma.EvalConfig()), ['precision'], [0.5]), ('_threshold_at_recall', + """, config_pb2.EvalConfig()), ['precision'], [0.5]), ('_threshold_at_recall', text_format.Parse( """ model_specs { @@ -111,7 +112,7 @@ class ObjectDetectionConfusionMatrixMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"thresholdatrecall"' } } - """, tfma.EvalConfig()), ['thresholdatrecall'], [0.3])) + """, config_pb2.EvalConfig()), ['thresholdatrecall'], [0.3])) def testObjectDetectionMetrics(self, eval_config, name_list, expected_results): diff --git a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py index b07bb8b9c1..f844f3fd3f 100644 --- a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py @@ -19,6 +19,7 @@ from apache_beam.testing import util import numpy as np import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis.metrics import metric_types from tensorflow_model_analysis.utils import test_util @@ -46,7 +47,7 @@ def testConfusionMatrixPlot(self): '"max_num_detections":100, "name":"iou0.5"' } } - """, tfma.EvalConfig()) + """, config_pb2.EvalConfig()) extracts = [ # The match at iou_threshold = 0.5 is # gt_matches: [[0]] dt_matches: [[0, -1]] diff --git a/tensorflow_model_analysis/metrics/object_detection_metrics_test.py b/tensorflow_model_analysis/metrics/object_detection_metrics_test.py index 8eef949e14..079685fea9 100644 --- a/tensorflow_model_analysis/metrics/object_detection_metrics_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_metrics_test.py @@ -19,6 +19,7 @@ from apache_beam.testing import util import numpy as np import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis.metrics import metric_types from google.protobuf import text_format @@ -60,7 +61,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"iou0.5"' } } - """, tfma.EvalConfig()), ['iou0.5'], [0.916]), + """, config_pb2.EvalConfig()), ['iou0.5'], [0.916]), ('_average_precision_iou0.75', text_format.Parse( """ @@ -78,7 +79,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"iou0.75"' } } - """, tfma.EvalConfig()), ['iou0.75'], [0.416]), + """, config_pb2.EvalConfig()), ['iou0.75'], [0.416]), ('_average_precision_ave', text_format.Parse( """ @@ -96,7 +97,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"iouave"' } } - """, tfma.EvalConfig()), ['iouave'], [0.666]), ('_average_recall_mdet1', + """, config_pb2.EvalConfig()), ['iouave'], [0.666]), ('_average_recall_mdet1', text_format.Parse( """ model_specs { @@ -113,7 +114,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"name":"mdet1"' } } - """, tfma.EvalConfig()), ['mdet1'], [0.375]), ('_average_recall_mdet10', + """, config_pb2.EvalConfig()), ['mdet1'], [0.375]), ('_average_recall_mdet10', text_format.Parse( """ model_specs { @@ -130,7 +131,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"name":"mdet10"' } } - """, tfma.EvalConfig()), ['mdet10'], [0.533]), + """, config_pb2.EvalConfig()), ['mdet10'], [0.533]), ('_average_recall_mdet100', text_format.Parse( """ @@ -148,7 +149,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"name":"mdet100"' } } - """, tfma.EvalConfig()), ['mdet100'], [0.533]), + """, config_pb2.EvalConfig()), ['mdet100'], [0.533]), ('_average_recall_arsmall', text_format.Parse( """ @@ -166,7 +167,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"arsmall"' } } - """, tfma.EvalConfig()), ['arsmall'], [0.500]), + """, config_pb2.EvalConfig()), ['arsmall'], [0.500]), ('_average_recall_armedium', text_format.Parse( """ @@ -184,7 +185,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"armedium"' } } - """, tfma.EvalConfig()), ['armedium'], [0.300]), + """, config_pb2.EvalConfig()), ['armedium'], [0.300]), ('_average_recall_arlarge', text_format.Parse( """ @@ -202,7 +203,7 @@ class ObjectDetectionMetricsTest(parameterized.TestCase): '"max_num_detections":100, "name":"arlarge"' } } - """, tfma.EvalConfig()), ['arlarge'], [0.700])) + """, config_pb2.EvalConfig()), ['arlarge'], [0.700])) def testMetricValuesWithLargerData(self, eval_config, name_list, expected_results): @@ -284,7 +285,7 @@ def check_result(got): '"predictions_to_stack":["bbox", "class_id", "scores"]' } } - """, tfma.EvalConfig()), ['iou0.5'], [0.916])) + """, config_pb2.EvalConfig()), ['iou0.5'], [0.916])) def testMetricValuesWithSplittedData(self, eval_config, name_list, expected_results): diff --git a/tensorflow_model_analysis/metrics/rouge_test.py b/tensorflow_model_analysis/metrics/rouge_test.py index c5f1f162aa..94a128079d 100644 --- a/tensorflow_model_analysis/metrics/rouge_test.py +++ b/tensorflow_model_analysis/metrics/rouge_test.py @@ -22,6 +22,7 @@ import numpy as np import tensorflow as tf import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis import constants from tensorflow_model_analysis.evaluators import metrics_plots_and_validations_evaluator from tensorflow_model_analysis.metrics import metric_types @@ -660,7 +661,7 @@ def testRougeEnd2End(self): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ) rouge_types = ['rouge1', 'rouge2', 'rougeL', 'rougeLsum'] example_weights = [0.5, 0.7] diff --git a/tensorflow_model_analysis/metrics/semantic_segmentation_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/semantic_segmentation_confusion_matrix_metrics_test.py index 74762e5596..b19af57e4b 100644 --- a/tensorflow_model_analysis/metrics/semantic_segmentation_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/semantic_segmentation_confusion_matrix_metrics_test.py @@ -22,6 +22,7 @@ import numpy as np from PIL import Image import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis import constants from tensorflow_model_analysis.contrib.aggregates import binary_confusion_matrices from tensorflow_model_analysis.metrics import metric_types @@ -102,7 +103,7 @@ def setUp(self): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), name='SegConfusionMatrix', expected_result={ @@ -133,7 +134,7 @@ def setUp(self): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), name='SegConfusionMatrix', expected_result={ @@ -164,7 +165,7 @@ def setUp(self): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), name='SegTruePositive', expected_result={ @@ -195,7 +196,7 @@ def setUp(self): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), name='SegFalsePositive', expected_result={ diff --git a/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py index 5a9de1ae50..961cf89291 100644 --- a/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py @@ -19,6 +19,7 @@ from apache_beam.testing import util import numpy as np import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis.metrics import metric_types from google.protobuf import text_format @@ -44,7 +45,7 @@ class SetMatchConfusionMatrixMetricsTest(parameterized.TestCase): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), ['set_match_precision'], [0.4], @@ -67,7 +68,7 @@ class SetMatchConfusionMatrixMetricsTest(parameterized.TestCase): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), ['recall'], [0.5], @@ -90,7 +91,7 @@ class SetMatchConfusionMatrixMetricsTest(parameterized.TestCase): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), ['precision'], [0.25], @@ -113,7 +114,7 @@ class SetMatchConfusionMatrixMetricsTest(parameterized.TestCase): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), ['recall'], [0.25], @@ -136,7 +137,7 @@ class SetMatchConfusionMatrixMetricsTest(parameterized.TestCase): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), ['recall'], [0.25], @@ -221,7 +222,7 @@ def check_result(got): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), ['set_match_precision'], [0.25], @@ -245,7 +246,7 @@ def check_result(got): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ), ['recall'], [0.294118], diff --git a/tensorflow_model_analysis/metrics/stats_test.py b/tensorflow_model_analysis/metrics/stats_test.py index 2abb60fe80..0f62819ed3 100644 --- a/tensorflow_model_analysis/metrics/stats_test.py +++ b/tensorflow_model_analysis/metrics/stats_test.py @@ -20,6 +20,7 @@ import numpy as np import tensorflow as tf import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis.metrics import metric_types from tensorflow_model_analysis.metrics import metric_util from tensorflow_model_analysis.metrics import stats @@ -324,7 +325,7 @@ def testMeanEnd2End(self): } , } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ) extractors = tfma.default_extractors(eval_config=eval_config) @@ -400,7 +401,7 @@ def testMeanEnd2EndWithoutExampleWeights(self): } , } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ) extractors = tfma.default_extractors(eval_config=eval_config) diff --git a/tensorflow_model_analysis/utils/example_keras_model_test.py b/tensorflow_model_analysis/utils/example_keras_model_test.py index 266f45ea71..750c85db8b 100644 --- a/tensorflow_model_analysis/utils/example_keras_model_test.py +++ b/tensorflow_model_analysis/utils/example_keras_model_test.py @@ -28,6 +28,7 @@ from tensorflow import keras import tensorflow.compat.v1 as tf import tensorflow_model_analysis as tfma +from tensorflow_model_analysis.proto import config_pb2 from tensorflow_model_analysis.utils import example_keras_model from google.protobuf import text_format @@ -128,7 +129,7 @@ def test_example_keras_model(self): } } """, - tfma.EvalConfig(), + config_pb2.EvalConfig(), ) validate_tf_file_path = self._write_tf_records(data) From 261a29fa0b4e2cd277a7168fe6ea686023393fd3 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:06:08 -0700 Subject: [PATCH 14/22] Add more `expectedFailure`s --- .../extractors/inference_base_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow_model_analysis/extractors/inference_base_test.py b/tensorflow_model_analysis/extractors/inference_base_test.py index a56a4f3998..a1dd0fc313 100644 --- a/tensorflow_model_analysis/extractors/inference_base_test.py +++ b/tensorflow_model_analysis/extractors/inference_base_test.py @@ -72,6 +72,8 @@ def _create_tfxio_and_feature_extractor( ) return tfx_io, feature_extractor + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testIsValidConfigForBulkInferencePass(self): saved_model_proto = text_format.Parse( """ @@ -131,6 +133,8 @@ def testIsValidConfigForBulkInferencePass(self): ) ) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testIsValidConfigForBulkInferencePassDefaultSignatureLookUp(self): saved_model_proto = text_format.Parse( """ @@ -186,6 +190,8 @@ def testIsValidConfigForBulkInferencePassDefaultSignatureLookUp(self): ) ) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testIsValidConfigForBulkInferenceFailNoSignatureFound(self): saved_model_proto = text_format.Parse( """ @@ -241,6 +247,8 @@ def testIsValidConfigForBulkInferenceFailNoSignatureFound(self): ) ) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testIsValidConfigForBulkInferenceFailKerasModel(self): saved_model_proto = text_format.Parse( """ @@ -298,6 +306,8 @@ def testIsValidConfigForBulkInferenceFailKerasModel(self): ) ) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testIsValidConfigForBulkInferenceFailWrongInputType(self): saved_model_proto = text_format.Parse( """ From 8363c74ccddf7040a5d0e0de5918cbf0dc508a52 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:06:33 -0700 Subject: [PATCH 15/22] Remove `unexpectedFailure` from unexpected success --- tensorflow_model_analysis/api/model_eval_lib_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow_model_analysis/api/model_eval_lib_test.py b/tensorflow_model_analysis/api/model_eval_lib_test.py index c54402bd32..07169bcd72 100644 --- a/tensorflow_model_analysis/api/model_eval_lib_test.py +++ b/tensorflow_model_analysis/api/model_eval_lib_test.py @@ -1123,8 +1123,6 @@ def testRunModelAnalysisWithQueryBasedMetrics(self): for k in expected_metrics[group]: self.assertIn(k, got_metrics[group]) - # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure def testRunModelAnalysisWithUncertainty(self): examples = [ self._makeExample(age=3.0, language='english', label=1.0), From 127493b69eeedb797810d0d561f7a79827f77f27 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:17:33 -0700 Subject: [PATCH 16/22] Add Python 3.11 to CI tests --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index a9487a9ea8..210970489b 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - python-version: ['3.9', '3.10'] + python-version: ['3.9', '3.10', '3.11'] steps: - name: Checkout repository From 00f0f291feee2f413c42e0bfbb3833297f1b8e6b Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:18:13 -0700 Subject: [PATCH 17/22] Remove `libprotobuf-c-dev` --- .github/workflows/ci-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 210970489b..ad5db6c8fd 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -34,7 +34,7 @@ jobs: - name: Install dependencies run: | sudo apt update - sudo apt install -y protobuf-compiler libprotobuf-c-dev + sudo apt install -y protobuf-compiler pip install . - name: Run unit tests From f00949bdc8b0a360cc3d774ecf6d3fa0e5c6eba4 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:25:27 -0700 Subject: [PATCH 18/22] Remove unnecessary import --- tensorflow_model_analysis/metrics/bleu_test.py | 1 - tensorflow_model_analysis/metrics/example_count_test.py | 1 - tensorflow_model_analysis/metrics/metric_specs_test.py | 1 - .../metrics/object_detection_confusion_matrix_metrics_test.py | 1 - .../metrics/object_detection_confusion_matrix_plot_test.py | 1 - .../metrics/object_detection_metrics_test.py | 1 - tensorflow_model_analysis/metrics/rouge_test.py | 1 - .../metrics/set_match_confusion_matrix_metrics_test.py | 1 - tensorflow_model_analysis/metrics/stats_test.py | 1 - tensorflow_model_analysis/utils/example_keras_model_test.py | 1 - 10 files changed, 10 deletions(-) diff --git a/tensorflow_model_analysis/metrics/bleu_test.py b/tensorflow_model_analysis/metrics/bleu_test.py index 04dee2e391..0cac537787 100644 --- a/tensorflow_model_analysis/metrics/bleu_test.py +++ b/tensorflow_model_analysis/metrics/bleu_test.py @@ -14,7 +14,6 @@ """Tests for BLEU metric.""" from absl.testing import parameterized -import unittest import apache_beam as beam from apache_beam.testing import util import numpy as np diff --git a/tensorflow_model_analysis/metrics/example_count_test.py b/tensorflow_model_analysis/metrics/example_count_test.py index 1cc3930391..5e11515743 100644 --- a/tensorflow_model_analysis/metrics/example_count_test.py +++ b/tensorflow_model_analysis/metrics/example_count_test.py @@ -14,7 +14,6 @@ """Tests for example count metric.""" from absl.testing import parameterized -import unittest import apache_beam as beam from apache_beam.testing import util import numpy as np diff --git a/tensorflow_model_analysis/metrics/metric_specs_test.py b/tensorflow_model_analysis/metrics/metric_specs_test.py index 0e18902977..9c307bc648 100644 --- a/tensorflow_model_analysis/metrics/metric_specs_test.py +++ b/tensorflow_model_analysis/metrics/metric_specs_test.py @@ -14,7 +14,6 @@ """Tests for metric specs.""" import json -import unittest import tensorflow as tf from tensorflow_model_analysis.metrics import calibration diff --git a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py index 288e8ba292..9d772e6e00 100644 --- a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_metrics_test.py @@ -13,7 +13,6 @@ # limitations under the License. """Tests for object detection related confusion matrix metrics.""" from absl.testing import absltest -import unittest from absl.testing import parameterized import apache_beam as beam from apache_beam.testing import util diff --git a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py index f844f3fd3f..3289cd5b15 100644 --- a/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_confusion_matrix_plot_test.py @@ -14,7 +14,6 @@ """Tests for object detection confusion matrix plot.""" from absl.testing import absltest -import unittest import apache_beam as beam from apache_beam.testing import util import numpy as np diff --git a/tensorflow_model_analysis/metrics/object_detection_metrics_test.py b/tensorflow_model_analysis/metrics/object_detection_metrics_test.py index 079685fea9..6cfa3e357e 100644 --- a/tensorflow_model_analysis/metrics/object_detection_metrics_test.py +++ b/tensorflow_model_analysis/metrics/object_detection_metrics_test.py @@ -13,7 +13,6 @@ # limitations under the License. """Tests for object detection related metrics.""" from absl.testing import absltest -import unittest from absl.testing import parameterized import apache_beam as beam from apache_beam.testing import util diff --git a/tensorflow_model_analysis/metrics/rouge_test.py b/tensorflow_model_analysis/metrics/rouge_test.py index 94a128079d..a645be58e1 100644 --- a/tensorflow_model_analysis/metrics/rouge_test.py +++ b/tensorflow_model_analysis/metrics/rouge_test.py @@ -18,7 +18,6 @@ from absl.testing import parameterized import apache_beam as beam from apache_beam.testing import util -import unittest import numpy as np import tensorflow as tf import tensorflow_model_analysis as tfma diff --git a/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py b/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py index 961cf89291..7c22a3b0ba 100644 --- a/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py +++ b/tensorflow_model_analysis/metrics/set_match_confusion_matrix_metrics_test.py @@ -14,7 +14,6 @@ """Tests for set match related confusion matrix metrics.""" from absl.testing import absltest from absl.testing import parameterized -import unittest import apache_beam as beam from apache_beam.testing import util import numpy as np diff --git a/tensorflow_model_analysis/metrics/stats_test.py b/tensorflow_model_analysis/metrics/stats_test.py index 0f62819ed3..c26dc633b2 100644 --- a/tensorflow_model_analysis/metrics/stats_test.py +++ b/tensorflow_model_analysis/metrics/stats_test.py @@ -16,7 +16,6 @@ from absl.testing import parameterized import apache_beam as beam from apache_beam.testing import util -import unittest import numpy as np import tensorflow as tf import tensorflow_model_analysis as tfma diff --git a/tensorflow_model_analysis/utils/example_keras_model_test.py b/tensorflow_model_analysis/utils/example_keras_model_test.py index 750c85db8b..8044668338 100644 --- a/tensorflow_model_analysis/utils/example_keras_model_test.py +++ b/tensorflow_model_analysis/utils/example_keras_model_test.py @@ -21,7 +21,6 @@ import datetime import os import tempfile -import unittest import numpy as np import six From 231c1570ba3160fc19bfa6ba5b3254f0faf74c47 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 22:15:54 -0700 Subject: [PATCH 19/22] Add `expectedFailure` --- tensorflow_model_analysis/api/model_eval_lib_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_model_analysis/api/model_eval_lib_test.py b/tensorflow_model_analysis/api/model_eval_lib_test.py index 07169bcd72..c54402bd32 100644 --- a/tensorflow_model_analysis/api/model_eval_lib_test.py +++ b/tensorflow_model_analysis/api/model_eval_lib_test.py @@ -1123,6 +1123,8 @@ def testRunModelAnalysisWithQueryBasedMetrics(self): for k in expected_metrics[group]: self.assertIn(k, got_metrics[group]) + # PR 189: Remove the `expectedFailure` mark if the test passes + @unittest.expectedFailure def testRunModelAnalysisWithUncertainty(self): examples = [ self._makeExample(age=3.0, language='english', label=1.0), From 1597b1987da3a835f2b2164c6e4b38aebc2ce55e Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 22:32:12 -0700 Subject: [PATCH 20/22] Use `skip` instead of `expectedFailure` --- tensorflow_model_analysis/api/model_eval_lib_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_model_analysis/api/model_eval_lib_test.py b/tensorflow_model_analysis/api/model_eval_lib_test.py index c54402bd32..8c580104c3 100644 --- a/tensorflow_model_analysis/api/model_eval_lib_test.py +++ b/tensorflow_model_analysis/api/model_eval_lib_test.py @@ -1124,7 +1124,7 @@ def testRunModelAnalysisWithQueryBasedMetrics(self): self.assertIn(k, got_metrics[group]) # PR 189: Remove the `expectedFailure` mark if the test passes - @unittest.expectedFailure + @unittest.skip('Fails for some versions of Python, including 3.9') def testRunModelAnalysisWithUncertainty(self): examples = [ self._makeExample(age=3.0, language='english', label=1.0), From 1023622b5238900be837183ebd1b217edd391bbb Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 22:46:43 -0700 Subject: [PATCH 21/22] Fix code comment --- tensorflow_model_analysis/api/model_eval_lib_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_model_analysis/api/model_eval_lib_test.py b/tensorflow_model_analysis/api/model_eval_lib_test.py index 8c580104c3..abf89f3259 100644 --- a/tensorflow_model_analysis/api/model_eval_lib_test.py +++ b/tensorflow_model_analysis/api/model_eval_lib_test.py @@ -1123,7 +1123,8 @@ def testRunModelAnalysisWithQueryBasedMetrics(self): for k in expected_metrics[group]: self.assertIn(k, got_metrics[group]) - # PR 189: Remove the `expectedFailure` mark if the test passes + # PR 189: Remove the `skip` mark if the test passes for all supported versions + # of python @unittest.skip('Fails for some versions of Python, including 3.9') def testRunModelAnalysisWithUncertainty(self): examples = [ From 7ba7ea8f88ca040ed5ef0e8001391a533dc07604 Mon Sep 17 00:00:00 2001 From: smokestacklightnin <125844868+smokestacklightnin@users.noreply.github.com> Date: Mon, 14 Jul 2025 22:47:40 -0700 Subject: [PATCH 22/22] Run for all users --- .github/workflows/ci-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index ad5db6c8fd..e527a1cce8 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -12,7 +12,6 @@ on: jobs: unit-tests: - if: github.actor != 'copybara-service[bot]' runs-on: ubuntu-latest strategy: