feature_01_testing

.gitignore

@@ -0,0 +1,6 @@
+__pycache__/
+metastore_db/
+.coverage
+.DS_Store
+gurobi.log
+derby.log

LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2021, Diptesh
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -37,7 +37,7 @@ It's good practice to follow accepted standards while coding in python:
 
 Also, it's a good idea to rate all our python scripts with [Pylint](https://www.pylint.org/). If we score anything less than 8/10, we should consider redesigning the code architecture.
 
-A composite pylint ratings for all the codes are automatically computed when we [run the tests](./bin/runTests.sh) and prepended on top of this file.
+A composite pylint ratings for all the codes are automatically computed when we [run the tests](./bin/run_tests.sh) and prepended on top of this file.
 
 #### Version control
 

bin/programs.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# =============================================================================
+# Python dependencies
+#
+# Objective: Install python dependencies from requirements.txt
+#
+# Version: 0.1.0
+#
+# Author: Diptesh
+#
+# Date: Mar 03, 2020
+#
+# =============================================================================
+
+# Set test directory
+path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)\
+/$(basename "${BASH_SOURCE[0]}")"
+
+proj_dir=$(sed -E 's/(.+)(\/bin\/.+)/\1/' <<< $path)
+
+if ! [[ $(hostname) =~ .+target\.com ]]; then
+    pip install -r $proj_dir/requirements.txt
+fi

bin/run_tests.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# =============================================================================
+# Python unit tests and code ratings
+#
+# Objective: Test all python test modules and rate all python scripts
+#
+# Version: 0.2.0
+#
+# Author: Diptesh
+#
+# Date: Jun 24, 2020
+#
+# =============================================================================
+
+# Set test directory
+path="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)\
+/$(basename "${BASH_SOURCE[0]}")"
+
+test_dir=$(sed -E 's/(.+)(\/bin\/.+)/\1\/tests/' <<< $path)
+proj_dir=$(sed -E 's/(.+)(\/bin\/.+)/\1/' <<< $path)
+
+if [[ -z $1 ]]
+then
+	module="-a"
+else
+	module=$1
+fi
+
+printf "=%.0s" {1..70}
+
+# Run unit tests
+if [[ $module == "-a" || $module == "-u" ]]
+then
+	printf "\nRunning unit & integration tests...\n\n"
+	coverage run -m unittest discover -v -s $test_dir -p "test_*.py"
+	coverage report -m --omit="*/tests/test_*,*/opt/spark-*" > "$proj_dir/log/cov.out"
+	COV_SCORE=`grep "TOTAL" $proj_dir/log/cov.out | tail -1 | awk '{ printf("%d", $4) }'`
+	COV_COLOR="red"
+	if [[ $COV_SCORE == "100" ]]
+	then
+		COV_COLOR="dagreen"
+	fi
+	sed -i "2s/.*/\[\!\[Coverage score\]\(\https\:\/\/img\.shields\.io\/badge\/coverage\-$COV_SCORE\%25\-$COV_COLOR.svg\)\]\(\.\/log\/cov\.out\)/" "$proj_dir/README.md"
+	printf "=%.0s" {1..70}
+	printf "\n"
+fi
+
+# Rate coding styles for all python scripts
+if [[ $module == "-a" || $module == "-r" ]]
+then
+    printf "\nRating code style...\n\n"
+    score=0
+    cnt=0
+    for i in $(find "$proj_dir" -name "*.py")
+    do
+        file=${i#$(dirname "$(dirname "$i")")/}
+        printf "%-67s %s" "$file"
+        file_dir=$(sed -E 's/(.+\/)(.+\.py)/\1/' <<< $i)
+        cd "$file_dir"
+        pylint "$i" > "$proj_dir/log/pylint/pylint.out"
+        PYLINT_SCORE=`grep "Your code has been rated" $proj_dir/log/pylint/pylint.out | cut -d" " -f7 | cut -d"." -f1`
+        file_name=$(sed -E 's/(\/)/-/' <<< $file)
+        file_name=$(sed -E 's/(\.)/-/' <<< $file_name)
+        cp "$proj_dir/log/pylint/pylint.out" "$proj_dir/log/pylint/$file_name.out"
+        score=$((score + PYLINT_SCORE))
+        cnt=$((cnt + 1))
+        printf "$PYLINT_SCORE\n"
+        cd "$proj_dir"
+    done
+    tot_score=$(echo "scale=1; $score/$cnt" | bc)
+    printf "\nTotal score: $tot_score\n"
+    # Add pylint badge to README.md
+    sed -i "1s/.*/\[\!\[pylint Score\]\(https\:\/\/mperlet\.github\.io\/pybadge\/badges\/$tot_score.svg\)\]\(\.\/log\/pylint\/\)/" "$proj_dir/README.md"
+    printf "=%.0s" {1..70}
+    printf "\n"
+fi
+
+pipreqs --force $proj_dir &> $proj_dir/log/pip.out
+
+exit 0

data/input/iris.csv

@@ -0,0 +1,151 @@
+x3,x4,x1,x2,y
+5.1,3.5,1.4,0.2,0
+4.9,3.0,1.4,0.2,0
+4.7,3.2,1.3,0.2,0
+4.6,3.1,1.5,0.2,0
+5.0,3.6,1.4,0.2,0
+5.4,3.9,1.7,0.4,0
+4.6,3.4,1.4,0.3,0
+5.0,3.4,1.5,0.2,0
+4.4,2.9,1.4,0.2,0
+4.9,3.1,1.5,0.1,0
+5.4,3.7,1.5,0.2,0
+4.8,3.4,1.6,0.2,0
+4.8,3.0,1.4,0.1,0
+4.3,3.0,1.1,0.1,0
+5.8,4.0,1.2,0.2,0
+5.7,4.4,1.5,0.4,0
+5.4,3.9,1.3,0.4,0
+5.1,3.5,1.4,0.3,0
+5.7,3.8,1.7,0.3,0
+5.1,3.8,1.5,0.3,0
+5.4,3.4,1.7,0.2,0
+5.1,3.7,1.5,0.4,0
+4.6,3.6,1.0,0.2,0
+5.1,3.3,1.7,0.5,0
+4.8,3.4,1.9,0.2,0
+5.0,3.0,1.6,0.2,0
+5.0,3.4,1.6,0.4,0
+5.2,3.5,1.5,0.2,0
+5.2,3.4,1.4,0.2,0
+4.7,3.2,1.6,0.2,0
+4.8,3.1,1.6,0.2,0
+5.4,3.4,1.5,0.4,0
+5.2,4.1,1.5,0.1,0
+5.5,4.2,1.4,0.2,0
+4.9,3.1,1.5,0.2,0
+5.0,3.2,1.2,0.2,0
+5.5,3.5,1.3,0.2,0
+4.9,3.6,1.4,0.1,0
+4.4,3.0,1.3,0.2,0
+5.1,3.4,1.5,0.2,0
+5.0,3.5,1.3,0.3,0
+4.5,2.3,1.3,0.3,0
+4.4,3.2,1.3,0.2,0
+5.0,3.5,1.6,0.6,0
+5.1,3.8,1.9,0.4,0
+4.8,3.0,1.4,0.3,0
+5.1,3.8,1.6,0.2,0
+4.6,3.2,1.4,0.2,0
+5.3,3.7,1.5,0.2,0
+5.0,3.3,1.4,0.2,0
+7.0,3.2,4.7,1.4,1
+6.4,3.2,4.5,1.5,1
+6.9,3.1,4.9,1.5,1
+5.5,2.3,4.0,1.3,1
+6.5,2.8,4.6,1.5,1
+5.7,2.8,4.5,1.3,1
+6.3,3.3,4.7,1.6,1
+4.9,2.4,3.3,1.0,1
+6.6,2.9,4.6,1.3,1
+5.2,2.7,3.9,1.4,1
+5.0,2.0,3.5,1.0,1
+5.9,3.0,4.2,1.5,1
+6.0,2.2,4.0,1.0,1
+6.1,2.9,4.7,1.4,1
+5.6,2.9,3.6,1.3,1
+6.7,3.1,4.4,1.4,1
+5.6,3.0,4.5,1.5,1
+5.8,2.7,4.1,1.0,1
+6.2,2.2,4.5,1.5,1
+5.6,2.5,3.9,1.1,1
+5.9,3.2,4.8,1.8,1
+6.1,2.8,4.0,1.3,1
+6.3,2.5,4.9,1.5,1
+6.1,2.8,4.7,1.2,1
+6.4,2.9,4.3,1.3,1
+6.6,3.0,4.4,1.4,1
+6.8,2.8,4.8,1.4,1
+6.7,3.0,5.0,1.7,1
+6.0,2.9,4.5,1.5,1
+5.7,2.6,3.5,1.0,1
+5.5,2.4,3.8,1.1,1
+5.5,2.4,3.7,1.0,1
+5.8,2.7,3.9,1.2,1
+6.0,2.7,5.1,1.6,1
+5.4,3.0,4.5,1.5,1
+6.0,3.4,4.5,1.6,1
+6.7,3.1,4.7,1.5,1
+6.3,2.3,4.4,1.3,1
+5.6,3.0,4.1,1.3,1
+5.5,2.5,4.0,1.3,1
+5.5,2.6,4.4,1.2,1
+6.1,3.0,4.6,1.4,1
+5.8,2.6,4.0,1.2,1
+5.0,2.3,3.3,1.0,1
+5.6,2.7,4.2,1.3,1
+5.7,3.0,4.2,1.2,1
+5.7,2.9,4.2,1.3,1
+6.2,2.9,4.3,1.3,1
+5.1,2.5,3.0,1.1,1
+5.7,2.8,4.1,1.3,1
+6.3,3.3,6.0,2.5,2
+5.8,2.7,5.1,1.9,2
+7.1,3.0,5.9,2.1,2
+6.3,2.9,5.6,1.8,2
+6.5,3.0,5.8,2.2,2
+7.6,3.0,6.6,2.1,2
+4.9,2.5,4.5,1.7,2
+7.3,2.9,6.3,1.8,2
+6.7,2.5,5.8,1.8,2
+7.2,3.6,6.1,2.5,2
+6.5,3.2,5.1,2.0,2
+6.4,2.7,5.3,1.9,2
+6.8,3.0,5.5,2.1,2
+5.7,2.5,5.0,2.0,2
+5.8,2.8,5.1,2.4,2
+6.4,3.2,5.3,2.3,2
+6.5,3.0,5.5,1.8,2
+7.7,3.8,6.7,2.2,2
+7.7,2.6,6.9,2.3,2
+6.0,2.2,5.0,1.5,2
+6.9,3.2,5.7,2.3,2
+5.6,2.8,4.9,2.0,2
+7.7,2.8,6.7,2.0,2
+6.3,2.7,4.9,1.8,2
+6.7,3.3,5.7,2.1,2
+7.2,3.2,6.0,1.8,2
+6.2,2.8,4.8,1.8,2
+6.1,3.0,4.9,1.8,2
+6.4,2.8,5.6,2.1,2
+7.2,3.0,5.8,1.6,2
+7.4,2.8,6.1,1.9,2
+7.9,3.8,6.4,2.0,2
+6.4,2.8,5.6,2.2,2
+6.3,2.8,5.1,1.5,2
+6.1,2.6,5.6,1.4,2
+7.7,3.0,6.1,2.3,2
+6.3,3.4,5.6,2.4,2
+6.4,3.1,5.5,1.8,2
+6.0,3.0,4.8,1.8,2
+6.9,3.1,5.4,2.1,2
+6.7,3.1,5.6,2.4,2
+6.9,3.1,5.1,2.3,2
+5.8,2.7,5.1,1.9,2
+6.8,3.2,5.9,2.3,2
+6.7,3.3,5.7,2.5,2
+6.7,3.0,5.2,2.3,2
+6.3,2.5,5.0,1.9,2
+6.5,3.0,5.2,2.0,2
+6.2,3.4,5.4,2.3,2
+5.9,3.0,5.1,1.8,2