diff --git a/api/cases/admin.py b/api/cases/admin.py index 4c40641..cbb3c12 100644 --- a/api/cases/admin.py +++ b/api/cases/admin.py @@ -1,5 +1,7 @@ from django.contrib import admin -from .models import Case, CaseDetails, CaseMatch, CasePhoto +from .models import Case, CaseContact, CaseDetails, CaseMatch, CasePhoto, PhotoEncoding -admin.site.register((Case, CaseDetails, CaseMatch, CasePhoto)) +admin.site.register( + (Case, CaseDetails, CaseMatch, CasePhoto, PhotoEncoding, CaseContact) +) diff --git a/api/cases/migrations/0005_photoencoding.py b/api/cases/migrations/0005_photoencoding.py new file mode 100644 index 0000000..396138a --- /dev/null +++ b/api/cases/migrations/0005_photoencoding.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.13 on 2022-06-04 15:32 + +import django.contrib.postgres.fields +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cases', '0004_casecontact'), + ] + + operations = [ + migrations.CreateModel( + name='PhotoEncoding', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('values', django.contrib.postgres.fields.ArrayField(base_field=models.FloatField(), size=None)), + ('photo', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='encoding', to='cases.casephoto')), + ], + ), + ] diff --git a/api/cases/models.py b/api/cases/models.py index cd1a5ca..fb2e97f 100644 --- a/api/cases/models.py +++ b/api/cases/models.py @@ -1,3 +1,4 @@ +from django.contrib.postgres.fields import ArrayField from django.core.validators import MaxValueValidator, MinValueValidator from django.db import models from django.utils import timezone @@ -111,6 +112,13 @@ class CasePhoto(models.Model): case = models.ForeignKey(Case, on_delete=models.CASCADE, related_name="photos") +class PhotoEncoding(models.Model): + photo = models.OneToOneField( + CasePhoto, on_delete=models.CASCADE, related_name="encoding" + ) + values = ArrayField(models.FloatField()) + + class CaseContact(models.Model): case = models.ForeignKey(Case, on_delete=models.CASCADE, related_name="contacts") user = models.ForeignKey(User, on_delete=models.CASCADE, related_name="contacts") diff --git a/api/cases/services.py b/api/cases/services.py index 819a056..071a691 100644 --- a/api/cases/services.py +++ b/api/cases/services.py @@ -1,6 +1,8 @@ from datetime import date +from pathlib import Path from typing import Dict, List, Optional +from django.conf import settings from django.core.exceptions import PermissionDenied from django.db import transaction from django.utils import timezone @@ -9,18 +11,23 @@ from firebase_admin.messaging import Notification as FirebaseNotification from rest_framework.exceptions import ValidationError +from api.cases.tasks import activate_case from api.common.utils import get_object from api.files.models import File +from api.integrations.ai.model import AIModel from api.locations.models import Location from api.locations.services import create_location from api.notifications.models import Notification from api.notifications.services import create_notification from api.users.models import User -from .models import Case, CaseContact, CaseDetails, CaseMatch, CasePhoto +from .models import Case, CaseContact, CaseDetails, CaseMatch, CasePhoto, PhotoEncoding + +# from pathlib import Path Gender = CaseDetails.Gender CaseType = Case.Types +ml_model = None def create_case_photo(*, case: Case, file: File) -> CasePhoto: @@ -64,10 +71,19 @@ def create_case( create_case_details(case=case, **details) - # TODO Factor out to an async function - activate_case(case) + create_notification( + case=case, + action=Notification.Action.DETAILS, + title="تم رفع الحاله بنجاح", + body="جارى البحث عن المفقود وسنقوم بإشعارك فى حاله العثور لأى نتائج", + level=Notification.Level.INFO, + sent_to=case.user, + ) + case.save() + activate_case.delay(case.id) + return case @@ -114,18 +130,100 @@ def create_case_match(*, missing: Case, found: Case, score: int) -> CaseMatch: return case_match -def process_case(case: Case) -> List[Dict[int, int]]: +def create_photo_encoding(*, photo: CasePhoto, values: List[float]): + photo_encoding = PhotoEncoding(photo=photo, values=values) + photo_encoding.full_clean() + photo_encoding.save() + + return photo_encoding + + +def process_case(case: Case) -> Dict[int, float]: """ - Send case id and photos to the machine learing model then - recives list of ids & scores that matched with the case + Send case photos to the machine learning model to find it's matches """ - return [] + # Instantiate the model + global ml_model + if ml_model is None: + all_encodings = PhotoEncoding.objects.all() + + encodings_data = () + encodings_labels = () + + for photo_encoding in all_encodings: + encodings_data, encodings_labels = zip( + *[ + (photo_encoding.values, photo_encoding.photo.case.id) + for photo_encoding in all_encodings + ] + ) + # data = np.load(settings.APPS_DIR / "integrations/ai/data.npz") + # encodings_data = data["arr_0"] + # encodings_labels = data["arr_1"] + + ml_model = AIModel( + facenet_path=settings.APPS_DIR / "integrations/ai/facenet_keras.h5", + knn_path=settings.APPS_DIR / "integrations/ai/knn_new.clf", + data=encodings_data, + labels=encodings_labels, + ) -def case_matching_binding(*, case: Case, matches_list: List[Dict[int, int]]) -> None: - """ """ - if not matches_list: - # TODO refactor notifications + matches = {} + new_photos_encodings = [] + valid_photos = 0 + # Fetch all case photos + photos = case.photos.all() + + # Test each photo in the case against our ML model + for photo in photos: + # Extract face encoding + encoding = ml_model.encode_photo(photo.file.url) + if not encoding: + continue + valid_photos += 1 + # Record encoding to the database + photo_encoding = create_photo_encoding(photo=photo, values=encoding) + # Temporary storing new encoding to train the model at the end + new_photos_encodings.append(photo_encoding) + # Run the model against our new encoding to find case matches + case_ids = ml_model.check_face_identity(encoding) + + # Record matches and their scores + for case_id in case_ids: + # Fetch case + match = get_object(Case, pk=case_id) + + # Safety check if case still exists or not + if match is None: + continue + + matches[match] = matches.get(match, 0) + case_ids[case_id] + + # Checks all photos are invalid + if not valid_photos: + return {} + + # Normalizing matches scores + for match in matches: + matches[match] = matches[match] / valid_photos + + # Add new case photo encodings to the model training data + new_case_encodings_data = [ + photo_encoding.values for photo_encoding in new_photos_encodings + ] + + # Retrain the model on the new data + ml_model.retrain_model(new_case_encodings_data, case.id, Path("api/common")) + + return matches + + +def case_matching_binding(*, case: Case, matches: Dict[int, int]) -> None: + """ + Bind the processed case with it's matches by instantiating CaseMatch objects + """ + if not matches: create_notification( case=case, action=Notification.Action.PUBLISH, @@ -146,13 +244,9 @@ def case_matching_binding(*, case: Case, matches_list: List[Dict[int, int]]) -> return - cases_ids = [match["id"] for match in matches_list] - cases_scores = [match["score"] for match in matches_list] - matches: List[Case] = Case.objects.filter(id__in=cases_ids) - missing = True if case.type == CaseType.MISSING else False - for match, score in zip(matches, cases_scores): + for match, score in matches.items(): if missing: create_case_match(missing=case, found=match, score=score) else: @@ -195,32 +289,6 @@ def case_matching_binding(*, case: Case, matches_list: List[Dict[int, int]]) -> device.send_message(msg) -@transaction.atomic -def activate_case(case: Case): - matches = process_case(case) - case_matching_binding(case=case, matches_list=matches) - case.activate() - # TODO success or failure notification - create_notification( - case=case, - action=Notification.Action.DETAILS, - title="تم رفع الحاله بنجاح", - body="جارى البحث عن المفقود وسنقوم بإشعارك فى حاله العثور لأى نتائج", - level=Notification.Level.INFO, - sent_to=case.user, - ) - - msg = Message( - notification=FirebaseNotification( - title="تم رفع الحاله بنجاح", - body="جارى البحث عن المفقود وسنقوم بإشعارك فى حاله العثور لأى نتائج", - ) - ) - - device = FCMDevice.objects.filter(user=case.user).first() - device.send_message(msg) - - # TODO refactor object permission on view level to some mixin or permission class def publish_case(*, case: Case, performed_by: User): if case.user != performed_by: @@ -231,7 +299,6 @@ def publish_case(*, case: Case, performed_by: User): if case.posted_at: raise ValidationError("Case already published") - case.publish() case.save() diff --git a/api/cases/tasks.py b/api/cases/tasks.py new file mode 100644 index 0000000..d47b250 --- /dev/null +++ b/api/cases/tasks.py @@ -0,0 +1,14 @@ +from api.cases.models import Case +from api.common.utils import get_object +from config import celery_app + + +@celery_app.task +def activate_case(case_id: int): + from api.cases.services import case_matching_binding, process_case + + case = get_object(Case, pk=case_id) + matches = process_case(case) + case_matching_binding(case=case, matches=matches) + case.activate() + case.save() diff --git a/api/integrations/ai/data.npz b/api/integrations/ai/data.npz new file mode 100644 index 0000000..449d6a1 Binary files /dev/null and b/api/integrations/ai/data.npz differ diff --git a/api/integrations/ai/facenet_keras.h5 b/api/integrations/ai/facenet_keras.h5 new file mode 100644 index 0000000..b181ac0 Binary files /dev/null and b/api/integrations/ai/facenet_keras.h5 differ diff --git a/api/integrations/ai/knn_new.clf b/api/integrations/ai/knn_new.clf new file mode 100644 index 0000000..e61c9dd Binary files /dev/null and b/api/integrations/ai/knn_new.clf differ diff --git a/api/integrations/ai/model.py b/api/integrations/ai/model.py new file mode 100644 index 0000000..3480883 --- /dev/null +++ b/api/integrations/ai/model.py @@ -0,0 +1,236 @@ +import pickle +from pathlib import Path +from typing import Dict, Optional +from urllib.request import Request, urlopen + +import cv2 +import joblib +import numpy as np +from deepface.detectors.FaceDetector import alignment_procedure +from mtcnn import MTCNN +from sklearn import neighbors +from sklearn.metrics import ( + accuracy_score, + classification_report, + f1_score, + precision_recall_fscore_support, + precision_score, + recall_score, +) +from sklearn.preprocessing import Normalizer +from tensorflow.keras.models import load_model + +# from time import time + + +class AIModel: + def __init__( + self, facenet_path: Path, knn_path: Path, data: np.ndarray, labels: np.ndarray + ) -> None: + self.knn_path = knn_path + self.detector = MTCNN() + self.facenet = load_model(facenet_path) + self.knn = joblib.load(open(knn_path, "rb")) + self.data = data + self.labels = labels + + def detect_align_face(self, img: np.ndarray) -> np.ndarray: + """Detect face with 4 landmarks and align + + Args: + img (np.ndarray): image array + + Returns: + numpy.ndarray, list: image array, list of face landmarks and bounding box data + """ + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + detections = self.detector.detect_faces(img) + if len(detections) == 0 or len(detections) > 1: + return None + + # Crop and align the face + x, y, w, h = detections[0]["box"] + x1, y1 = int(x), int(y) + x2, y2 = int(x + w), int(y + h) + face = img[y1:y2, x1:x2] + keypoints = detections[0]["keypoints"] + face_align = alignment_procedure( + face, keypoints["left_eye"], keypoints["right_eye"] + ) + + # resize the face and convert to numpy array + face_align = cv2.resize(face_align, (160, 160)) + face_array = np.asarray(face_align) + + return face_array + + def feature_encoding(self, img: np.ndarray) -> np.ndarray: + """Extract 128 numerical feature from face array + + Args: + img (numpy.ndarray): face array + + Returns: + numpy.ndarray: array of 128 numerica feature + """ + img = img.astype("float32") + mean, std = img.mean(), img.std() + img = (img - mean) / std + samples = np.expand_dims(img, axis=0) + yhat = self.facenet.predict(samples) + + return yhat[0] + + def encode_photo(self, url: str) -> np.ndarray: + """Extract 128 numerical feature from face array + + Args: + url (str): url to image + + Returns: + numpy.ndarray: array of 128 numerical feature + """ + req = Request(url, headers={"User-Agent": "XYZ/3.0"}) + data = urlopen(req) + arr = np.asarray(bytearray(data.read()), dtype=np.uint8) + img = cv2.imdecode(arr, -1) + face = self.detect_align_face(img) + if face is None: + return None + + embedding = self.feature_encoding(face).reshape(1, -1) + + return embedding + + def normalize_images(self, *, kind: Optional[str] = "l2") -> np.ndarray: + """Normalize training data + + Args: + kind (str): type of normalization. Defaults to "l2". + Returns: + numpy.ndarray: X normalized + """ + in_encoder = Normalizer(norm=kind) + X = in_encoder.transform(self.data) + return X + + def KNN_Classifier(self, n_neighbors: Optional[int] = 3): + """KNN classifier to classify faces + + Args: + n_neighbors (int, optional):number of nearest neighbors Defaults to 1. + + Returns: + sklearn_model: KNN model + """ + X = self.normalize_images() + knn_clf = neighbors.KNeighborsClassifier( + n_neighbors=n_neighbors, algorithm="ball_tree", weights="distance" + ) + knn_clf.fit(X, self.labels) + return knn_clf + + def evaluate(self, X_test, y_test): + """Evaluate model + Args: + model (tensorflow.python.keras.engine.functional.Functional): acenet keras model + X_test (numpy.ndarray): array of all faces for each person + y_test (numpy.ndarray): array of labels for each face + + Returns: + accuracy (float): model accuracy + precision (float): model precision + recall (float): model recall + f1 (float): model f1_score""" + X_test = self.normalize_images(X_test) + y_pred = self.knn.predict(X_test) + accuracy = accuracy_score(y_test, y_pred) + precision = precision_score(y_test, y_pred, average="weighted", zero_division=0) + recall = recall_score(y_test, y_pred, average="weighted", zero_division=0) + f1 = f1_score(y_test, y_pred, average="weighted", zero_division=0) + print("as") + print("Accuracy : %.3f" % accuracy) + print("Precision : %.3f" % precision) + print("Recall : %.3f" % recall) + print("F1-Score : %.3f" % f1) + print( + "\nPrecision Recall F1-Score Support Per Class : \n", + precision_recall_fscore_support( + y_test, y_pred, average="weighted", zero_division=0 + ), + ) + print("\nClassification Report : ") + print(classification_report(y_test, y_pred, zero_division=0)) + return accuracy, precision, recall, f1 + + def dump_model(self): + """Dump model to file""" + if self.knn_path is not None: + with open(self.knn_path, "wb") as f: + pickle.dump(self.knn, f) + + def check_face_identity( + self, + encodings: np.ndarray, + *, + threshold: Optional[int] = 15, + n_neighbors: Optional[int] = 9 + ) -> Dict[int, int]: + """Check face identity + + Args: + encodings (numpy.darray): face encodings + threshold (int, optional): threshold for face identification Defaults to 0.5. + + Returns: + identity (Dict[int, int]): return ids with number of images. + """ + closest_distances = self.knn.kneighbors( + encodings.reshape(1, -1), n_neighbors=n_neighbors + ) + ids = {} + distances = closest_distances[0][0] + indices = closest_distances[1][0] + for idx, distance in zip(indices, distances): + print(idx, self.labels[idx], distance) + if distance <= threshold: + if not ids.__contains__(self.labels[idx]): + ids[self.labels[idx]] = (1, distance) + else: + num_photo = ids[self.labels[idx]][0] + 1 + dist = ids[self.labels[idx]][1] + distance + ids[self.labels[idx]] = (num_photo, dist) + + if len(ids) == 0: + return {-1: 0} + else: + ids_distances = [[], [], []] + + for id, val in ids.items(): + ids_distances[val[0] - 1].append((id, val[1] / val[0])) + + for id_distance in ids_distances: + id_distance.sort(key=lambda i: i[1]) + + ids_confidence = {} + + for num_photos in range(0, len(ids_distances)): + for rank in range(0, len(ids_distances[num_photos])): + eqn = (num_photos + 1) / 3.0 - (rank + 1) * 0.03 + ids_confidence[ids_distances[num_photos][rank][0]] = eqn + + return ids_confidence + + def retrain_model(self, new_encodings: np.ndarray, identity: int): + """Retrain model on new images + + Args: + new_encodings (numpy.darray): new face encodings + identity (int): identity of the face + """ + for new_encoding in new_encodings: + self.data = np.vstack([self.data, new_encoding]) + self.labels = np.append(self.labels, identity) + + self.knn = self.KNN_Classifier() + self.dump_model() diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 5fed6c5..4bbeca0 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -18,6 +18,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ # Requirements are installed here to ensure they will be cached. COPY ./requirements . +# Upgrade pip +RUN python -m pip install --upgrade pip + # Create Python Dependency and Sub-Dependency Wheels. RUN pip wheel --wheel-dir /usr/src/app/wheels \ -r ${BUILD_ENVIRONMENT}.txt @@ -41,6 +44,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ libpq-dev \ # Translations dependencies gettext \ + ffmpeg libsm6 libxext6 \ # cleaning up unused files && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ && rm -rf /var/lib/apt/lists/* diff --git a/requirements/base.txt b/requirements/base.txt index 5981ca8..86efad1 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -29,3 +29,12 @@ django-fsm==2.8.0 # https://github.com/viewflow/django-fsm boto3==1.21.45 # https://boto3.amazonaws.com/v1/documentation/api/latest/index.html django-filter==21.1 # https://github.com/carltongibson/django-filter fcm-django==1.0.11 # https://pypi.org/project/fcm-django/1.0.11/ +# Machine Learning Model +opencv-python==4.5.5.62 # https://github.com/opencv/opencv +deepface==0.0.73 # https://pypi.org/project/deepface/ +mtcnn==0.1.1 # https://github.com/ipazc/mtcnn +scikit-learn==0.24.2 #https://pypi.org/project/scikit-learn/ +# tensorflow==2.3.0 # https://pypi.org/project/tensorflow/ +# keras==2.4.0 # https://pypi.org/project/keras/ +joblib==1.1.0 # https://joblib.readthedocs.io/en/latest/installing.html +keras-facenet==0.3.2