From b6179462d6e0bf2e3639022b8f6fabee8cd3e68b Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Fri, 16 Jul 2021 18:32:33 +0530 Subject: [PATCH 1/9] synthetic data module --- source/pic2card/.pylintrc | 2 +- .../commands/generate_synthetic_dataset.py | 157 +++++++++++++++ .../synthetic/card_elements/actionset/1.png | Bin 0 -> 2189 bytes source/pic2card/datagen/__init__.py | 0 .../pic2card/datagen/generate_annotations.py | 164 ++++++++++++++++ .../datagen/generate_synthetic_image.py | 184 ++++++++++++++++++ source/pic2card/datagen/utils.py | 81 ++++++++ source/pic2card/mystique/config.py | 24 +++ 8 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 source/pic2card/commands/generate_synthetic_dataset.py create mode 100644 source/pic2card/data/synthetic/card_elements/actionset/1.png create mode 100644 source/pic2card/datagen/__init__.py create mode 100644 source/pic2card/datagen/generate_annotations.py create mode 100644 source/pic2card/datagen/generate_synthetic_image.py create mode 100644 source/pic2card/datagen/utils.py diff --git a/source/pic2card/.pylintrc b/source/pic2card/.pylintrc index 794554bba6..058dc27a5d 100644 --- a/source/pic2card/.pylintrc +++ b/source/pic2card/.pylintrc @@ -9,7 +9,7 @@ extension-pkg-allow-list= # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. (This is an alternative name to extension-pkg-allow-list # for backward compatibility.) -extension-pkg-whitelist=cv2 +extension-pkg-whitelist=cv2,lxml # Specify a score threshold to be exceeded before program exits with error. fail-under=10.0 diff --git a/source/pic2card/commands/generate_synthetic_dataset.py b/source/pic2card/commands/generate_synthetic_dataset.py new file mode 100644 index 0000000000..6a77ee8137 --- /dev/null +++ b/source/pic2card/commands/generate_synthetic_dataset.py @@ -0,0 +1,157 @@ +#!/usr/bin/python + +# pip install lxml +""" +Command to generate adaptive cards as training image dataset in bulk. + +USAGE : + +Arguments : card_elements_dir_path , number_of_elements , bg_color, bulk_img \ + save_as_zip + +command : python -m commands.generate_synthetic_dataset \ + --number_of_elements=number of elements desired \ + --card_elements_dir_path=/path/to/input/card_elements \ + --bg_color= any one of the list ["WHITE","GREY","CYAN","ROSE","GOLD"] \ + --bulk_img=number of images needs to be generated \ + --save_as_zip=True if you want to save files to zip + + python -m commands.generate_synthetic_dataset -n \ + -z true \ + -c -e + +""" +import argparse +from typing import Any +import cv2 +from mystique import config +from datagen import generate_synthetic_image as synthetic +from datagen.generate_annotations import get_annotation_file +from datagen.utils import ( + get_zip_paths, + zip_and_remove_file, + get_image_name, + get_outgoing_file_paths, +) + + +def save_img_and_annotations( + image_with_canvas: Any, annotation_xml: Any +) -> None: + """ + Function is used to save the generated images and annotation files + in the respective paths defined in config or given in args while using + command + """ + image_name = get_image_name() + # saving image and annotation xml + cv2.imwrite( + filename=f"{config.GENERATED_IMG_DIR}/{image_name}.png", + img=image_with_canvas, + ) + + annotation_xml.write( + f"{config.GENERATED_ANNOTATION_DIR}/{image_name}.xml", pretty_print=True + ) + synthetic.logger.debug( + "Saved image and annotation - %s.png and %s.xml in destination folder", + image_name, + image_name, + ) + + +def main( + card_elements_dir_path=None, + number_of_elements=None, + background_colour=None, + bulk_img: int = None, + save_as_zip: bool = None, +) -> None: + """ + Command generates cards with random elements from the + card_elements directory at bulk + + @param bulk_img: number of images needs to be generated + """ + generating_file_paths, generating_xml_paths = get_outgoing_file_paths( + bulk_img + ) + layout = synthetic.CardElements(number_of_elements, card_elements_dir_path) + padded_image_element = synthetic.add_padding_to_img_elements( + layout.elements_with_path + ) + generated_image = synthetic.generate_image(padded_image_element) + + for _ in range(bulk_img): + annotation_xml = get_annotation_file( + layout, generated_image, padded_image_element + ) + image_with_canvas = synthetic.add_background_colour_to_generated_image( + generated_image, background_colour + ) + save_img_and_annotations(image_with_canvas, annotation_xml) + if save_as_zip: + zip_path, annotation_zip_path = get_zip_paths() + zip_and_remove_file( + config.GENERATED_IMG_DIR, zip_path, generating_file_paths + ) + zip_and_remove_file( + config.GENERATED_ANNOTATION_DIR, + annotation_zip_path, + generating_xml_paths, + ) + + +def set_args() -> Any: + """ + Returns parser object for the command + @return: parser + """ + parser = argparse.ArgumentParser(description="Generate adaptive cards") + parser.add_argument( + "-p", + "--card_elements_dir_path", + default=config.ELEMENTS_DIR, + help="Enter Card elements directory path", + ) + parser.add_argument( + "-c", + "--bg_color", + default=config.BACKGROUND_COLOR, + help="Enter the background color desired for the generated image", + ) + parser.add_argument( + "-e", + "--number_of_elements", + type=int, + default=config.ELEMENT_COUNT_THRESHOLD, + help="Enter number of Card elements desired in the generated image", + ) + + parser.add_argument( + "-n", + "--bulk_img", + default=config.BULK_IMAGES_NEEDED, + type=int, + help="Number of images needed for bulk generation", + ) + parser.add_argument( + "-z", + "--save_as_zip", + default=False, + type=bool, + help="Set True to save generated images in zip format", + ) + return parser + + +if __name__ == "__main__": + parser_object = set_args() + args = parser_object.parse_args() + main( + card_elements_dir_path=args.card_elements_dir_path, + number_of_elements=args.number_of_elements, + background_colour=args.bg_color, + bulk_img=args.bulk_img, + save_as_zip=args.save_as_zip, + ) diff --git a/source/pic2card/data/synthetic/card_elements/actionset/1.png b/source/pic2card/data/synthetic/card_elements/actionset/1.png new file mode 100644 index 0000000000000000000000000000000000000000..f25eb57886970e2898132fd6581adcfe110bf403 GIT binary patch literal 2189 zcmb`Jc{tST7r?($SuUX(WJ|jkWKTq6$(AL|Vu{gIMwS{G8Vr?`L{W-jM6%CVvKwPa zwtEr6WV_kNGIM9fa;-z|^w;m7-+%YM&vV{$&U?;3=XuUKpZA%ggO!+woCp8_Vm8*7 zoB;r|&bJMP1^IVz@PSRf+J`fE zrz^cR)iIW9K!N)HZ2_)tNk6A`8E4;#(m)2aeaaeCEkov>G1fZ%2;b&OLbzB^g-b+< zt;FFYg=*wBDEZ#j&MEnfFwAV~Wa~0tFZyH%=9~5x#bgp^e z80FYSvYV8q0u%Ry@e`)CfeyrT+}G4K807)M#GCbE*)Vf?oY|Lm63zPX_%kOmJYZGP z-tC79SBkuh*|tJp6nU44epJrlvkB={S`Sbr2K)Tg5Tp{-HjHTHMq3pxnTo%-!jNJvP#t>FfF7!o44 zT6QoD`j-=nu_24cAfu{(hG&f!iP9ZsK0!d+y$3laasm>HF1TJb?}F>}6+bsE+b4+^ zX3{bWTRS@{{Tcfy3bK&Y)9;U1YZWQRlS)~&ay3VhI|M6w#Hpg#D%G=wA3syj0^R#r z0euA~GhZIoQxiK@IB5DE?zgrpJt~@$Gm@51XZ)dA#e@WZf1aCwWpQ|atF0x$n4~{K z-eqY`qhgfC;=eFsGYMu5{cW?YAU&>~0@v5Lu-kD;8CcL~q%}bjS9m)?)F0oaA5y_E zR_16`xqsJu6*MwR0@8~u&!TT1Q*jbrv^o<~)aK0#c)Pa0KWzjRNg5KwvM>_LD4A)ItsC?7` zZ$9ef40CcF`^rP~>zy#jE8MC&OfT2fxsRH+PK4;MlXVs*Kg>K4@FQzDW`zxZs~|

u|^l?e5>kd)^wfX+W?+@&%nOth&2(tD)8c(zHw@9(!#T;eXuiv zynSWpkaX=wc>BT;l!o((p7w#^F^%yT#>|^dVA79Q!fWZ2Fu(ASD0d{IH6J+9)K4Pp z<|yGmo@8!JjJx{_QqxK9v{0j=kWG0yjmS^;@$t8f^sT2I%BD4F5a9BexOWB9@mNU} zRb(eIccfg`cF9XC7_Ks@7S(mI@Qy@zc{84k*pC3xU1|V@ny{I@|37XphnhzLrghzfZEmv#se29K0sa z74bdV{@CwdOeai_{TWqxS9#Dw(L|aly4=WhC4HfWHYkboUW96H1O)3UJ~e7u`vFar$Q7CCFz>--Xmh}xXLSnq4U$EMMCVe1>s=5|>HAveCE!=MX; zmR?gUKX{u?@9mi{sU9|inB&kz6aLVRoJW*>mb;vYN?Ko;*_?f6II=S`aMz=X8{C1r z88rB_j4IR7lZlQj7HS3~d#w@#28P1T#!75OlvcMzI}gu)nlnh28H`V7x=c5g4m=fmfcp|6R+;$e4t56Zjs6V$U2sUSCoBRE&b&$BMo_4%%!`bZSXXy@hR6AKAO;xOgStp^(ZUr-?7IW@Y;sJzYo?2|v!w zS4FRS#wt$*7A~fCg$|8_SAIsLpKx3V0H=c;l*WhH1R-YEk List[List[tuple]]: + """ + Calculates the annotations for a given list of elements. + + @param elements_with_path: list of elements path from elements directory + @param elements_dimensions: list of elements dimensions + @param padded_image_element: list of image elements after padding + @return: annotations + """ + annotations = [] + padded_elements_height = [ + element.shape[0] for element in padded_image_element + ] + number_of_elements = len(elements_with_path) + for index in range(number_of_elements): + xmin = 10 + if index == 0: + ymin = 10 + else: + ymin = ( + sum([padded_elements_height[height] for height in range(index)]) + + 10 + ) + xmax = elements_dimensions[index][1] + 10 + ymax = ( + sum([padded_elements_height[height] for height in range(index + 1)]) + - 10 + ) + annotations.append([(xmin, ymin), (xmax, ymax)]) + return annotations + + +def generate_annotation_xml( + annotations: List[List[tuple]], + img_prop: List[Any], + element_types: List[str], +) -> Any: + """ + Returns an xml tree object with annotations for the generated image + @param annotations: List of coordinates of the image elements + @return tree + """ + root = et.Element("annotation") + et.SubElement(root, "folder").text = os.path.basename( + os.path.abspath(config.GENERATED_ANNOTATION_DIR) + ) + et.SubElement(root, "filename").text = f"{img_prop[-1]}.png" + et.SubElement(root, "path").text = ( + os.path.abspath(config.GENERATED_ANNOTATION_DIR) + + f"\\{img_prop[-1]}.xml" + ) + source = et.SubElement(root, "source") + et.SubElement(source, "database").text = "Unknown" + size = et.SubElement(root, "size") + et.SubElement(size, "height").text = str(img_prop[0][0]) + et.SubElement(size, "width").text = str(img_prop[0][1]) + et.SubElement(size, "depth").text = str(img_prop[0][2]) + et.SubElement(root, "segmented").text = "0" + for index, element_type in enumerate(element_types): + element = et.SubElement(root, "object") + et.SubElement(element, "name").text = os.path.basename(element_type) + et.SubElement(element, "pose").text = "Unspecified" + et.SubElement(element, "truncated").text = "0" + et.SubElement(element, "difficult").text = "0" + bndbox = et.SubElement(element, "bndbox") + et.SubElement(bndbox, "xmin").text = str(annotations[index][0][0]) + et.SubElement(bndbox, "ymin").text = str(annotations[index][0][1]) + et.SubElement(bndbox, "xmax").text = str(annotations[index][1][0]) + et.SubElement(bndbox, "ymax").text = str(annotations[index][1][1]) + + tree = et.ElementTree(root) + return tree + + +def run_annotator( + elements_with_path: List[str], + elements_dimensions: List[tuple], + padded_image_element: List[Sequence], +) -> List[List[tuple]]: + """ + Returns a list of x and y coords of the elements in the generated image + for both above and below threshold element numbers + @param elements_with_path: list of elements path from elements directory + @param elements_dimensions: list of elements dimensions + @param padded_image_element: list of image elements after padding + @return: annotations + """ + number_of_elements = len(elements_with_path) + if number_of_elements <= config.ELEMENT_COUNT_THRESHOLD: + annotations = calculate_annotation( + elements_with_path, elements_dimensions, padded_image_element + ) + else: + mid_value = number_of_elements // 2 + left_elements_with_path = elements_with_path[:mid_value] + left_elements_dimensions = elements_dimensions[:mid_value] + left_padded_image_element = padded_image_element[:mid_value] + + left_elements_annotations = calculate_annotation( + left_elements_with_path, + left_elements_dimensions, + left_padded_image_element, + ) + + right_elements_with_path = elements_with_path[mid_value:] + right_elements_dimensions = elements_dimensions[mid_value:] + right_padded_image_element = padded_image_element[mid_value:] + + right_elements_annotations = calculate_annotation( + right_elements_with_path, + right_elements_dimensions, + right_padded_image_element, + ) + + padded_pixels_for_right_element = ( + max([element[1] for element in elements_dimensions]) + 20 + ) + right_elements_annotations = np.array(right_elements_annotations) + right_elements_annotations = right_elements_annotations + [ + padded_pixels_for_right_element, + 0, + ] + right_elements_annotations = right_elements_annotations.tolist() + right_elements_annotations = [ + [(*coord,) for coord in element] + for element in right_elements_annotations + ] + annotations = left_elements_annotations + right_elements_annotations + return annotations + + +def get_annotation_file( + layout: Any, generated_image: Any, padded_image_element: List[Sequence] +) -> Any: + """Returns the annotation file in xml format""" + # properties of generated image + image_name = get_image_name() + element_type = [os.path.dirname(path) for path in layout.elements_with_path] + generated_image_prop = [np.shape(generated_image), image_name] + + # run annotations + annotations = run_annotator( + layout.elements_with_path, + layout.element_dimensions, + padded_image_element, + ) + # get annotation xml + annotation_xml = generate_annotation_xml( + annotations, generated_image_prop, element_type + ) + return annotation_xml diff --git a/source/pic2card/datagen/generate_synthetic_image.py b/source/pic2card/datagen/generate_synthetic_image.py new file mode 100644 index 0000000000..35961949fa --- /dev/null +++ b/source/pic2card/datagen/generate_synthetic_image.py @@ -0,0 +1,184 @@ +#!/usr/bin/python + +# pip install lxml +""" +Module contains functions needed for generating synthetic image dataset. +""" +import random +import os +import glob +import logging +from typing import List, Sequence, Any +import cv2 +import numpy as np +from mystique import config + + +logger = logging.getLogger("commands.generate_bulk_data") +LOGFORMAT = "%(asctime)s - [%(filename)s:%(lineno)s - %(funcName)20s()] - \ + %(levelname)s - %(message)s" +logging.basicConfig(level=logging.DEBUG, format=LOGFORMAT) + + +class CardElements: + """ + This class is responsible for the card elements properties + and its related functionalities + The card_elements directory default path is defined in the init + function and can also be passed as argument during runtime + which is not compulsory + """ + + def __init__(self, number_of_elements: int, elements_dir: str) -> None: + self.number_of_elements = number_of_elements + self.elements_dir = elements_dir + self.elements_with_path = self.get_elements_path() + self.element_dimensions = self.get_elements_dimensions( + self.elements_with_path + ) + + def get_elements_path(self) -> List[str]: + """ + Returns a list of complete path of card_elements selected at random + @param self: CardElements object + @return: elements_with_path + """ + elements = glob.glob(self.elements_dir + "/**/*.*", recursive=True) + elements_exist = [os.path.isfile(filepath) for filepath in elements] + if elements_exist: + elements_with_path = random.choices( + elements, k=self.number_of_elements + ) + else: + error_msg = "No image elements found under card_elements directory" + logger.error(error_msg) + raise Exception(error_msg) + return elements_with_path + + @staticmethod + def get_elements_dimensions(elements_with_path: List[str]) -> List[tuple]: + """ + Returns a list of dimensions for the selected elements + @param elements_with_path : list of selected element paths + @return : elements_dimensions + """ + elements_dimensions = [] + for element in elements_with_path: + element_img = cv2.imread(element) + dimension = element_img.shape + elements_dimensions.append(dimension) + return elements_dimensions + + +def add_padding_to_img_elements( + elements_with_path: List[str], +) -> List[Sequence]: + """ + Returns a list of elements in image format padded + along width of the image + @param elements_with_path: list of elements path from elements directory + @return: reshaped_image_elements + """ + + image_elements = [cv2.imread(element) for element in elements_with_path] + reference_canvas_width = max( + [element.shape[1] for element in image_elements] + ) + reshaped_image_elements = [] + for image_element in image_elements: + image_element_width = image_element.shape[1] + pixel_diff = reference_canvas_width - image_element_width + padded_image_element = cv2.copyMakeBorder( + image_element, + top=10, + bottom=10, + left=10, + right=pixel_diff + 10, + borderType=cv2.BORDER_CONSTANT, + value=config.CANVAS_COLOR["WHITE"], + ) + reshaped_image_elements.append(padded_image_element) + return reshaped_image_elements + + +def generate_image(reshaped_image_elements: List[Sequence]) -> List[Sequence]: + """ + Stacks the image elements along an axis and return a list of them + @param reshaped_image_elements: list of image elements after padding + @return: stacked image elements in one or two columns respectively + """ + number_of_elements = len(reshaped_image_elements) + # to stack elements vertically set number_of_elements less than threshold + if number_of_elements <= config.ELEMENT_COUNT_THRESHOLD: + stacked_image_elements = np.vstack(reshaped_image_elements) + else: + # stacks to form another column of elements in the generated image + left_elements = np.vstack( + reshaped_image_elements[: number_of_elements // 2] + ) + right_elements = np.vstack( + reshaped_image_elements[number_of_elements // 2 :] + ) + pixel_diff = abs(left_elements.shape[0] - right_elements.shape[0]) + + if left_elements.shape[0] < right_elements.shape[0]: + padded_image_element = cv2.copyMakeBorder( + left_elements, + top=0, + bottom=pixel_diff, + left=0, + right=0, + borderType=cv2.BORDER_CONSTANT, + value=config.CANVAS_COLOR["WHITE"], + ) + stacked_image_elements = np.hstack( + [padded_image_element, right_elements] + ) + else: + padded_image_element = cv2.copyMakeBorder( + right_elements, + top=0, + bottom=pixel_diff, + left=0, + right=0, + borderType=cv2.BORDER_CONSTANT, + value=config.CANVAS_COLOR["WHITE"], + ) + stacked_image_elements = np.hstack( + [left_elements, padded_image_element] + ) + + return stacked_image_elements + + +def add_background_colour_to_generated_image( + generated_image: Any, background_colour: str +) -> List[Sequence]: + """ + Returns an image with desired color added to background of the image + generated + + @ param generated_image: the image generated + @ param background_colour: the default or selected colour + @ return: overlayed_img + """ + height, width, channels = generated_image.shape + # creating a canvas with white background + canvas = np.ones((height, width, channels), np.uint8) * 255 + canvas[:] = config.CANVAS_COLOR[background_colour] + foreground_gray_image = cv2.cvtColor(generated_image, cv2.COLOR_RGB2GRAY) + mask = cv2.adaptiveThreshold( + src=foreground_gray_image, + maxValue=255, + adaptiveMethod=cv2.ADAPTIVE_THRESH_GAUSSIAN_C, + thresholdType=cv2.THRESH_BINARY, + blockSize=199, + C=5, + ) + mask_inv = cv2.bitwise_not(mask) + background = cv2.bitwise_and(canvas, canvas, mask=mask) + foreground = cv2.bitwise_and( + generated_image, generated_image, mask=mask_inv + ) + overlayed_img = cv2.add(foreground, background) + return overlayed_img diff --git a/source/pic2card/datagen/utils.py b/source/pic2card/datagen/utils.py new file mode 100644 index 0000000000..0fea9fb559 --- /dev/null +++ b/source/pic2card/datagen/utils.py @@ -0,0 +1,81 @@ +"""Contains utilities for the datagen package""" + +import os +import time +import glob +import zipfile +import logging +from typing import Tuple, List +from mystique import config + +logger = logging.getLogger("commands.generate_bulk_data") + + +def get_image_name() -> int: + """ + Returns an image number for the file name to save + @return: new_img_name + """ + generated_images = glob.glob(config.GENERATED_IMG_DIR + "/*") + if generated_images: + latest_file = max(generated_images, key=os.path.getctime) + latest_filename = os.path.basename(latest_file) + new_img_name = int(latest_filename.split(".")[0]) + 1 + else: + new_img_name = int(1) + return new_img_name + + +def get_outgoing_file_paths(bulk_img: int) -> Tuple[List, List]: + """ + Returns list of image and annotations file names that will be generated + @param bulk_img: number of images needed for bulk generation + @retrun img_files, xml_files + """ + generating_latest_file = get_image_name() + generating_last_file = generating_latest_file + bulk_img + img_files = [ + f"{file}.png" + for file in range(generating_latest_file, generating_last_file) + ] + xml_files = [ + f"{file}.xml" + for file in range(generating_latest_file, generating_last_file) + ] + return img_files, xml_files + + +def get_zip_paths() -> Tuple[str, str]: + """ + Function is used to construct the path for the zip files + to be stored for generating synthetic images + @return : Tuple of the zip path and annotation file zip path + """ + if not os.path.exists(config.GENERATED_ZIP_DIR): + os.makedirs(config.GENERATED_ZIP_DIR) + creation_time = time.strftime("%Y%m%d_%H%M%S", time.localtime()) + zip_path = f"{config.GENERATED_ZIP_DIR}/generated_{creation_time}.zip" + annotation_zip_path = ( + f"{config.GENERATED_ZIP_DIR}/annotation_{creation_time}.zip" + ) + return zip_path, annotation_zip_path + + +def zip_and_remove_file(folder_path: str, zip_path: str, files: List) -> None: + """ + Saves a zip file of the generated images and annotations to folder + @param folder_path: the folder path needed to be zipped + @param zip_path: path where zipfile needs to be saved + @param files: list of files that will be generated + @retrun None + """ + with zipfile.ZipFile(zip_path, mode="a") as zipf: + len_dir_path = len(folder_path) + for root, _, _ in os.walk(folder_path): + for file in files: + file_path = os.path.join(root, file) + zipf.write(file_path, file_path[len_dir_path:]) + os.remove(file_path) + logger.debug( + "Saved %s and removed duplicate files", os.path.basename(zip_path) + ) diff --git a/source/pic2card/mystique/config.py b/source/pic2card/mystique/config.py index baef354739..b7d31bdb92 100644 --- a/source/pic2card/mystique/config.py +++ b/source/pic2card/mystique/config.py @@ -138,3 +138,27 @@ # Multi Process flag to run card-layout and properties extraction as a # parallel or sequential tasks, True by default MULTI_PROC = True + +# synthetic module config values +CANVAS_COLOR = { + "WHITE": [255, 255, 255], + "GREY": [211, 211, 211], + "CYAN": [238, 238, 175], + "ROSE": [255, 228, 255], + "GOLD": [0, 255, 255], +} +BACKGROUND_COLOR = "WHITE" +ELEMENT_COUNT_THRESHOLD = 5 +BULK_IMAGES_NEEDED = 10 +ELEMENTS_DIR = os.path.join( + os.path.dirname(__file__), "../data/synthetic/card_elements/" +) +GENERATED_IMG_DIR = os.path.join( + os.path.dirname(__file__), "../data/synthetic/generated_dataset/" +) +GENERATED_ANNOTATION_DIR = os.path.join( + os.path.dirname(__file__), "../data/synthetic/generated_annotations/" +) +GENERATED_ZIP_DIR = os.path.join( + os.path.dirname(__file__), "../data/synthetic/generated_zipfiles/" +) From 4ce04a032334a5c415b23a72a914cc36a8e1e6eb Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Sat, 17 Jul 2021 01:40:06 +0530 Subject: [PATCH 2/9] Added a function to get synthetic image properties --- .../commands/generate_synthetic_dataset.py | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/source/pic2card/commands/generate_synthetic_dataset.py b/source/pic2card/commands/generate_synthetic_dataset.py index 6a77ee8137..17bad22986 100644 --- a/source/pic2card/commands/generate_synthetic_dataset.py +++ b/source/pic2card/commands/generate_synthetic_dataset.py @@ -22,7 +22,7 @@ """ import argparse -from typing import Any +from typing import Any, Sequence, List import cv2 from mystique import config from datagen import generate_synthetic_image as synthetic @@ -59,6 +59,25 @@ def save_img_and_annotations( image_name, ) +def get_synthetic_image_properties(number_of_elements:int, card_elements_dir_path:List)-> Sequence: + """ + To create the synthetic image from the card elements arguments given + @param number_of_elements + @param card_elements_dir_path + return generated_image + """ + layout = synthetic.CardElements(number_of_elements, card_elements_dir_path) + padded_image_element = synthetic.add_padding_to_img_elements( + layout.elements_with_path + ) + generated_image = synthetic.generate_image(padded_image_element) + synthetic_image_properties = { + 'layout': layout, + 'padded_image': padded_image_element, + 'generated_image': generated_image + } + return synthetic_image_properties + def main( card_elements_dir_path=None, @@ -76,18 +95,15 @@ def main( generating_file_paths, generating_xml_paths = get_outgoing_file_paths( bulk_img ) - layout = synthetic.CardElements(number_of_elements, card_elements_dir_path) - padded_image_element = synthetic.add_padding_to_img_elements( - layout.elements_with_path - ) - generated_image = synthetic.generate_image(padded_image_element) - for _ in range(bulk_img): + synthetic_image_property = get_synthetic_image_properties(number_of_elements, card_elements_dir_path) annotation_xml = get_annotation_file( - layout, generated_image, padded_image_element + synthetic_image_property['layout'], + synthetic_image_property['generated_image'], + synthetic_image_property['padded_image'] ) image_with_canvas = synthetic.add_background_colour_to_generated_image( - generated_image, background_colour + synthetic_image_property['generated_image'], background_colour ) save_img_and_annotations(image_with_canvas, annotation_xml) if save_as_zip: From 34ea5f3e65aba921bb2034ad8d76d08dc89ae4c4 Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Sat, 17 Jul 2021 01:43:32 +0530 Subject: [PATCH 3/9] applied pylint format --- .../commands/generate_synthetic_dataset.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/source/pic2card/commands/generate_synthetic_dataset.py b/source/pic2card/commands/generate_synthetic_dataset.py index 17bad22986..e38f5ce4d4 100644 --- a/source/pic2card/commands/generate_synthetic_dataset.py +++ b/source/pic2card/commands/generate_synthetic_dataset.py @@ -59,7 +59,10 @@ def save_img_and_annotations( image_name, ) -def get_synthetic_image_properties(number_of_elements:int, card_elements_dir_path:List)-> Sequence: + +def get_synthetic_image_properties( + number_of_elements: int, card_elements_dir_path: List +) -> Sequence: """ To create the synthetic image from the card elements arguments given @param number_of_elements @@ -72,10 +75,10 @@ def get_synthetic_image_properties(number_of_elements:int, card_elements_dir_pat ) generated_image = synthetic.generate_image(padded_image_element) synthetic_image_properties = { - 'layout': layout, - 'padded_image': padded_image_element, - 'generated_image': generated_image - } + "layout": layout, + "padded_image": padded_image_element, + "generated_image": generated_image, + } return synthetic_image_properties @@ -96,14 +99,16 @@ def main( bulk_img ) for _ in range(bulk_img): - synthetic_image_property = get_synthetic_image_properties(number_of_elements, card_elements_dir_path) + synthetic_image_property = get_synthetic_image_properties( + number_of_elements, card_elements_dir_path + ) annotation_xml = get_annotation_file( - synthetic_image_property['layout'], - synthetic_image_property['generated_image'], - synthetic_image_property['padded_image'] + synthetic_image_property["layout"], + synthetic_image_property["generated_image"], + synthetic_image_property["padded_image"], ) image_with_canvas = synthetic.add_background_colour_to_generated_image( - synthetic_image_property['generated_image'], background_colour + synthetic_image_property["generated_image"], background_colour ) save_img_and_annotations(image_with_canvas, annotation_xml) if save_as_zip: From 55c52fdd36f13540aaa0e0c6cc31362ef31ab8e0 Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Sat, 17 Jul 2021 02:35:30 +0530 Subject: [PATCH 4/9] disabled pylint errors on multiple files --- source/pic2card/app/resources.py | 2 ++ source/pic2card/commands/generate_bleu_score.py | 1 + source/pic2card/commands/inference_from_tf_serving.py | 2 ++ source/pic2card/datagen/generate_annotations.py | 2 ++ source/pic2card/mystique/metrics/cosine_similarity.py | 3 +++ source/pic2card/tests/test_sample_images.py | 2 +- 6 files changed, 11 insertions(+), 1 deletion(-) diff --git a/source/pic2card/app/resources.py b/source/pic2card/app/resources.py index d907320895..049dc9bf5f 100644 --- a/source/pic2card/app/resources.py +++ b/source/pic2card/app/resources.py @@ -1,4 +1,6 @@ """ resources for api """ +# pylint: disable=consider-using-with + import sys import os import io diff --git a/source/pic2card/commands/generate_bleu_score.py b/source/pic2card/commands/generate_bleu_score.py index 285338702c..5f2fc31f23 100644 --- a/source/pic2card/commands/generate_bleu_score.py +++ b/source/pic2card/commands/generate_bleu_score.py @@ -42,6 +42,7 @@ """ # pylint: disable=too-many-locals # pylint: disable=too-many-statements +# pylint: disable=consider-using-with import argparse import base64 diff --git a/source/pic2card/commands/inference_from_tf_serving.py b/source/pic2card/commands/inference_from_tf_serving.py index b56e175e8d..91d39017b7 100644 --- a/source/pic2card/commands/inference_from_tf_serving.py +++ b/source/pic2card/commands/inference_from_tf_serving.py @@ -4,6 +4,8 @@ We have loaded the saved model into the tf-serve """ # pylint: disable=no-value-for-parameter +# pylint: disable=consider-using-with + import os import base64 import click diff --git a/source/pic2card/datagen/generate_annotations.py b/source/pic2card/datagen/generate_annotations.py index 19b4f77207..cbd0c21eba 100644 --- a/source/pic2card/datagen/generate_annotations.py +++ b/source/pic2card/datagen/generate_annotations.py @@ -1,4 +1,6 @@ """module that consists of functions to create the annotation for datagen""" +# pylint: disable=import-error + import os from typing import List, Sequence, Any import numpy as np diff --git a/source/pic2card/mystique/metrics/cosine_similarity.py b/source/pic2card/mystique/metrics/cosine_similarity.py index b2de3071bf..d93e5c090c 100644 --- a/source/pic2card/mystique/metrics/cosine_similarity.py +++ b/source/pic2card/mystique/metrics/cosine_similarity.py @@ -1,6 +1,9 @@ """Module to Calculate Cosine similarity between the testing and generated card json """ +# pylint: disable=consider-using-with +# pylint: disable=import-error + import os import re import argparse diff --git a/source/pic2card/tests/test_sample_images.py b/source/pic2card/tests/test_sample_images.py index cecf5e8f5b..270fc4b596 100644 --- a/source/pic2card/tests/test_sample_images.py +++ b/source/pic2card/tests/test_sample_images.py @@ -32,6 +32,6 @@ def test_response(self): self.client, self.api, self.headers, self.data ) self.output = json.loads(self.response.data) - self.assertEqual(self.response.status_code, 200) + self.assertEqual(self.response.status_code, 200) # pylint: disable=no-member self.assertEqual(bool(self.output), True) self.assertIsNone(self.output["error"]) From ee5d93060550705a352a6b275a1a8e3ebeb73511 Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Sat, 17 Jul 2021 02:39:20 +0530 Subject: [PATCH 5/9] reformated test_sample_images.py --- source/pic2card/tests/test_sample_images.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/pic2card/tests/test_sample_images.py b/source/pic2card/tests/test_sample_images.py index 270fc4b596..997a7a4083 100644 --- a/source/pic2card/tests/test_sample_images.py +++ b/source/pic2card/tests/test_sample_images.py @@ -32,6 +32,8 @@ def test_response(self): self.client, self.api, self.headers, self.data ) self.output = json.loads(self.response.data) - self.assertEqual(self.response.status_code, 200) # pylint: disable=no-member + self.assertEqual( + self.response.status_code, 200 + ) # pylint: disable=no-member self.assertEqual(bool(self.output), True) self.assertIsNone(self.output["error"]) From 960f9e7a14d36169ddba377232346a1f1d929de3 Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Sat, 17 Jul 2021 02:44:02 +0530 Subject: [PATCH 6/9] disabled pylint no member error --- source/pic2card/tests/test_sample_images.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/pic2card/tests/test_sample_images.py b/source/pic2card/tests/test_sample_images.py index 997a7a4083..cfc05be654 100644 --- a/source/pic2card/tests/test_sample_images.py +++ b/source/pic2card/tests/test_sample_images.py @@ -32,8 +32,7 @@ def test_response(self): self.client, self.api, self.headers, self.data ) self.output = json.loads(self.response.data) - self.assertEqual( - self.response.status_code, 200 - ) # pylint: disable=no-member + # pylint: disable=no-member + self.assertEqual(self.response.status_code, 200) self.assertEqual(bool(self.output), True) self.assertIsNone(self.output["error"]) From 8d7d36272508d2713f92196b8ac68b49cf78a31a Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Fri, 5 Nov 2021 13:24:32 +0530 Subject: [PATCH 7/9] Added positions for the image elements. --- .../commands/generate_synthetic_dataset.py | 4 +- .../datagen/generate_synthetic_image.py | 107 ++++++++++++------ source/pic2card/mystique/config.py | 10 +- 3 files changed, 86 insertions(+), 35 deletions(-) diff --git a/source/pic2card/commands/generate_synthetic_dataset.py b/source/pic2card/commands/generate_synthetic_dataset.py index e38f5ce4d4..fa90c7568d 100644 --- a/source/pic2card/commands/generate_synthetic_dataset.py +++ b/source/pic2card/commands/generate_synthetic_dataset.py @@ -70,8 +70,8 @@ def get_synthetic_image_properties( return generated_image """ layout = synthetic.CardElements(number_of_elements, card_elements_dir_path) - padded_image_element = synthetic.add_padding_to_img_elements( - layout.elements_with_path + padded_image_element = layout.add_padding_to_img_elements( + layout.elements_with_path, layout.elements_type ) generated_image = synthetic.generate_image(padded_image_element) synthetic_image_properties = { diff --git a/source/pic2card/datagen/generate_synthetic_image.py b/source/pic2card/datagen/generate_synthetic_image.py index 35961949fa..e4d1932899 100644 --- a/source/pic2card/datagen/generate_synthetic_image.py +++ b/source/pic2card/datagen/generate_synthetic_image.py @@ -8,7 +8,7 @@ import os import glob import logging -from typing import List, Sequence, Any +from typing import List, Sequence, Any, Dict import cv2 import numpy as np from mystique import config @@ -36,8 +36,9 @@ def __init__(self, number_of_elements: int, elements_dir: str) -> None: self.element_dimensions = self.get_elements_dimensions( self.elements_with_path ) + self.elements_type = self.get_elements_type(self.elements_with_path) - def get_elements_path(self) -> List[str]: + def get_elements_path(self) -> Dict[str, str]: """ Returns a list of complete path of card_elements selected at random @param self: CardElements object @@ -55,6 +56,17 @@ def get_elements_path(self) -> List[str]: raise Exception(error_msg) return elements_with_path + @staticmethod + def get_elements_type(elements_with_path: List[str]) -> Dict[int, str]: + """ + Returns the list of element types of card_elements + @params self: CardElements object + @return: element_types + """ + element_type = [os.path.basename(os.path.dirname(element)) for element in elements_with_path] + element_type = {k: v for k, v in enumerate(element_type)} + return element_type + @staticmethod def get_elements_dimensions(elements_with_path: List[str]) -> List[tuple]: """ @@ -69,36 +81,67 @@ def get_elements_dimensions(elements_with_path: List[str]) -> List[tuple]: elements_dimensions.append(dimension) return elements_dimensions - -def add_padding_to_img_elements( - elements_with_path: List[str], -) -> List[Sequence]: - """ - Returns a list of elements in image format padded - along width of the image - @param elements_with_path: list of elements path from elements directory - @return: reshaped_image_elements - """ - - image_elements = [cv2.imread(element) for element in elements_with_path] - reference_canvas_width = max( - [element.shape[1] for element in image_elements] - ) - reshaped_image_elements = [] - for image_element in image_elements: - image_element_width = image_element.shape[1] - pixel_diff = reference_canvas_width - image_element_width - padded_image_element = cv2.copyMakeBorder( - image_element, - top=10, - bottom=10, - left=10, - right=pixel_diff + 10, - borderType=cv2.BORDER_CONSTANT, - value=config.CANVAS_COLOR["WHITE"], + def add_padding_to_img_elements(self, elements_with_path: List[str], + elements_type: Dict[int, str]) -> List[Sequence]: + """ + Returns a list of elements in image format padded + along width of the image + @param elements_with_path: list of elements path from elements directory + @param elements_type: list of element categories + @return: reshaped_image_elements + """ + sorted_elements_with_path = position_elements_path(elements_with_path, elements_type) + # get the updated elements type for the sorted elements path + elements_type = self.get_elements_type(sorted_elements_with_path) + # updating the sorted list for annotator + self.elements_with_path = sorted_elements_with_path + self.elements_type = elements_type + image_elements = [cv2.imread(element) for element in sorted_elements_with_path] + reference_canvas_width = max( + [element.shape[1] for element in image_elements] ) - reshaped_image_elements.append(padded_image_element) - return reshaped_image_elements + reshaped_image_elements = [] + for e_index, image_element in enumerate(image_elements): + image_element_width = image_element.shape[1] + pixel_diff = reference_canvas_width - image_element_width + # To place at proper position - left, right + e_type = elements_type[e_index] + e_position = config.ELEMENT_POSITION[e_type] + if 'right' in e_position: + padded_image_element = cv2.copyMakeBorder( + image_element, + top=10, + bottom=10, + left=pixel_diff + 10, + right=10, + borderType=cv2.BORDER_CONSTANT, + value=config.CANVAS_COLOR["WHITE"], + ) + elif 'left' in e_position: + padded_image_element = cv2.copyMakeBorder( + image_element, + top=10, + bottom=10, + left=10, + right=pixel_diff + 10, + borderType=cv2.BORDER_CONSTANT, + value=config.CANVAS_COLOR["WHITE"], + ) + else: + raise Exception('Position configuration for the elements are not provided') + reshaped_image_elements.append(padded_image_element) + return reshaped_image_elements + + +def position_elements_path(elements_with_path, elements_type): + elements_position_key = config.ELEMENT_POSITION + for path_index, element_path in enumerate(elements_with_path): + e_path_type = elements_type[path_index] + elements_position = elements_position_key[e_path_type] + if 'bottom' in elements_position: + elements_with_path.pop(path_index) + elements_with_path.append(element_path) + return elements_with_path def generate_image(reshaped_image_elements: List[Sequence]) -> List[Sequence]: @@ -117,7 +160,7 @@ def generate_image(reshaped_image_elements: List[Sequence]) -> List[Sequence]: reshaped_image_elements[: number_of_elements // 2] ) right_elements = np.vstack( - reshaped_image_elements[number_of_elements // 2 :] + reshaped_image_elements[number_of_elements // 2:] ) pixel_diff = abs(left_elements.shape[0] - right_elements.shape[0]) diff --git a/source/pic2card/mystique/config.py b/source/pic2card/mystique/config.py index b7d31bdb92..250fc1ae0c 100644 --- a/source/pic2card/mystique/config.py +++ b/source/pic2card/mystique/config.py @@ -148,7 +148,7 @@ "GOLD": [0, 255, 255], } BACKGROUND_COLOR = "WHITE" -ELEMENT_COUNT_THRESHOLD = 5 +ELEMENT_COUNT_THRESHOLD = 4 BULK_IMAGES_NEEDED = 10 ELEMENTS_DIR = os.path.join( os.path.dirname(__file__), "../data/synthetic/card_elements/" @@ -162,3 +162,11 @@ GENERATED_ZIP_DIR = os.path.join( os.path.dirname(__file__), "../data/synthetic/generated_zipfiles/" ) + +ELEMENT_POSITION = { + "images": "top_right", + "actionset": "bottom_right", + "radiobutton": "top_left", + "textbox": "top_left", + "checkbox": "top_left", +} \ No newline at end of file From cadb6fd3e6c93bbbbae000b9d1b3ba679434ca5b Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Mon, 31 Jan 2022 16:07:58 +0530 Subject: [PATCH 8/9] Added support for image merges on cards --- .../pic2card/datagen/generate_annotations.py | 45 ++++- .../datagen/generate_synthetic_image.py | 157 ++++++++++++++++-- source/pic2card/mystique/config.py | 16 +- 3 files changed, 193 insertions(+), 25 deletions(-) diff --git a/source/pic2card/datagen/generate_annotations.py b/source/pic2card/datagen/generate_annotations.py index cbd0c21eba..acf5654976 100644 --- a/source/pic2card/datagen/generate_annotations.py +++ b/source/pic2card/datagen/generate_annotations.py @@ -7,12 +7,15 @@ from lxml import etree as et from mystique import config from .utils import get_image_name +import cv2 def calculate_annotation( elements_with_path: List[str], elements_dimensions: List[tuple], padded_image_element: List[Sequence], + has_merged_image_element: List[bool], + element_positions: List[str], ) -> List[List[tuple]]: """ Calculates the annotations for a given list of elements. @@ -20,15 +23,23 @@ def calculate_annotation( @param elements_with_path: list of elements path from elements directory @param elements_dimensions: list of elements dimensions @param padded_image_element: list of image elements after padding + @param has_merged_image_element: boolean True if the image has any merged elements + @param element_positions: list of the image element postions on canvas wrt config @return: annotations """ annotations = [] padded_elements_height = [ element.shape[0] for element in padded_image_element ] + padded_elements_width = padded_image_element[0].shape[1] number_of_elements = len(elements_with_path) + for index in range(number_of_elements): - xmin = 10 + if 'right' in element_positions[index]: + xmin = padded_elements_width - (elements_dimensions[index][1] + 10) + else: + xmin = 10 + if index == 0: ymin = 10 else: @@ -36,11 +47,20 @@ def calculate_annotation( sum([padded_elements_height[height] for height in range(index)]) + 10 ) - xmax = elements_dimensions[index][1] + 10 + if 'right' in element_positions[index]: + xmax = padded_elements_width - 10 + else: + xmax = elements_dimensions[index][1] + 10 ymax = ( sum([padded_elements_height[height] for height in range(index + 1)]) - 10 ) + + if has_merged_image_element: + if has_merged_image_element[index] and 'right' in element_positions[index]: + ymin = annotations[-1][0][1] + ymax = annotations[-1][1][1] + annotations.append([(xmin, ymin), (xmax, ymax)]) return annotations @@ -91,6 +111,8 @@ def run_annotator( elements_with_path: List[str], elements_dimensions: List[tuple], padded_image_element: List[Sequence], + has_merged_image_element: List[bool], + element_positions: List[str], ) -> List[List[tuple]]: """ Returns a list of x and y coords of the elements in the generated image @@ -98,12 +120,15 @@ def run_annotator( @param elements_with_path: list of elements path from elements directory @param elements_dimensions: list of elements dimensions @param padded_image_element: list of image elements after padding + @param has_merged_image_element: boolean True if the image has any merged elements + @param element_positions: list of the image element postions on canvas wrt config @return: annotations """ number_of_elements = len(elements_with_path) if number_of_elements <= config.ELEMENT_COUNT_THRESHOLD: annotations = calculate_annotation( - elements_with_path, elements_dimensions, padded_image_element + elements_with_path, elements_dimensions, padded_image_element, + has_merged_image_element, element_positions ) else: mid_value = number_of_elements // 2 @@ -115,6 +140,8 @@ def run_annotator( left_elements_with_path, left_elements_dimensions, left_padded_image_element, + has_merged_image_element, + element_positions, ) right_elements_with_path = elements_with_path[mid_value:] @@ -125,6 +152,8 @@ def run_annotator( right_elements_with_path, right_elements_dimensions, right_padded_image_element, + has_merged_image_element, + element_positions, ) padded_pixels_for_right_element = ( @@ -158,7 +187,17 @@ def get_annotation_file( layout.elements_with_path, layout.element_dimensions, padded_image_element, + layout.has_merged_elements, + layout.element_positions, ) + + #check annotations + for annotation in annotations: + cv2.rectangle(generated_image, annotation[0], annotation[1], (255, 0, 0), 2) + cv2.imshow('annotated_image', generated_image) + cv2.waitKey(0) + cv2.destroyAllWindows() + # get annotation xml annotation_xml = generate_annotation_xml( annotations, generated_image_prop, element_type diff --git a/source/pic2card/datagen/generate_synthetic_image.py b/source/pic2card/datagen/generate_synthetic_image.py index e4d1932899..0988bbdf4a 100644 --- a/source/pic2card/datagen/generate_synthetic_image.py +++ b/source/pic2card/datagen/generate_synthetic_image.py @@ -37,6 +37,27 @@ def __init__(self, number_of_elements: int, elements_dir: str) -> None: self.elements_with_path ) self.elements_type = self.get_elements_type(self.elements_with_path) + self.attach_mandatory_element = self.get_mandatory_element(self.elements_with_path) + self.has_merged_elements = None + self.element_positions = None + + def get_mandatory_element(self, elements_with_path): + """ + Replaces an mandatory element like textbox or image which is configured in + MANDATORY_CARD_ELEMENTS to the list of elements paths. + @param self: CardElements object + @param elements_with_path: List of elements path + @return elements_with_path: List of elements path + """ + if config.MANDATORY_CARD_ELEMENTS: + for index, mandatory_element in enumerate(config.MANDATORY_CARD_ELEMENTS): + if any(mandatory_element in path for path in elements_with_path): + continue + random_mandatory_element_path = random.sample(glob.glob(self.elements_dir + f"{mandatory_element}/*.*"), + k=1) + elements_with_path.pop(index) + elements_with_path.insert(index, random_mandatory_element_path[0]) + return elements_with_path def get_elements_path(self) -> Dict[str, str]: """ @@ -47,9 +68,11 @@ def get_elements_path(self) -> Dict[str, str]: elements = glob.glob(self.elements_dir + "/**/*.*", recursive=True) elements_exist = [os.path.isfile(filepath) for filepath in elements] if elements_exist: - elements_with_path = random.choices( + elements_with_path = random.sample( elements, k=self.number_of_elements ) + elements_with_path = self.get_mandatory_element(elements_with_path) + else: error_msg = "No image elements found under card_elements directory" logger.error(error_msg) @@ -91,57 +114,159 @@ def add_padding_to_img_elements(self, elements_with_path: List[str], @return: reshaped_image_elements """ sorted_elements_with_path = position_elements_path(elements_with_path, elements_type) - # get the updated elements type for the sorted elements path - elements_type = self.get_elements_type(sorted_elements_with_path) - # updating the sorted list for annotator + + # updating parameters necessary for annotations self.elements_with_path = sorted_elements_with_path + elements_type = self.get_elements_type(self.elements_with_path) self.elements_type = elements_type + updated_element_dimensions = self.get_elements_dimensions(sorted_elements_with_path) + self.element_dimensions = updated_element_dimensions + + # selecting random element positions from the available positions for elements in config + element_random_positions = get_random_elements_positions(elements_type) + self.element_positions = element_random_positions image_elements = [cv2.imread(element) for element in sorted_elements_with_path] + + # check for element merging + element_merge = check_possible_element_merge(element_random_positions, sorted_elements_with_path) + reference_canvas_width = max( [element.shape[1] for element in image_elements] ) + reshaped_image_elements = [] for e_index, image_element in enumerate(image_elements): image_element_width = image_element.shape[1] - pixel_diff = reference_canvas_width - image_element_width - # To place at proper position - left, right - e_type = elements_type[e_index] - e_position = config.ELEMENT_POSITION[e_type] + pixel_diff_width = reference_canvas_width - image_element_width + e_position = element_random_positions[e_index] + + merge = element_merge[e_index] + first_element_for_merging = element_merge[e_index] and element_merge[e_index + 1 if len(image_elements)-1 != e_index else 0] + if first_element_for_merging: + first_image_height_with_merge = image_elements[e_index].shape[0] + second_image_height = image_elements[e_index + 1 if len(image_elements) - 1 != e_index else 0].shape[0] + first_image_width = image_elements[e_index].shape[1] + second_image_width = image_elements[e_index + 1 if len(image_elements) - 1 != e_index else 0].shape[1] + pixel_diff_height = abs(first_image_height_with_merge - second_image_height) + if 'right' in e_position: + top_padding_for_merge = 10 + left_padding = pixel_diff_width + 10 + if merge: + left_padding = 10 + if first_image_height_with_merge > second_image_height: + top_padding_for_merge = pixel_diff_height + 10 + padded_image_element = cv2.copyMakeBorder( image_element, - top=10, + top=top_padding_for_merge, bottom=10, - left=pixel_diff + 10, + left=left_padding, right=10, borderType=cv2.BORDER_CONSTANT, value=config.CANVAS_COLOR["WHITE"], ) elif 'left' in e_position: + top_padding_for_merge = 10 + right_padding = pixel_diff_width + 10 + if merge: + right_padding = reference_canvas_width - (first_image_width + 10 + second_image_width) + if first_image_height_with_merge < second_image_height: + top_padding_for_merge = pixel_diff_height + 10 + padded_image_element = cv2.copyMakeBorder( image_element, - top=10, + top=top_padding_for_merge, bottom=10, left=10, - right=pixel_diff + 10, + right=right_padding, borderType=cv2.BORDER_CONSTANT, value=config.CANVAS_COLOR["WHITE"], ) else: raise Exception('Position configuration for the elements are not provided') reshaped_image_elements.append(padded_image_element) + + # replacing the merged image with the existing one and adding dummy white image inplace of second image + if any(element_merge): + for e_index, merge in enumerate(element_merge): + merge = element_merge[e_index] and element_merge[ + e_index + 1 if len(image_elements) - 1 != e_index else 0] + if merge: + first_img = reshaped_image_elements[e_index] + second_img = reshaped_image_elements[e_index + 1] + # second_img[:first_img.shape[0], :first_img.shape[1]] = second_img + merged_img = np.concatenate((first_img, second_img), axis=1) + dummy_img = np.ones(merged_img.shape, np.uint8) * 255 + reshaped_image_elements.pop(e_index + 1) + reshaped_image_elements.pop(e_index) + reshaped_image_elements.insert(e_index, merged_img) + reshaped_image_elements.insert(e_index + 1, dummy_img) + + # passing info for annotator + self.has_merged_elements = element_merge + return reshaped_image_elements +def get_random_elements_positions(elements_type): + random_element_positions = {} + elements_position_key = config.ELEMENT_POSITION + + for index, element in elements_type.items(): + random_pos = random.choice(elements_position_key[element]) + random_element_positions.update({index: random_pos}) + return random_element_positions + + def position_elements_path(elements_with_path, elements_type): + elements_position_key = config.ELEMENT_POSITION + sorted_elements_path = {'top': [], 'mid': [], 'bottom': []} for path_index, element_path in enumerate(elements_with_path): e_path_type = elements_type[path_index] elements_position = elements_position_key[e_path_type] - if 'bottom' in elements_position: - elements_with_path.pop(path_index) - elements_with_path.append(element_path) - return elements_with_path + bottom = [position for position in elements_position if 'bottom' in position] + top = [position for position in elements_position if 'top' in position] + mid = [position for position in elements_position if 'mid' in position] + if top: + sorted_elements_path.get('top').append(element_path) + elif bottom: + sorted_elements_path.get('bottom').append(element_path) + elif mid: + sorted_elements_path.get('mid').append(element_path) + else: + pass + sorted_elements_path = sorted_elements_path.get('top') + sorted_elements_path.get('mid') +\ + sorted_elements_path.get('bottom') + return sorted_elements_path + + +def check_possible_element_merge(element_positions, elements_path): + """ + Returns a list of boolean values that specify which elements are + capable of merging + """ + e_merge = [] + dimensions = CardElements.get_elements_dimensions(elements_path) + canvas_width = max([dimension[1] for dimension in dimensions]) + for index, position in element_positions.items(): + if index == 0: + e_merge.insert(index, False) + continue + + prev_position = element_positions[index-1] + if 'left' in prev_position and 'right' in position and prev_position.split('_')[0] == position.split('_')[0]: + if dimensions[index][1]+dimensions[index-1][1] < canvas_width: + e_merge.insert(index-1, True) + e_merge.pop(index) + e_merge.insert(index, True) + else: + e_merge.insert(index, False) + else: + e_merge.insert(index, False) + + return e_merge def generate_image(reshaped_image_elements: List[Sequence]) -> List[Sequence]: diff --git a/source/pic2card/mystique/config.py b/source/pic2card/mystique/config.py index 250fc1ae0c..9338fd1f8f 100644 --- a/source/pic2card/mystique/config.py +++ b/source/pic2card/mystique/config.py @@ -164,9 +164,13 @@ ) ELEMENT_POSITION = { - "images": "top_right", - "actionset": "bottom_right", - "radiobutton": "top_left", - "textbox": "top_left", - "checkbox": "top_left", -} \ No newline at end of file + "images": ["top_right", "mid_left", "mid_right"], + "textbox": ["top_left", "top_right", ], # "mid_left", "mid_right"], + "radiobutton": ["mid_left", ], + "checkbox": ["mid_left", "mid_right"], + "actionset": ["bottom_right", "bottom_left"], +} + +MANDATORY_CARD_ELEMENTS = [ + "textbox", +] \ No newline at end of file From dfc7767292a0fff64155898409780de9defc6d23 Mon Sep 17 00:00:00 2001 From: amitramachandran Date: Mon, 7 Feb 2022 08:13:12 +0530 Subject: [PATCH 9/9] removing extra space --- source/pic2card/datagen/generate_synthetic_image.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/pic2card/datagen/generate_synthetic_image.py b/source/pic2card/datagen/generate_synthetic_image.py index 0988bbdf4a..3018e44e79 100644 --- a/source/pic2card/datagen/generate_synthetic_image.py +++ b/source/pic2card/datagen/generate_synthetic_image.py @@ -141,7 +141,8 @@ def add_padding_to_img_elements(self, elements_with_path: List[str], e_position = element_random_positions[e_index] merge = element_merge[e_index] - first_element_for_merging = element_merge[e_index] and element_merge[e_index + 1 if len(image_elements)-1 != e_index else 0] + first_element_for_merging = element_merge[e_index] and\ + element_merge[e_index + 1 if len(image_elements)-1 != e_index else 0] if first_element_for_merging: first_image_height_with_merge = image_elements[e_index].shape[0] second_image_height = image_elements[e_index + 1 if len(image_elements) - 1 != e_index else 0].shape[0] @@ -237,8 +238,8 @@ def position_elements_path(elements_with_path, elements_type): sorted_elements_path.get('mid').append(element_path) else: pass - sorted_elements_path = sorted_elements_path.get('top') + sorted_elements_path.get('mid') +\ - sorted_elements_path.get('bottom') + sorted_elements_path = sorted_elements_path.get('top') + sorted_elements_path.get('mid')\ + + sorted_elements_path.get('bottom') return sorted_elements_path