From 0b877d27d645051cb6700c4d887f37a1d4d91d14 Mon Sep 17 00:00:00 2001 From: pini-girit Date: Mon, 11 May 2020 18:01:06 +0300 Subject: [PATCH 01/30] v1.12.0,CLOUDINARY-269: Improved product-gallery-api - now supports background processing using cronjobs. --- Command/ProductGalleryApiQueueProcess.php | 51 ++++ Cron/ProductGalleryApiQueue.php | 213 ++++++++++++++++ Model/Api/ProductGalleryManagement.php | 241 +++++++++++------- Model/Configuration.php | 62 ++++- Model/Logger.php | 6 + Model/Logger/CloudinaryHandler.php | 19 ++ Model/ProductGalleryApiQueue.php | 11 + .../ResourceModel/ProductGalleryApiQueue.php | 16 ++ .../ProductGalleryApiQueue/Collection.php | 17 ++ Setup/InstallSchema.php | 2 +- Setup/UpgradeSchema.php | 83 ++++++ etc/adminhtml/system.xml | 19 ++ etc/config.xml | 3 + etc/crontab.xml | 3 + etc/di.xml | 14 + 15 files changed, 659 insertions(+), 101 deletions(-) create mode 100644 Command/ProductGalleryApiQueueProcess.php create mode 100644 Cron/ProductGalleryApiQueue.php create mode 100644 Model/Logger.php create mode 100644 Model/Logger/CloudinaryHandler.php create mode 100644 Model/ProductGalleryApiQueue.php create mode 100644 Model/ResourceModel/ProductGalleryApiQueue.php create mode 100644 Model/ResourceModel/ProductGalleryApiQueue/Collection.php diff --git a/Command/ProductGalleryApiQueueProcess.php b/Command/ProductGalleryApiQueueProcess.php new file mode 100644 index 0000000..86717a0 --- /dev/null +++ b/Command/ProductGalleryApiQueueProcess.php @@ -0,0 +1,51 @@ +job = $job; + } + + /** + * Configure the command + * + * @return void + */ + protected function configure() + { + $this->setName('cloudinary:product-gallery-api-queue:process'); + $this->setDescription('Process queued items for product gallery API'); + } + + /** + * @param InputInterface $input + * @param OutputInterface $output + * + * @return void + */ + protected function execute(InputInterface $input, OutputInterface $output) + { + return $this->job + ->setOutput($output) + ->execute(); + } +} diff --git a/Cron/ProductGalleryApiQueue.php b/Cron/ProductGalleryApiQueue.php new file mode 100644 index 0000000..0cc37ed --- /dev/null +++ b/Cron/ProductGalleryApiQueue.php @@ -0,0 +1,213 @@ +configuration = $configuration; + $this->cldProductGalleryManagement = $cldProductGalleryManagement; + $this->jsonHelper = $jsonHelper; + $this->productVideoFactory = $productVideoFactory; + $this->productGalleryApiQueueFactory = $productGalleryApiQueueFactory; + $this->appState = $appState; + $this->notifierPool = $notifierPool; + } + + public function execute() + { + if ($this->configuration->isEnabled() && $this->configuration->isEnabledProductgalleryApiQueue()) { + try { + $this->setCrontabAreaCode(); + $queuedItems = $this->productGalleryApiQueueFactory->create()->getCollection() + ->addFieldToFilter("success", 0) + ->addFieldToFilter("tryouts", ['lt' => $this->configuration->getProductgalleryApiQueueMaxTryouts()]) + ->setOrder('created_at', 'asc') + ->setPageSize($this->configuration->getProductgalleryApiQueueLimit()); + + foreach ($queuedItems as $item) { + try { + $fullItemData = $this->jsonHelper->jsonDecode($item->getFullItemData()); + $this->processOutput("ProductGalleryApiQueue::execute() - Processing item ID: {$item->getId()} ...", "debug", ['full_item_data' => $fullItemData]); + $item->setTryouts($item->getTryouts() + 1); + $this->cldProductGalleryManagement->addGalleryItem( + $fullItemData["url"], + $fullItemData["sku"], + $fullItemData["publicId"], + $fullItemData["roles"], + $fullItemData["label"], + $fullItemData["disabled"] + ); + $item->setSuccess(1); + $item->setSuccessAt(date('Y-m-d H:i:s')); + $item->setMessage('success'); + $item->setHasErrors(0); + } catch (\Exception $e) { + $item->setSuccess(0); + $item->setMessage("[ERROR]\n" . $e->getMessage() . "\n" . $e->getTraceAsString()); + $item->setHasErrors(1); + + $this->processOutput("ProductGalleryApiQueue::execute() - Exception during product-gallery API queued item processing: " . $e->getMessage(), 'error', ['trace' => $e->getTraceAsString(), 'queued_item' => $item->getData()]); + if (!($this->output instanceof OutputInterface) && $item->getTryouts() >= $this->configuration->getProductgalleryApiQueueMaxTryouts() && count($this->adminNotificationErrors) < 7) { + $this->adminNotificationErrors[] = [ + "message" => $e->getMessage(), + "tryouts" => $item->getTryouts(), + "item_data" => $fullItemData + ]; + } + } + + $item->save(); + $this->processOutput("ProductGalleryApiQueue::execute() - Processing item ID: {$item->getId()} - Done.", "debug"); + } + } catch (\Exception $e) { + $this->processOutput("ProductGalleryApiQueue::execute() - Exception during product-gallery API queue processing: " . $e->getMessage(), 'error', ['trace' => $e->getTraceAsString()]); + if (!($this->output instanceof OutputInterface) && $item->getTryouts() >= $this->configuration->getProductgalleryApiQueueMaxTryouts() && count($this->adminNotificationErrors) < 7) { + $this->adminNotificationErrors[] = [ + "message" => $e->getMessage(), + "details" => $e->getTraceAsString() + ]; + } + } + if ($this->adminNotificationErrors) { + $adminNotificationErrors = $this->jsonHelper->jsonEncode(array_slice($this->adminNotificationErrors, 0, 5)); + if (count($this->adminNotificationErrors) > 5) { + $adminNotificationErrors .= " ... [this message is too long, check the log for the rest] "; + } + $this->addAdminNotification("[Cloudinary] An error occurred during the background processing of the product-gallery API queue! *More detailes can be found on the Cloudinary log file (var/log/cloudinary_cloudinary.log)", $adminNotificationErrors, 'critical'); + } + } + + return $this; + } + + /** + * @method setOutput + * @param OutputInterface $output + */ + public function setOutput(OutputInterface $output) + { + $this->output = $output; + return $this; + } + + /** + * @method getOutput + * @return OutputInterface + */ + public function getOutput() + { + return $this->output; + } + + /** + * Process output messages (log to system.log / output to terminal) + * @method _processOutput + * @return $this + */ + protected function processOutput($message, $type = "info", $data = []) + { + if ($this->output instanceof OutputInterface) { + //Output to terminal + $outputType = ($type === "error") ? $type : "info"; + $this->output->writeln('<' . $outputType . '>' . json_encode($message) . ''); + if ($data) { + $this->output->writeln('' . json_encode($data) . ''); + } + } else { + //Log to var/log/cloudinary_cloudinary.log + $this->configuration->log($message, $data); + } + + return $this; + } + + private function addAdminNotification(string $title, $description = "", $type = 'critical') + { + $method = 'add' . ucfirst($type); + $this->notifierPool->{$method}($title, $description); + return $this; + } + + private function setCrontabAreaCode() + { + try { + $this->appState->setAreaCode(\Magento\Framework\App\Area::AREA_CRONTAB); + } catch (\Exception $e) { + $this->processOutput("ProductGalleryApiQueue::setCrontabAreaCode() - Exception: " . $e->getMessage() . "\n" . $e->getTraceAsString(), "debug"); + } + return $this; + } +} diff --git a/Model/Api/ProductGalleryManagement.php b/Model/Api/ProductGalleryManagement.php index e8ab78d..0b7add5 100644 --- a/Model/Api/ProductGalleryManagement.php +++ b/Model/Api/ProductGalleryManagement.php @@ -6,6 +6,7 @@ use Cloudinary\Cloudinary\Core\ConfigurationInterface; use Cloudinary\Cloudinary\Model\MediaLibraryMap; use Cloudinary\Cloudinary\Model\MediaLibraryMapFactory; +use Cloudinary\Cloudinary\Model\ProductGalleryApiQueueFactory; use Cloudinary\Cloudinary\Model\ProductImageFinder; use Cloudinary\Cloudinary\Model\TransformationFactory; use Magento\Catalog\Api\ProductRepositoryInterface; @@ -143,6 +144,11 @@ class ProductGalleryManagement implements \Cloudinary\Cloudinary\Api\ProductGall */ private $mediaLibraryMapFactory; + /** + * @var ProductGalleryApiQueueFactory + */ + private $productGalleryApiQueueFactory; + /** * @var AppEmulation */ @@ -150,26 +156,27 @@ class ProductGalleryManagement implements \Cloudinary\Cloudinary\Api\ProductGall /** * @method __construct - * @param ConfigurationInterface $configuration - * @param Http $request - * @param JsonHelper $jsonHelper - * @param ProductRepositoryInterface $productRepository - * @param ProductMediaConfig $mediaConfig - * @param Filesystem $fileSystem - * @param ImageAdapterFactory $imageAdapterFactory - * @param Curl $curl - * @param FileUtility $fileUtility - * @param FileProcessor $fileProcessor - * @param AllowedProtocols $protocolValidator - * @param NotProtectedExtension $extensionValidator - * @param StoreManagerInterface $storeManager - * @param ProductImageFinder $productImageFinder - * @param CloudinaryImageManager $cloudinaryImageManager - * @param ResourcesManagement $cloudinaryResourcesManagement - * @param TransformationFactory $transformationFactory - * @param Processor $mediaGalleryProcessor - * @param MediaLibraryMapFactory $mediaLibraryMapFactory - * @param AppEmulation $appEmulation + * @param ConfigurationInterface $configuration + * @param Http $request + * @param JsonHelper $jsonHelper + * @param ProductRepositoryInterface $productRepository + * @param ProductMediaConfig $mediaConfig + * @param Filesystem $fileSystem + * @param ImageAdapterFactory $imageAdapterFactory + * @param Curl $curl + * @param FileUtility $fileUtility + * @param FileProcessor $fileProcessor + * @param AllowedProtocols $protocolValidator + * @param NotProtectedExtension $extensionValidator + * @param StoreManagerInterface $storeManager + * @param ProductImageFinder $productImageFinder + * @param CloudinaryImageManager $cloudinaryImageManager + * @param ResourcesManagement $cloudinaryResourcesManagement + * @param TransformationFactory $transformationFactory + * @param Processor $mediaGalleryProcessor + * @param MediaLibraryMapFactory $mediaLibraryMapFactory + * @param ProductGalleryApiQueueFactory $productGalleryApiQueueFactory + * @param AppEmulation $appEmulation */ public function __construct( ConfigurationInterface $configuration, @@ -191,6 +198,7 @@ public function __construct( TransformationFactory $transformationFactory, Processor $mediaGalleryProcessor, MediaLibraryMapFactory $mediaLibraryMapFactory, + ProductGalleryApiQueueFactory $productGalleryApiQueueFactory, AppEmulation $appEmulation ) { $this->configuration = $configuration; @@ -212,6 +220,7 @@ public function __construct( $this->transformationFactory = $transformationFactory; $this->mediaGalleryProcessor = $mediaGalleryProcessor; $this->mediaLibraryMapFactory = $mediaLibraryMapFactory; + $this->productGalleryApiQueueFactory = $productGalleryApiQueueFactory; $this->appEmulation = $appEmulation; } @@ -233,12 +242,11 @@ public function addProductMedia($sku, $urls) __("Cloudinary module is disabled. Please enable it first in order to use this API.") ); } - $this->emulateAdminhtmlArea(); $urls = (array)$urls; foreach ($urls as $i => $url) { try { $url = (array)$url; - $this->addGalleryItem( + $this->processOrQueue( $url["url"], $sku, (isset($url["publicId"])) ? $url["publicId"] : null, @@ -253,12 +261,15 @@ public function addProductMedia($sku, $urls) $result["failed"]["urls"][] = $url; } } - $this->stopEnvironmentEmulation(); } catch (\Exception $e) { $result["error"] = 1; $result["message"] = $e->getMessage(); } + if ($result["passed"] && !$result["failed"]["count"]) { + $result["message"] = $this->configuration->isEnabledProductgalleryApiQueue() ? "All items have been added to queue." : "success"; + } + return $this->jsonHelper->jsonEncode($result); } @@ -293,14 +304,13 @@ public function addItems($items) __("Cloudinary module is disabled. Please enable it first in order to use this API.") ); } - $this->emulateAdminhtmlArea(); $items = (array)$items; foreach ($items as $i => $item) { try { $item = $result["items"][$i] = (array)$item; $result["items"][$i]["error"] = 0; - $result["items"][$i]["message"] = "success"; - $this->addGalleryItem( + $result["items"][$i]["message"] = $this->configuration->isEnabledProductgalleryApiQueue() ? "The item was added to the queue." : "success"; + $this->processOrQueue( $item["url"], $item["sku"], (isset($item["publicId"])) ? $item["publicId"] : null, @@ -317,14 +327,13 @@ public function addItems($items) } } } - $this->stopEnvironmentEmulation(); } catch (\Exception $e) { $result["errors"]++; $result["message"] = "\n{$e->getMessage()}"; } if (!$result["errors"]) { - $result["message"] = "success"; + $result["message"] = $this->configuration->isEnabledProductgalleryApiQueue() ? "All items have been added to queue." : "success"; } else { $result["message"] = "error" . $result["message"]; } @@ -332,6 +341,34 @@ public function addItems($items) return $this->jsonHelper->jsonEncode($result); } + /** + * @method processOrQueue + * @param string $url + * @param string $sku + * @param string|null $publicId + * @param string|null $roles + * @param string|null $label + * @param bool|int|null $disabled + */ + public function processOrQueue($url, $sku, $publicId = null, $roles = null, $label = null, $disabled = 0) + { + if ($this->configuration->isEnabledProductgalleryApiQueue()) { + $fullItemData = $this->jsonHelper->jsonEncode([ + "url" => $url, + "sku" => $sku, + "publicId" => $publicId, + "roles" => $roles, + "label" => $label, + "disabled" => $disabled, + ]); + return $this->productGalleryApiQueueFactory->create() + ->setSku($sku) + ->setFullItemData($fullItemData) + ->save(); + } else { + return $this->addGalleryItem($url, $sku, $publicId, $roles, $label, $disabled); + } + } /** * @method addGalleryItem * @param string $url @@ -340,92 +377,98 @@ public function addItems($items) * @param string|null $roles * @param string|null $label * @param bool|int|null $disabled + * @return $this */ - private function addGalleryItem($url, $sku, $publicId = null, $roles = null, $label = null, $disabled = 0) + public function addGalleryItem($url, $sku, $publicId = null, $roles = null, $label = null, $disabled = 0) { - $this->cldUniqid = $this->mapped = null; - $this->parsedRemoteFileUrl = $this->configuration->parseCloudinaryUrl($url, $publicId); + try { + $this->emulateAdminhtmlArea(); - if (!$this->parsedRemoteFileUrl["version"] && !$publicId) { - throw new LocalizedException( - __("The `publicId` field is mandatory for Cloudinary URLs that doesn't contain a version number.") + $this->cldUniqid = $this->mapped = null; + $this->parsedRemoteFileUrl = $this->configuration->parseCloudinaryUrl($url, $publicId); + + if (!$this->parsedRemoteFileUrl["version"] && !$publicId) { + throw new LocalizedException( + __("The `publicId` field is mandatory for Cloudinary URLs that doesn't contain a version number.") + ); + } + + $roles = ($roles) ? array_map('trim', (is_string($roles) ? explode(',', $roles) : (array) $roles)) : null; + $product = $this->productRepository->get($sku); + + $result = $this->retrieveImage($this->parsedRemoteFileUrl['thumbnail_url'] ?: $this->parsedRemoteFileUrl['transformationless_url']); + $result["file"] = $this->mediaGalleryProcessor->addImage( + $product, + $result["tmp_name"], + $roles, + true, + false ); - } - $roles = ($roles) ? array_map('trim', (is_string($roles) ? explode(',', $roles) : (array) $roles)) : null; - $product = $this->productRepository->get($sku); - - $result = $this->retrieveImage($this->parsedRemoteFileUrl['thumbnail_url'] ?: $this->parsedRemoteFileUrl['transformationless_url']); - $result["file"] = $this->mediaGalleryProcessor->addImage( - $product, - $result["tmp_name"], - $roles, - true, - false - ); - - $mediaGalleryData = $product->getMediaGallery(); - $galItem = array_pop($mediaGalleryData["images"]); - - if ($this->parsedRemoteFileUrl["type"] === "video") { - $videoData = (array) $this->jsonHelper->jsonDecode($this->cloudinaryResourcesManagement->setId($this->parsedRemoteFileUrl["publicId"])->getVideo()); - $videoData["title"] = $label; - $videoData["description"] = ""; - if (!$videoData["error"]) { - $videoData["context"] = new DataObject((isset($videoData["data"]["context"])) ? (array)$videoData["data"]["context"] : []); - $videoData["title"] = $videoData["title"] ? $videoData["title"] : ($videoData["context"]->getData('caption') ?: $videoData["context"]->getData('alt')); - $videoData["description"] = $videoData["context"]->getData('description') ?: $videoData["context"]->getData('alt'); + $mediaGalleryData = $product->getMediaGallery(); + $galItem = array_pop($mediaGalleryData["images"]); + + if ($this->parsedRemoteFileUrl["type"] === "video") { + $videoData = (array) $this->jsonHelper->jsonDecode($this->cloudinaryResourcesManagement->setId($this->parsedRemoteFileUrl["publicId"])->getVideo()); + $videoData["title"] = $label; + $videoData["description"] = ""; + if (!$videoData["error"]) { + $videoData["context"] = new DataObject((isset($videoData["data"]["context"])) ? (array)$videoData["data"]["context"] : []); + $videoData["title"] = $videoData["title"] ? $videoData["title"] : ($videoData["context"]->getData('caption') ?: $videoData["context"]->getData('alt')); + $videoData["description"] = $videoData["context"]->getData('description') ?: $videoData["context"]->getData('alt'); + } + $videoData["title"] = $videoData["title"] ?: $this->parsedRemoteFileUrl["publicId"]; + $videoData["description"] = preg_replace('/( |<([^>]+)>)/i', '', $videoData["description"] ?: $videoData["title"]); + + $galItem = array_merge($galItem, [ + "media_type" => "external-video", + "video_provider" => "cloudinary", + "disabled" => $disabled ? 1 : 0, + "label" => $videoData["title"], + "video_url" => $this->parsedRemoteFileUrl["orig_url"], + "video_title" => $videoData["title"], + "video_description" => $videoData["description"], + ]); } - $videoData["title"] = $videoData["title"] ?: $this->parsedRemoteFileUrl["publicId"]; - $videoData["description"] = preg_replace('/( |<([^>]+)>)/i', '', $videoData["description"] ?: $videoData["title"]); - - $galItem = array_merge($galItem, [ - "media_type" => "external-video", - "video_provider" => "cloudinary", - "disabled" => $disabled ? 1 : 0, - "label" => $videoData["title"], - "video_url" => $this->parsedRemoteFileUrl["orig_url"], - "video_title" => $videoData["title"], - "video_description" => $videoData["description"], - ]); - } - if ($this->parsedRemoteFileUrl["type"] === "image") { - if (!$label) { - $imageData = (array) $this->jsonHelper->jsonDecode($this->cloudinaryResourcesManagement->setId($this->parsedRemoteFileUrl["publicId"])->getImage()); - if (!$imageData["error"]) { - $imageData["context"] = new DataObject((isset($imageData["data"]["context"])) ? (array)$imageData["data"]["context"] : []); - $label = $imageData["context"]->getData('caption') ?: $imageData["context"]->getData('alt'); + if ($this->parsedRemoteFileUrl["type"] === "image") { + if (!$label) { + $imageData = (array) $this->jsonHelper->jsonDecode($this->cloudinaryResourcesManagement->setId($this->parsedRemoteFileUrl["publicId"])->getImage()); + if (!$imageData["error"]) { + $imageData["context"] = new DataObject((isset($imageData["data"]["context"])) ? (array)$imageData["data"]["context"] : []); + $label = $imageData["context"]->getData('caption') ?: $imageData["context"]->getData('alt'); + } + $label = $label ?: ""; } - $label = $label ?: ""; + $galItem = array_merge($galItem, [ + "disabled" => $disabled ? 1 : 0, + "label" => $label, + ]); } - $galItem = array_merge($galItem, [ - "disabled" => $disabled ? 1 : 0, - "label" => $label, - ]); - } - $mediaGalleryData["images"][] = $galItem; - $product->setData('media_gallery', $mediaGalleryData); + $mediaGalleryData["images"][] = $galItem; + $product->setData('media_gallery', $mediaGalleryData); - $product->save(); - $mediaGalleryData = $product->getMediaGallery(); - $galItem = array_pop($mediaGalleryData["images"]); + $product->save(); + $mediaGalleryData = $product->getMediaGallery(); + $galItem = array_pop($mediaGalleryData["images"]); - /*foreach ($this->productImageFinder->findNewImages($product) as $image) { - $this->cloudinaryImageManager->uploadAndSynchronise($image); - }*/ + if ($this->parsedRemoteFileUrl["type"] === "image" && $this->parsedRemoteFileUrl['transformations_string']) { + $this->transformationFactory->create() + ->setImageName($galItem["file"]) + ->setFreeTransformation($this->parsedRemoteFileUrl['transformations_string']) + ->save(); + } - if ($this->parsedRemoteFileUrl["type"] === "image" && $this->parsedRemoteFileUrl['transformations_string']) { - $this->transformationFactory->create() - ->setImageName($galItem["file"]) - ->setFreeTransformation($this->parsedRemoteFileUrl['transformations_string']) - ->save(); + if ($this->configuration->isEnabledLocalMapping()) { + $this->saveCloudinaryMapping(); + } + } catch (\Exception $e) { + $this->stopEnvironmentEmulation(); + throw $e; } - if ($this->configuration->isEnabledLocalMapping()) { - $this->saveCloudinaryMapping(); - } + return $this; } /** diff --git a/Model/Configuration.php b/Model/Configuration.php index e2fa854..bc4cfda 100644 --- a/Model/Configuration.php +++ b/Model/Configuration.php @@ -15,6 +15,7 @@ use Cloudinary\Cloudinary\Core\Image\Transformation\Quality; use Cloudinary\Cloudinary\Core\Security\CloudinaryEnvironmentVariable; use Cloudinary\Cloudinary\Core\UploadConfig; +use Cloudinary\Cloudinary\Model\Logger as CloudinaryLogger; use Magento\Framework\App\Config\ScopeConfigInterface; use Magento\Framework\App\Config\Storage\WriterInterface; use Magento\Framework\App\Filesystem\DirectoryList; @@ -56,6 +57,9 @@ class Configuration implements ConfigurationInterface const CONFIG_PATH_USE_SIGNED_URLS = 'cloudinary/advanced/use_signed_urls'; const CONFIG_PATH_ENABLE_LOCAL_MAPPING = 'cloudinary/advanced/enable_local_mapping'; const CONFIG_PATH_SCHEDULED_VIDEO_DATA_IMPORT_LIMIT = 'cloudinary/advanced/cloudinary_scheduled_video_data_import_limit'; + const CONFIG_PATH_PG_API_QUEUE_ENABLED = 'cloudinary/advanced/product_gallery_api_queue_enabled'; + const CONFIG_PATH_PG_API_QUEUE_LIMIT = 'cloudinary/advanced/product_gallery_api_queue_limit'; + const CONFIG_PATH_PG_API_QUEUE_MAX_TRYOUTS = 'cloudinary/advanced/product_gallery_api_queue_max_tryouts'; //= Product Gallery const CONFIG_PATH_PG_ALL = 'cloudinary/product_gallery'; @@ -144,6 +148,11 @@ class Configuration implements ConfigurationInterface */ private $productMetadata; + /** + * @var CloudinaryLogger + */ + private $cloudinaryLogger; + /** * @method __construct * @param ScopeConfigInterface $configReader @@ -154,6 +163,7 @@ class Configuration implements ConfigurationInterface * @param StoreManagerInterface $storeManager * @param ModuleListInterface $moduleList * @param ProductMetadataInterface $productMetadata + * @param CloudinaryLogger $cloudinaryLogger */ public function __construct( ScopeConfigInterface $configReader, @@ -163,7 +173,8 @@ public function __construct( LoggerInterface $logger, StoreManagerInterface $storeManager, ModuleListInterface $moduleList, - ProductMetadataInterface $productMetadata + ProductMetadataInterface $productMetadata, + CloudinaryLogger $cloudinaryLogger ) { $this->configReader = $configReader; $this->configWriter = $configWriter; @@ -173,6 +184,7 @@ public function __construct( $this->storeManager = $storeManager; $this->moduleList = $moduleList; $this->productMetadata = $productMetadata; + $this->cloudinaryLogger = $cloudinaryLogger; } /** @@ -461,6 +473,41 @@ public function getScheduledVideoDataImportLimit() return (int) $this->configReader->getValue(self::CONFIG_PATH_SCHEDULED_VIDEO_DATA_IMPORT_LIMIT); } + /** + * @return bool + */ + public function isEnabledProductgalleryApiQueue() + { + return (bool) $this->configReader->getValue(self::CONFIG_PATH_PG_API_QUEUE_ENABLED); + } + + /** + * @return bool + */ + public function getProductgalleryApiQueueLimit() + { + $return = (int) $this->configReader->getValue(self::CONFIG_PATH_PG_API_QUEUE_LIMIT); + if ($return < 0) { + return 0; + } + return $return; + } + + /** + * @return bool + */ + public function getProductgalleryApiQueueMaxTryouts() + { + $return = (int) $this->configReader->getValue(self::CONFIG_PATH_PG_API_QUEUE_MAX_TRYOUTS); + if ($return > 20) { + return 20; + } + if ($return < 1) { + return 5; + } + return $return; + } + /** * @method getMediaBaseUrl * @return string @@ -604,4 +651,17 @@ public function addUniquePrefixToBasename($filename, $uniqid = null) $uniqid = $uniqid ? $uniqid : $this->generateCLDuniqid(); return dirname($filename) . '/' . $uniqid . basename($filename); } + + /** + * Log to var/log/cloudinary_cloudinary.log + * @method log + * @param mixed $message + * @param array $data + * @return $this + */ + public function log($message, $data = [], $prefix = '[Cloudinary Log] ') + { + $this->cloudinaryLogger->info($prefix . json_encode($message), $data); + return $this; + } } diff --git a/Model/Logger.php b/Model/Logger.php new file mode 100644 index 0000000..26ad638 --- /dev/null +++ b/Model/Logger.php @@ -0,0 +1,6 @@ +_init(\Cloudinary\Cloudinary\Model\ResourceModel\ProductGalleryApiQueue::class); + } +} diff --git a/Model/ResourceModel/ProductGalleryApiQueue.php b/Model/ResourceModel/ProductGalleryApiQueue.php new file mode 100644 index 0000000..70d3230 --- /dev/null +++ b/Model/ResourceModel/ProductGalleryApiQueue.php @@ -0,0 +1,16 @@ +_init('cloudinary_product_gallery_api_queue', 'id'); + } +} diff --git a/Model/ResourceModel/ProductGalleryApiQueue/Collection.php b/Model/ResourceModel/ProductGalleryApiQueue/Collection.php new file mode 100644 index 0000000..ab916fc --- /dev/null +++ b/Model/ResourceModel/ProductGalleryApiQueue/Collection.php @@ -0,0 +1,17 @@ +_init( + \Cloudinary\Cloudinary\Model\ProductGalleryApiQueue::class, + \Cloudinary\Cloudinary\Model\ResourceModel\ProductGalleryApiQueue::class + ); + } +} diff --git a/Setup/InstallSchema.php b/Setup/InstallSchema.php index 4763ec1..03482ef 100644 --- a/Setup/InstallSchema.php +++ b/Setup/InstallSchema.php @@ -2,10 +2,10 @@ namespace Cloudinary\Cloudinary\Setup; +use Magento\Framework\DB\Ddl\Table; use Magento\Framework\Setup\InstallSchemaInterface; use Magento\Framework\Setup\ModuleContextInterface; use Magento\Framework\Setup\SchemaSetupInterface; -use Magento\Framework\DB\Ddl\Table; class InstallSchema implements InstallSchemaInterface { diff --git a/Setup/UpgradeSchema.php b/Setup/UpgradeSchema.php index 7697443..a8f9804 100644 --- a/Setup/UpgradeSchema.php +++ b/Setup/UpgradeSchema.php @@ -25,6 +25,10 @@ public function upgrade(SchemaSetupInterface $setup, ModuleContextInterface $con $this->createMediaLibraryMapTable($setup); } + if (version_compare($context->getVersion(), '1.12.0', '<')) { + $this->createProductGalleryApiQueueTable($setup); + } + $setup->endSetup(); } @@ -95,4 +99,83 @@ private function createMediaLibraryMapTable(SchemaSetupInterface $setup) $setup->getConnection()->createTable($table); } + + /** + * + * @param SchemaSetupInterface $setup + */ + private function createProductGalleryApiQueueTable(SchemaSetupInterface $setup) + { + $table = $setup->getConnection()->newTable( + $setup->getTable('cloudinary_product_gallery_api_queue') + )->addColumn( + 'id', + \Magento\Framework\DB\Ddl\Table::TYPE_INTEGER, + null, + ['identity' => true, 'unsigned' => true, 'nullable' => false, 'primary' => true], + 'ID' + )->addColumn( + 'sku', + Table::TYPE_TEXT, + 255, + ['nullable' => false], + 'Product SKU' + )->addColumn( + 'full_item_data', + \Magento\Framework\DB\Ddl\Table::TYPE_TEXT, + 3000, + ['nullable' => true], + 'Prepared Schema' + ) + ->addColumn( + 'created_at', + \Magento\Framework\DB\Ddl\Table::TYPE_TIMESTAMP, + null, + ['nullable' => false, 'default' => \Magento\Framework\DB\Ddl\Table::TIMESTAMP_INIT], + 'Created At' + ) + ->addColumn( + 'updated_at', + \Magento\Framework\DB\Ddl\Table::TYPE_TIMESTAMP, + null, + ['nullable' => false, 'default' => \Magento\Framework\DB\Ddl\Table::TIMESTAMP_INIT_UPDATE], + 'Created At' + ) + ->addColumn( + 'success', + \Magento\Framework\DB\Ddl\Table::TYPE_INTEGER, + 1, + ['unsigned' => true, 'nullable' => false, 'default' => '0'], + 'Success' + ) + ->addColumn( + 'success_at', + \Magento\Framework\DB\Ddl\Table::TYPE_DATETIME, + null, + ['nullable' => true], + 'Success At' + ) + ->addColumn( + 'message', + \Magento\Framework\DB\Ddl\Table::TYPE_TEXT, + 3000, + ['nullable' => true], + 'Message' + ) + ->addColumn( + 'has_errors', + \Magento\Framework\DB\Ddl\Table::TYPE_INTEGER, + 1, + ['unsigned' => true, 'nullable' => true, 'default' => '0'], + 'Has Errors' + ) + ->addColumn( + 'tryouts', + \Magento\Framework\DB\Ddl\Table::TYPE_INTEGER, + 11, + ['unsigned' => true, 'nullable' => true, 'default' => '0'], + 'Tryouts' + ); + $setup->getConnection()->createTable($table); + } } diff --git a/etc/adminhtml/system.xml b/etc/adminhtml/system.xml index 32486b8..5b9d80a 100644 --- a/etc/adminhtml/system.xml +++ b/etc/adminhtml/system.xml @@ -337,6 +337,25 @@ Map Cloudinary assets to their original location to prevent duplication in your Cloudinary Media Library. If set to 'No', assets will be synchronized to a new Cloudinary location based on the local Magento path. Magento\Config\Model\Config\Source\Yesno + + + When using the rest/V1/cloudinary/products API endpoint, add the requests to queue and process later using a cronjob. + Magento\Config\Model\Config\Source\Yesno + + + + Maximum queue items to process on each run (0 = no limit). + + 1 + + + + + How many times to try processing the queued item before stopping and adding error notification on admin (Minimum: 1, Maximum: 20). + + 1 + + diff --git a/etc/config.xml b/etc/config.xml index b03019d..fe73989 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -23,6 +23,9 @@ 0 1 10 + 1 + 20 + 5 0 diff --git a/etc/crontab.xml b/etc/crontab.xml index a13acb1..4f50aa5 100644 --- a/etc/crontab.xml +++ b/etc/crontab.xml @@ -3,6 +3,9 @@ * * * * * + + + * * * * * diff --git a/etc/di.xml b/etc/di.xml index 8784694..98fc0e2 100644 --- a/etc/di.xml +++ b/etc/di.xml @@ -8,6 +8,7 @@ Cloudinary\Cloudinary\Command\UploadImages Cloudinary\Cloudinary\Command\StopMigration Cloudinary\Cloudinary\Command\ResetAll + Cloudinary\Cloudinary\Command\ProductGalleryApiQueueProcess @@ -94,4 +95,17 @@ + + + Magento\Framework\Filesystem\Driver\File + + + + + cloudinaryLogger + + Cloudinary\Cloudinary\Model\Logger\CloudinaryHandler + + + From 06d6acfa62e578cb2d735de6b894b9ef5e8c9783 Mon Sep 17 00:00:00 2001 From: pini-girit Date: Thu, 14 May 2020 17:12:00 +0300 Subject: [PATCH 02/30] v1.12.0,CLOUDINARY-269: fixes on ProductGalleryApiQueue process --- Command/ProductGalleryApiQueueProcess.php | 13 ++++++++++++- Cron/ProductGalleryApiQueue.php | 20 -------------------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/Command/ProductGalleryApiQueueProcess.php b/Command/ProductGalleryApiQueueProcess.php index 86717a0..65d2961 100644 --- a/Command/ProductGalleryApiQueueProcess.php +++ b/Command/ProductGalleryApiQueueProcess.php @@ -3,25 +3,35 @@ namespace Cloudinary\Cloudinary\Command; use Cloudinary\Cloudinary\Cron\ProductGalleryApiQueue; +use Magento\Framework\App\State as AppState; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; class ProductGalleryApiQueueProcess extends Command { + /** + * @var AppState + */ + private $appState; + /** * @var ProductGalleryApiQueue */ private $job; /** - * @param ProductGalleryApiQueue $job + * @method __construct + * @param AppState $appState + * @param ProductGalleryApiQueue $job */ public function __construct( + AppState $appState, ProductGalleryApiQueue $job ) { parent::__construct('cloudinary:product-gallery-api-queue:process'); + $this->appState = $appState; $this->job = $job; } @@ -44,6 +54,7 @@ protected function configure() */ protected function execute(InputInterface $input, OutputInterface $output) { + $this->appState->setAreaCode(\Magento\Framework\App\Area::AREA_CRONTAB); return $this->job ->setOutput($output) ->execute(); diff --git a/Cron/ProductGalleryApiQueue.php b/Cron/ProductGalleryApiQueue.php index 0cc37ed..85c8218 100644 --- a/Cron/ProductGalleryApiQueue.php +++ b/Cron/ProductGalleryApiQueue.php @@ -6,7 +6,6 @@ use Cloudinary\Cloudinary\Model\Api\ProductGalleryManagement; use Cloudinary\Cloudinary\Model\ProductGalleryApiQueueFactory; use Cloudinary\Cloudinary\Model\ProductVideoFactory; -use Magento\Framework\App\State as AppState; use Magento\Framework\Json\Helper\Data as JsonHelper; use Magento\Framework\Notification\NotifierInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -48,11 +47,6 @@ class ProductGalleryApiQueue */ private $productGalleryApiQueueFactory; - /** - * @var AppState - */ - private $appState; - /** * @var NotifierInterface */ @@ -65,7 +59,6 @@ class ProductGalleryApiQueue * @param JsonHelper $jsonHelper * @param ProductVideoFactory $productVideoFactory * @param ProductGalleryApiQueueFactory $productGalleryApiQueueFactory - * @param AppState $appState * @param NotifierInterface $notifierPool */ public function __construct( @@ -74,7 +67,6 @@ public function __construct( JsonHelper $jsonHelper, ProductVideoFactory $productVideoFactory, ProductGalleryApiQueueFactory $productGalleryApiQueueFactory, - AppState $appState, NotifierInterface $notifierPool ) { $this->configuration = $configuration; @@ -82,7 +74,6 @@ public function __construct( $this->jsonHelper = $jsonHelper; $this->productVideoFactory = $productVideoFactory; $this->productGalleryApiQueueFactory = $productGalleryApiQueueFactory; - $this->appState = $appState; $this->notifierPool = $notifierPool; } @@ -90,7 +81,6 @@ public function execute() { if ($this->configuration->isEnabled() && $this->configuration->isEnabledProductgalleryApiQueue()) { try { - $this->setCrontabAreaCode(); $queuedItems = $this->productGalleryApiQueueFactory->create()->getCollection() ->addFieldToFilter("success", 0) ->addFieldToFilter("tryouts", ['lt' => $this->configuration->getProductgalleryApiQueueMaxTryouts()]) @@ -200,14 +190,4 @@ private function addAdminNotification(string $title, $description = "", $type = $this->notifierPool->{$method}($title, $description); return $this; } - - private function setCrontabAreaCode() - { - try { - $this->appState->setAreaCode(\Magento\Framework\App\Area::AREA_CRONTAB); - } catch (\Exception $e) { - $this->processOutput("ProductGalleryApiQueue::setCrontabAreaCode() - Exception: " . $e->getMessage() . "\n" . $e->getTraceAsString(), "debug"); - } - return $this; - } } From ee6f7707d2e7f8df770e71189d32550ff3160353 Mon Sep 17 00:00:00 2001 From: pini-girit Date: Thu, 14 May 2020 17:27:51 +0300 Subject: [PATCH 03/30] v1.12.0,CLOUDINARY-269: fixes on ProductGalleryApiQueue process --- etc/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/config.xml b/etc/config.xml index fe73989..0954855 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -25,7 +25,7 @@ 10 1 20 - 5 + 5 0 From e7efe99d9a9b989d20d6329a123802fa934c1b92 Mon Sep 17 00:00:00 2001 From: pini-girit Date: Thu, 14 May 2020 17:28:31 +0300 Subject: [PATCH 04/30] v1.12.0,CLOUDINARY-270: Added CSP whitelist --- etc/config.xml | 10 +++++ etc/csp_whitelist.xml | 92 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 etc/csp_whitelist.xml diff --git a/etc/config.xml b/etc/config.xml index 0954855..7db980a 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -54,5 +54,15 @@ {} + + + + 1 + + + 1 + + + diff --git a/etc/csp_whitelist.xml b/etc/csp_whitelist.xml new file mode 100644 index 0000000..cb62c34 --- /dev/null +++ b/etc/csp_whitelist.xml @@ -0,0 +1,92 @@ + + + + + + cloudinary.com + www.cloudinary.com + api.cloudinary.com + media-library.cloudinary.com + product-gallery.cloudinary.com + cdnjs.cloudflare.com + res.cloudinary.com + res-1.cloudinary.com + res-2.cloudinary.com + res-3.cloudinary.com + res-4.cloudinary.com + + + + + cloudinary.com + www.cloudinary.com + p.cloudinary.com + media-library.cloudinary.com + product-gallery.cloudinary.com + cdnjs.cloudflare.com + res.cloudinary.com + res-1.cloudinary.com + res-2.cloudinary.com + res-3.cloudinary.com + res-4.cloudinary.com + + + + + cloudinary.com + www.cloudinary.com + p.cloudinary.com + media-library.cloudinary.com + product-gallery.cloudinary.com + cdnjs.cloudflare.com + res.cloudinary.com + res-1.cloudinary.com + res-2.cloudinary.com + res-3.cloudinary.com + res-4.cloudinary.com + + + + + cloudinary.com + www.cloudinary.com + p.cloudinary.com + media-library.cloudinary.com + product-gallery.cloudinary.com + res.cloudinary.com + res-1.cloudinary.com + res-2.cloudinary.com + res-3.cloudinary.com + res-4.cloudinary.com + + + + + cloudinary.com + www.cloudinary.com + p.cloudinary.com + media-library.cloudinary.com + product-gallery.cloudinary.com + res.cloudinary.com + res-1.cloudinary.com + res-2.cloudinary.com + res-3.cloudinary.com + res-4.cloudinary.com + + + + + cloudinary.com + www.cloudinary.com + p.cloudinary.com + media-library.cloudinary.com + product-gallery.cloudinary.com + res.cloudinary.com + res-1.cloudinary.com + res-2.cloudinary.com + res-3.cloudinary.com + res-4.cloudinary.com + + + + From 8399260d78810aae88d1dc6c714555ce1b770ae3 Mon Sep 17 00:00:00 2001 From: pini-girit Date: Sun, 17 May 2020 14:14:36 +0300 Subject: [PATCH 05/30] v1.12.0: Change version to 1.12.0 --- composer.json | 2 +- etc/module.xml | 2 +- marketplace.composer.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/composer.json b/composer.json index 76eb03b..fe5cab7 100644 --- a/composer.json +++ b/composer.json @@ -2,7 +2,7 @@ "name": "cloudinary/cloudinary-magento2", "description": "Cloudinary Magento 2 Integration.", "type": "magento2-module", - "version": "1.11.3", + "version": "1.12.0", "license": "MIT", "require": { "cloudinary/cloudinary_php": "*" diff --git a/etc/module.xml b/etc/module.xml index 86c6b13..07093e5 100644 --- a/etc/module.xml +++ b/etc/module.xml @@ -1,6 +1,6 @@ - + diff --git a/marketplace.composer.json b/marketplace.composer.json index 2ba9f63..5c32b47 100644 --- a/marketplace.composer.json +++ b/marketplace.composer.json @@ -2,7 +2,7 @@ "name": "cloudinary/cloudinary", "description": "Cloudinary Magento 2 Integration.", "type": "magento2-module", - "version": "1.11.3", + "version": "1.12.0", "license": "MIT", "require": { "cloudinary/cloudinary_php": "*" From 9ed2cd635130b4991f66a0d7a240fceff048eb96 Mon Sep 17 00:00:00 2001 From: pini-girit Date: Sun, 17 May 2020 15:17:32 +0300 Subject: [PATCH 06/30] v1.12.0: Changed CSP to restrict mode --- etc/config.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/config.xml b/etc/config.xml index 7db980a..6440507 100644 --- a/etc/config.xml +++ b/etc/config.xml @@ -57,10 +57,10 @@ - 1 + 0 - 1 + 0 From e7497002129595c8fddc03163bf10e6c8a5a33ca Mon Sep 17 00:00:00 2001 From: pini-girit Date: Sun, 17 May 2020 15:22:47 +0300 Subject: [PATCH 07/30] v1.13.0,CLOUDINARY-261: Add support for spinset asset types - step 1 --- view/adminhtml/requirejs-config.js | 3 +- .../catalog/product/helper/gallery.phtml | 17 + view/adminhtml/web/js/product-gallery.js | 664 ++++++++++++++++++ 3 files changed, 683 insertions(+), 1 deletion(-) create mode 100644 view/adminhtml/web/js/product-gallery.js diff --git a/view/adminhtml/requirejs-config.js b/view/adminhtml/requirejs-config.js index 930fbd6..851407f 100644 --- a/view/adminhtml/requirejs-config.js +++ b/view/adminhtml/requirejs-config.js @@ -4,7 +4,8 @@ var config = { cloudinaryFreeTransform: 'Cloudinary_Cloudinary/js/cloudinary-free', newVideoDialog: 'Cloudinary_Cloudinary/js/new-video-dialog', 'Magento_ProductVideo/js/get-video-information': 'Cloudinary_Cloudinary/js/get-video-information', - cloudinaryMediaLibraryModal: 'Cloudinary_Cloudinary/js/cloudinary-media-library-modal' + cloudinaryMediaLibraryModal: 'Cloudinary_Cloudinary/js/cloudinary-media-library-modal', + productGallery: 'Cloudinary_Cloudinary/js/product-gallery.js', } }, paths: { diff --git a/view/adminhtml/templates/catalog/product/helper/gallery.phtml b/view/adminhtml/templates/catalog/product/helper/gallery.phtml index 77b0c3a..4acd36a 100644 --- a/view/adminhtml/templates/catalog/product/helper/gallery.phtml +++ b/view/adminhtml/templates/catalog/product/helper/gallery.phtml @@ -119,6 +119,10 @@ $elementToggleCode = $element->getToggleCode() ? $element->getToggleCode() : 'to name="[<%- data.file_id %>][label]" value="<%- data.label %>" data-form-part=""/> + getToggleCode() ? $element->getToggleCode() : 'to +
+ + +
+ +
+
+
@@ -326,6 +334,41 @@ $elementToggleCode = $element->getToggleCode() ? $element->getToggleCode() : 'to getChildHtml('new-video') ?> + + +