From 50b61fb44d4e4efabac181de7458249721a88656 Mon Sep 17 00:00:00 2001 From: Dieter Holvoet Date: Fri, 20 Nov 2020 12:02:49 +0100 Subject: [PATCH 1/4] Make StorageInterface::getByTags return full cache objects --- src/Storage/Database.php | 19 +++++++++++++++---- src/Storage/StorageInterface.php | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/Storage/Database.php b/src/Storage/Database.php index a3bdc05..1ff9310 100644 --- a/src/Storage/Database.php +++ b/src/Storage/Database.php @@ -127,7 +127,7 @@ public function getExpired($amount) return $q->execute()->fetchAll(\PDO::FETCH_COLUMN); } - public function getByTags(array $tags) + public function getByTags(array $tags): array { if (!$tags) { return []; @@ -142,13 +142,24 @@ public function getByTags(array $tags) } $q = $this->db->select(self::TABLE_ENTRIES, 'c') - ->fields('c', ['id']); + ->fields('c', []); $q->condition('c.expiry', time(), '>='); $q->innerJoin(self::TABLE_TAGS, 't', 't.id = c.id'); $q->condition('t.tag', $tags, 'IN'); - $ids = $q->execute()->fetchAll(\PDO::FETCH_COLUMN); - return $ids; + return array_map( + static function (array $data) { + return new Cache( + $data['id'], + $data['uri'], + $data['method'], + $data['content'], + unserialize($data['headers'], ['allowed_classes' => false]), + $data['expiry'] + ); + }, + $q->execute()->fetchAll(\PDO::FETCH_ASSOC) + ); } public function remove(array $ids) diff --git a/src/Storage/StorageInterface.php b/src/Storage/StorageInterface.php index c9a4495..574074a 100644 --- a/src/Storage/StorageInterface.php +++ b/src/Storage/StorageInterface.php @@ -36,9 +36,9 @@ public function set(Cache $item, array $tags); * * @param string[] $tags * - * @return string[] The cache ids + * @return Cache[] The cache ids */ - public function getByTags(array $tags); + public function getByTags(array $tags): array; /** * Remove expired items from storage. From 904694cca024f03340fc5593c4f407bd398a1dc0 Mon Sep 17 00:00:00 2001 From: Dieter Holvoet Date: Fri, 20 Nov 2020 12:04:02 +0100 Subject: [PATCH 2/4] WIP: Cache warmup using queue --- src/CacheHeater.php | 31 +++++++++++ src/CacheHeaterInterface.php | 11 ++++ src/Invalidator.php | 20 +++++-- .../QueueWorker/CacheHeaterQueueWorker.php | 53 +++++++++++++++++++ wmpage_cache.services.yml | 17 ++++++ 5 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 src/CacheHeater.php create mode 100644 src/CacheHeaterInterface.php create mode 100644 src/Plugin/QueueWorker/CacheHeaterQueueWorker.php diff --git a/src/CacheHeater.php b/src/CacheHeater.php new file mode 100644 index 0000000..0aca931 --- /dev/null +++ b/src/CacheHeater.php @@ -0,0 +1,31 @@ +queueFactory = $queueFactory; + } + + public function warmup(array $entries): void + { + $queue = $this->queueFactory->get(CacheHeaterQueueWorker::ID); + + foreach ($entries as $entry) { + $queue->createItem($entry->getUri()); + } + } + + public function warmupAll(): void + { + } +} diff --git a/src/CacheHeaterInterface.php b/src/CacheHeaterInterface.php new file mode 100644 index 0000000..57264b4 --- /dev/null +++ b/src/CacheHeaterInterface.php @@ -0,0 +1,11 @@ +storage = $storage; + $this->heater = $heater; } public function invalidateCacheTags(array $tags) { - $this->storage->remove( - $this->storage->getByTags($tags) + $entries = $this->storage->getByTags($tags); + $ids = array_map( + function (Cache $entry) { return $entry->getId(); }, + $entries ); + + $this->storage->remove($ids); + $this->heater->warmup($entries); } } diff --git a/src/Plugin/QueueWorker/CacheHeaterQueueWorker.php b/src/Plugin/QueueWorker/CacheHeaterQueueWorker.php new file mode 100644 index 0000000..c6ca03e --- /dev/null +++ b/src/Plugin/QueueWorker/CacheHeaterQueueWorker.php @@ -0,0 +1,53 @@ +client = $container->get('wmpage_cache.heater.client'); + $instance->hostName = $container->getParameter('wmpage_cache.heater.host_name'); + + return $instance; + } + + public function processItem($uri) + { + try { + $this->client->get($uri, [ + 'headers' => [ + // @see https://github.com/guzzle/guzzle/issues/1297 + 'Host' => $this->hostName, + ], + ]); + } catch (GuzzleException $e) { + watchdog_exception('wmpage_cache.heater', $e); + } + } +} diff --git a/wmpage_cache.services.yml b/wmpage_cache.services.yml index 500741a..0c40eb7 100644 --- a/wmpage_cache.services.yml +++ b/wmpage_cache.services.yml @@ -119,6 +119,9 @@ parameters: # use a CDN that charges for invalidations. Set this number much lower. wmpage_cache.max_purges_per_invalidation: 100000 + wmpage_cache.heater.host_name: '' + wmpage_cache.heater.base_url: 'https://localhost' + services: wmpage_cache.storage: class: Drupal\wmpage_cache\Storage\StorageInterface @@ -219,6 +222,7 @@ services: class: Drupal\wmpage_cache\Invalidator arguments: - '@wmpage_cache.storage' + - '@wmpage_cache.heater' wmpage_cache.maxage.default: class: Drupal\wmpage_cache\MaxAgeDecider @@ -255,3 +259,16 @@ services: class: Drupal\wmpage_cache\GlobalCacheableMetadata arguments: - '@renderer' + + wmpage_cache.heater: + class: Drupal\wmpage_cache\CacheHeater + arguments: + - '@queue' + + wmpage_cache.heater.client: + class: GuzzleHttp\Client + arguments: + - base_uri: '%wmpage_cache.heater.base_url%' + headers: + host: '%wmpage_cache.heater.host_name%' + verify: false From 27f2f91f5642d89fd5c69202230296c35b6fae51 Mon Sep 17 00:00:00 2001 From: Dieter Holvoet Date: Fri, 20 Nov 2020 18:45:04 +0100 Subject: [PATCH 3/4] Warm up all configured pages after a cache flush --- src/CacheHeater.php | 54 ++++++++++++++++++++++++++++++++++++++- src/Manager.php | 5 ++++ wmpage_cache.services.yml | 10 ++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/CacheHeater.php b/src/CacheHeater.php index 0aca931..35c45fd 100644 --- a/src/CacheHeater.php +++ b/src/CacheHeater.php @@ -2,18 +2,29 @@ namespace Drupal\wmpage_cache; +use Drupal\Core\Entity\EntityTypeManagerInterface; use Drupal\Core\Queue\QueueFactory; +use Drupal\Core\Url; +use Drupal\user\Entity\User; use Drupal\wmpage_cache\Plugin\QueueWorker\CacheHeaterQueueWorker; class CacheHeater implements CacheHeaterInterface { + /** @var EntityTypeManagerInterface */ + protected $entityTypeManager; /** @var QueueFactory */ protected $queueFactory; + /** @var array */ + protected $includedPages; public function __construct( - QueueFactory $queueFactory + EntityTypeManagerInterface $entityTypeManager, + QueueFactory $queueFactory, + array $includedPages ) { + $this->entityTypeManager = $entityTypeManager; $this->queueFactory = $queueFactory; + $this->includedPages = $includedPages; } public function warmup(array $entries): void @@ -27,5 +38,46 @@ public function warmup(array $entries): void public function warmupAll(): void { + $urls = []; + + foreach ($this->includedPages['entities'] ?? [] as $entityTypeId => $bundles) { + $definition = $this->entityTypeManager->getDefinition($entityTypeId); + + if (!$definition->hasLinkTemplate('canonical')) { + throw new \InvalidArgumentException( + "Entity type with id ${$entityTypeId} does not have a canonical route" + ); + } + + $query = $this->entityTypeManager + ->getStorage($entityTypeId) + ->getQuery(); + + if (!empty($bundles)) { + $query->condition($definition->getKey('bundle'), $bundles, 'IN'); + } + + foreach ($query->execute() as $id) { + $urls[] = Url::fromRoute( + "entity.${entityTypeId}.canonical", + [$entityTypeId => $id] + ); + } + } + + foreach ($this->includedPages['routes'] as $routeName => $routeParameters) { + $urls[] = Url::fromRoute($routeName, $routeParameters); + } + + $queue = $this->queueFactory->get(CacheHeaterQueueWorker::ID); + $anonymousUser = User::load(0); + + foreach ($urls as $url) { + if (!$url->access($anonymousUser)) { + continue; + } + + $queue->createItem($url->setAbsolute(false)->toString()); + } } } diff --git a/src/Manager.php b/src/Manager.php index 3aa8e24..c2b2310 100644 --- a/src/Manager.php +++ b/src/Manager.php @@ -22,6 +22,8 @@ class Manager implements CacheTagsInvalidatorInterface protected $cacheKeyGenerator; /** @var CacheBuilderInterface */ protected $cacheBuilder; + /** @var CacheHeaterInterface */ + protected $heater; /** @var bool */ protected $storeCache; /** @var bool */ @@ -39,6 +41,7 @@ public function __construct( InvalidatorInterface $invalidator, CacheKeyGeneratorInterface $cacheKeyGenerator, CacheBuilderInterface $cacheBuilder, + CacheHeaterInterface $heater, $storeCache, $storeTags, $maxPurgesPerInvalidation, @@ -50,6 +53,7 @@ public function __construct( $this->invalidator = $invalidator; $this->cacheKeyGenerator = $cacheKeyGenerator; $this->cacheBuilder = $cacheBuilder; + $this->heater = $heater; $this->storeCache = $storeCache && $storeTags; $this->storeTags = $storeTags; $this->maxPurgesPerInvalidation = $maxPurgesPerInvalidation; @@ -112,6 +116,7 @@ public function invalidateTags(array $tags) foreach ($this->flushTriggerTags as $re) { if (preg_match('#' . $re . '#', $tag)) { $this->storage->flush(); + $this->heater->warmupAll(); return; } } diff --git a/wmpage_cache.services.yml b/wmpage_cache.services.yml index 0c40eb7..7d79651 100644 --- a/wmpage_cache.services.yml +++ b/wmpage_cache.services.yml @@ -121,6 +121,13 @@ parameters: wmpage_cache.heater.host_name: '' wmpage_cache.heater.base_url: 'https://localhost' + wmpage_cache.heater.included_pages: + entities: + node: { } + routes: + my_module.custom_route: + route_parameter_key: route_parameter_value + my_module.other_custom_route: { } services: wmpage_cache.storage: @@ -179,6 +186,7 @@ services: - '@wmpage_cache.invalidator' - '@wmpage_cache.keygenerator' - '@wmpage_cache.builder' + - '@wmpage_cache.heater' - '%wmpage_cache.store%' - '%wmpage_cache.tags%' - '%wmpage_cache.max_purges_per_invalidation%' @@ -263,7 +271,9 @@ services: wmpage_cache.heater: class: Drupal\wmpage_cache\CacheHeater arguments: + - '@entity_type.manager' - '@queue' + - '%wmpage_cache.heater.included_pages%' wmpage_cache.heater.client: class: GuzzleHttp\Client From fbd9290c3a26bda710288586f14cc18d01747b0c Mon Sep 17 00:00:00 2001 From: Dieter Holvoet Date: Fri, 20 Nov 2020 18:45:22 +0100 Subject: [PATCH 4/4] Add a command to warm up all configured pages after a cache flush --- drush.services.yml | 9 ++++++++- src/Commands/CacheWarmupCommands.php | 29 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 src/Commands/CacheWarmupCommands.php diff --git a/drush.services.yml b/drush.services.yml index af95866..e9bc7fb 100644 --- a/drush.services.yml +++ b/drush.services.yml @@ -1,5 +1,12 @@ services: wmpage_cache.commands.cache-clear: class: Drupal\wmpage_cache\Commands\CacheClearCommands - arguments: ['@wmpage_cache.storage'] tags: [{ name: drush.command }] + arguments: + - '@wmpage_cache.storage' + + wmpage_cache.commands.cache-warmup: + class: Drupal\wmpage_cache\Commands\CacheWarmupCommands + tags: [{ name: drush.command }] + arguments: + - '@wmpage_cache.heater' diff --git a/src/Commands/CacheWarmupCommands.php b/src/Commands/CacheWarmupCommands.php new file mode 100644 index 0000000..554801d --- /dev/null +++ b/src/Commands/CacheWarmupCommands.php @@ -0,0 +1,29 @@ +heater = $heater; + } + + /** + * Warm up the page cache of all configured pages + * + * @command wmpage_cache:warmup + * @aliases wmpage-cache-warmup,wmpcw + */ + public function cacheWarmup(): void + { + $this->heater->warmupAll(); + } +}