From 478f78ad873636aad430a9e39a45a325446f6108 Mon Sep 17 00:00:00 2001 From: Alex Tharp Date: Sat, 1 Jul 2017 14:18:45 -0500 Subject: [PATCH 1/4] fix(Webpage Feed): try out regex solution for agnostic URLs --- app/interactors/get_webpage_feed.rb | 10 +--------- app/models/webpage.rb | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/app/interactors/get_webpage_feed.rb b/app/interactors/get_webpage_feed.rb index 294b55456..f92bec10f 100644 --- a/app/interactors/get_webpage_feed.rb +++ b/app/interactors/get_webpage_feed.rb @@ -4,15 +4,7 @@ class GetWebpageFeed def call webpage = ::Webpage webpage = webpage.find_by_tenant_id(context.tenant) if context.tenant - webpage = webpage.find_by_protocol_agnostic_url(protocol_agnostic_url(context.params.url)).first + webpage = webpage.agnostic_find_by_url(context.params.url).first context.webpage = webpage end - - private - - def protocol_agnostic_url(url) - uri = Addressable::URI.parse(url) - path = uri.path == '/' ? uri.path : uri.path.chomp('/') - "://#{uri.authority}#{path}" - end end diff --git a/app/models/webpage.rb b/app/models/webpage.rb index 6c815ec61..26faaf88c 100644 --- a/app/models/webpage.rb +++ b/app/models/webpage.rb @@ -5,7 +5,7 @@ class Webpage < ApplicationRecord serialize :tables_widget serialize :charts_widget - scope :find_by_protocol_agnostic_url, ->(suffix) { where('url LIKE :suffix', suffix: "%#{suffix}") } + scope :agnostic_find_by_url, ->(url) { where('url ~* ?', "(?:[-\w]+\.)+[-\w]+(?::\d+)?(?:\/[^#?]*)") } acts_as_paranoid acts_as_taggable_on :seo_keywords From 4b5ff8658606d00c31bc1bf325a29cbde8eb505c Mon Sep 17 00:00:00 2001 From: Alex Tharp Date: Sat, 1 Jul 2017 14:34:32 -0500 Subject: [PATCH 2/4] fix(Webpage Feed): solve agnostic URL issue by 'guessing' a few possible URLs, then enumerating through each to find an exact match --- app/interactors/get_webpage_feed.rb | 11 ++++++++++- app/models/webpage.rb | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/app/interactors/get_webpage_feed.rb b/app/interactors/get_webpage_feed.rb index f92bec10f..4c50a76e9 100644 --- a/app/interactors/get_webpage_feed.rb +++ b/app/interactors/get_webpage_feed.rb @@ -4,7 +4,16 @@ class GetWebpageFeed def call webpage = ::Webpage webpage = webpage.find_by_tenant_id(context.tenant) if context.tenant - webpage = webpage.agnostic_find_by_url(context.params.url).first + webpage = webpage.agnostic_find_by_url(context.params.url) + context.webpage = webpage end + + private + + def protocol_agnostic_url(url) + uri = Addressable::URI.parse(url) + path = uri.path == '/' ? uri.path : uri.path.chomp('/') + "://#{uri.authority}#{path}" + end end diff --git a/app/models/webpage.rb b/app/models/webpage.rb index 26faaf88c..f6ef6a42a 100644 --- a/app/models/webpage.rb +++ b/app/models/webpage.rb @@ -5,7 +5,7 @@ class Webpage < ApplicationRecord serialize :tables_widget serialize :charts_widget - scope :agnostic_find_by_url, ->(url) { where('url ~* ?', "(?:[-\w]+\.)+[-\w]+(?::\d+)?(?:\/[^#?]*)") } + scope :agnostic_guess_by_url, ->(url) { where('url LIKE :url', url: "%#{url}%") } acts_as_paranoid acts_as_taggable_on :seo_keywords @@ -16,6 +16,11 @@ class Webpage < ApplicationRecord accepts_nested_attributes_for :snippets + def self.agnostic_find_by_url(url) + url = protocol_agnostic_url url + agnostic_guess_by_url.find { |webpage| protocol_agnostic_url(webpage.url) == url } + end + def tables_widget_yaml tables_widget.to_yaml end @@ -63,4 +68,12 @@ def accordion_group_widget_json def accordion_group_widget_json= p self.accordion_group_widget = JSON.parse(p, quirks_mode: true) # Quirks mode will let us parse a null JSON object end + + private + + def self.protocol_agnostic_url(url) + uri = Addressable::URI.parse(url) + path = uri.path == '/' ? uri.path : uri.path.chomp('/') + "://#{uri.authority}#{path}" + end end From b681e009817d446b8fa108586ec3c696142bde1e Mon Sep 17 00:00:00 2001 From: Alex Tharp Date: Sun, 2 Jul 2017 15:33:11 -0500 Subject: [PATCH 3/4] fix(Semaphore): remove manual ElasticSearch install step, opting instead for Semaphore's default ES install --- semaphore_ci/setup.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/semaphore_ci/setup.sh b/semaphore_ci/setup.sh index ab8f4123b..9676e380b 100755 --- a/semaphore_ci/setup.sh +++ b/semaphore_ci/setup.sh @@ -16,18 +16,6 @@ bundle exec rake bower:install:development echo "bundle exec rake webpack:ensure_assets_compiled" bundle exec rake webpack:ensure_assets_compiled -echo "ES Install" -sudo service elasticsearch stop -if ! [ -e .semaphore-cache/elasticsearch-2.4.1.deb ]; then (cd .semaphore-cache; curl -OL https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-2.4.1.deb); fi -sudo dpkg -i --force-confnew .semaphore-cache/elasticsearch-2.4.1.deb -sudo service elasticsearch start - -echo "sleep 5" -sleep 5 - -echo "ES Version Check" -curl -XGET 'http://localhost:9200' - echo "bundle exec rake db:setup" bundle exec rake db:setup From 4a1fc9bf83e681357cab5cec631a3c6e1bf8a0bb Mon Sep 17 00:00:00 2001 From: Alex Tharp Date: Mon, 3 Jul 2017 00:50:53 -0500 Subject: [PATCH 4/4] fix(Webpage): invalid params for agnostic_guess_by_url usage --- app/models/webpage.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/webpage.rb b/app/models/webpage.rb index f6ef6a42a..26124304e 100644 --- a/app/models/webpage.rb +++ b/app/models/webpage.rb @@ -17,8 +17,8 @@ class Webpage < ApplicationRecord accepts_nested_attributes_for :snippets def self.agnostic_find_by_url(url) - url = protocol_agnostic_url url - agnostic_guess_by_url.find { |webpage| protocol_agnostic_url(webpage.url) == url } + url = protocol_agnostic_url(url) + agnostic_guess_by_url(url).find { |webpage| protocol_agnostic_url(webpage.url) == url } end def tables_widget_yaml