From 80dec99ed5239f37ce14c9c47f5e7b5a1e395c9c Mon Sep 17 00:00:00 2001 From: Alex Tymchenko Date: Mon, 18 Mar 2024 17:17:50 +0000 Subject: [PATCH 1/3] Exclude `/dart/` subtree from crawling. --- robots.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/robots.txt b/robots.txt index 19cb6906..8661a56c 100644 --- a/robots.txt +++ b/robots.txt @@ -6,4 +6,5 @@ User-agent: * {% for node in site.pages %}{% if node.noindex %}{% assign isset = true %}Disallow: {{ node.url }} {% endif %}{% endfor %}{% if isset != true %}Disallow: {% endif %} +Disallow: /dart/ Sitemap: {{site.url}}{{ site.baseurl }}/sitemap.xml From b6d0c9b53e4bfcb6269c7652dfb73db813c94334 Mon Sep 17 00:00:00 2001 From: Alex Tymchenko Date: Mon, 18 Mar 2024 17:41:36 +0000 Subject: [PATCH 2/3] Remove the default `Disallow: ` entry to avoid any confusion, as we now have an explicit `Disallow: /dart/`. --- robots.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/robots.txt b/robots.txt index 8661a56c..be7b1cc3 100644 --- a/robots.txt +++ b/robots.txt @@ -4,7 +4,6 @@ layout: null User-agent: * {% for node in site.pages %}{% if node.noindex %}{% assign isset = true %}Disallow: {{ node.url }} -{% endif %}{% endfor %}{% if isset != true %}Disallow: -{% endif %} +{% endif %}{% endfor %} Disallow: /dart/ Sitemap: {{site.url}}{{ site.baseurl }}/sitemap.xml From c6fd5122144dfcad6b3fd51bc75cd771bba027c6 Mon Sep 17 00:00:00 2001 From: Alex Tymchenko Date: Mon, 18 Mar 2024 17:42:34 +0000 Subject: [PATCH 3/3] Move `Disallow: /dart/` right under the user-agent. --- robots.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/robots.txt b/robots.txt index be7b1cc3..275c866a 100644 --- a/robots.txt +++ b/robots.txt @@ -3,7 +3,8 @@ layout: null --- User-agent: * +Disallow: /dart/ {% for node in site.pages %}{% if node.noindex %}{% assign isset = true %}Disallow: {{ node.url }} {% endif %}{% endfor %} -Disallow: /dart/ + Sitemap: {{site.url}}{{ site.baseurl }}/sitemap.xml