diff --git a/robots.txt b/robots.txt index 7b725b6..0543ee9 100644 --- a/robots.txt +++ b/robots.txt @@ -18,6 +18,8 @@ Disallow: /*.tar.gz$ Disallow: /metrics Disallow: /*/*/blame/* Disallow: /*/*/log/* +Disallow: /*/*/tree/* +Disallow: /*/*/item/* # Too aggressive, marketing/SEO User-agent: SemrushBot