feat: add crawling suggestions for robots
1 file changed, 4 insertions(+), 0 deletions(-)
changed files
A frontend/static/robots.txt
@@ -0,0 +1,4 @@ +User-Agent: * +Disallow: /man/ +Disallow: /*?*page=0 +Crawl-Delay: 2