User-Agent: * Disallow: /admin/ # disable crawling django admin Disallow: /cms/ # and wagtail admin Disallow: /cite-check/r/ # cite check reports: the unguessable URL is the capability Disallow: /newsletter/confirm/ # newsletter capability URLs: tokens are the identity Disallow: /newsletter/manage/ Disallow: /newsletter/u/ Sitemap: https://semantis.ai/sitemap.xml