add datasets wildcard to robots disallow list

nih-sparc · Sep 30, 2024 · c4d1788 · c4d1788
1 parent c4e6bb0
commit c4d1788
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/nuxt.config.ts b/nuxt.config.ts
@@ -226,6 +226,7 @@ export default defineNuxtConfig({
     // provide simple disallow rules for all robots `user-agent: *`
     // disallowing certain pages that are either redirects, authticated routes, or causing bots to recursively crawl
     disallow: [
+      '/datasets/*?*',
       '/welcome', 
       '/user', 
       '/contact-us',