Index: openacs-4/www/robots.txt
===================================================================
RCS file: /usr/local/cvsroot/openacs-4/www/robots.txt,v
diff -u -r1.1.4.1.2.1 -r1.1.4.1.2.2
--- openacs-4/www/robots.txt	1 Dec 2004 10:42:08 -0000	1.1.4.1.2.1
+++ openacs-4/www/robots.txt	18 May 2005 10:46:42 -0000	1.1.4.1.2.2
@@ -1,7 +1,5 @@
 User-agent: *
-Disallow: /
-Disallow: /doc/
-Disallow: /api-doc/
+Disallow: /register
 Disallow: /register/
 Disallow: /comments/
 Disallow: /notifications/
@@ -12,5 +10,17 @@
 Disallow: /acs-admin/
 Disallow: /RPC2/
 
-
-
+# googlebot knows wildcards so block all register
+User-agent: googlebot
+Disallow: /register
+Disallow: /register/
+Disallow: /*/register
+Disallow: /*/register/
+Disallow: /comments/
+Disallow: /notifications/
+Disallow: /test/
+Disallow: /workflow/
+Disallow: /acs-service-contract/
+Disallow: /acs-lang/
+Disallow: /acs-admin/
+Disallow: /RPC2/