# added Drupal robots.txt rules to bottom of this file - TJY 18mar2009
User-agent: *
Disallow: /cgi-bin

User-agent: *
Crawl-delay: 10

User-Agent: *
#Disallow: /jparc/
Disallow: /internal/
Disallow: /pi/nobots/
Disallow: /pi/technical/
Disallow: /pi/explorer/
Disallow: /web_aggregates/
Disallow: /pi/image/
Disallow: /glopad/images/
#Disallow: /pi/fr/
Disallow: /internal/
Disallow: /pi/en/nobots/
Disallow: /pi/en/explorer/
Disallow: /pi/en/image/
Disallow: /pi/en/record/bibliography/
Disallow: /pi/en/search_results.php
Disallow: /pi/ru/nobots/
Disallow: /pi/ru/explorer/
Disallow: /pi/ru/image/
Disallow: /pi/ru/record/bibliography/
Disallow: /pi/ru/search_results.php
Disallow: /pi/ja/nobots/
Disallow: /pi/ja/explorer/
Disallow: /pi/ja/image/
Disallow: /pi/ja/record/bibliography/
Disallow: /pi/ja/search_results.php
Disallow: /pi/de/nobots/
Disallow: /pi/de/explorer/
Disallow: /pi/de/image/
Disallow: /pi/de/record/bibliography/
Disallow: /pi/de/search_results.php
Disallow: /pi/zh/nobots/
Disallow: /pi/zh/explorer/
Disallow: /pi/zh/image/
Disallow: /pi/zh/record/bibliography/
Disallow: /pi/zh/search_results.php
Disallow: /pi/fr/nobots/
Disallow: /pi/fr/explorer/
Disallow: /pi/fr/image/
Disallow: /pi/fr/record/bibliography/
Disallow: /pi/fr/search_results.php

# robots.txt for jparc - from Drupal 5 installation 
# added 18mar2009 by TJY when opened jparc to crawls
# see http://tips.webdesign10.com/robots-txt-and-drupal for extra rules
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt (thus copied contents from /jparc/robots.txt) 
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/wc/robots.html
#
# For syntax checking, see:
# http://www.sxw.org.uk/computing/robots/check.html

# Directories
Disallow: /jparc/database/
Disallow: /jparc/includes/
Disallow: /jparc/misc/
Disallow: /jparc/modules/
Disallow: /jparc/sites/
Disallow: /jparc/themes/
Disallow: /jparc/scripts/
Disallow: /jparc/updates/
Disallow: /jparc/profiles/
# Files
Disallow: /jparc/xmlrpc.php
Disallow: /jparc/cron.php
Disallow: /jparc/update.php
Disallow: /jparc/install.php
Disallow: /jparc/INSTALL.mysql.txt
Disallow: /jparc/INSTALL.pgsql.txt
Disallow: /jparc/CHANGELOG.txt
Disallow: /jparc/MAINTAINERS.txt
Disallow: /jparc/LICENSE.txt
Disallow: /jparc/UPGRADE.txt
# Paths (clean URLs)
Disallow: /jparc/en/admin/
Disallow: /jparc/en/aggregator
Disallow: /jparc/en/comment/reply/
Disallow: /jparc/en/contact
Disallow: /jparc/en/logout
Disallow: /jparc/en/node/add
Disallow: /jparc/en/search/
Disallow: /jparc/en/user/register
Disallow: /jparc/en/user/password
Disallow: /jparc/en/user/login
Disallow: /jparc/en/print/
Disallow: /jparc/en/glossary/
Disallow: /jparc/en/glossary
Disallow: /jparc/en/taxonomy/
# Paths (no clean URLs)
Disallow: /jparc/?q=en/admin/
Disallow: /jparc/?q=en/aggregator
Disallow: /jparc/?q=en/comment/reply/
Disallow: /jparc/?q=en/contact
Disallow: /jparc/?q=en/logout
Disallow: /jparc/?q=en/node/add
Disallow: /jparc/?q=en/search/
Disallow: /jparc/?q=en/user/password
Disallow: /jparc/?q=en/user/register
Disallow: /jparc/?q=en/user/login
# added disallows per modules
Disallow: /jparc/?q=en/node$
Disallow: /jparc/?q=en/user
Disallow: /jparc/?q=en/*sort=
Disallow: /jparc/?q=en/search$
Disallow: /jparc/?q=en/*/feed$
Disallow: /jparc/?q=en/*/track$
Disallow: /jparc/?q=en/tracker?
Disallow: /jparc/?q=en/print/
Disallow: /jparc/?q=en/glossary/
Disallow: /jparc/?q=en/glossary$
Disallow: /jparc/?q=en/taxonomy/

# Japanese Paths (clean URLs)
Disallow: /jparc/ja/admin/
Disallow: /jparc/ja/aggregator
Disallow: /jparc/ja/comment/reply/
Disallow: /jparc/ja/contact
Disallow: /jparc/ja/logout
Disallow: /jparc/ja/node/add
Disallow: /jparc/ja/search/
Disallow: /jparc/ja/user/register
Disallow: /jparc/ja/user/password
Disallow: /jparc/ja/user/login
Disallow: /jparc/ja/print/
Disallow: /jparc/ja/glossary/
Disallow: /jparc/ja/glossary$
Disallow: /jparc/ja/taxonomy/
# Japanese Paths (no clean URLs)
Disallow: /jparc/?q=ja/admin/
Disallow: /jparc/?q=ja/aggregator
Disallow: /jparc/?q=ja/comment/reply/
Disallow: /jparc/?q=ja/contact
Disallow: /jparc/?q=ja/logout
Disallow: /jparc/?q=ja/node/add
Disallow: /jparc/?q=ja/search/
Disallow: /jparc/?q=ja/user/password
Disallow: /jparc/?q=ja/user/register
Disallow: /jparc/?q=ja/user/login
# Japanese added disallows per modules
Disallow: /jparc/?q=ja/node$
Disallow: /jparc/?q=ja/user
Disallow: /jparc/?q=ja/*sort=
Disallow: /jparc/?q=ja/search$
Disallow: /jparc/?q=ja/*/feed$
Disallow: /jparc/?q=ja/*/track$
Disallow: /jparc/?q=ja/tracker?
Disallow: /jparc/?q=ja/print/
Disallow: /jparc/?q=ja/glossary/
Disallow: /jparc/?q=ja/glossary$
Disallow: /jparc/?q=ja/taxonomy/

#End jparc portion 

Allow: /$