Sitemap: /sitemap.xml.gz

# Define access-restrictions for robots/spiders
# http://www.robotstxt.org/wc/norobots.html

# By default we allow robots to access all areas of our site 
# already accessible to anonymous users

# Yandex bot
User-agent: Yandex
Disallow: /

# Dicoveryengine.com
User-agent: discobot
Disallow: / 

# Blekkobot
User-agent: Blekkobot
Disallow: /

# SEOkicks
User-agent: SEOkicks
Disallow: /

# Exabot
User-agent: Exabot
Disallow: /

# Google Image bot
User-agent: Googlebot-Image
Disallow: /*image* 
Disallow: /*leadImage* 

User-agent: *
Crawl-delay: 5
Disallow: /nonmobile
Disallow: /search
Disallow: /search_rss
Disallow: /m/
Disallow: /nonmobile
Disallow: /sandbox
Disallow: /intranet
Disallow: /private
Disallow: /*ics_view$
Disallow: /*vcs_view$
Disallow: /*sendto_form$
Disallow: /*folder_factories$
Disallow: /*folder_contents$
Disallow: /*?searchterm=*
Disallow: /*image_view_fullscreen$
Disallow: /*newsletter_print
Disallow: /*/order$
Disallow: /4-h/horses/cloverbud-program/cloverbud-activity-book-a
Disallow: /4-h/horses/program-information/advisory-committee-minutes-april-2007
Disallow: /4-h/horses/program-information/development-committee-minutes-november-2006
Disallow: /about/history/distinguished-alumni-dairy-science
Disallow: /about/history/history.pdf
Disallow: /about/history/horses
Disallow: /alumni/archives/hoofprints/articles/hoofprints-200806-07
Disallow: /alumni/archives/hoofprints/articles/hoofprints-200912-04
Disallow: /alumni/archives/hoofprints/articles/hoofprints-201012-02
Disallow: /alumni/archives/hoofprints/pdf/hoofprints-200512.pdf
Disallow: /alumni/archives/hoofprints/pdf/hoofprints-200606.pdf
Disallow: /alumni/archives/hoofprints/pdf/hoofprints-200912.pdf
Disallow: /alumni/archives/poultry-science-update/2002-poultry-alumni-newsletter.pdf
Disallow: /alumni/archives/poultry-science-update/2004-poultry-alumni-newsletter.pdf
Disallow: /events/seminars/jerry-bouma-tbd
Disallow: /files/pdf/cellular-localization-and-function.pdf
Disallow: /files/pdf/cellular-localization.pdf
Disallow: /files/pdf/fall-2014-undergraduate-handbook
Disallow: /files/pdf/rainbowbookweb1.pdf
Disallow: /files/pdf/rainbowbookweb4.09.pdf
Disallow: /files/pdf/regulation-of-interferon-stimulated-genes.pdf
Disallow: /images
Disallow: /news/2008/13-states-have-topped-the-50-percent-premises-registrations-pennsylvania-is-one
Disallow: /news/2012/penn-state-das-students-win-quiz-bowl-at-national-academic-quadrathlon
Disallow: /news/2013/penn-state-animal-science-students-take-beef-issues-to-washington
Disallow: /news/2013/penn-state-animal-science-students-top-national-beef-quiz-bowl
Disallow: /news/2014/penn-states-extension-livestock-team-offers-beef-producer-webinars
Disallow: /research/centers/crbh/related-links/colorado-state-university-animal-reproduction-and-biotechnology-laboratory
Disallow: /students/teams/livestock-judging/livestock-judging