#robots.txt file
#Created 08/15/2024
#Chris Carpenter

#A robots.txt file tells search engine crawlers which URLs the crawler can access on your site. This is used mainly to avoid overloading your site with requests; it is not a mechanism for keeping a web page out of Google. 

# Ensure that this file is always at the root. This file will be ignored within any other directory.

User-agent: *

Sitemap: https://www.com.edu/sitemap.xml

#Sections not allowed
Disallow: /_dev
Disallow: /_fetcher
Disallow: /_request
Disallow: /_resources/data
Disallow: /_resources/dmc
Disallow: /_resources/email
Disallow: /_resources/fonts
Disallow: /_resources/includes
Disallow: /_resources/js
Disallow: /_resources/ldp
Disallow: /_resources/ou
Disallow: /_resources/php
Disallow: /_resources/search
Disallow: /_resources/xls
Disallow: /_resources/xls-62600
Disallow: /2020
Disallow: */_props.html$