# disallow all 
User-agent: *
Disallow: /
# set the crawl delay 
Crawl-delay: 600
Sitemap: http://www.iwmi.cgiar.org/page-sitemap.xml
Sitemap: http://www.iwmi.cgiar.org/post-sitemap.xml
# but allow only important bots
User-agent: Googlebot
Crawl-delay: 600
User-agent: Googlebot-Image
Crawl-delay: 600
User-agent: Googlebot-Video
Crawl-delay: 600
User-agent: Googlebot-News
Crawl-delay: 600
User-agent: Mediapartners-Google
Crawl-delay: 600
User-agent: bingbot
Crawl-delay: 600
User-agent: bing
Crawl-delay: 600
User-agent: msnbot
Crawl-delay: 600
User-agent: msnbot-media
Crawl-delay: 600
User-agent: Slurp
Crawl-delay: 600
User-agent: Yahoo-Blogs
Crawl-delay: 600
User-agent: Yahoo-MMCrawler
Crawl-delay: 600
User-agent: DuckDuckBot
Crawl-delay: 600
User-agent: YandexBot
Crawl-delay: 600
User-agent: Exabot
Crawl-delay: 600
User-agent: facebot
Crawl-delay: 600
User-agent: facebookexternalhit
Crawl-delay: 600
User-agent: ia_archiver
Crawl-delay: 600
# Index these directories
Allow: /Publications/*/
Allow: /wp-content/images/*/
Allow: /wp-content/uploads/*/
Allow: /about/staff-list/*/
Allow: /wp-includes/*/
# Do not index directories
Disallow: /wp-admin/
Disallow: /wp-content/mu-plugins/
Disallow: /wp-content/upgrade/
Disallow: /events/
Disallow: /flipcards/
Disallow: /search-results/
Disallow: /search_gcse/
# Files
Disallow: /wp-login.php
Disallow: /wp-cron.php
Disallow: /xmlrpc.php