#                                                    
#                ..;coxkOOOOOOkxoc;'.                
#             .:d0NWMMMMMMMMMMMMMMWN0xc'             
#          .:kXMMMMMMMMMMMMMMMMMMMMMMMXl.            
#        .c0WMMMMMMMMMMMMMMMMMMMMMMMXd'              
#       ,OWMMMMMMMMMMMMMMMMMMMMMMMXo'       ..       
#      cXMMMMMMXo::::::::::::::col.       .lKXl.     
#     lNMMMMMMM0'                       .lKWMMNo     
#    :XMMMMMMMM0'                     .l0WMMMMMNc    
#   .OMMMMMMMMM0'     .ccccccc;.      ,KMMMMMMMMO.   
#   :NMMMMMMMMM0'     oWMMMMMMWKc.     oWMMMMMMMN:   
#   lWMMMMMMMMM0'     oWMMMMMMMMX:     ,KMMMMMMMMo   
#   oMMMMMMMMMM0'     oWMMMMMMMMNc     ,KMMMMMMMMd   
#   cNMMMMMMMMM0'     oWMMMMMMMNd.     lWMMMMMMMWl   
#   '0MMMMMMMMWk.     ,oooooooc'      ,0MMMMMMMMK,   
#    oWMMMMMMXo.                     ,0MMMMMMMMWo    
#    .xWMMMXd'                     ,dXMMMMMMMMWk.    
#     .xWNx'       .',''''''',,;coONMMMMMMMMMWk.     
#      .:,       .l0WWWWWWWWWWWMMMMMMMMMMMMMNd.      
#              .lKWMMMMMMMMMMMMMMMMMMMMMMMWk;        
#            .lKWMMMMMMMMMMMMMMMMMMMMMMMNk;.         
#            .ckXWMMMMMMMMMMMMMMMMMMWXkl'            
#               .;ldO0XNWWWWWWNXKOxl;.               
#                    ..'',,,,''..                    
#                                                    
#
User-agent: Swiftbot
Disallow:
Crawl-delay: 2

User-agent: *
Disallow: /admin/
Disallow: /trending/tweet/
Disallow: /newsletter/
Disallow: /healthcheck/
Disallow: /subpage/
Disallow: /ckeditor/
Disallow: /feeds/search/
Disallow: /jobs/edit/
Disallow: /wishfeedback/
Disallow: /api/
Disallow: /static/images/

# no deep queries to search
Disallow: /search/*
Allow: /search/$

# don't index our dynamic images
Disallow: /user_media/
Allow: /google_news_sitemap.xml

User-agent: Twitterbot
Disallow:

# Keep google images from indexing entire site
User-agent: Googlebot-Image
Disallow: /

# Facebook crawler user-agent
User-agent: facebookexternalhit/1.1
Disallow:

# trend reports
User-agent: Googlebot
Disallow: /news/*/541893/