#
# robots.txt for www.torontopubliclibrary.ca
# Updated: 2015-01-15

# "Allow" and "Crawl-delay" are "non-standard"

# Not supported by all robots, but requests a 10s delay between page loads:
User-agent: *
Crawl-delay: 10

User-agent: *
Disallow: /kidsstop/
Disallow: /eblast/
Disallow: /components/
Disallow: /config/
Disallow: /branch-computer/
Disallow: /research-computer/
Disallow: /kids-computer/
Disallow: /kids-computer-rr/
Disallow: /kiosk/
Disallow: /xml/
Disallow: /placehold
Disallow: /share-item-detail.jsp
Disallow: /rss.jsp

# WebLogic pages:
Disallow: /hou_az_ab.jsp
Disallow: /hou_az_acd.jsp
Disallow: /hou_az_ad.jsp
Disallow: /hou_az_ag.jsp
Disallow: /hou_az_ah.jsp
Disallow: /hou_az_an.jsp
Disallow: /hou_az_ap.jsp
Disallow: /hou_az_bb.jsp
Disallow: /hou_az_bc.jsp
Disallow: /hou_az_bd.jsp
Disallow: /hou_az_be.jsp
Disallow: /hou_az_bf.jsp
Disallow: /hou_az_bl.jsp
Disallow: /hou_az_br.jsp
Disallow: /hou_az_brw.jsp
Disallow: /hou_az_bur.jsp
Disallow: /hou_az_by.jsp
Disallow: /hou_az_cc.jsp
Disallow: /hou_az_ce.jsp
Disallow: /hou_az_ced.jsp
Disallow: /hou_az_ch.jsp
Disallow: /hou_az_cs.jsp
Disallow: /hou_az_da.jsp
Disallow: /hou_az_dm.jsp
Disallow: /hou_az_do.jsp
Disallow: /hou_az_dp.jsp
Disallow: /hou_az_dr.jsp
Disallow: /hou_az_dt.jsp
Disallow: /hou_az_du.jsp
Disallow: /hou_az_ea.jsp
Disallow: /hou_az_eb.jsp
Disallow: /hou_az_eg.jsp
Disallow: /hou_az_es.jsp
Disallow: /hou_az_fh.jsp
Disallow: /hou_az_fp.jsp
Disallow: /hou_az_fv.jsp
Disallow: /hou_az_ge.jsp
Disallow: /hou_az_ghp.jsp
Disallow: /hou_az_gw.jsp
Disallow: /hou_az_hb.jsp
Disallow: /hou_az_hc.jsp
Disallow: /hou_az_hil.jsp
Disallow: /hou_az_hp.jsp
Disallow: /hou_az_hs.jsp
Disallow: /hou_az_hw.jsp
Disallow: /hou_az_index.jsp
Disallow: /hou_az_jd.jsp
Disallow: /hou_az_jo.jsp
Disallow: /hou_az_js.jsp
Disallow: /hou_az_ke.jsp
Disallow: /hou_az_lb.jsp
Disallow: /hou_az_le.jsp
Disallow: /hou_az_lo.jsp
Disallow: /hou_az_ls.jsp
Disallow: /hou_az_ma.jsp
Disallow: /hou_az_mal.jsp
Disallow: /hou_az_mas.jsp
Disallow: /hou_az_mcg.jsp
Disallow: /hou_az_md.jsp
Disallow: /hou_az_mi.jsp
Disallow: /hou_az_mp.jsp
Disallow: /hou_az_mrv.jsp
Disallow: /hou_az_ms.jsp
Disallow: /hou_az_nd.jsp
Disallow: /hou_az_ne.jsp
Disallow: /hou_az_nt.jsp
Disallow: /hou_az_nycl.jsp
Disallow: /hou_az_ov.jsp
Disallow: /hou_az_pa.jsp
Disallow: /hou_az_pe.jsp
Disallow: /hou_az_pk.jsp
Disallow: /hou_az_pl.jsp
Disallow: /hou_az_pm.jsp
Disallow: /hou_az_pu.jsp
Disallow: /hou_az_pv.jsp
Disallow: /hou_az_qs.jsp
Disallow: /hou_az_rd.jsp
Disallow: /hou_az_ri.jsp
Disallow: /hou_az_rn.jsp
Disallow: /hou_az_rx.jsp
Disallow: /hou_az_sa.jsp
Disallow: /hou_az_si.jsp
Disallow: /hou_az_sj.jsp
Disallow: /hou_az_sl.jsp
Disallow: /hou_az_sp.jsp
Disallow: /hou_az_st.jsp
Disallow: /hou_az_sw.jsp
Disallow: /hou_az_sws.jsp
Disallow: /hou_az_ta.jsp
Disallow: /hou_az_th.jsp
Disallow: /hou_az_tod.jsp
Disallow: /hou_az_trl.jsp
Disallow: /hou_az_urb.jsp
Disallow: /hou_az_vv.jsp
Disallow: /hou_az_we.jsp
Disallow: /hou_az_wp.jsp
Disallow: /hou_az_ws.jsp
Disallow: /hou_az_wy.jsp
Disallow: /hou_az_yo.jsp
Disallow: /hou_az_yw.jsp
Disallow: /hou_boo_index.jsp
Disallow: /hou_loc_index.jsp
Disallow: /hou_sun_index.jsp
Disallow: /hel_cat_due.jsp
Disallow: /hel_cat_inf.jsp
Disallow: /hel_cat_man_hld.jsp
Disallow: /hel_cat_missed_holds.jsp
Disallow: /hel_cat_mke_hld.jsp
Disallow: /hel_cat_ren.jsp
Disallow: /hel_cat_sch.jsp
Disallow: /hel_cat_signin.jsp
Disallow: /hel_index.jsp
Disallow: /hel_new_catalogue.jsp
Disallow: /hel_tro_index.jsp
Disallow: /mul_index.jsp
Disallow: /mul_ser_citizenship.jsp
Disallow: /mul_ser_citizenship_fr_test.jsp
Disallow: /mul_ser_citizenship_test.jsp
Disallow: /pro_index.jsp
Disallow: /pro_nycl.jsp
Disallow: /pro_trl.jsp
Disallow: /rec_clu_index.jsp
Disallow: /rec_index.jsp
Disallow: /spe_lea_computer.jsp
Disallow: /spe_ser_index.jsp
Disallow: /spe_ser_photo.jsp
Disallow: /spe_ser_rent.jsp
Disallow: /spe_ser_wir.jsp
Disallow: /sup_index.jsp
Disallow: /sup_vol_index.jsp


# Wikipedia bot exclusions that seem like a good idea:
User-agent: Mediapartners-Google*
Disallow: /

User-agent: UbiCrawler
Disallow: /

User-agent: DOC
Disallow: /

User-agent: Zao
Disallow: /

User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: Zealbot
Disallow: /

User-agent: MSIECrawler
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: Fetch
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: linko
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: Microsoft.URL.Control
Disallow: /

User-agent: Xenu
Disallow: /

User-agent: larbin
Disallow: /

User-agent: libwww
Disallow: /

User-agent: ZyBORG
Disallow: /

User-agent: Download Ninja
Disallow: /

User-agent: wget
Disallow: /

User-agent: grub-client
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: WebReaper
Disallow: /