# please do not crawl these pages
User-agent: *
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /includes/
Disallow: /dev/
Disallow: /download/
Disallow: /images/
Disallow: /removed/
Disallow: /dev/
Disallow: /latest.php
Disallow: /product_info.php
Disallow: /popup_image.php

Disallow: /photos_libres_de_droits.php
Disallow: /editeur_images.php

# seconds to wait between successive requests = 30
User-agent: *
Crawl-delay: 30

# maximum rate is one page every 20 seconds
User-agent: *
Request-rate: 1/20 

# only visit between 1:00 AM and 8:00 AM UT (GMT)
# Visit-time: 0100-0900 

# these agents is not allowed to crawl pages
User-Agent: MJ12bot
Disallow:   /

User-Agent: Yanga
Disallow:   /

User-Agent: Yanga WorldSearch Bot
Disallow:   /

User-Agent: YodaoBot
Disallow:   /

User-agent: ia_archiver
Disallow: /