#!/usr/bin/env python # encoding: utf-8 # # Copyright (c) 2009 Doug Hellmann All rights reserved. # """ """ #end_pymotw_header import robotparser import time import urlparse AGENT_NAME = 'PyMOTW' parser = robotparser.RobotFileParser() # Using the local copy parser.set_url('robots.txt') parser.read() parser.modified() PATHS = [ '/', '/PyMOTW/', '/admin/', '/downloads/PyMOTW-1.92.tar.gz', ] for n, path in enumerate(PATHS): print age = int(time.time() - parser.mtime()) print 'age:', age, if age > 1: print 're-reading robots.txt' parser.read() parser.modified() else: print print '%6s : %s' % (parser.can_fetch(AGENT_NAME, path), path) # Simulate a delay in processing time.sleep(1)