svnno****@sourc*****
svnno****@sourc*****
2008年 7月 1日 (火) 15:36:20 JST
Revision: 682 http://svn.sourceforge.jp/cgi-bin/viewcvs.cgi?root=slashdotjp&view=rev&rev=682 Author: tach Date: 2008-07-01 15:36:20 +0900 (Tue, 01 Jul 2008) Log Message: ----------- simplify robots.txt rule Modified Paths: -------------- images/robots.txt -------------- next part -------------- Modified: images/robots.txt =================================================================== --- images/robots.txt 2008-07-01 05:58:10 UTC (rev 681) +++ images/robots.txt 2008-07-01 06:36:20 UTC (rev 682) @@ -7,79 +7,24 @@ User-agent: Mediapartners-Google Disallow: -User-agent: Googlebot -Crawl-delay: 100 -Disallow: /firehose.pl -Disallow: /submit.pl -Disallow: /users.pl -Disallow: /zoo.pl -Disallow: firehose.pl -Disallow: submit.pl -Disallow: users.pl -Disallow: zoo.pl - -User-agent: Slurp -Crawl-delay: 100 -Disallow: /firehose.pl -Disallow: /submit.pl -Disallow: /users.pl -Disallow: /zoo.pl -Disallow: firehose.pl -Disallow: submit.pl -Disallow: users.pl -Disallow: zoo.pl - -User-agent: Yahoo-NewsCrawler -Disallow: /firehose.pl -Disallow: /submit.pl -Disallow: /users.pl -Disallow: /zoo.pl -Disallow: firehose.pl -Disallow: submit.pl -Disallow: users.pl -Disallow: zoo.pl - -User-Agent: msnbot -Crawl-delay: 100 -Disallow: /firehose.pl -Disallow: /submit.pl -Disallow: /users.pl -Disallow: /zoo.pl -Disallow: firehose.pl -Disallow: submit.pl -Disallow: users.pl -Disallow: zoo.pl - User-agent: * -Crawl-delay: 100 -Disallow: /authors.pl -Disallow: /index.pl -Disallow: /article.pl -Disallow: /comments.pl +Crawl-delay: 5 +Disallow: /404.pl +Disallow: /admin.pl +Disallow: /ajax.pl +Disallow: /blob.pl +Disallow: /bookmark.pl +Disallow: /fileadmin.pl Disallow: /firehose.pl -Disallow: /journal.pl +Disallow: /hof.pl +Disallow: /login.pl Disallow: /messages.pl Disallow: /metamod.pl -Disallow: /users.pl -Disallow: /search.pl -Disallow: /submit.pl -Disallow: /pollBooth.pl Disallow: /pubkey.pl -Disallow: /topics.pl +Disallow: /relocate.pl +Disallow: /slashdot-it.pl +Disallow: /stats.pl +Disallow: /submit.pl +Disallow: /tags.pl +Disallow: /users.pl Disallow: /zoo.pl -Disallow: authors.pl -Disallow: index.pl -Disallow: article.pl -Disallow: comments.pl -Disallow: firehose.pl -Disallow: journal.pl -Disallow: messages.pl -Disallow: metamod.pl -Disallow: users.pl -Disallow: search.pl -Disallow: submit.pl -Disallow: pollBooth.pl -Disallow: pubkey.pl -Disallow: topics.pl -Disallow: zoo.pl -Disallow: /i