Summerschool Aachen 2005/Challenges/verBOTen

verBOTen is a dictionary of robots.txt contents. We use a simple database structure:

CREATE TABLE `archive_hosts` (
 `id` int(11) NOT NULL auto_increment,
 `host` varchar(255) default NULL,
 `ip` bigint(11) unsigned default NULL,
 `port` int(5) unsigned default '80',
 PRIMARY KEY  (`id`),
 UNIQUE KEY `host` (`host`,`port`)
);
CREATE TABLE `archive_robots_disallows` (
 `id` int(11) NOT NULL auto_increment,
 `archive_host_id` int(11) default NULL,
 `path` text,
 PRIMARY KEY  (`id`)
);