#! /bin/bash

# wget wget http://myip.ms/files/bots/live_webcrawlers.txt

tail -n +12 live_webcrawlers.txt | grep -e "^#" | cut -d" " -f5- > bots.txt
echo "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)" >> bots.txt
