cd /opt/nutch
export JAVA_HOME=/usr/lib/java/

# Update the blacklist
/opt/nutch/update_url_filter.pl > /opt/nutch/conf/crawl-urlfilter.txt

# First get the latest seeds
wget http://www.nitrc.org/search/nutch_seeds.php -O /var/lib/nutch/projectmap.txt

# Now split out the URL from the seed file
awk '/(.*)\t/ {print $1}' /var/lib/nutch/projectmap.txt > /var/lib/nutch/seed/urls

# Delete the old redirect DB
rm -f /tmp/redirect_db.txt

# Do initial crawl, recreating link DB
rm -Rf /var/lib/nutch/crawl2/; ./bin/nutch crawl /var/lib/nutch/seed/ -dir /var/lib/nutch/crawl2 -depth 5 -threads 50 -topN 40000

# Now dump the linkdb to a file
./bin/nutch readlinkdb /var/lib/nutch/crawl2/linkdb/ -dump /var/lib/nutch/crawl2/linkdump

# Now update the index with the new link DB
rm -Rf /var/lib/nutch/crawl2/indexes/; ./bin/nutch index /var/lib/nutch/crawl2/indexes/ /var/lib/nutch/crawl2/crawldb/ /var/lib/nutch/crawl2/linkdb/ /var/lib/nutch/crawl2/segments/*
./bin/nutch dedup /var/lib/nutch/crawl2/indexes/
rm -Rf /var/lib/nutch/crawl2/index/; ./bin/nutch merge /var/lib/nutch/crawl2/index/ /var/lib/nutch/crawl2/indexes/

# Check that our crawl is large enough (bigger than 200 meg)
if [ `du -ms /var/lib/nutch/crawl2/ | awk '{print $1}'` -gt "50" ]; then
 # Now delete the old index if there
 rm -Rf /var/lib/nutch/crawl.old
 mv /var/lib/nutch/crawl /var/lib/nutch/crawl.old
 mv /var/lib/nutch/crawl2 /var/lib/nutch/crawl

 # Now restart Tomcat
 export JAVA_HOME=/usr/lib/java/
 /usr/local/tomcat/bin/shutdown.sh
 # Ensure Java is dead (Note, this will kill ALL java processes
 killall -w -9 -q java
 sleep 10
 /usr/local/tomcat/bin/startup.sh

else
  echo "NITRC Nutch Crawl Failed" | mail -s "Nutch Crawl Failed" butch@tcg.com;
fi
