cd /opt/nutch
export JAVA_HOME=/usr/lib/java/

# First get the latest seeds
wget http://www.nitrc.org/search/nutch_seeds.php -O /var/lib/nutch/projectmap.txt

# Now split out the URL from the seed file
awk '/(.*)\t/ {print $1}' /var/lib/nutch/projectmap.txt > /var/lib/nutch/seed/urls

# Delete the old link DB
rm -f /tmp/link_db.txt

# Do initial crawl, recreating link DB
rm -Rf /var/lib/nutch/crawl2/; ./bin/nutch crawl /var/lib/nutch/seed/ -dir /var/lib/nutch/crawl2 -depth 1 -threads 50 -topN 20000

# Now update the index with the new link DB
rm -Rf /var/lib/nutch/crawl2/indexes/; ./bin/nutch index /var/lib/nutch/crawl2/indexes/ /var/lib/nutch/crawl2/crawldb/ /var/lib/nutch/crawl2/linkdb/ /var/lib/nutch/crawl2/segments/*
./bin/nutch dedup /var/lib/nutch/crawl2/indexes/
rm -Rf /var/lib/nutch/crawl2/index/; ./bin/nutch merge /var/lib/nutch/crawl2/index/ /var/lib/nutch/crawl2/indexes/

# Now delete the old index if there
rm -Rf /var/lib/nutch/crawl.old
mv /var/lib/nutch/crawl /var/lib/nutch/crawl.old
mv /var/lib/nutch/crawl2 /var/lib/nutch/crawl

# Now restart Tomcat
export JAVA_HOME=/usr/lib/java/
/usr/local/tomcat/bin/shutdown.sh
# Ensure Java is dead (Note, this will kill ALL java processes
killall -w -9 -q java
sleep 10
/usr/local/tomcat/bin/startup.sh


