8 lines
510 B
Text
8 lines
510 B
Text
# run the crawler twice a day
|
|
# logs sent to /var/log/mirrormanager/crawler.log and crawl/* by default
|
|
# 27 threads with 32GB of RAM seems to work so far
|
|
#
|
|
# [ "`hostname -s`" == "mm-crawler02" ] && sleep 2h is used to start the crawl
|
|
# later on the second crawler to reduce the number of parallel accesses to
|
|
# the database
|
|
0 */12 * * * mirrormanager [ "`hostname -s`" == "mm-crawler02" ] && sleep 2h; /usr/bin/mm2_crawler --timeout-minutes 180 --threads 27 `/usr/local/bin/run_crawler.sh 2` > /dev/null 2>&1
|