Infrastructure/ansible
1
0
Fork 0
Code Issues Pull requests 6 Projects Releases Packages Wiki Activity Actions

Use the new script to better distribute the load between the crawlers.

Browse source
This commit is contained in:
Adrian Reber 2015-06-28 10:24:56 +00:00
parent 106562e2ca
commit 0b76e6721a
2 changed files with 4 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

20
roles/mirrormanager/crawler/files/run_crawler.sh
View file

@ -6,27 +6,11 @@ fi
NUMBER_OF_CRAWLERS=$1
HOST=`hostname -s`
MAX_HOST=`/usr/local/bin/mm2_get-highest-active-host-id`
START_STOP="/usr/local/bin/mm2_get-mirrors-to-crawl"
# make MAX_HOST a multiple of NUMBER_OF_CRAWLERS
let FIX=${MAX_HOST}%${NUMBER_OF_CRAWLERS}
if [ "${FIX}" -ne "0" ]; then
let MAX_HOST=${MAX_HOST}+${NUMBER_OF_CRAWLERS}-${FIX}
fi
let PART=${MAX_HOST}/${NUMBER_OF_CRAWLERS}
STARTID=0
STOPID=${PART}
for i in `seq 1 ${NUMBER_OF_CRAWLERS}`; do
if [ "${HOST}" == "mm-crawler0${i}" ]; then
echo "--startid=${STARTID} --stopid=${STOPID}"
${START_STOP} -f ${i}:${NUMBER_OF_CRAWLERS}
fi
let STARTID=${STARTID}+${PART}
let STOPID=${STOPID}+${PART}
if [ "${STOPID}" -eq "${MAX_HOST}" ]; then
let STOPID=${STOPID}+1
fi
done

4
roles/mirrormanager/crawler/tasks/main.yml
View file

@ -35,8 +35,8 @@
- config
when: env != 'staging'
- name: install the script to get the highest mirror id
copy: src=mm2_get-highest-active-host-id dest=/usr/local/bin/mm2_get-highest-active-host-id mode=0755
- name: install the script to distribute the mirrors to crawl
copy: src=mm2_get-mirrors-to-crawl dest=/usr/local/bin/mm2_get-mirrors-to-crawl mode=0755
- name: install the crawler distribute script
copy: src=run_crawler.sh dest=/usr/local/bin/run_crawler.sh mode=0755