iicd/xautodl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update ImageNet training

Browse Source
This commit is contained in:
Xuanyi Dong
2019-04-04 20:29:41 +08:00
parent 666c105f51
commit 4121d1719f
8 changed files with 64 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
scripts-cnn/train-imagenet.sh
View File

@@ -1,7 +1,7 @@
#!/usr/bin/env sh
if [ "$#" -ne 3 ] ;then
if [ "$#" -ne 5 ] ;then
echo "Input illegal number of parameters " $#
echo "Need 3 parameters for the architecture, and the channel and the layers"
echo "Need 5 parameters for the architecture, and the channel, and the layers, and the batch-size, and the seed"
exit 1
fi
if [ "$TORCH_HOME" = "" ]; then
@@ -15,7 +15,9 @@ arch=$1
dataset=imagenet
channels=$2
layers=$3
SAVED=./output/NAS-CNN/${arch}-${dataset}-C${channels}-L${layers}-E250
BATCH=$4
seed=$5
SAVED=./output/NAS-CNN/${arch}-${dataset}-C${channels}-L${layers}-${BATCH}-E250
PY_C="./env/bin/python"
#PY_C="$CONDA_PYTHON_EXE"
@@ -27,8 +29,8 @@ else
echo "Cluster Run with Python: "${PY_C}
echo "Unzip ILSVRC2012"
tar --version
#tar xf ./hadoop-data/ILSVRC2012.tar -C ${TORCH_HOME}
commands="./data/data/get_imagenet.sh"
tar -xf ./hadoop-data/ILSVRC2012.tar -C ${TORCH_HOME}
#commands="./data/data/get_imagenet.sh"
#${PY_C} ./data/decompress.py ./hadoop-data/ILSVRC2012-TAR ./data/data/ILSVRC2012 tar > ${commands}
#${PY_C} ./data/decompress.py ./hadoop-data/ILSVRC2012-ZIP ./data/data/ILSVRC2012 zip > ./data/data/get_imagenet.sh
#bash ./data/data/get_imagenet.sh
@@ -42,16 +44,16 @@ else
# free -g
#done < "${commands}"
#wget http://10.127.2.44:8000/ILSVRC2012.tar --directory-prefix=${TORCH_HOME}
${PY_C} ./data/decompress.py ./data/classes.txt ${TORCH_HOME}/ILSVRC2012 wget > ${commands}
count=0
while read -r line; do
temp_file="./data/data/TEMP-${count}.sh"
echo "${line}" > ${temp_file}
bash ${temp_file}
count=$((count+1))
#${PY_C} ./data/decompress.py ./data/classes.txt ${TORCH_HOME}/ILSVRC2012 wget > ${commands}
#count=0
#while read -r line; do
# temp_file="./data/data/TEMP-${count}.sh"
# echo "${line}" > ${temp_file}
# bash ${temp_file}
# count=$((count+1))
#${PY_C} ./data/ps_mem.py -p $$
# free -g
done < "${commands}"
#done < "${commands}"
#echo "Copy ILSVRC2012 done"
#tar -xvf ${TORCH_HOME}/ILSVRC2012.tar -C ${TORCH_HOME}
#rm ${TORCH_HOME}/ILSVRC2012.tar
@@ -66,5 +68,6 @@ ${PY_C} ./exps-cnn/train_base.py \
--save_path ${SAVED} \
--grad_clip 5 \
--init_channels ${channels} --layers ${layers} \
--model_config ./configs/nas-imagenet.config \
--model_config ./configs/nas-imagenet-${BATCH}.config \
--manualSeed ${seed} \
--print_freq 200 --workers 20