diff --git a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/env.sh b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/env.sh index bc171a7524d990d1e224e540ff64d27213461ea9..560102bd3d368ee110034b173f22ede41938d909 100644 --- a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/env.sh +++ b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/test/env.sh @@ -42,7 +42,6 @@ export COMBINED_ENABLE=1 export HCCL_WHITELIST_DISABLE=1 export SCALAR_TO_HOST_MEM=1 -export TRI_COMBINED_ENABLE=1 #设置Device侧日志等级为error ${install_path}/driver/tools/msnpureport -d 0 -g error diff --git a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/tokenize_en_zh.sh b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/tokenize_en_zh.sh index 2fadc5b72c59c526a7743c16c0a481f1f929425d..9bbf29c23a38db4d1d8db22b1d95eaddfb8f984d 100644 --- a/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/tokenize_en_zh.sh +++ b/PyTorch/built-in/nlp/XLM_ID0740_for_PyTorch/tokenize_en_zh.sh @@ -14,24 +14,21 @@ cat data/wiki/txt/zh.valid | ./tools/tokenize.sh zh > data/wiki/txt/token_zh.val shuf -r -n 10000000 data/wiki/txt/token_en.train >> $OUTPATH/bpe.train.en shuf -r -n 10000000 data/wiki/txt/token_zh.train >> $OUTPATH/bpe.train.zh -$FASTBPE learnbpe 50000 $OUTPATH/bpe.train.en > $OUTPATH/codes +$FASTBPE learnbpe 50000 $OUTPATH/bpe.train.en $OUTPATH/bpe.train.zh > $OUTPATH/codes $FASTBPE applybpe $OUTPATH/train.en data/wiki/txt/token_en.train $OUTPATH/codes $FASTBPE applybpe $OUTPATH/test.en data/wiki/txt/token_en.test $OUTPATH/codes $FASTBPE applybpe $OUTPATH/valid.en data/wiki/txt/token_en.valid $OUTPATH/codes -rm -rf $OUTPATH/codes - -$FASTBPE learnbpe 50000 $OUTPATH/bpe.train.zh > $OUTPATH/codes $FASTBPE applybpe $OUTPATH/train.zh data/wiki/txt/token_zh.train $OUTPATH/codes $FASTBPE applybpe $OUTPATH/test.zh data/wiki/txt/token_zh.test $OUTPATH/codes $FASTBPE applybpe $OUTPATH/valid.zh data/wiki/txt/token_zh.valid $OUTPATH/codes $FASTBPE getvocab $OUTPATH/train.en $OUTPATH/train.zh > $OUTPATH/vocab -python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/train.en.pth -python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/test.en.pth -python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/valid.en.pth +python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/train.en +python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/test.en +python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/valid.en -python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/train.zh.pth -python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/test.zh.pth -python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/valid.zh.pth +python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/train.zh +python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/test.zh +python3.7 preprocess.py $OUTPATH/vocab $OUTPATH/valid.zh