WMT16 en-de test records

1:data preprocess

TEXT=wmt16_en_de_bpe32k


fairseq-preprocess --source-lang en --target-lang de \
--trainpref $TEXT/train.tok.clean.bpe.32000 \
--validpref $TEXT/newstest2013.tok.bpe.32000 \
--testpref $TEXT/newstest2014.tok.bpe.32000 \
--destdir data-bin/wmt16_en_de_bpe32k \
--nwordssrc 32768 --nwordstgt 32768 \
--joined-dictionary \
--srcdict /wmt16.en-de.joined-dict.transformer/dict.en.txt

2:exam bleu

CUDA_VISIBLE_DEVICES=0,1,2,3 fairseq-generate \
data-bin/wmt16_en_de_bpe32k/ --path ./model/model.pt \
--remove-bpe --beam 4 --batch-size 64 --lenpen 0.6 \
--max-len-a 1 --max-len-b 50|tee generate.out

grep ^T generate.out | cut -f2- | perl -ple 's{(\S)-(\S)}{$1 ##AT##-##AT## $2}g' > generate.ref

grep ^H generate.out |cut -f3- | perl -ple 's{(\S)-(\S)}{$1 ##AT##-##AT## $2}g' > generate.sys

fairseq-score --sys generate.sys --ref generate.ref

3: genarate attention img

CUDA_VISIBLE_DEVICES=0 fairseq-train data-bin/wmt16_en_de_bpe32k \
  --arch transformer_vaswani_wmt_en_de_big --share-all-embeddings \
  --optimizer adam --adam-betas '(0.9, 0.98)' --clip-norm 0.0 \
  --lr-scheduler inverse_sqrt --warmup-init-lr 1e-07 --warmup-updates 4000 \
  --lr 0.0005 --min-lr 1e-09 \
  --dropout 0.3 --weight-decay 0.0 --criterion label_smoothed_cross_entropy --label-smoothing 0.1 \
  --max-tokens 3584 \
  --fp16\
  --save-attn

4: add attn loss

CUDA_VISIBLE_DEVICES=0,1,2,3 fairseq-train ../data-bin/wmt16_en_de_bpe32k \
  --arch transformer_vaswani_wmt_en_de_big --share-all-embeddings \
  --optimizer adam --adam-betas '(0.9, 0.98)' --clip-norm 0.0 \
  --lr-scheduler inverse_sqrt --warmup-init-lr 1e-07 --warmup-updates 4000 \
  --lr 0.0005 --min-lr 1e-09 \
  --dropout 0.3 --weight-decay 0.0 --criterion label_smoothed_cross_entropy_attn_loss --label-smoothing 0.1 \
  --max-tokens 3584 \
  --save-interval-updates 1000 --reset-optimizer --save-dir ch > r_train