6135863
yangsaisai 7 years ago
50 changed file(s) with 1025 addition(s) and 14 deletion(s). Raw diff Collapse all Expand all
+2
-2
.ipynb_checkpoints/config_poem-checkpoint.py less more
99 # help='data file')
1010
1111 parser.add_argument('--data_path', type=str,
12 default='./data/poem/',
12 default='./datasets/yangsaisai-poetrydatasets-0_0_1/',
1313 help='data path')
1414
1515
3838 help='which model to use (rnn, lstm or gru).')
3939
4040 # Parameters to control the training.
41 parser.add_argument('--num_epochs', type=int, default=8,
41 parser.add_argument('--num_epochs', type=int, default=15,
4242 help='number of epochs')
4343 parser.add_argument('--batch_size', type=int, default=16,
4444 help='minibatch size')
+6
-5
.ipynb_checkpoints/train-checkpoint.py less more
2929 # Create necessary directories.
3030 if len(args.init_dir) != 0:
3131 args.output_dir = args.init_dir
32 else:
33 if os.path.exists(args.output_dir):
34 shutil.rmtree(args.output_dir)
35 for paths in [args.save_model, args.save_best_model, args.tb_log_dir]:
36 os.makedirs(os.path.dirname(paths))
32
33 # else:
34 # if os.path.exists(args.output_dir):
35 # shutil.rmtree(args.output_dir)
36 # for paths in [args.save_model, args.save_best_model, args.tb_log_dir]:
37 # os.makedirs(os.path.dirname(paths))
3738
3839 logging.basicConfig(stream=sys.stdout,
3940 format='%(asctime)s %(levelname)s:%(message)s',
+2
-2
config_poem.py less more
99 # help='data file')
1010
1111 parser.add_argument('--data_path', type=str,
12 default='./data/poem/',
12 default='./datasets/yangsaisai-poetrydatasets-0_0_1/',
1313 help='data path')
1414
1515
3838 help='which model to use (rnn, lstm or gru).')
3939
4040 # Parameters to control the training.
41 parser.add_argument('--num_epochs', type=int, default=8,
41 parser.add_argument('--num_epochs', type=int, default=15,
4242 help='number of epochs')
4343 parser.add_argument('--batch_size', type=int, default=16,
4444 help='minibatch size')
+13
-0
job_logs/job-gpu-5c00ae391afd942b83265f05.log less more
0 2018-11-30T03:28:28.428857569Z SYSTEM: Preparing env...
1 2018-11-30T03:28:28.935041545Z SYSTEM: Running...
2 2018-11-30T03:28:33.423454716Z Writing to /home/jovyan/work/results/tb_results/tensorboard_log/1543548513
3 2018-11-30T03:28:33.423485247Z
4 2018-11-30T03:28:33.431874605Z Traceback (most recent call last):
5 2018-11-30T03:28:33.431891675Z File "train.py", line 196, in <module>
6 2018-11-30T03:28:33.431899303Z main(args)
7 2018-11-30T03:28:33.431904418Z File "train.py", line 37, in main
8 2018-11-30T03:28:33.431909705Z os.makedirs(os.path.dirname(paths))
9 2018-11-30T03:28:33.431914742Z File "/home/jovyan/.virtualenvs/5c00ae391afd942b83265f05/lib/python3.5/os.py", line 241, in makedirs
10 2018-11-30T03:28:33.43192371Z mkdir(name, mode)
11 2018-11-30T03:28:33.431928773Z FileExistsError: [Errno 17] File exists: '/home/jovyan/work/results/tb_results/tensorboard_log'
12 2018-11-30T03:28:33.993854401Z SYSTEM: Finishing...
+13
-0
job_logs/job-gpu-5c00bbe61afd942cd74d7c6e.log less more
0 2018-11-30T04:26:39.43498957Z SYSTEM: Preparing env...
1 2018-11-30T04:26:39.934421776Z SYSTEM: Running...
2 2018-11-30T04:26:45.550945851Z Writing to /home/jovyan/work/results/tb_results/tensorboard_log/1543552005
3 2018-11-30T04:26:45.550968326Z
4 2018-11-30T04:26:45.569260182Z Traceback (most recent call last):
5 2018-11-30T04:26:45.569303389Z File "train.py", line 196, in <module>
6 2018-11-30T04:26:45.569310617Z main(args)
7 2018-11-30T04:26:45.56931561Z File "train.py", line 37, in main
8 2018-11-30T04:26:45.56932072Z os.makedirs(os.path.dirname(paths))
9 2018-11-30T04:26:45.5693256Z File "/home/jovyan/.virtualenvs/5c00bbe61afd942cd74d7c6e/lib/python3.5/os.py", line 241, in makedirs
10 2018-11-30T04:26:45.569333699Z mkdir(name, mode)
11 2018-11-30T04:26:45.569338847Z FileExistsError: [Errno 17] File exists: '/home/jovyan/work/results/tb_results/tensorboard_log'
12 2018-11-30T04:26:46.164187505Z SYSTEM: Finishing...
+13
-0
job_logs/job-gpu-5c00be431afd942b66b36bad.log less more
0 2018-11-30T04:36:45.372548117Z SYSTEM: Preparing env...
1 2018-11-30T04:36:46.007082923Z SYSTEM: Running...
2 2018-11-30T04:36:51.251278513Z Writing to /home/jovyan/work/results/tb_results/tensorboard_log/1543552611
3 2018-11-30T04:36:51.251311019Z
4 2018-11-30T04:36:51.271397883Z Traceback (most recent call last):
5 2018-11-30T04:36:51.271435868Z File "train.py", line 196, in <module>
6 2018-11-30T04:36:51.271657251Z main(args)
7 2018-11-30T04:36:51.271674104Z File "train.py", line 37, in main
8 2018-11-30T04:36:51.276856336Z os.makedirs(os.path.dirname(paths))
9 2018-11-30T04:36:51.276871243Z File "/home/jovyan/.virtualenvs/5c00be431afd942b66b36bad/lib/python3.5/os.py", line 241, in makedirs
10 2018-11-30T04:36:51.276884231Z mkdir(name, mode)
11 2018-11-30T04:36:51.276888696Z FileExistsError: [Errno 17] File exists: '/home/jovyan/work/results/tb_results/tensorboard_log'
12 2018-11-30T04:36:51.975847222Z SYSTEM: Finishing...
+78
-0
job_logs/job-gpu-5c00bed61afd943016b45263.log less more
0 2018-11-30T04:39:18.771169882Z SYSTEM: Preparing env...
1 2018-11-30T04:39:19.230566549Z SYSTEM: Running...
2 2018-11-30T04:39:24.792998787Z Writing to /home/jovyan/work/results/tb_results/tensorboard_log/1543552764
3 2018-11-30T04:39:24.793042757Z
4 2018-11-30T04:39:24.793049271Z ============================================================
5 2018-11-30T04:39:24.793054456Z All final and intermediate outputs will be stored in ./results/output_poem/
6 2018-11-30T04:39:24.793059361Z ============================================================
7 2018-11-30T04:39:24.793064116Z
8 2018-11-30T04:39:24.793068359Z 12:39:24 INFO:args are:
9 2018-11-30T04:39:24.793078562Z Namespace(batch_size=16, best_model='', best_valid_ppl=inf, cell_type='lstm', data_path='./datasets/yangsaisai-poetrydatasets-0_0_1/', debug=False, dropout=0.0, embedding_size=128, encoding='utf-8', hidden_size=128, init_dir='', init_model='', input_dropout=0.0, learning_rate=0.005, max_grad_norm=5.0, num_epochs=8, num_layers=2, num_unrollings=64, output_dir='./results/output_poem', progress_freq=100, save_best_model='./results/output_poem/best_model/model', save_model='./results/output_poem/save_model/model', tb_log_dir='/home/jovyan/work/results/tb_results/tensorboard_log/1543552764', test=False, train_frac=0.9, valid_frac=0.05, verbose=0)
10 2018-11-30T04:39:24.793087349Z 12:39:24 INFO:Parameters are:
11 2018-11-30T04:39:24.793092061Z {
12 2018-11-30T04:39:24.793095972Z "batch_size": 16,
13 2018-11-30T04:39:24.793100611Z "cell_type": "lstm",
14 2018-11-30T04:39:24.793105184Z "dropout": 0.0,
15 2018-11-30T04:39:24.793109809Z "embedding_size": 128,
16 2018-11-30T04:39:24.793114104Z "hidden_size": 128,
17 2018-11-30T04:39:24.79312085Z "input_dropout": 0.0,
18 2018-11-30T04:39:24.793125397Z "learning_rate": 0.005,
19 2018-11-30T04:39:24.793129944Z "max_grad_norm": 5.0,
20 2018-11-30T04:39:24.79313419Z "num_layers": 2,
21 2018-11-30T04:39:24.793138499Z "num_unrollings": 64
22 2018-11-30T04:39:24.793143017Z }
23 2018-11-30T04:39:24.793147129Z
24 2018-11-30T04:39:25.009827544Z tensor_file:./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
25 2018-11-30T04:39:25.010589006Z Loading dataset from ./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
26 2018-11-30T04:39:25.359910285Z file maxSeqLen = 64
27 2018-11-30T04:39:25.359961327Z Loaded ./datasets/yangsaisai-poetrydatasets-0_0_1/: training samples:65235 ,validationSamples:3837,testingSamples:7676
28 2018-11-30T04:39:26.276017793Z tensor_file:./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
29 2018-11-30T04:39:26.278968445Z Loading dataset from ./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
30 2018-11-30T04:39:26.821024692Z file maxSeqLen = 64
31 2018-11-30T04:39:26.821074339Z Loaded ./datasets/yangsaisai-poetrydatasets-0_0_1/: training samples:65235 ,validationSamples:3837,testingSamples:7676
32 2018-11-30T04:39:26.883903582Z tensor_file:./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
33 2018-11-30T04:39:26.883949908Z Loading dataset from ./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
34 2018-11-30T04:39:27.70684711Z file maxSeqLen = 64
35 2018-11-30T04:39:27.706886182Z Loaded ./datasets/yangsaisai-poetrydatasets-0_0_1/: training samples:65235 ,validationSamples:3837,testingSamples:7676
36 2018-11-30T04:39:27.732969217Z 12:39:27 INFO:Creating graph
37 2018-11-30T04:39:44.21809571Z 12:39:44 INFO:Start training
38 2018-11-30T04:39:44.218139835Z
39 2018-11-30T04:39:44.222225853Z 2018-11-30 12:39:44.221828: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
40 2018-11-30T04:39:44.814174911Z 2018-11-30 12:39:44.813206: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:964] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
41 2018-11-30T04:39:44.817549103Z 2018-11-30 12:39:44.816320: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 0 with properties:
42 2018-11-30T04:39:44.81756827Z name: Tesla P100-PCIE-16GB major: 6 minor: 0 memoryClockRate(GHz): 1.3285
43 2018-11-30T04:39:44.81757418Z pciBusID: 0000:00:07.0
44 2018-11-30T04:39:44.817578953Z totalMemory: 15.90GiB freeMemory: 15.61GiB
45 2018-11-30T04:39:44.817583447Z 2018-11-30 12:39:44.816366: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0
46 2018-11-30T04:39:46.742354307Z 2018-11-30 12:39:46.736966: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix:
47 2018-11-30T04:39:46.742386365Z 2018-11-30 12:39:46.737031: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988] 0
48 2018-11-30T04:39:46.742393292Z 2018-11-30 12:39:46.737042: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0: N
49 2018-11-30T04:39:46.742400066Z 2018-11-30 12:39:46.737548: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 15129 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0)
50 2018-11-30T04:39:56.505560041Z 12:39:56 INFO:=================== Epoch 0 ===================
51 2018-11-30T04:39:56.505607493Z
52 2018-11-30T04:39:56.50561525Z 12:39:56 INFO:Training on training set
53 2018-11-30T04:40:13.433002658Z 12:40:13 INFO:2.5%, step:99, perplexity: 603.783, speed: 6159 words
54 2018-11-30T04:40:24.835022537Z 12:40:24 INFO:4.9%, step:199, perplexity: 382.204, speed: 7306 words
55 2018-11-30T04:40:37.11490721Z 12:40:37 INFO:7.4%, step:299, perplexity: 294.070, speed: 7621 words
56 2018-11-30T04:40:46.951976568Z 12:40:46 INFO:9.8%, step:399, perplexity: 246.033, speed: 8168 words
57 2018-11-30T04:40:54.273857463Z 12:40:54 INFO:12.3%, step:499, perplexity: 215.291, speed: 8910 words
58 2018-11-30T04:41:01.898883206Z 12:41:01 INFO:14.7%, step:599, perplexity: 193.340, speed: 9439 words
59 2018-11-30T04:41:09.470843498Z 12:41:09 INFO:17.2%, step:699, perplexity: 176.586, speed: 9864 words
60 2018-11-30T04:41:16.48887215Z 12:41:16 INFO:19.6%, step:799, perplexity: 163.220, speed: 10281 words
61 2018-11-30T04:41:23.899031045Z 12:41:23 INFO:22.1%, step:899, perplexity: 152.391, speed: 10583 words
62 2018-11-30T04:41:30.936227426Z 12:41:30 INFO:24.5%, step:999, perplexity: 143.515, speed: 10878 words
63 2018-11-30T04:41:38.123902254Z 12:41:38 INFO:27.0%, step:1099, perplexity: 136.097, speed: 11117 words
64 2018-11-30T04:41:45.061928369Z 12:41:45 INFO:29.4%, step:1199, perplexity: 129.725, speed: 11351 words
65 2018-11-30T04:41:52.005683983Z 12:41:52 INFO:31.9%, step:1299, perplexity: 124.212, speed: 11556 words
66 2018-11-30T04:42:02.165205709Z 12:42:02 INFO:34.3%, step:1399, perplexity: 119.386, speed: 11439 words
67 2018-11-30T04:42:14.464941867Z 12:42:14 INFO:36.8%, step:1499, perplexity: 115.124, speed: 11158 words
68 2018-11-30T04:42:26.401843448Z 12:42:26 INFO:39.2%, step:1599, perplexity: 111.343, speed: 10952 words
69 2018-11-30T04:42:35.551908158Z 12:42:35 INFO:41.7%, step:1699, perplexity: 107.962, speed: 10966 words
70 2018-11-30T04:42:43.651462585Z 12:42:43 INFO:44.1%, step:1799, perplexity: 104.918, speed: 11047 words
71 2018-11-30T04:42:50.724941217Z 12:42:50 INFO:46.6%, step:1899, perplexity: 102.144, speed: 11187 words
72 2018-11-30T04:42:59.457983904Z 12:42:59 INFO:49.0%, step:1999, perplexity: 99.596, speed: 11212 words
73 2018-11-30T04:43:07.632098487Z 12:43:07 INFO:51.5%, step:2099, perplexity: 97.275, speed: 11269 words
74 2018-11-30T04:43:15.221914296Z 12:43:15 INFO:53.9%, step:2199, perplexity: 95.154, speed: 11354 words
75 2018-11-30T04:43:22.536957817Z 12:43:22 INFO:56.4%, step:2299, perplexity: 93.198, speed: 11448 words
76 2018-11-30T04:43:29.994808334Z 12:43:29 INFO:58.9%, step:2399, perplexity: 91.377, speed: 11528 words
77 2018-11-30T04:43:36.589198501Z 12:43:36 INFO:61.3%, step:2499, perplexity: 89.682, speed: 11648 words
+862
-0
job_logs/job-gpu-5c00c00d1afd942b54c88504.log less more
0 2018-11-30T04:44:30.098430332Z SYSTEM: Preparing env...
1 2018-11-30T04:44:30.874874757Z SYSTEM: Running...
2 2018-11-30T04:44:35.628433623Z Writing to /home/jovyan/work/results/tb_results/tensorboard_log/1543553075
3 2018-11-30T04:44:35.628475928Z
4 2018-11-30T04:44:35.628481949Z ============================================================
5 2018-11-30T04:44:35.628487801Z All final and intermediate outputs will be stored in ./results/output_poem/
6 2018-11-30T04:44:35.628493017Z ============================================================
7 2018-11-30T04:44:35.62849837Z
8 2018-11-30T04:44:35.628503018Z 12:44:35 INFO:args are:
9 2018-11-30T04:44:35.628512249Z Namespace(batch_size=16, best_model='', best_valid_ppl=inf, cell_type='lstm', data_path='./datasets/yangsaisai-poetrydatasets-0_0_1/', debug=False, dropout=0.0, embedding_size=128, encoding='utf-8', hidden_size=128, init_dir='', init_model='', input_dropout=0.0, learning_rate=0.005, max_grad_norm=5.0, num_epochs=15, num_layers=2, num_unrollings=64, output_dir='./results/output_poem', progress_freq=100, save_best_model='./results/output_poem/best_model/model', save_model='./results/output_poem/save_model/model', tb_log_dir='/home/jovyan/work/results/tb_results/tensorboard_log/1543553075', test=False, train_frac=0.9, valid_frac=0.05, verbose=0)
10 2018-11-30T04:44:35.628521048Z 12:44:35 INFO:Parameters are:
11 2018-11-30T04:44:35.628527126Z {
12 2018-11-30T04:44:35.628532666Z "batch_size": 16,
13 2018-11-30T04:44:35.628538309Z "cell_type": "lstm",
14 2018-11-30T04:44:35.628543774Z "dropout": 0.0,
15 2018-11-30T04:44:35.628548874Z "embedding_size": 128,
16 2018-11-30T04:44:35.628553914Z "hidden_size": 128,
17 2018-11-30T04:44:35.628558899Z "input_dropout": 0.0,
18 2018-11-30T04:44:35.628563834Z "learning_rate": 0.005,
19 2018-11-30T04:44:35.628568724Z "max_grad_norm": 5.0,
20 2018-11-30T04:44:35.628573371Z "num_layers": 2,
21 2018-11-30T04:44:35.628578617Z "num_unrollings": 64
22 2018-11-30T04:44:35.628583471Z }
23 2018-11-30T04:44:35.628588821Z
24 2018-11-30T04:44:35.909728717Z tensor_file:./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
25 2018-11-30T04:44:35.925025451Z Loading dataset from ./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
26 2018-11-30T04:44:36.409730893Z file maxSeqLen = 64
27 2018-11-30T04:44:36.409800263Z Loaded ./datasets/yangsaisai-poetrydatasets-0_0_1/: training samples:65235 ,validationSamples:3837,testingSamples:7676
28 2018-11-30T04:44:37.230019521Z tensor_file:./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
29 2018-11-30T04:44:37.231228692Z Loading dataset from ./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
30 2018-11-30T04:44:37.845702909Z file maxSeqLen = 64
31 2018-11-30T04:44:37.845751537Z Loaded ./datasets/yangsaisai-poetrydatasets-0_0_1/: training samples:65235 ,validationSamples:3837,testingSamples:7676
32 2018-11-30T04:44:37.916491824Z tensor_file:./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
33 2018-11-30T04:44:37.916544899Z Loading dataset from ./datasets/yangsaisai-poetrydatasets-0_0_1/poem_ids.txt
34 2018-11-30T04:44:38.526782397Z file maxSeqLen = 64
35 2018-11-30T04:44:38.52683197Z Loaded ./datasets/yangsaisai-poetrydatasets-0_0_1/: training samples:65235 ,validationSamples:3837,testingSamples:7676
36 2018-11-30T04:44:38.592585066Z 12:44:38 INFO:Creating graph
37 2018-11-30T04:44:55.350007534Z 12:44:55 INFO:Start training
38 2018-11-30T04:44:55.350054157Z
39 2018-11-30T04:44:55.354432285Z 2018-11-30 12:44:55.353917: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
40 2018-11-30T04:44:55.568409737Z 2018-11-30 12:44:55.567367: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:964] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
41 2018-11-30T04:44:55.568439468Z 2018-11-30 12:44:55.568064: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 0 with properties:
42 2018-11-30T04:44:55.568446992Z name: Tesla P100-PCIE-16GB major: 6 minor: 0 memoryClockRate(GHz): 1.3285
43 2018-11-30T04:44:55.568452164Z pciBusID: 0000:00:07.0
44 2018-11-30T04:44:55.568456864Z totalMemory: 15.90GiB freeMemory: 15.61GiB
45 2018-11-30T04:44:55.568461565Z 2018-11-30 12:44:55.568102: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0
46 2018-11-30T04:44:57.270963764Z 2018-11-30 12:44:57.268304: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix:
47 2018-11-30T04:44:57.271005914Z 2018-11-30 12:44:57.269656: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988] 0
48 2018-11-30T04:44:57.271022928Z 2018-11-30 12:44:57.269672: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0: N
49 2018-11-30T04:44:57.271030126Z 2018-11-30 12:44:57.270156: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 15129 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0)
50 2018-11-30T04:45:04.920920876Z 12:45:04 INFO:=================== Epoch 0 ===================
51 2018-11-30T04:45:04.920971912Z
52 2018-11-30T04:45:04.920979912Z 12:45:04 INFO:Training on training set
53 2018-11-30T04:45:20.642664868Z 12:45:20 INFO:2.5%, step:99, perplexity: 597.051, speed: 6575 words
54 2018-11-30T04:45:27.398939821Z 12:45:27 INFO:4.9%, step:199, perplexity: 378.881, speed: 9168 words
55 2018-11-30T04:45:34.597749921Z 12:45:34 INFO:7.4%, step:299, perplexity: 291.397, speed: 10400 words
56 2018-11-30T04:45:41.451073767Z 12:45:41 INFO:9.8%, step:399, perplexity: 243.438, speed: 11261 words
57 2018-11-30T04:45:48.859105407Z 12:45:48 INFO:12.3%, step:499, perplexity: 212.490, speed: 11690 words
58 2018-11-30T04:45:56.335908055Z 12:45:56 INFO:14.7%, step:599, perplexity: 190.407, speed: 11982 words
59 2018-11-30T04:46:03.340016139Z 12:46:03 INFO:17.2%, step:699, perplexity: 173.602, speed: 12300 words
60 2018-11-30T04:46:10.403947829Z 12:46:10 INFO:19.6%, step:799, perplexity: 160.248, speed: 12537 words
61 2018-11-30T04:46:17.004407075Z 12:46:16 INFO:22.1%, step:899, perplexity: 149.473, speed: 12811 words
62 2018-11-30T04:46:24.264249136Z 12:46:24 INFO:24.5%, step:999, perplexity: 140.668, speed: 12929 words
63 2018-11-30T04:46:31.071145174Z 12:46:31 INFO:27.0%, step:1099, perplexity: 133.325, speed: 13096 words
64 2018-11-30T04:46:37.586931593Z 12:46:37 INFO:29.4%, step:1199, perplexity: 127.034, speed: 13280 words
65 2018-11-30T04:46:44.793397205Z 12:46:44 INFO:31.9%, step:1299, perplexity: 121.599, speed: 13348 words
66 2018-11-30T04:46:51.780899282Z 12:46:51 INFO:34.3%, step:1399, perplexity: 116.847, speed: 13433 words
67 2018-11-30T04:46:59.069548743Z 12:46:59 INFO:36.8%, step:1499, perplexity: 112.656, speed: 13473 words
68 2018-11-30T04:47:07.095971198Z 12:47:07 INFO:39.2%, step:1599, perplexity: 108.944, speed: 13426 words
69 2018-11-30T04:47:14.116929028Z 12:47:14 INFO:41.7%, step:1699, perplexity: 105.627, speed: 13489 words
70 2018-11-30T04:47:20.626616443Z 12:47:20 INFO:44.1%, step:1799, perplexity: 102.644, speed: 13596 words
71 2018-11-30T04:47:28.062087507Z 12:47:28 INFO:46.6%, step:1899, perplexity: 99.928, speed: 13605 words
72 2018-11-30T04:47:35.219894499Z 12:47:35 INFO:49.0%, step:1999, perplexity: 97.435, speed: 13639 words
73 2018-11-30T04:47:42.336843689Z 12:47:42 INFO:51.5%, step:2099, perplexity: 95.166, speed: 13673 words
74 2018-11-30T04:47:49.430225644Z 12:47:49 INFO:53.9%, step:2199, perplexity: 93.094, speed: 13706 words
75 2018-11-30T04:47:56.267199304Z 12:47:56 INFO:56.4%, step:2299, perplexity: 91.187, speed: 13756 words
76 2018-11-30T04:48:04.621567998Z 12:48:04 INFO:58.9%, step:2399, perplexity: 89.412, speed: 13687 words
77 2018-11-30T04:48:12.013966688Z 12:48:11 INFO:61.3%, step:2499, perplexity: 87.761, speed: 13694 words
78 2018-11-30T04:48:18.485738433Z 12:48:18 INFO:63.8%, step:2599, perplexity: 86.205, speed: 13764 words
79 2018-11-30T04:48:25.552724766Z 12:48:25 INFO:66.2%, step:2699, perplexity: 84.735, speed: 13790 words
80 2018-11-30T04:48:32.526875677Z 12:48:32 INFO:68.7%, step:2799, perplexity: 83.347, speed: 13821 words
81 2018-11-30T04:48:39.37521386Z 12:48:39 INFO:71.1%, step:2899, perplexity: 82.032, speed: 13856 words
82 2018-11-30T04:48:46.531790039Z 12:48:46 INFO:73.6%, step:2999, perplexity: 80.782, speed: 13871 words
83 2018-11-30T04:48:53.405020076Z 12:48:53 INFO:76.0%, step:3099, perplexity: 79.600, speed: 13902 words
84 2018-11-30T04:49:00.831453622Z 12:49:00 INFO:78.5%, step:3199, perplexity: 78.480, speed: 13898 words
85 2018-11-30T04:49:07.916507249Z 12:49:07 INFO:80.9%, step:3299, perplexity: 77.410, speed: 13915 words
86 2018-11-30T04:49:14.647921252Z 12:49:14 INFO:83.4%, step:3399, perplexity: 76.388, speed: 13949 words
87 2018-11-30T04:49:21.887778037Z 12:49:21 INFO:85.8%, step:3499, perplexity: 75.412, speed: 13955 words
88 2018-11-30T04:49:28.715236028Z 12:49:28 INFO:88.3%, step:3599, perplexity: 74.483, speed: 13982 words
89 2018-11-30T04:49:35.892043914Z 12:49:35 INFO:90.7%, step:3699, perplexity: 73.597, speed: 13990 words
90 2018-11-30T04:49:42.949935386Z 12:49:42 INFO:93.2%, step:3799, perplexity: 72.747, speed: 14003 words
91 2018-11-30T04:49:49.65803061Z 12:49:49 INFO:95.6%, step:3899, perplexity: 71.931, speed: 14032 words
92 2018-11-30T04:49:56.8939061Z 12:49:56 INFO:98.1%, step:3999, perplexity: 71.150, speed: 14035 words
93 2018-11-30T04:50:02.153168175Z 12:50:02 INFO:Perplexity: 40.376, speed: 14056 words per sec
94 2018-11-30T04:50:04.509660809Z 12:50:04 INFO:Latest model saved in ./results/output_poem/save_model/model-4078
95 2018-11-30T04:50:04.509710969Z
96 2018-11-30T04:50:04.509718342Z 12:50:04 INFO:Evaluate on validation set
97 2018-11-30T04:50:09.786403126Z 12:50:09 INFO:41.7%, step:99, perplexity: 26.796, speed: 22472 words
98 2018-11-30T04:50:12.645946345Z 12:50:12 INFO:83.3%, step:199, perplexity: 26.486, speed: 27588 words
99 2018-11-30T04:50:13.780408211Z 12:50:13 INFO:Perplexity: 27.093, speed: 28719 words per sec
100 2018-11-30T04:50:16.252919441Z 12:50:16 INFO:Best model is saved in ./results/output_poem/best_model/model-4078
101 2018-11-30T04:50:16.252975233Z 12:50:16 INFO:Best validation ppl is 27.093090
102 2018-11-30T04:50:16.252980862Z
103 2018-11-30T04:50:16.269899173Z 12:50:16 INFO:=================== Epoch 1 ===================
104 2018-11-30T04:50:16.269938843Z
105 2018-11-30T04:50:16.269944675Z 12:50:16 INFO:Training on training set
106 2018-11-30T04:50:23.182920256Z 12:50:23 INFO:2.5%, step:99, perplexity: 31.086, speed: 14816 words
107 2018-11-30T04:50:30.379948524Z 12:50:30 INFO:4.9%, step:199, perplexity: 30.167, speed: 14517 words
108 2018-11-30T04:50:37.104407394Z 12:50:37 INFO:7.4%, step:299, perplexity: 29.983, speed: 14745 words
109 2018-11-30T04:50:44.050592024Z 12:50:44 INFO:9.8%, step:399, perplexity: 29.939, speed: 14747 words
110 2018-11-30T04:50:50.813973191Z 12:50:50 INFO:12.3%, step:499, perplexity: 29.911, speed: 14827 words
111 2018-11-30T04:50:57.763540549Z 12:50:57 INFO:14.7%, step:599, perplexity: 29.828, speed: 14809 words
112 2018-11-30T04:51:04.93688358Z 12:51:04 INFO:17.2%, step:699, perplexity: 29.668, speed: 14730 words
113 2018-11-30T04:51:11.818223767Z 12:51:11 INFO:19.6%, step:799, perplexity: 29.435, speed: 14748 words
114 2018-11-30T04:51:18.608248086Z 12:51:18 INFO:22.1%, step:899, perplexity: 29.226, speed: 14784 words
115 2018-11-30T04:51:25.327574995Z 12:51:25 INFO:24.5%, step:999, perplexity: 29.075, speed: 14829 words
116 2018-11-30T04:51:32.322663754Z 12:51:32 INFO:27.0%, step:1099, perplexity: 28.958, speed: 14811 words
117 2018-11-30T04:51:39.374021023Z 12:51:39 INFO:29.4%, step:1199, perplexity: 28.837, speed: 14786 words
118 2018-11-30T04:51:46.155604519Z 12:51:46 INFO:31.9%, step:1299, perplexity: 28.729, speed: 14810 words
119 2018-11-30T04:51:53.011944105Z 12:51:53 INFO:34.3%, step:1399, perplexity: 28.632, speed: 14819 words
120 2018-11-30T04:51:59.627906575Z 12:51:59 INFO:36.8%, step:1499, perplexity: 28.547, speed: 14861 words
121 2018-11-30T04:52:06.865040522Z 12:52:06 INFO:39.2%, step:1599, perplexity: 28.480, speed: 14814 words
122 2018-11-30T04:52:13.998871187Z 12:52:13 INFO:41.7%, step:1699, perplexity: 28.424, speed: 14787 words
123 2018-11-30T04:52:20.83655581Z 12:52:20 INFO:44.1%, step:1799, perplexity: 28.379, speed: 14798 words
124 2018-11-30T04:52:27.978281074Z 12:52:27 INFO:46.6%, step:1899, perplexity: 28.332, speed: 14772 words
125 2018-11-30T04:52:35.068923205Z 12:52:35 INFO:49.0%, step:1999, perplexity: 28.281, speed: 14756 words
126 2018-11-30T04:52:41.958870114Z 12:52:41 INFO:51.5%, step:2099, perplexity: 28.244, speed: 14761 words
127 2018-11-30T04:52:48.969435456Z 12:52:48 INFO:53.9%, step:2199, perplexity: 28.220, speed: 14753 words
128 2018-11-30T04:52:55.887845431Z 12:52:55 INFO:56.4%, step:2299, perplexity: 28.201, speed: 14755 words
129 2018-11-30T04:53:02.733308671Z 12:53:02 INFO:58.9%, step:2399, perplexity: 28.180, speed: 14764 words
130 2018-11-30T04:53:10.031465322Z 12:53:10 INFO:61.3%, step:2499, perplexity: 28.161, speed: 14733 words
131 2018-11-30T04:53:17.543377915Z 12:53:17 INFO:63.8%, step:2599, perplexity: 28.135, speed: 14687 words
132 2018-11-30T04:53:24.607928277Z 12:53:24 INFO:66.2%, step:2699, perplexity: 28.102, speed: 14680 words
133 2018-11-30T04:53:31.178106031Z 12:53:31 INFO:68.7%, step:2799, perplexity: 28.066, speed: 14711 words
134 2018-11-30T04:53:38.199119688Z 12:53:38 INFO:71.1%, step:2899, perplexity: 28.026, speed: 14707 words
135 2018-11-30T04:53:45.613813247Z 12:53:45 INFO:73.6%, step:2999, perplexity: 27.983, speed: 14675 words
136 2018-11-30T04:53:52.212935942Z 12:53:52 INFO:76.0%, step:3099, perplexity: 27.942, speed: 14700 words
137 2018-11-30T04:53:59.36615765Z 12:53:59 INFO:78.5%, step:3199, perplexity: 27.902, speed: 14688 words
138 2018-11-30T04:54:06.487182875Z 12:54:06 INFO:80.9%, step:3299, perplexity: 27.859, speed: 14678 words
139 2018-11-30T04:54:13.336202846Z 12:54:13 INFO:83.4%, step:3399, perplexity: 27.815, speed: 14686 words
140 2018-11-30T04:54:20.568008827Z 12:54:20 INFO:85.8%, step:3499, perplexity: 27.771, speed: 14671 words
141 2018-11-30T04:54:27.292987477Z 12:54:27 INFO:88.3%, step:3599, perplexity: 27.730, speed: 14686 words
142 2018-11-30T04:54:34.218477811Z 12:54:34 INFO:90.7%, step:3699, perplexity: 27.692, speed: 14688 words
143 2018-11-30T04:54:40.646602522Z 12:54:40 INFO:93.2%, step:3799, perplexity: 27.653, speed: 14718 words
144 2018-11-30T04:54:47.74184093Z 12:54:47 INFO:95.6%, step:3899, perplexity: 27.615, speed: 14711 words
145 2018-11-30T04:54:54.490552469Z 12:54:54 INFO:98.1%, step:3999, perplexity: 27.579, speed: 14722 words
146 2018-11-30T04:54:59.866367824Z 12:54:59 INFO:Perplexity: 26.181, speed: 14725 words per sec
147 2018-11-30T04:55:01.712990487Z 12:55:01 INFO:Latest model saved in ./results/output_poem/save_model/model-8156
148 2018-11-30T04:55:01.713034965Z
149 2018-11-30T04:55:01.713041517Z 12:55:01 INFO:Evaluate on validation set
150 2018-11-30T04:55:06.909890183Z 12:55:06 INFO:41.7%, step:99, perplexity: 22.658, speed: 23190 words
151 2018-11-30T04:55:09.73766637Z 12:55:09 INFO:83.3%, step:199, perplexity: 22.534, speed: 28281 words
152 2018-11-30T04:55:10.785907546Z 12:55:10 INFO:Perplexity: 23.182, speed: 29632 words per sec
153 2018-11-30T04:55:12.455380543Z 12:55:12 INFO:Best model is saved in ./results/output_poem/best_model/model-8156
154 2018-11-30T04:55:12.455431399Z 12:55:12 INFO:Best validation ppl is 23.182232
155 2018-11-30T04:55:12.455438777Z
156 2018-11-30T04:55:12.455444057Z 12:55:12 INFO:=================== Epoch 2 ===================
157 2018-11-30T04:55:12.455449532Z
158 2018-11-30T04:55:12.45545442Z 12:55:12 INFO:Training on training set
159 2018-11-30T04:55:19.32794826Z 12:55:19 INFO:2.5%, step:99, perplexity: 25.949, speed: 14908 words
160 2018-11-30T04:55:25.956089772Z 12:55:25 INFO:4.9%, step:199, perplexity: 25.157, speed: 15177 words
161 2018-11-30T04:55:32.585537853Z 12:55:32 INFO:7.4%, step:299, perplexity: 24.978, speed: 15267 words
162 2018-11-30T04:55:39.45697273Z 12:55:39 INFO:9.8%, step:399, perplexity: 24.938, speed: 15171 words
163 2018-11-30T04:55:46.355875307Z 12:55:46 INFO:12.3%, step:499, perplexity: 24.924, speed: 15104 words
164 2018-11-30T04:55:53.420614113Z 12:55:53 INFO:14.7%, step:599, perplexity: 24.870, speed: 14999 words
165 2018-11-30T04:55:59.93292938Z 12:55:59 INFO:17.2%, step:699, perplexity: 24.752, speed: 15100 words
166 2018-11-30T04:56:07.061982325Z 12:56:07 INFO:19.6%, step:799, perplexity: 24.576, speed: 15003 words
167 2018-11-30T04:56:13.83203003Z 12:56:13 INFO:22.1%, step:899, perplexity: 24.420, speed: 15018 words
168 2018-11-30T04:56:20.862144729Z 12:56:20 INFO:24.5%, step:999, perplexity: 24.314, speed: 14970 words
169 2018-11-30T04:56:27.645940271Z 12:56:27 INFO:27.0%, step:1099, perplexity: 24.235, speed: 14981 words
170 2018-11-30T04:56:34.056850637Z 12:56:34 INFO:29.4%, step:1199, perplexity: 24.153, speed: 15059 words
171 2018-11-30T04:56:40.893092114Z 12:56:40 INFO:31.9%, step:1299, perplexity: 24.083, speed: 15053 words
172 2018-11-30T04:56:47.961311814Z 12:56:47 INFO:34.3%, step:1399, perplexity: 24.022, speed: 15011 words
173 2018-11-30T04:56:55.149329071Z 12:56:55 INFO:36.8%, step:1499, perplexity: 23.971, speed: 14958 words
174 2018-11-30T04:57:02.456092192Z 12:57:02 INFO:39.2%, step:1599, perplexity: 23.934, speed: 14896 words
175 2018-11-30T04:57:09.476118045Z 12:57:09 INFO:41.7%, step:1699, perplexity: 23.909, speed: 14876 words
176 2018-11-30T04:57:16.045842326Z 12:57:16 INFO:44.1%, step:1799, perplexity: 23.892, speed: 14914 words
177 2018-11-30T04:57:22.594088094Z 12:57:22 INFO:46.6%, step:1899, perplexity: 23.874, speed: 14951 words
178 2018-11-30T04:57:29.623168287Z 12:57:29 INFO:49.0%, step:1999, perplexity: 23.853, speed: 14931 words
179 2018-11-30T04:57:36.973949514Z 12:57:36 INFO:51.5%, step:2099, perplexity: 23.843, speed: 14880 words
180 2018-11-30T04:57:43.649471347Z 12:57:43 INFO:53.9%, step:2199, perplexity: 23.842, speed: 14900 words
181 2018-11-30T04:57:50.542320293Z 12:57:50 INFO:56.4%, step:2299, perplexity: 23.846, speed: 14898 words
182 2018-11-30T04:57:57.465966036Z 12:57:57 INFO:58.9%, step:2399, perplexity: 23.848, speed: 14894 words
183 2018-11-30T04:58:04.679580492Z 12:58:04 INFO:61.3%, step:2499, perplexity: 23.850, speed: 14865 words
184 2018-11-30T04:58:11.742470381Z 12:58:11 INFO:63.8%, step:2599, perplexity: 23.847, speed: 14850 words
185 2018-11-30T04:58:17.903301968Z 12:58:17 INFO:66.2%, step:2699, perplexity: 23.838, speed: 14909 words
186 2018-11-30T04:58:24.554498938Z 12:58:24 INFO:68.7%, step:2799, perplexity: 23.826, speed: 14926 words
187 2018-11-30T04:58:31.154002848Z 12:58:31 INFO:71.1%, step:2899, perplexity: 23.811, speed: 14945 words
188 2018-11-30T04:58:38.030818807Z 12:58:38 INFO:73.6%, step:2999, perplexity: 23.793, speed: 14944 words
189 2018-11-30T04:58:44.735861705Z 12:58:44 INFO:76.0%, step:3099, perplexity: 23.777, speed: 14954 words
190 2018-11-30T04:58:51.229840442Z 12:58:51 INFO:78.5%, step:3199, perplexity: 23.761, speed: 14978 words
191 2018-11-30T04:58:58.148909917Z 12:58:58 INFO:80.9%, step:3299, perplexity: 23.743, speed: 14973 words
192 2018-11-30T04:59:05.85115759Z 12:59:05 INFO:83.4%, step:3399, perplexity: 23.723, speed: 14918 words
193 2018-11-30T04:59:13.042844973Z 12:59:13 INFO:85.8%, step:3499, perplexity: 23.703, speed: 14897 words
194 2018-11-30T04:59:20.272957177Z 12:59:20 INFO:88.3%, step:3599, perplexity: 23.686, speed: 14876 words
195 2018-11-30T04:59:27.145565287Z 12:59:27 INFO:90.7%, step:3699, perplexity: 23.670, speed: 14876 words
196 2018-11-30T04:59:34.279430045Z 12:59:34 INFO:93.2%, step:3799, perplexity: 23.653, speed: 14862 words
197 2018-11-30T04:59:41.015871224Z 12:59:41 INFO:95.6%, step:3899, perplexity: 23.638, speed: 14871 words
198 2018-11-30T04:59:47.973821696Z 12:59:47 INFO:98.1%, step:3999, perplexity: 23.623, speed: 14867 words
199 2018-11-30T04:59:53.672180381Z 12:59:53 INFO:Perplexity: 23.105, speed: 14849 words per sec
200 2018-11-30T04:59:56.127146406Z 12:59:56 INFO:Latest model saved in ./results/output_poem/save_model/model-12234
201 2018-11-30T04:59:56.127189999Z
202 2018-11-30T04:59:56.127197567Z 12:59:56 INFO:Evaluate on validation set
203 2018-11-30T05:00:01.317251452Z 01:00:01 INFO:41.7%, step:99, perplexity: 22.012, speed: 22788 words
204 2018-11-30T05:00:04.570819983Z 01:00:04 INFO:83.3%, step:199, perplexity: 21.878, speed: 26442 words
205 2018-11-30T05:00:05.866881353Z 01:00:05 INFO:Perplexity: 22.550, speed: 27176 words per sec
206 2018-11-30T05:00:07.533799706Z 01:00:07 INFO:Best model is saved in ./results/output_poem/best_model/model-12234
207 2018-11-30T05:00:07.533863056Z 01:00:07 INFO:Best validation ppl is 22.550182
208 2018-11-30T05:00:07.533870874Z
209 2018-11-30T05:00:07.536077263Z 01:00:07 INFO:=================== Epoch 3 ===================
210 2018-11-30T05:00:07.536122725Z
211 2018-11-30T05:00:07.536128901Z 01:00:07 INFO:Training on training set
212 2018-11-30T05:00:14.297945668Z 01:00:14 INFO:2.5%, step:99, perplexity: 23.972, speed: 15148 words
213 2018-11-30T05:00:20.938083478Z 01:00:20 INFO:4.9%, step:199, perplexity: 23.241, speed: 15285 words
214 2018-11-30T05:00:27.805888369Z 01:00:27 INFO:7.4%, step:299, perplexity: 23.072, speed: 15161 words
215 2018-11-30T05:00:34.551002168Z 01:00:34 INFO:9.8%, step:399, perplexity: 23.038, speed: 15164 words
216 2018-11-30T05:00:41.155884074Z 01:00:41 INFO:12.3%, step:499, perplexity: 23.025, speed: 15231 words
217 2018-11-30T05:00:47.837683243Z 01:00:47 INFO:14.7%, step:599, perplexity: 22.976, speed: 15248 words
218 2018-11-30T05:00:54.679879948Z 01:00:54 INFO:17.2%, step:699, perplexity: 22.872, speed: 15206 words
219 2018-11-30T05:01:01.388838594Z 01:01:01 INFO:19.6%, step:799, perplexity: 22.718, speed: 15213 words
220 2018-11-30T05:01:09.146072917Z 01:01:09 INFO:22.1%, step:899, perplexity: 22.584, speed: 14960 words
221 2018-11-30T05:01:15.899010181Z 01:01:15 INFO:24.5%, step:999, perplexity: 22.495, speed: 14980 words
222 2018-11-30T05:01:23.372931746Z 01:01:23 INFO:27.0%, step:1099, perplexity: 22.432, speed: 14854 words
223 2018-11-30T05:01:30.452859118Z 01:01:30 INFO:29.4%, step:1199, perplexity: 22.364, speed: 14820 words
224 2018-11-30T05:01:38.82588389Z 01:01:38 INFO:31.9%, step:1299, perplexity: 22.307, speed: 14582 words
225 2018-11-30T05:01:45.547925898Z 01:01:45 INFO:34.3%, step:1399, perplexity: 22.258, speed: 14627 words
226 2018-11-30T05:01:52.42089121Z 01:01:52 INFO:36.8%, step:1499, perplexity: 22.217, speed: 14645 words
227 2018-11-30T05:01:59.254834834Z 01:01:59 INFO:39.2%, step:1599, perplexity: 22.190, speed: 14666 words
228 2018-11-30T05:02:07.061817315Z 01:02:07 INFO:41.7%, step:1699, perplexity: 22.172, speed: 14565 words
229 2018-11-30T05:02:13.8239193Z 01:02:13 INFO:44.1%, step:1799, perplexity: 22.163, speed: 14596 words
230 2018-11-30T05:02:20.467906948Z 01:02:20 INFO:46.6%, step:1899, perplexity: 22.154, speed: 14637 words
231 2018-11-30T05:02:27.690456195Z 01:02:27 INFO:49.0%, step:1999, perplexity: 22.143, speed: 14613 words
232 2018-11-30T05:02:34.494205916Z 01:02:34 INFO:51.5%, step:2099, perplexity: 22.141, speed: 14634 words
233 2018-11-30T05:02:42.496934138Z 01:02:42 INFO:53.9%, step:2199, perplexity: 22.148, speed: 14538 words
234 2018-11-30T05:02:49.606329159Z 01:02:49 INFO:56.4%, step:2299, perplexity: 22.158, speed: 14532 words
235 2018-11-30T05:02:56.196907046Z 01:02:56 INFO:58.9%, step:2399, perplexity: 22.166, speed: 14572 words
236 2018-11-30T05:03:03.493909434Z 01:03:03 INFO:61.3%, step:2499, perplexity: 22.176, speed: 14550 words
237 2018-11-30T05:03:10.667044978Z 01:03:10 INFO:63.8%, step:2599, perplexity: 22.180, speed: 14538 words
238 2018-11-30T05:03:17.964917689Z 01:03:17 INFO:66.2%, step:2699, perplexity: 22.178, speed: 14519 words
239 2018-11-30T05:03:24.831911662Z 01:03:24 INFO:68.7%, step:2799, perplexity: 22.174, speed: 14533 words
240 2018-11-30T05:03:31.652639076Z 01:03:31 INFO:71.1%, step:2899, perplexity: 22.167, speed: 14549 words
241 2018-11-30T05:03:38.604742578Z 01:03:38 INFO:73.6%, step:2999, perplexity: 22.157, speed: 14555 words
242 2018-11-30T05:03:45.590932078Z 01:03:45 INFO:76.0%, step:3099, perplexity: 22.148, speed: 14558 words
243 2018-11-30T05:03:52.561745122Z 01:03:52 INFO:78.5%, step:3199, perplexity: 22.139, speed: 14562 words
244 2018-11-30T05:03:59.283620476Z 01:03:59 INFO:80.9%, step:3299, perplexity: 22.128, speed: 14582 words
245 2018-11-30T05:04:06.504941236Z 01:04:06 INFO:83.4%, step:3399, perplexity: 22.116, speed: 14569 words
246 2018-11-30T05:04:13.845824155Z 01:04:13 INFO:85.8%, step:3499, perplexity: 22.103, speed: 14551 words
247 2018-11-30T05:04:20.350215035Z 01:04:20 INFO:88.3%, step:3599, perplexity: 22.092, speed: 14582 words
248 2018-11-30T05:04:27.077870317Z 01:04:27 INFO:90.7%, step:3699, perplexity: 22.083, speed: 14598 words
249 2018-11-30T05:04:33.789621559Z 01:04:33 INFO:93.2%, step:3799, perplexity: 22.073, speed: 14615 words
250 2018-11-30T05:04:40.317297973Z 01:04:40 INFO:95.6%, step:3899, perplexity: 22.063, speed: 14641 words
251 2018-11-30T05:04:47.665124397Z 01:04:47 INFO:98.1%, step:3999, perplexity: 22.055, speed: 14622 words
252 2018-11-30T05:04:52.853038153Z 01:04:52 INFO:Perplexity: 21.774, speed: 14636 words per sec
253 2018-11-30T05:04:54.724611813Z 01:04:54 INFO:Latest model saved in ./results/output_poem/save_model/model-16312
254 2018-11-30T05:04:54.724651312Z
255 2018-11-30T05:04:54.724656363Z 01:04:54 INFO:Evaluate on validation set
256 2018-11-30T05:05:00.117830884Z 01:05:00 INFO:41.7%, step:99, perplexity: 21.484, speed: 22217 words
257 2018-11-30T05:05:03.228889168Z 01:05:03 INFO:83.3%, step:199, perplexity: 21.357, speed: 26523 words
258 2018-11-30T05:05:04.410253568Z 01:05:04 INFO:Perplexity: 22.081, speed: 27603 words per sec
259 2018-11-30T05:05:06.242269719Z 01:05:06 INFO:Best model is saved in ./results/output_poem/best_model/model-16312
260 2018-11-30T05:05:06.242317858Z 01:05:06 INFO:Best validation ppl is 22.081152
261 2018-11-30T05:05:06.242325131Z
262 2018-11-30T05:05:06.266830104Z 01:05:06 INFO:=================== Epoch 4 ===================
263 2018-11-30T05:05:06.266865558Z
264 2018-11-30T05:05:06.266873335Z 01:05:06 INFO:Training on training set
265 2018-11-30T05:05:13.189967603Z 01:05:13 INFO:2.5%, step:99, perplexity: 22.791, speed: 14781 words
266 2018-11-30T05:05:20.214432348Z 01:05:20 INFO:4.9%, step:199, perplexity: 22.105, speed: 14689 words
267 2018-11-30T05:05:26.628891398Z 01:05:26 INFO:7.4%, step:299, perplexity: 21.943, speed: 15085 words
268 2018-11-30T05:05:33.218825707Z 01:05:33 INFO:9.8%, step:399, perplexity: 21.925, speed: 15196 words
269 2018-11-30T05:05:39.587389663Z 01:05:39 INFO:12.3%, step:499, perplexity: 21.928, speed: 15364 words
270 2018-11-30T05:05:46.735723733Z 01:05:46 INFO:14.7%, step:599, perplexity: 21.895, speed: 15180 words
271 2018-11-30T05:05:53.335250147Z 01:05:53 INFO:17.2%, step:699, perplexity: 21.808, speed: 15227 words
272 2018-11-30T05:05:59.819745477Z 01:05:59 INFO:19.6%, step:799, perplexity: 21.670, speed: 15296 words
273 2018-11-30T05:06:06.903625455Z 01:06:06 INFO:22.1%, step:899, perplexity: 21.549, speed: 15198 words
274 2018-11-30T05:06:13.584289441Z 01:06:13 INFO:24.5%, step:999, perplexity: 21.469, speed: 15210 words
275 2018-11-30T05:06:20.078794545Z 01:06:20 INFO:27.0%, step:1099, perplexity: 21.413, speed: 15259 words
276 2018-11-30T05:06:26.462399842Z 01:06:26 INFO:29.4%, step:1199, perplexity: 21.353, speed: 15322 words
277 2018-11-30T05:06:32.987086721Z 01:06:32 INFO:31.9%, step:1299, perplexity: 21.304, speed: 15350 words
278 2018-11-30T05:06:39.565193043Z 01:06:39 INFO:34.3%, step:1399, perplexity: 21.263, speed: 15367 words
279 2018-11-30T05:06:46.8740217Z 01:06:46 INFO:36.8%, step:1499, perplexity: 21.230, speed: 15267 words
280 2018-11-30T05:06:53.675893365Z 01:06:53 INFO:39.2%, step:1599, perplexity: 21.209, speed: 15253 words
281 2018-11-30T05:07:00.494076093Z 01:07:00 INFO:41.7%, step:1699, perplexity: 21.196, speed: 15239 words
282 2018-11-30T05:07:07.850846973Z 01:07:07 INFO:44.1%, step:1799, perplexity: 21.190, speed: 15159 words
283 2018-11-30T05:07:14.49324276Z 01:07:14 INFO:46.6%, step:1899, perplexity: 21.184, speed: 15173 words
284 2018-11-30T05:07:21.306115013Z 01:07:21 INFO:49.0%, step:1999, perplexity: 21.176, speed: 15165 words
285 2018-11-30T05:07:28.58204599Z 01:07:28 INFO:51.5%, step:2099, perplexity: 21.176, speed: 15110 words
286 2018-11-30T05:07:34.885914404Z 01:07:34 INFO:53.9%, step:2199, perplexity: 21.186, speed: 15158 words
287 2018-11-30T05:07:41.894222696Z 01:07:41 INFO:56.4%, step:2299, perplexity: 21.199, speed: 15133 words
288 2018-11-30T05:07:48.632029777Z 01:07:48 INFO:58.9%, step:2399, perplexity: 21.209, speed: 15136 words
289 2018-11-30T05:07:55.006924696Z 01:07:55 INFO:61.3%, step:2499, perplexity: 21.221, speed: 15171 words
290 2018-11-30T05:08:02.225662972Z 01:08:02 INFO:63.8%, step:2599, perplexity: 21.228, speed: 15131 words
291 2018-11-30T05:08:09.131320376Z 01:08:09 INFO:66.2%, step:2699, perplexity: 21.229, speed: 15119 words
292 2018-11-30T05:08:16.298226565Z 01:08:16 INFO:68.7%, step:2799, perplexity: 21.227, speed: 15089 words
293 2018-11-30T05:08:23.269507093Z 01:08:23 INFO:71.1%, step:2899, perplexity: 21.223, speed: 15074 words
294 2018-11-30T05:08:29.763897926Z 01:08:29 INFO:73.6%, step:2999, perplexity: 21.215, speed: 15096 words
295 2018-11-30T05:08:36.722060757Z 01:08:36 INFO:76.0%, step:3099, perplexity: 21.209, speed: 15083 words
296 2018-11-30T05:08:43.369938515Z 01:08:43 INFO:78.5%, step:3199, perplexity: 21.203, speed: 15093 words
297 2018-11-30T05:08:50.160809653Z 01:08:50 INFO:80.9%, step:3299, perplexity: 21.194, speed: 15093 words
298 2018-11-30T05:08:57.582559501Z 01:08:57 INFO:83.4%, step:3399, perplexity: 21.184, speed: 15051 words
299 2018-11-30T05:09:05.172010032Z 01:09:05 INFO:85.8%, step:3499, perplexity: 21.174, speed: 15002 words
300 2018-11-30T05:09:12.42411949Z 01:09:12 INFO:88.3%, step:3599, perplexity: 21.166, speed: 14976 words
301 2018-11-30T05:09:18.816095566Z 01:09:18 INFO:90.7%, step:3699, perplexity: 21.159, speed: 15002 words
302 2018-11-30T05:09:25.477255685Z 01:09:25 INFO:93.2%, step:3799, perplexity: 21.151, speed: 15012 words
303 2018-11-30T05:09:32.480370654Z 01:09:32 INFO:95.6%, step:3899, perplexity: 21.144, speed: 15001 words
304 2018-11-30T05:09:39.20038995Z 01:09:39 INFO:98.1%, step:3999, perplexity: 21.139, speed: 15007 words
305 2018-11-30T05:09:44.689857535Z 01:09:44 INFO:Perplexity: 20.963, speed: 14998 words per sec
306 2018-11-30T05:09:46.310941834Z 01:09:46 INFO:Latest model saved in ./results/output_poem/save_model/model-20390
307 2018-11-30T05:09:46.310994809Z
308 2018-11-30T05:09:46.311001745Z 01:09:46 INFO:Evaluate on validation set
309 2018-11-30T05:09:51.055952364Z 01:09:51 INFO:41.7%, step:99, perplexity: 21.626, speed: 25200 words
310 2018-11-30T05:09:53.929604089Z 01:09:53 INFO:83.3%, step:199, perplexity: 21.471, speed: 29524 words
311 2018-11-30T05:09:55.453903534Z 01:09:55 INFO:Perplexity: 22.178, speed: 29112 words per sec
312 2018-11-30T05:09:55.453946296Z 01:09:55 INFO:Decay the learning rate: 0.0025
313 2018-11-30T05:09:55.462918712Z 01:09:55 INFO:Best model is saved in ./results/output_poem/best_model/model-16312
314 2018-11-30T05:09:55.462936179Z 01:09:55 INFO:Best validation ppl is 22.081152
315 2018-11-30T05:09:55.462941943Z
316 2018-11-30T05:09:55.465201853Z 01:09:55 INFO:=================== Epoch 5 ===================
317 2018-11-30T05:09:55.465216766Z
318 2018-11-30T05:09:55.465222712Z 01:09:55 INFO:Training on training set
319 2018-11-30T05:10:02.40753574Z 01:10:02 INFO:2.5%, step:99, perplexity: 22.137, speed: 14754 words
320 2018-11-30T05:10:09.36044283Z 01:10:09 INFO:4.9%, step:199, perplexity: 21.425, speed: 14741 words
321 2018-11-30T05:10:16.271511692Z 01:10:16 INFO:7.4%, step:299, perplexity: 21.220, speed: 14768 words
322 2018-11-30T05:10:23.135980682Z 01:10:23 INFO:9.8%, step:399, perplexity: 21.158, speed: 14804 words
323 2018-11-30T05:10:30.51287084Z 01:10:30 INFO:12.3%, step:499, perplexity: 21.125, speed: 14610 words
324 2018-11-30T05:10:36.995533366Z 01:10:36 INFO:14.7%, step:599, perplexity: 21.067, speed: 14799 words
325 2018-11-30T05:10:43.709170086Z 01:10:43 INFO:17.2%, step:699, perplexity: 20.960, speed: 14860 words
326 2018-11-30T05:10:50.252568349Z 01:10:50 INFO:19.6%, step:799, perplexity: 20.809, speed: 14954 words
327 2018-11-30T05:10:57.234711945Z 01:10:57 INFO:22.1%, step:899, perplexity: 20.678, speed: 14921 words
328 2018-11-30T05:11:04.905816689Z 01:11:04 INFO:24.5%, step:999, perplexity: 20.586, speed: 14748 words
329 2018-11-30T05:11:11.73893905Z 01:11:11 INFO:27.0%, step:1099, perplexity: 20.518, speed: 14770 words
330 2018-11-30T05:11:18.543687996Z 01:11:18 INFO:29.4%, step:1199, perplexity: 20.447, speed: 14792 words
331 2018-11-30T05:11:25.006189073Z 01:11:25 INFO:31.9%, step:1299, perplexity: 20.387, speed: 14868 words
332 2018-11-30T05:11:32.06386711Z 01:11:32 INFO:34.3%, step:1399, perplexity: 20.337, speed: 14841 words
333 2018-11-30T05:11:38.968368817Z 01:11:38 INFO:36.8%, step:1499, perplexity: 20.294, speed: 14842 words
334 2018-11-30T05:11:45.53662154Z 01:11:45 INFO:39.2%, step:1599, perplexity: 20.264, speed: 14885 words
335 2018-11-30T05:11:52.377540066Z 01:11:52 INFO:41.7%, step:1699, perplexity: 20.242, speed: 14890 words
336 2018-11-30T05:11:59.345725551Z 01:11:59 INFO:44.1%, step:1799, perplexity: 20.227, speed: 14879 words
337 2018-11-30T05:12:06.850685115Z 01:12:06 INFO:46.6%, step:1899, perplexity: 20.211, speed: 14809 words
338 2018-11-30T05:12:13.607850128Z 01:12:13 INFO:49.0%, step:1999, perplexity: 20.194, speed: 14826 words
339 2018-11-30T05:12:20.250887065Z 01:12:20 INFO:51.5%, step:2099, perplexity: 20.185, speed: 14852 words
340 2018-11-30T05:12:27.311499354Z 01:12:27 INFO:53.9%, step:2199, perplexity: 20.184, speed: 14836 words
341 2018-11-30T05:12:33.659256284Z 01:12:33 INFO:56.4%, step:2299, perplexity: 20.186, speed: 14889 words
342 2018-11-30T05:12:41.262969747Z 01:12:41 INFO:58.9%, step:2399, perplexity: 20.187, speed: 14823 words
343 2018-11-30T05:12:47.727816597Z 01:12:47 INFO:61.3%, step:2499, perplexity: 20.189, speed: 14862 words
344 2018-11-30T05:12:54.528919008Z 01:12:54 INFO:63.8%, step:2599, perplexity: 20.186, speed: 14869 words
345 2018-11-30T05:13:01.356896345Z 01:13:01 INFO:66.2%, step:2699, perplexity: 20.180, speed: 14873 words
346 2018-11-30T05:13:08.618609623Z 01:13:08 INFO:68.7%, step:2799, perplexity: 20.170, speed: 14844 words
347 2018-11-30T05:13:15.065375652Z 01:13:15 INFO:71.1%, step:2899, perplexity: 20.159, speed: 14878 words
348 2018-11-30T05:13:21.387953716Z 01:13:21 INFO:73.6%, step:2999, perplexity: 20.145, speed: 14918 words
349 2018-11-30T05:13:28.385005493Z 01:13:28 INFO:76.0%, step:3099, perplexity: 20.132, speed: 14909 words
350 2018-11-30T05:13:35.221913078Z 01:13:35 INFO:78.5%, step:3199, perplexity: 20.119, speed: 14911 words
351 2018-11-30T05:13:41.986615281Z 01:13:41 INFO:80.9%, step:3299, perplexity: 20.105, speed: 14918 words
352 2018-11-30T05:13:48.696327677Z 01:13:48 INFO:83.4%, step:3399, perplexity: 20.090, speed: 14928 words
353 2018-11-30T05:13:55.649956457Z 01:13:55 INFO:85.8%, step:3499, perplexity: 20.074, speed: 14922 words
354 2018-11-30T05:14:02.733557524Z 01:14:02 INFO:88.3%, step:3599, perplexity: 20.061, speed: 14909 words
355 2018-11-30T05:14:10.352043563Z 01:14:10 INFO:90.7%, step:3699, perplexity: 20.048, speed: 14865 words
356 2018-11-30T05:14:16.963579533Z 01:14:16 INFO:93.2%, step:3799, perplexity: 20.036, speed: 14881 words
357 2018-11-30T05:14:23.668020164Z 01:14:23 INFO:95.6%, step:3899, perplexity: 20.024, speed: 14891 words
358 2018-11-30T05:14:30.381455739Z 01:14:30 INFO:98.1%, step:3999, perplexity: 20.013, speed: 14899 words
359 2018-11-30T05:14:35.755193098Z 01:14:35 INFO:Perplexity: 19.637, speed: 14898 words per sec
360 2018-11-30T05:14:37.583411621Z 01:14:37 INFO:Latest model saved in ./results/output_poem/save_model/model-24468
361 2018-11-30T05:14:37.583459468Z
362 2018-11-30T05:14:37.583464853Z 01:14:37 INFO:Evaluate on validation set
363 2018-11-30T05:14:42.713945924Z 01:14:42 INFO:41.7%, step:99, perplexity: 20.878, speed: 22235 words
364 2018-11-30T05:14:45.69710335Z 01:14:45 INFO:83.3%, step:199, perplexity: 20.728, speed: 26980 words
365 2018-11-30T05:14:46.800749418Z 01:14:46 INFO:Perplexity: 21.441, speed: 28286 words per sec
366 2018-11-30T05:14:48.425390085Z 01:14:48 INFO:Best model is saved in ./results/output_poem/best_model/model-24468
367 2018-11-30T05:14:48.425436375Z 01:14:48 INFO:Best validation ppl is 21.440775
368 2018-11-30T05:14:48.425443312Z
369 2018-11-30T05:14:48.427462506Z 01:14:48 INFO:=================== Epoch 6 ===================
370 2018-11-30T05:14:48.427479421Z
371 2018-11-30T05:14:48.427484859Z 01:14:48 INFO:Training on training set
372 2018-11-30T05:14:55.60105068Z 01:14:55 INFO:2.5%, step:99, perplexity: 21.021, speed: 14279 words
373 2018-11-30T05:15:02.397089721Z 01:15:02 INFO:4.9%, step:199, perplexity: 20.387, speed: 14666 words
374 2018-11-30T05:15:09.680083898Z 01:15:09 INFO:7.4%, step:299, perplexity: 20.211, speed: 14458 words
375 2018-11-30T05:15:16.366919619Z 01:15:16 INFO:9.8%, step:399, perplexity: 20.173, speed: 14661 words
376 2018-11-30T05:15:22.832747566Z 01:15:22 INFO:12.3%, step:499, perplexity: 20.159, speed: 14884 words
377 2018-11-30T05:15:30.446924304Z 01:15:30 INFO:14.7%, step:599, perplexity: 20.121, speed: 14622 words
378 2018-11-30T05:15:37.1483245Z 01:15:37 INFO:17.2%, step:699, perplexity: 20.037, speed: 14713 words
379 2018-11-30T05:15:44.319934827Z 01:15:44 INFO:19.6%, step:799, perplexity: 19.910, speed: 14657 words
380 2018-11-30T05:15:50.933938007Z 01:15:50 INFO:22.1%, step:899, perplexity: 19.799, speed: 14744 words
381 2018-11-30T05:15:58.328704232Z 01:15:58 INFO:24.5%, step:999, perplexity: 19.724, speed: 14654 words
382 2018-11-30T05:16:05.482345462Z 01:16:05 INFO:27.0%, step:1099, perplexity: 19.671, speed: 14619 words
383 2018-11-30T05:16:12.544006024Z 01:16:12 INFO:29.4%, step:1199, perplexity: 19.615, speed: 14609 words
384 2018-11-30T05:16:19.387903274Z 01:16:19 INFO:31.9%, step:1299, perplexity: 19.567, speed: 14636 words
385 2018-11-30T05:16:26.100183865Z 01:16:26 INFO:34.3%, step:1399, perplexity: 19.527, speed: 14678 words
386 2018-11-30T05:16:32.708988528Z 01:16:32 INFO:36.8%, step:1499, perplexity: 19.495, speed: 14730 words
387 2018-11-30T05:16:39.620618845Z 01:16:39 INFO:39.2%, step:1599, perplexity: 19.473, speed: 14735 words
388 2018-11-30T05:16:46.209190024Z 01:16:46 INFO:41.7%, step:1699, perplexity: 19.459, speed: 14780 words
389 2018-11-30T05:16:53.161918464Z 01:16:53 INFO:44.1%, step:1799, perplexity: 19.452, speed: 14777 words
390 2018-11-30T05:17:00.099867672Z 01:17:00 INFO:46.6%, step:1899, perplexity: 19.444, speed: 14777 words
391 2018-11-30T05:17:07.281849978Z 01:17:07 INFO:49.0%, step:1999, perplexity: 19.433, speed: 14750 words
392 2018-11-30T05:17:14.649985116Z 01:17:14 INFO:51.5%, step:2099, perplexity: 19.431, speed: 14707 words
393 2018-11-30T05:17:21.648879927Z 01:17:21 INFO:53.9%, step:2199, perplexity: 19.436, speed: 14704 words
394 2018-11-30T05:17:28.586224799Z 01:17:28 INFO:56.4%, step:2299, perplexity: 19.444, speed: 14706 words
395 2018-11-30T05:17:34.888479241Z 01:17:34 INFO:58.9%, step:2399, perplexity: 19.451, speed: 14764 words
396 2018-11-30T05:17:41.55616057Z 01:17:41 INFO:61.3%, step:2499, perplexity: 19.458, speed: 14787 words
397 2018-11-30T05:17:48.363684923Z 01:17:48 INFO:63.8%, step:2599, perplexity: 19.461, speed: 14797 words
398 2018-11-30T05:17:54.921563874Z 01:17:54 INFO:66.2%, step:2699, perplexity: 19.460, speed: 14826 words
399 2018-11-30T05:18:02.227485649Z 01:18:02 INFO:68.7%, step:2799, perplexity: 19.456, speed: 14795 words
400 2018-11-30T05:18:09.096859469Z 01:18:09 INFO:71.1%, step:2899, perplexity: 19.451, speed: 14799 words
401 2018-11-30T05:18:15.742103605Z 01:18:15 INFO:73.6%, step:2999, perplexity: 19.442, speed: 14818 words
402 2018-11-30T05:18:22.848865159Z 01:18:22 INFO:76.0%, step:3099, perplexity: 19.435, speed: 14805 words
403 2018-11-30T05:18:29.237471725Z 01:18:29 INFO:78.5%, step:3199, perplexity: 19.428, speed: 14840 words
404 2018-11-30T05:18:35.680502671Z 01:18:35 INFO:80.9%, step:3299, perplexity: 19.419, speed: 14870 words
405 2018-11-30T05:18:42.560888993Z 01:18:42 INFO:83.4%, step:3399, perplexity: 19.408, speed: 14871 words
406 2018-11-30T05:18:49.642503256Z 01:18:49 INFO:85.8%, step:3499, perplexity: 19.398, speed: 14858 words
407 2018-11-30T05:18:57.08493735Z 01:18:57 INFO:88.3%, step:3599, perplexity: 19.389, speed: 14825 words
408 2018-11-30T05:19:03.895388267Z 01:19:03 INFO:90.7%, step:3699, perplexity: 19.382, speed: 14831 words
409 2018-11-30T05:19:10.893966151Z 01:19:10 INFO:93.2%, step:3799, perplexity: 19.374, speed: 14826 words
410 2018-11-30T05:19:17.309053327Z 01:19:17 INFO:95.6%, step:3899, perplexity: 19.367, speed: 14853 words
411 2018-11-30T05:19:24.08104614Z 01:19:24 INFO:98.1%, step:3999, perplexity: 19.361, speed: 14859 words
412 2018-11-30T05:19:29.278925134Z 01:19:29 INFO:Perplexity: 19.171, speed: 14869 words per sec
413 2018-11-30T05:19:31.23693894Z 01:19:31 INFO:Latest model saved in ./results/output_poem/save_model/model-28546
414 2018-11-30T05:19:31.236990298Z
415 2018-11-30T05:19:31.236997807Z 01:19:31 INFO:Evaluate on validation set
416 2018-11-30T05:19:36.224943016Z 01:19:36 INFO:41.7%, step:99, perplexity: 20.870, speed: 23805 words
417 2018-11-30T05:19:38.769693561Z 01:19:38 INFO:83.3%, step:199, perplexity: 20.720, speed: 29923 words
418 2018-11-30T05:19:39.733175858Z 01:19:39 INFO:Perplexity: 21.444, speed: 31481 words per sec
419 2018-11-30T05:19:39.733229978Z 01:19:39 INFO:Decay the learning rate: 0.00125
420 2018-11-30T05:19:39.735913756Z 01:19:39 INFO:Best model is saved in ./results/output_poem/best_model/model-24468
421 2018-11-30T05:19:39.735942901Z 01:19:39 INFO:Best validation ppl is 21.440775
422 2018-11-30T05:19:39.735948496Z
423 2018-11-30T05:19:39.749905437Z 01:19:39 INFO:=================== Epoch 7 ===================
424 2018-11-30T05:19:39.749949413Z
425 2018-11-30T05:19:39.749954828Z 01:19:39 INFO:Training on training set
426 2018-11-30T05:19:46.66973716Z 01:19:46 INFO:2.5%, step:99, perplexity: 20.639, speed: 14797 words
427 2018-11-30T05:19:53.100443962Z 01:19:53 INFO:4.9%, step:199, perplexity: 19.989, speed: 15346 words
428 2018-11-30T05:20:00.707858884Z 01:20:00 INFO:7.4%, step:299, perplexity: 19.798, speed: 14657 words
429 2018-11-30T05:20:08.174921932Z 01:20:08 INFO:9.8%, step:399, perplexity: 19.749, speed: 14409 words
430 2018-11-30T05:20:14.800830641Z 01:20:14 INFO:12.3%, step:499, perplexity: 19.728, speed: 14607 words
431 2018-11-30T05:20:21.648962859Z 01:20:21 INFO:14.7%, step:599, perplexity: 19.685, speed: 14664 words
432 2018-11-30T05:20:28.110479632Z 01:20:28 INFO:17.2%, step:699, perplexity: 19.597, speed: 14823 words
433 2018-11-30T05:20:34.818065625Z 01:20:34 INFO:19.6%, step:799, perplexity: 19.467, speed: 14876 words
434 2018-11-30T05:20:41.287467866Z 01:20:41 INFO:22.1%, step:899, perplexity: 19.353, speed: 14977 words
435 2018-11-30T05:20:47.746212099Z 01:20:47 INFO:24.5%, step:999, perplexity: 19.275, speed: 15059 words
436 2018-11-30T05:20:54.194935136Z 01:20:54 INFO:27.0%, step:1099, perplexity: 19.219, speed: 15130 words
437 2018-11-30T05:21:01.080919874Z 01:21:01 INFO:29.4%, step:1199, perplexity: 19.159, speed: 15108 words
438 2018-11-30T05:21:08.371053141Z 01:21:08 INFO:31.9%, step:1299, perplexity: 19.109, speed: 15022 words
439 2018-11-30T05:21:15.084318823Z 01:21:15 INFO:34.3%, step:1399, perplexity: 19.066, speed: 15037 words
440 2018-11-30T05:21:21.9198413Z 01:21:21 INFO:36.8%, step:1499, perplexity: 19.030, speed: 15038 words
441 2018-11-30T05:21:28.685081452Z 01:21:28 INFO:39.2%, step:1599, perplexity: 19.005, speed: 15040 words
442 2018-11-30T05:21:35.160106603Z 01:21:35 INFO:41.7%, step:1699, perplexity: 18.987, speed: 15084 words
443 2018-11-30T05:21:41.955070363Z 01:21:41 INFO:44.1%, step:1799, perplexity: 18.976, speed: 15084 words
444 2018-11-30T05:21:48.41992538Z 01:21:48 INFO:46.6%, step:1899, perplexity: 18.964, speed: 15121 words
445 2018-11-30T05:21:55.09784153Z 01:21:55 INFO:49.0%, step:1999, perplexity: 18.950, speed: 15132 words
446 2018-11-30T05:22:02.410791818Z 01:22:02 INFO:51.5%, step:2099, perplexity: 18.944, speed: 15073 words
447 2018-11-30T05:22:09.449966994Z 01:22:09 INFO:53.9%, step:2199, perplexity: 18.945, speed: 15049 words
448 2018-11-30T05:22:16.594937477Z 01:22:16 INFO:56.4%, step:2299, perplexity: 18.948, speed: 15016 words
449 2018-11-30T05:22:23.214920451Z 01:22:23 INFO:58.9%, step:2399, perplexity: 18.951, speed: 15034 words
450 2018-11-30T05:22:29.996059742Z 01:22:29 INFO:61.3%, step:2499, perplexity: 18.954, speed: 15037 words
451 2018-11-30T05:22:36.585892534Z 01:22:36 INFO:63.8%, step:2599, perplexity: 18.953, speed: 15056 words
452 2018-11-30T05:22:43.683935464Z 01:22:43 INFO:66.2%, step:2699, perplexity: 18.948, speed: 15032 words
453 2018-11-30T05:22:51.003953146Z 01:22:51 INFO:68.7%, step:2799, perplexity: 18.941, speed: 14991 words
454 2018-11-30T05:22:57.48140677Z 01:22:57 INFO:71.1%, step:2899, perplexity: 18.932, speed: 15018 words
455 2018-11-30T05:23:05.189393976Z 01:23:05 INFO:73.6%, step:2999, perplexity: 18.920, speed: 14953 words
456 2018-11-30T05:23:12.542861901Z 01:23:12 INFO:76.0%, step:3099, perplexity: 18.910, speed: 14918 words
457 2018-11-30T05:23:19.096038335Z 01:23:19 INFO:78.5%, step:3199, perplexity: 18.899, speed: 14940 words
458 2018-11-30T05:23:25.869940859Z 01:23:25 INFO:80.9%, step:3299, perplexity: 18.888, speed: 14944 words
459 2018-11-30T05:23:32.507955388Z 01:23:32 INFO:83.4%, step:3399, perplexity: 18.875, speed: 14958 words
460 2018-11-30T05:23:39.621899008Z 01:23:39 INFO:85.8%, step:3499, perplexity: 18.862, speed: 14941 words
461 2018-11-30T05:23:46.636282049Z 01:23:46 INFO:88.3%, step:3599, perplexity: 18.850, speed: 14932 words
462 2018-11-30T05:23:53.038746554Z 01:23:53 INFO:90.7%, step:3699, perplexity: 18.840, speed: 14958 words
463 2018-11-30T05:23:59.971058976Z 01:23:59 INFO:93.2%, step:3799, perplexity: 18.830, speed: 14954 words
464 2018-11-30T05:24:07.572971239Z 01:24:07 INFO:95.6%, step:3899, perplexity: 18.820, speed: 14911 words
465 2018-11-30T05:24:14.560944232Z 01:24:14 INFO:98.1%, step:3999, perplexity: 18.811, speed: 14905 words
466 2018-11-30T05:24:19.384944012Z 01:24:19 INFO:Perplexity: 18.509, speed: 14933 words per sec
467 2018-11-30T05:24:21.26590519Z 01:24:21 INFO:Latest model saved in ./results/output_poem/save_model/model-32624
468 2018-11-30T05:24:21.265945727Z
469 2018-11-30T05:24:21.265950947Z 01:24:21 INFO:Evaluate on validation set
470 2018-11-30T05:24:26.364649957Z 01:24:26 INFO:41.7%, step:99, perplexity: 20.678, speed: 22615 words
471 2018-11-30T05:24:29.170986704Z 01:24:29 INFO:83.3%, step:199, perplexity: 20.501, speed: 27941 words
472 2018-11-30T05:24:30.162020679Z 01:24:30 INFO:Perplexity: 21.198, speed: 29530 words per sec
473 2018-11-30T05:24:32.101327836Z 01:24:32 INFO:Best model is saved in ./results/output_poem/best_model/model-32624
474 2018-11-30T05:24:32.101380853Z 01:24:32 INFO:Best validation ppl is 21.198051
475 2018-11-30T05:24:32.101388053Z
476 2018-11-30T05:24:32.106104723Z 01:24:32 INFO:=================== Epoch 8 ===================
477 2018-11-30T05:24:32.10616528Z
478 2018-11-30T05:24:32.106171938Z 01:24:32 INFO:Training on training set
479 2018-11-30T05:24:38.886009312Z 01:24:38 INFO:2.5%, step:99, perplexity: 20.137, speed: 15114 words
480 2018-11-30T05:24:45.898919635Z 01:24:45 INFO:4.9%, step:199, perplexity: 19.513, speed: 14858 words
481 2018-11-30T05:24:52.508883905Z 01:24:52 INFO:7.4%, step:299, perplexity: 19.322, speed: 15058 words
482 2018-11-30T05:24:59.348878836Z 01:24:59 INFO:9.8%, step:399, perplexity: 19.273, speed: 15038 words
483 2018-11-30T05:25:06.916306525Z 01:25:06 INFO:12.3%, step:499, perplexity: 19.254, speed: 14709 words
484 2018-11-30T05:25:13.99023448Z 01:25:13 INFO:14.7%, step:599, perplexity: 19.215, speed: 14670 words
485 2018-11-30T05:25:20.811880469Z 01:25:20 INFO:17.2%, step:699, perplexity: 19.132, speed: 14717 words
486 2018-11-30T05:25:27.505269417Z 01:25:27 INFO:19.6%, step:799, perplexity: 19.009, speed: 14788 words
487 2018-11-30T05:25:34.209219853Z 01:25:34 INFO:22.1%, step:899, perplexity: 18.901, speed: 14841 words
488 2018-11-30T05:25:41.580957524Z 01:25:41 INFO:24.5%, step:999, perplexity: 18.829, speed: 14739 words
489 2018-11-30T05:25:48.307579256Z 01:25:48 INFO:27.0%, step:1099, perplexity: 18.777, speed: 14783 words
490 2018-11-30T05:25:54.960376868Z 01:25:54 INFO:29.4%, step:1199, perplexity: 18.723, speed: 14834 words
491 2018-11-30T05:26:01.736527928Z 01:26:01 INFO:31.9%, step:1299, perplexity: 18.678, speed: 14852 words
492 2018-11-30T05:26:09.401904759Z 01:26:09 INFO:34.3%, step:1399, perplexity: 18.639, speed: 14735 words
493 2018-11-30T05:26:16.151209597Z 01:26:16 INFO:36.8%, step:1499, perplexity: 18.607, speed: 14763 words
494 2018-11-30T05:26:22.980005892Z 01:26:22 INFO:39.2%, step:1599, perplexity: 18.586, speed: 14778 words
495 2018-11-30T05:26:29.596846431Z 01:26:29 INFO:41.7%, step:1699, perplexity: 18.572, speed: 14817 words
496 2018-11-30T05:26:36.320448724Z 01:26:36 INFO:44.1%, step:1799, perplexity: 18.564, speed: 14839 words
497 2018-11-30T05:26:43.384603189Z 01:26:43 INFO:46.6%, step:1899, perplexity: 18.555, speed: 14821 words
498 2018-11-30T05:26:49.984509162Z 01:26:49 INFO:49.0%, step:1999, perplexity: 18.544, speed: 14854 words
499 2018-11-30T05:26:56.88501Z 01:26:56 INFO:51.5%, step:2099, perplexity: 18.541, speed: 14853 words
500 2018-11-30T05:27:04.342270256Z 01:27:04 INFO:53.9%, step:2199, perplexity: 18.545, speed: 14798 words
501 2018-11-30T05:27:11.712436994Z 01:27:11 INFO:56.4%, step:2299, perplexity: 18.552, speed: 14757 words
502 2018-11-30T05:27:18.568897589Z 01:27:18 INFO:58.9%, step:2399, perplexity: 18.557, speed: 14764 words
503 2018-11-30T05:27:25.322866471Z 01:27:25 INFO:61.3%, step:2499, perplexity: 18.563, speed: 14780 words
504 2018-11-30T05:27:32.20805167Z 01:27:32 INFO:63.8%, step:2599, perplexity: 18.565, speed: 14783 words
505 2018-11-30T05:27:38.930116206Z 01:27:38 INFO:66.2%, step:2699, perplexity: 18.563, speed: 14800 words
506 2018-11-30T05:27:45.670940675Z 01:27:45 INFO:68.7%, step:2799, perplexity: 18.558, speed: 14813 words
507 2018-11-30T05:27:53.172784992Z 01:27:53 INFO:71.1%, step:2899, perplexity: 18.552, speed: 14770 words
508 2018-11-30T05:27:59.978636749Z 01:27:59 INFO:73.6%, step:2999, perplexity: 18.544, speed: 14778 words
509 2018-11-30T05:28:07.190004162Z 01:28:07 INFO:76.0%, step:3099, perplexity: 18.536, speed: 14759 words
510 2018-11-30T05:28:14.441593551Z 01:28:14 INFO:78.5%, step:3199, perplexity: 18.528, speed: 14738 words
511 2018-11-30T05:28:21.16236891Z 01:28:21 INFO:80.9%, step:3299, perplexity: 18.519, speed: 14753 words
512 2018-11-30T05:28:27.777920476Z 01:28:27 INFO:83.4%, step:3399, perplexity: 18.509, speed: 14773 words
513 2018-11-30T05:28:34.473937842Z 01:28:34 INFO:85.8%, step:3499, perplexity: 18.499, speed: 14788 words
514 2018-11-30T05:28:40.753475067Z 01:28:40 INFO:88.3%, step:3599, perplexity: 18.490, speed: 14826 words
515 2018-11-30T05:28:47.338934043Z 01:28:47 INFO:90.7%, step:3699, perplexity: 18.483, speed: 14845 words
516 2018-11-30T05:28:53.660848938Z 01:28:53 INFO:93.2%, step:3799, perplexity: 18.475, speed: 14877 words
517 2018-11-30T05:29:00.89985364Z 01:29:00 INFO:95.6%, step:3899, perplexity: 18.468, speed: 14858 words
518 2018-11-30T05:29:08.637703334Z 01:29:08 INFO:98.1%, step:3999, perplexity: 18.462, speed: 14813 words
519 2018-11-30T05:29:13.766597182Z 01:29:13 INFO:Perplexity: 18.262, speed: 14826 words per sec
520 2018-11-30T05:29:15.586878046Z 01:29:15 INFO:Latest model saved in ./results/output_poem/save_model/model-36702
521 2018-11-30T05:29:15.586924792Z
522 2018-11-30T05:29:15.586932417Z 01:29:15 INFO:Evaluate on validation set
523 2018-11-30T05:29:20.987823464Z 01:29:20 INFO:41.7%, step:99, perplexity: 20.649, speed: 21864 words
524 2018-11-30T05:29:23.810703769Z 01:29:23 INFO:83.3%, step:199, perplexity: 20.471, speed: 27277 words
525 2018-11-30T05:29:25.029846436Z 01:29:25 INFO:Perplexity: 21.174, speed: 28187 words per sec
526 2018-11-30T05:29:27.016935332Z 01:29:27 INFO:Best model is saved in ./results/output_poem/best_model/model-36702
527 2018-11-30T05:29:27.016979815Z 01:29:27 INFO:Best validation ppl is 21.173891
528 2018-11-30T05:29:27.016984611Z
529 2018-11-30T05:29:27.019306564Z 01:29:27 INFO:=================== Epoch 9 ===================
530 2018-11-30T05:29:27.01933541Z
531 2018-11-30T05:29:27.019341371Z 01:29:27 INFO:Training on training set
532 2018-11-30T05:29:33.498030237Z 01:29:33 INFO:2.5%, step:99, perplexity: 19.875, speed: 15832 words
533 2018-11-30T05:29:39.845084862Z 01:29:39 INFO:4.9%, step:199, perplexity: 19.259, speed: 15978 words
534 2018-11-30T05:29:46.527009883Z 01:29:46 INFO:7.4%, step:299, perplexity: 19.068, speed: 15754 words
535 2018-11-30T05:29:53.13992507Z 01:29:53 INFO:9.8%, step:399, perplexity: 19.020, speed: 15692 words
536 2018-11-30T05:29:59.93641151Z 01:29:59 INFO:12.3%, step:499, perplexity: 19.002, speed: 15557 words
537 2018-11-30T05:30:06.787986612Z 01:30:06 INFO:14.7%, step:599, perplexity: 18.964, speed: 15450 words
538 2018-11-30T05:30:13.431415482Z 01:30:13 INFO:17.2%, step:699, perplexity: 18.883, speed: 15447 words
539 2018-11-30T05:30:20.102037601Z 01:30:20 INFO:19.6%, step:799, perplexity: 18.763, speed: 15435 words
540 2018-11-30T05:30:26.538888134Z 01:30:26 INFO:22.1%, step:899, perplexity: 18.657, speed: 15486 words
541 2018-11-30T05:30:33.196943043Z 01:30:33 INFO:24.5%, step:999, perplexity: 18.587, speed: 15474 words
542 2018-11-30T05:30:39.484978353Z 01:30:39 INFO:27.0%, step:1099, perplexity: 18.537, speed: 15545 words
543 2018-11-30T05:30:46.422446167Z 01:30:46 INFO:29.4%, step:1199, perplexity: 18.485, speed: 15476 words
544 2018-11-30T05:30:52.604302692Z 01:30:52 INFO:31.9%, step:1299, perplexity: 18.441, speed: 15555 words
545 2018-11-30T05:30:59.672911679Z 01:30:59 INFO:34.3%, step:1399, perplexity: 18.403, speed: 15473 words
546 2018-11-30T05:31:06.611911813Z 01:31:06 INFO:36.8%, step:1499, perplexity: 18.373, speed: 15423 words
547 2018-11-30T05:31:13.091016246Z 01:31:13 INFO:39.2%, step:1599, perplexity: 18.352, speed: 15447 words
548 2018-11-30T05:31:20.022035459Z 01:31:20 INFO:41.7%, step:1699, perplexity: 18.339, speed: 15407 words
549 2018-11-30T05:31:26.689686305Z 01:31:26 INFO:44.1%, step:1799, perplexity: 18.332, speed: 15403 words
550 2018-11-30T05:31:33.493734052Z 01:31:33 INFO:46.6%, step:1899, perplexity: 18.324, speed: 15384 words
551 2018-11-30T05:31:39.949843714Z 01:31:39 INFO:49.0%, step:1999, perplexity: 18.315, speed: 15407 words
552 2018-11-30T05:31:46.559060466Z 01:31:46 INFO:51.5%, step:2099, perplexity: 18.313, speed: 15412 words
553 2018-11-30T05:31:53.027076497Z 01:31:53 INFO:53.9%, step:2199, perplexity: 18.317, speed: 15430 words
554 2018-11-30T05:32:00.095228606Z 01:32:00 INFO:56.4%, step:2299, perplexity: 18.325, speed: 15386 words
555 2018-11-30T05:32:07.21114997Z 01:32:07 INFO:58.9%, step:2399, perplexity: 18.330, speed: 15342 words
556 2018-11-30T05:32:13.751811343Z 01:32:13 INFO:61.3%, step:2499, perplexity: 18.337, speed: 15354 words
557 2018-11-30T05:32:20.295366404Z 01:32:20 INFO:63.8%, step:2599, perplexity: 18.340, speed: 15366 words
558 2018-11-30T05:32:26.893935531Z 01:32:26 INFO:66.2%, step:2699, perplexity: 18.339, speed: 15371 words
559 2018-11-30T05:32:33.589837268Z 01:32:33 INFO:68.7%, step:2799, perplexity: 18.335, speed: 15369 words
560 2018-11-30T05:32:40.451753229Z 01:32:40 INFO:71.1%, step:2899, perplexity: 18.330, speed: 15352 words
561 2018-11-30T05:32:47.034720628Z 01:32:47 INFO:73.6%, step:2999, perplexity: 18.322, speed: 15359 words
562 2018-11-30T05:32:53.874908111Z 01:32:53 INFO:76.0%, step:3099, perplexity: 18.315, speed: 15346 words
563 2018-11-30T05:33:00.94322438Z 01:33:00 INFO:78.5%, step:3199, perplexity: 18.309, speed: 15318 words
564 2018-11-30T05:33:07.936178765Z 01:33:07 INFO:80.9%, step:3299, perplexity: 18.301, speed: 15297 words
565 2018-11-30T05:33:15.429210638Z 01:33:15 INFO:83.4%, step:3399, perplexity: 18.292, speed: 15243 words
566 2018-11-30T05:33:22.591074581Z 01:33:22 INFO:85.8%, step:3499, perplexity: 18.282, speed: 15214 words
567 2018-11-30T05:33:28.879916847Z 01:33:28 INFO:88.3%, step:3599, perplexity: 18.274, speed: 15242 words
568 2018-11-30T05:33:35.578538637Z 01:33:35 INFO:90.7%, step:3699, perplexity: 18.268, speed: 15243 words
569 2018-11-30T05:33:42.381806111Z 01:33:42 INFO:93.2%, step:3799, perplexity: 18.261, speed: 15238 words
570 2018-11-30T05:33:48.723268362Z 01:33:48 INFO:95.6%, step:3899, perplexity: 18.255, speed: 15260 words
571 2018-11-30T05:33:55.484932446Z 01:33:55 INFO:98.1%, step:3999, perplexity: 18.249, speed: 15257 words
572 2018-11-30T05:34:01.505897579Z 01:34:01 INFO:Perplexity: 18.087, speed: 15214 words per sec
573 2018-11-30T05:34:03.707439818Z 01:34:03 INFO:Latest model saved in ./results/output_poem/save_model/model-40780
574 2018-11-30T05:34:03.707482155Z
575 2018-11-30T05:34:03.707486898Z 01:34:03 INFO:Evaluate on validation set
576 2018-11-30T05:34:09.80983305Z 01:34:09 INFO:41.7%, step:99, perplexity: 20.800, speed: 19092 words
577 2018-11-30T05:34:12.846948897Z 01:34:12 INFO:83.3%, step:199, perplexity: 20.594, speed: 24383 words
578 2018-11-30T05:34:13.813027924Z 01:34:13 INFO:Perplexity: 21.285, speed: 26246 words per sec
579 2018-11-30T05:34:13.81307202Z 01:34:13 INFO:Decay the learning rate: 0.000625
580 2018-11-30T05:34:13.813079394Z 01:34:13 INFO:Best model is saved in ./results/output_poem/best_model/model-36702
581 2018-11-30T05:34:13.813084548Z 01:34:13 INFO:Best validation ppl is 21.173891
582 2018-11-30T05:34:13.813089111Z
583 2018-11-30T05:34:13.823201691Z 01:34:13 INFO:=================== Epoch 10 ===================
584 2018-11-30T05:34:13.823216928Z
585 2018-11-30T05:34:13.823223289Z 01:34:13 INFO:Training on training set
586 2018-11-30T05:34:20.306076944Z 01:34:20 INFO:2.5%, step:99, perplexity: 19.691, speed: 15786 words
587 2018-11-30T05:34:27.096310873Z 01:34:27 INFO:4.9%, step:199, perplexity: 19.076, speed: 15430 words
588 2018-11-30T05:34:34.185027528Z 01:34:34 INFO:7.4%, step:299, perplexity: 18.882, speed: 15085 words
589 2018-11-30T05:34:40.987677815Z 01:34:40 INFO:9.8%, step:399, perplexity: 18.831, speed: 15077 words
590 2018-11-30T05:34:48.098088206Z 01:34:48 INFO:12.3%, step:499, perplexity: 18.810, speed: 14938 words
591 2018-11-30T05:34:54.810669561Z 01:34:54 INFO:14.7%, step:599, perplexity: 18.771, speed: 14991 words
592 2018-11-30T05:35:01.818701316Z 01:35:01 INFO:17.2%, step:699, perplexity: 18.689, speed: 14934 words
593 2018-11-30T05:35:09.215580484Z 01:35:09 INFO:19.6%, step:799, perplexity: 18.568, speed: 14788 words
594 2018-11-30T05:35:16.167749623Z 01:35:16 INFO:22.1%, step:899, perplexity: 18.462, speed: 14783 words
595 2018-11-30T05:35:22.90413841Z 01:35:22 INFO:24.5%, step:999, perplexity: 18.390, speed: 14823 words
596 2018-11-30T05:35:29.197753383Z 01:35:29 INFO:27.0%, step:1099, perplexity: 18.338, speed: 14944 words
597 2018-11-30T05:35:35.815160385Z 01:35:35 INFO:29.4%, step:1199, perplexity: 18.284, speed: 14986 words
598 2018-11-30T05:35:43.132417226Z 01:35:43 INFO:31.9%, step:1299, perplexity: 18.239, speed: 14905 words
599 2018-11-30T05:35:49.846097016Z 01:35:49 INFO:34.3%, step:1399, perplexity: 18.199, speed: 14929 words
600 2018-11-30T05:35:56.846946788Z 01:35:56 INFO:36.8%, step:1499, perplexity: 18.167, speed: 14909 words
601 2018-11-30T05:36:04.100144716Z 01:36:04 INFO:39.2%, step:1599, perplexity: 18.144, speed: 14856 words
602 2018-11-30T05:36:11.679157258Z 01:36:11 INFO:41.7%, step:1699, perplexity: 18.129, speed: 14770 words
603 2018-11-30T05:36:18.82292009Z 01:36:18 INFO:44.1%, step:1799, perplexity: 18.120, speed: 14745 words
604 2018-11-30T05:36:25.939187416Z 01:36:25 INFO:46.6%, step:1899, perplexity: 18.110, speed: 14726 words
605 2018-11-30T05:36:32.895768393Z 01:36:32 INFO:49.0%, step:1999, perplexity: 18.098, speed: 14727 words
606 2018-11-30T05:36:39.837683369Z 01:36:39 INFO:51.5%, step:2099, perplexity: 18.094, speed: 14728 words
607 2018-11-30T05:36:46.889124344Z 01:36:46 INFO:53.9%, step:2199, perplexity: 18.096, speed: 14718 words
608 2018-11-30T05:36:54.108337971Z 01:36:54 INFO:56.4%, step:2299, perplexity: 18.101, speed: 14694 words
609 2018-11-30T05:37:00.783936329Z 01:37:00 INFO:58.9%, step:2399, perplexity: 18.105, speed: 14720 words
610 2018-11-30T05:37:08.462619489Z 01:37:08 INFO:61.3%, step:2499, perplexity: 18.109, speed: 14659 words
611 2018-11-30T05:37:15.491653378Z 01:37:15 INFO:63.8%, step:2599, perplexity: 18.110, speed: 14655 words
612 2018-11-30T05:37:22.105787461Z 01:37:22 INFO:66.2%, step:2699, perplexity: 18.107, speed: 14685 words
613 2018-11-30T05:37:28.989052172Z 01:37:28 INFO:68.7%, step:2799, perplexity: 18.101, speed: 14691 words
614 2018-11-30T05:37:35.494448027Z 01:37:35 INFO:71.1%, step:2899, perplexity: 18.094, speed: 14725 words
615 2018-11-30T05:37:42.848475064Z 01:37:42 INFO:73.6%, step:2999, perplexity: 18.085, speed: 14697 words
616 2018-11-30T05:37:49.802469358Z 01:37:49 INFO:76.0%, step:3099, perplexity: 18.076, speed: 14698 words
617 2018-11-30T05:37:56.997900089Z 01:37:56 INFO:78.5%, step:3199, perplexity: 18.068, speed: 14682 words
618 2018-11-30T05:38:04.567965976Z 01:38:04 INFO:80.9%, step:3299, perplexity: 18.058, speed: 14644 words
619 2018-11-30T05:38:12.314995084Z 01:38:12 INFO:83.4%, step:3399, perplexity: 18.047, speed: 14599 words
620 2018-11-30T05:38:19.48452443Z 01:38:19 INFO:85.8%, step:3499, perplexity: 18.036, speed: 14589 words
621 2018-11-30T05:38:26.845874523Z 01:38:26 INFO:88.3%, step:3599, perplexity: 18.027, speed: 14569 words
622 2018-11-30T05:38:33.952407946Z 01:38:33 INFO:90.7%, step:3699, perplexity: 18.019, speed: 14565 words
623 2018-11-30T05:38:41.053900057Z 01:38:41 INFO:93.2%, step:3799, perplexity: 18.010, speed: 14561 words
624 2018-11-30T05:38:48.552871975Z 01:38:48 INFO:95.6%, step:3899, perplexity: 18.002, speed: 14536 words
625 2018-11-30T05:38:55.404093562Z 01:38:55 INFO:98.1%, step:3999, perplexity: 17.995, speed: 14546 words
626 2018-11-30T05:39:01.888036175Z 01:39:01 INFO:Perplexity: 17.762, speed: 14496 words per sec
627 2018-11-30T05:39:04.252735774Z 01:39:04 INFO:Latest model saved in ./results/output_poem/save_model/model-44858
628 2018-11-30T05:39:04.252799765Z
629 2018-11-30T05:39:04.252807539Z 01:39:04 INFO:Evaluate on validation set
630 2018-11-30T05:39:11.692964446Z 01:39:11 INFO:41.7%, step:99, perplexity: 20.692, speed: 15011 words
631 2018-11-30T05:39:14.831751582Z 01:39:14 INFO:83.3%, step:199, perplexity: 20.501, speed: 20562 words
632 2018-11-30T05:39:16.202377685Z 01:39:16 INFO:Perplexity: 21.213, speed: 21695 words per sec
633 2018-11-30T05:39:16.202435252Z 01:39:16 INFO:Decay the learning rate: 0.0003125
634 2018-11-30T05:39:16.202669338Z 01:39:16 INFO:Best model is saved in ./results/output_poem/best_model/model-36702
635 2018-11-30T05:39:16.20268337Z 01:39:16 INFO:Best validation ppl is 21.173891
636 2018-11-30T05:39:16.202689134Z
637 2018-11-30T05:39:16.219828622Z 01:39:16 INFO:=================== Epoch 11 ===================
638 2018-11-30T05:39:16.219851636Z
639 2018-11-30T05:39:16.219871067Z 01:39:16 INFO:Training on training set
640 2018-11-30T05:39:25.574910169Z 01:39:25 INFO:2.5%, step:99, perplexity: 19.474, speed: 10944 words
641 2018-11-30T05:39:33.896930518Z 01:39:33 INFO:4.9%, step:199, perplexity: 18.871, speed: 11592 words
642 2018-11-30T05:39:42.030045186Z 01:39:42 INFO:7.4%, step:299, perplexity: 18.679, speed: 11904 words
643 2018-11-30T05:39:49.262991649Z 01:39:49 INFO:9.8%, step:399, perplexity: 18.629, speed: 12397 words
644 2018-11-30T05:39:56.216369052Z 01:39:56 INFO:12.3%, step:499, perplexity: 18.610, speed: 12805 words
645 2018-11-30T05:40:03.863591763Z 01:40:03 INFO:14.7%, step:599, perplexity: 18.571, speed: 12895 words
646 2018-11-30T05:40:11.911328598Z 01:40:11 INFO:17.2%, step:699, perplexity: 18.492, speed: 12871 words
647 2018-11-30T05:40:18.640833614Z 01:40:18 INFO:19.6%, step:799, perplexity: 18.373, speed: 13124 words
648 2018-11-30T05:40:25.441411405Z 01:40:25 INFO:22.1%, step:899, perplexity: 18.269, speed: 13315 words
649 2018-11-30T05:40:32.258074282Z 01:40:32 INFO:24.5%, step:999, perplexity: 18.199, speed: 13467 words
650 2018-11-30T05:40:39.443962449Z 01:40:39 INFO:27.0%, step:1099, perplexity: 18.149, speed: 13534 words
651 2018-11-30T05:40:46.716285375Z 01:40:46 INFO:29.4%, step:1199, perplexity: 18.096, speed: 13579 words
652 2018-11-30T05:40:53.418927823Z 01:40:53 INFO:31.9%, step:1299, perplexity: 18.051, speed: 13696 words
653 2018-11-30T05:41:00.585017264Z 01:41:00 INFO:34.3%, step:1399, perplexity: 18.012, speed: 13736 words
654 2018-11-30T05:41:08.085930166Z 01:41:08 INFO:36.8%, step:1499, perplexity: 17.980, speed: 13731 words
655 2018-11-30T05:41:14.793818022Z 01:41:14 INFO:39.2%, step:1599, perplexity: 17.958, speed: 13819 words
656 2018-11-30T05:41:21.355882184Z 01:41:21 INFO:41.7%, step:1699, perplexity: 17.944, speed: 13911 words
657 2018-11-30T05:41:28.749029185Z 01:41:28 INFO:44.1%, step:1799, perplexity: 17.934, speed: 13908 words
658 2018-11-30T05:41:35.539914319Z 01:41:35 INFO:46.6%, step:1899, perplexity: 17.925, speed: 13966 words
659 2018-11-30T05:41:42.785792712Z 01:41:42 INFO:49.0%, step:1999, perplexity: 17.913, speed: 13973 words
660 2018-11-30T05:41:49.811079309Z 01:41:49 INFO:51.5%, step:2099, perplexity: 17.909, speed: 14001 words
661 2018-11-30T05:41:57.125781972Z 01:41:57 INFO:53.9%, step:2199, perplexity: 17.911, speed: 14001 words
662 2018-11-30T05:42:04.595728769Z 01:42:04 INFO:56.4%, step:2299, perplexity: 17.916, speed: 13989 words
663 2018-11-30T05:42:11.58688784Z 01:42:11 INFO:58.9%, step:2399, perplexity: 17.920, speed: 14014 words
664 2018-11-30T05:42:18.617538862Z 01:42:18 INFO:61.3%, step:2499, perplexity: 17.924, speed: 14035 words
665 2018-11-30T05:42:25.282372583Z 01:42:25 INFO:63.8%, step:2599, perplexity: 17.924, speed: 14082 words
666 2018-11-30T05:42:33.085912534Z 01:42:33 INFO:66.2%, step:2699, perplexity: 17.921, speed: 14044 words
667 2018-11-30T05:42:39.71280379Z 01:42:39 INFO:68.7%, step:2799, perplexity: 17.916, speed: 14090 words
668 2018-11-30T05:42:46.705952943Z 01:42:46 INFO:71.1%, step:2899, perplexity: 17.909, speed: 14108 words
669 2018-11-30T05:42:54.445962543Z 01:42:54 INFO:73.6%, step:2999, perplexity: 17.899, speed: 14077 words
670 2018-11-30T05:43:01.667221212Z 01:43:01 INFO:76.0%, step:3099, perplexity: 17.891, speed: 14080 words
671 2018-11-30T05:43:09.156325734Z 01:43:09 INFO:78.5%, step:3199, perplexity: 17.883, speed: 14067 words
672 2018-11-30T05:43:17.344076313Z 01:43:17 INFO:80.9%, step:3299, perplexity: 17.873, speed: 14014 words
673 2018-11-30T05:43:24.501887831Z 01:43:24 INFO:83.4%, step:3399, perplexity: 17.862, speed: 14023 words
674 2018-11-30T05:43:31.539920121Z 01:43:31 INFO:85.8%, step:3499, perplexity: 17.851, speed: 14037 words
675 2018-11-30T05:43:38.637703971Z 01:43:38 INFO:88.3%, step:3599, perplexity: 17.842, speed: 14048 words
676 2018-11-30T05:43:45.687862092Z 01:43:45 INFO:90.7%, step:3699, perplexity: 17.833, speed: 14060 words
677 2018-11-30T05:43:52.60397661Z 01:43:52 INFO:93.2%, step:3799, perplexity: 17.825, speed: 14079 words
678 2018-11-30T05:43:59.4659729Z 01:43:59 INFO:95.6%, step:3899, perplexity: 17.817, speed: 14099 words
679 2018-11-30T05:44:07.162878379Z 01:44:07 INFO:98.1%, step:3999, perplexity: 17.810, speed: 14078 words
680 2018-11-30T05:44:12.834833306Z 01:44:12 INFO:Perplexity: 17.570, speed: 14079 words per sec
681 2018-11-30T05:44:14.898021743Z 01:44:14 INFO:Latest model saved in ./results/output_poem/save_model/model-48936
682 2018-11-30T05:44:14.898063554Z
683 2018-11-30T05:44:14.898069124Z 01:44:14 INFO:Evaluate on validation set
684 2018-11-30T05:44:19.752161592Z 01:44:19 INFO:41.7%, step:99, perplexity: 20.631, speed: 24717 words
685 2018-11-30T05:44:22.836733927Z 01:44:22 INFO:83.3%, step:199, perplexity: 20.449, speed: 28346 words
686 2018-11-30T05:44:24.197663785Z 01:44:24 INFO:Perplexity: 21.183, speed: 28652 words per sec
687 2018-11-30T05:44:24.197715816Z 01:44:24 INFO:Decay the learning rate: 0.00015625
688 2018-11-30T05:44:24.197724329Z 01:44:24 INFO:Best model is saved in ./results/output_poem/best_model/model-36702
689 2018-11-30T05:44:24.197729903Z 01:44:24 INFO:Best validation ppl is 21.173891
690 2018-11-30T05:44:24.197734888Z
691 2018-11-30T05:44:24.19773943Z 01:44:24 INFO:=================== Epoch 12 ===================
692 2018-11-30T05:44:24.197744304Z
693 2018-11-30T05:44:24.197748608Z 01:44:24 INFO:Training on training set
694 2018-11-30T05:44:30.562863498Z 01:44:30 INFO:2.5%, step:99, perplexity: 19.357, speed: 16077 words
695 2018-11-30T05:44:37.226003251Z 01:44:37 INFO:4.9%, step:199, perplexity: 18.760, speed: 15714 words
696 2018-11-30T05:44:44.28114939Z 01:44:44 INFO:7.4%, step:299, perplexity: 18.568, speed: 15294 words
697 2018-11-30T05:44:51.424640364Z 01:44:51 INFO:9.8%, step:399, perplexity: 18.518, speed: 15045 words
698 2018-11-30T05:44:58.723330406Z 01:44:58 INFO:12.3%, step:499, perplexity: 18.498, speed: 14828 words
699 2018-11-30T05:45:06.279321872Z 01:45:06 INFO:14.7%, step:599, perplexity: 18.460, speed: 14599 words
700 2018-11-30T05:45:13.810880227Z 01:45:13 INFO:17.2%, step:699, perplexity: 18.381, speed: 14447 words
701 2018-11-30T05:45:21.046836839Z 01:45:21 INFO:19.6%, step:799, perplexity: 18.264, speed: 14411 words
702 2018-11-30T05:45:27.506147483Z 01:45:27 INFO:22.1%, step:899, perplexity: 18.161, speed: 14557 words
703 2018-11-30T05:45:34.569100629Z 01:45:34 INFO:24.5%, step:999, perplexity: 18.091, speed: 14551 words
704 2018-11-30T05:45:41.372894294Z 01:45:41 INFO:27.0%, step:1099, perplexity: 18.042, speed: 14594 words
705 2018-11-30T05:45:48.536715072Z 01:45:48 INFO:29.4%, step:1199, perplexity: 17.989, speed: 14569 words
706 2018-11-30T05:45:55.297995959Z 01:45:55 INFO:31.9%, step:1299, perplexity: 17.945, speed: 14613 words
707 2018-11-30T05:46:02.129977494Z 01:46:02 INFO:34.3%, step:1399, perplexity: 17.907, speed: 14639 words
708 2018-11-30T05:46:09.76291006Z 01:46:09 INFO:36.8%, step:1499, perplexity: 17.875, speed: 14550 words
709 2018-11-30T05:46:16.455847572Z 01:46:16 INFO:39.2%, step:1599, perplexity: 17.854, speed: 14594 words
710 2018-11-30T05:46:23.68183162Z 01:46:23 INFO:41.7%, step:1699, perplexity: 17.839, speed: 14569 words
711 2018-11-30T05:46:30.722260258Z 01:46:30 INFO:44.1%, step:1799, perplexity: 17.830, speed: 14568 words
712 2018-11-30T05:46:37.351342603Z 01:46:37 INFO:46.6%, step:1899, perplexity: 17.821, speed: 14611 words
713 2018-11-30T05:46:44.619287562Z 01:46:44 INFO:49.0%, step:1999, perplexity: 17.809, speed: 14584 words
714 2018-11-30T05:46:51.206917901Z 01:46:51 INFO:51.5%, step:2099, perplexity: 17.805, speed: 14627 words
715 2018-11-30T05:46:58.146479171Z 01:46:58 INFO:53.9%, step:2199, perplexity: 17.808, speed: 14633 words
716 2018-11-30T05:47:05.36560365Z 01:47:05 INFO:56.4%, step:2299, perplexity: 17.812, speed: 14614 words
717 2018-11-30T05:47:12.784860545Z 01:47:12 INFO:58.9%, step:2399, perplexity: 17.816, speed: 14578 words
718 2018-11-30T05:47:19.684348751Z 01:47:19 INFO:61.3%, step:2499, perplexity: 17.820, speed: 14588 words
719 2018-11-30T05:47:26.723246861Z 01:47:26 INFO:63.8%, step:2599, perplexity: 17.821, speed: 14586 words
720 2018-11-30T05:47:33.510979347Z 01:47:33 INFO:66.2%, step:2699, perplexity: 17.818, speed: 14604 words
721 2018-11-30T05:47:40.931667143Z 01:47:40 INFO:68.7%, step:2799, perplexity: 17.813, speed: 14574 words
722 2018-11-30T05:47:47.992895745Z 01:47:47 INFO:71.1%, step:2899, perplexity: 17.806, speed: 14571 words
723 2018-11-30T05:47:55.237862614Z 01:47:55 INFO:73.6%, step:2999, perplexity: 17.797, speed: 14557 words
724 2018-11-30T05:48:02.522650792Z 01:48:02 INFO:76.0%, step:3099, perplexity: 17.788, speed: 14540 words
725 2018-11-30T05:48:09.649953845Z 01:48:09 INFO:78.5%, step:3199, perplexity: 17.780, speed: 14534 words
726 2018-11-30T05:48:16.930474952Z 01:48:16 INFO:80.9%, step:3299, perplexity: 17.771, speed: 14519 words
727 2018-11-30T05:48:23.292722374Z 01:48:23 INFO:83.4%, step:3399, perplexity: 17.760, speed: 14561 words
728 2018-11-30T05:48:30.194621966Z 01:48:30 INFO:85.8%, step:3499, perplexity: 17.749, speed: 14569 words
729 2018-11-30T05:48:37.3246439Z 01:48:37 INFO:88.3%, step:3599, perplexity: 17.740, speed: 14564 words
730 2018-11-30T05:48:44.374703689Z 01:48:44 INFO:90.7%, step:3699, perplexity: 17.731, speed: 14562 words
731 2018-11-30T05:48:51.091099688Z 01:48:51 INFO:93.2%, step:3799, perplexity: 17.723, speed: 14579 words
732 2018-11-30T05:48:57.769827033Z 01:48:57 INFO:95.6%, step:3899, perplexity: 17.715, speed: 14598 words
733 2018-11-30T05:49:05.205888049Z 01:49:05 INFO:98.1%, step:3999, perplexity: 17.708, speed: 14576 words
734 2018-11-30T05:49:10.760990315Z 01:49:10 INFO:Perplexity: 17.469, speed: 14572 words per sec
735 2018-11-30T05:49:13.424895939Z 01:49:13 INFO:Latest model saved in ./results/output_poem/save_model/model-53014
736 2018-11-30T05:49:13.424946139Z
737 2018-11-30T05:49:13.424954117Z 01:49:13 INFO:Evaluate on validation set
738 2018-11-30T05:49:18.556905514Z 01:49:18 INFO:41.7%, step:99, perplexity: 20.555, speed: 23179 words
739 2018-11-30T05:49:21.426047352Z 01:49:21 INFO:83.3%, step:199, perplexity: 20.378, speed: 28097 words
740 2018-11-30T05:49:22.778662449Z 01:49:22 INFO:Perplexity: 21.122, speed: 28450 words per sec
741 2018-11-30T05:49:24.805776128Z 01:49:24 INFO:Best model is saved in ./results/output_poem/best_model/model-53014
742 2018-11-30T05:49:24.805802016Z 01:49:24 INFO:Best validation ppl is 21.121807
743 2018-11-30T05:49:24.805808914Z
744 2018-11-30T05:49:24.81973173Z 01:49:24 INFO:=================== Epoch 13 ===================
745 2018-11-30T05:49:24.81976437Z
746 2018-11-30T05:49:24.819771786Z 01:49:24 INFO:Training on training set
747 2018-11-30T05:49:31.726869326Z 01:49:31 INFO:2.5%, step:99, perplexity: 19.298, speed: 14844 words
748 2018-11-30T05:49:38.536932673Z 01:49:38 INFO:4.9%, step:199, perplexity: 18.697, speed: 14935 words
749 2018-11-30T05:49:45.587990924Z 01:49:45 INFO:7.4%, step:299, perplexity: 18.502, speed: 14798 words
750 2018-11-30T05:49:52.628930511Z 01:49:52 INFO:9.8%, step:399, perplexity: 18.448, speed: 14729 words
751 2018-11-30T05:49:59.923922759Z 01:49:59 INFO:12.3%, step:499, perplexity: 18.426, speed: 14587 words
752 2018-11-30T05:50:07.163921688Z 01:50:07 INFO:14.7%, step:599, perplexity: 18.387, speed: 14512 words
753 2018-11-30T05:50:14.36112288Z 01:50:14 INFO:17.2%, step:699, perplexity: 18.308, speed: 14470 words
754 2018-11-30T05:50:21.563997014Z 01:50:21 INFO:19.6%, step:799, perplexity: 18.190, speed: 14438 words
755 2018-11-30T05:50:28.043208551Z 01:50:28 INFO:22.1%, step:899, perplexity: 18.087, speed: 14579 words
756 2018-11-30T05:50:35.044854506Z 01:50:35 INFO:24.5%, step:999, perplexity: 18.018, speed: 14582 words
757 2018-11-30T05:50:41.660966772Z 01:50:41 INFO:27.0%, step:1099, perplexity: 17.968, speed: 14659 words
758 2018-11-30T05:50:48.61411206Z 01:50:48 INFO:29.4%, step:1199, perplexity: 17.916, speed: 14665 words
759 2018-11-30T05:50:55.559668411Z 01:50:55 INFO:31.9%, step:1299, perplexity: 17.872, speed: 14671 words
760 2018-11-30T05:51:02.72956301Z 01:51:02 INFO:34.3%, step:1399, perplexity: 17.834, speed: 14643 words
761 2018-11-30T05:51:10.37289443Z 01:51:10 INFO:36.8%, step:1499, perplexity: 17.803, speed: 14552 words
762 2018-11-30T05:51:16.492511294Z 01:51:16 INFO:39.2%, step:1599, perplexity: 17.781, speed: 14672 words
763 2018-11-30T05:51:23.346378994Z 01:51:23 INFO:41.7%, step:1699, perplexity: 17.767, speed: 14687 words
764 2018-11-30T05:51:30.170897474Z 01:51:30 INFO:44.1%, step:1799, perplexity: 17.758, speed: 14705 words
765 2018-11-30T05:51:36.789583226Z 01:51:36 INFO:46.6%, step:1899, perplexity: 17.749, speed: 14743 words
766 2018-11-30T05:51:43.705834604Z 01:51:43 INFO:49.0%, step:1999, perplexity: 17.738, speed: 14746 words
767 2018-11-30T05:51:50.4829828Z 01:51:50 INFO:51.5%, step:2099, perplexity: 17.735, speed: 14764 words
768 2018-11-30T05:51:57.579040503Z 01:51:57 INFO:53.9%, step:2199, perplexity: 17.737, speed: 14748 words
769 2018-11-30T05:52:04.892707035Z 01:52:04 INFO:56.4%, step:2299, perplexity: 17.742, speed: 14714 words
770 2018-11-30T05:52:12.303484597Z 01:52:12 INFO:58.9%, step:2399, perplexity: 17.746, speed: 14674 words
771 2018-11-30T05:52:19.003798242Z 01:52:18 INFO:61.3%, step:2499, perplexity: 17.751, speed: 14698 words
772 2018-11-30T05:52:25.911065448Z 01:52:25 INFO:63.8%, step:2599, perplexity: 17.752, speed: 14703 words
773 2018-11-30T05:52:32.906450794Z 01:52:32 INFO:66.2%, step:2699, perplexity: 17.750, speed: 14700 words
774 2018-11-30T05:52:39.663934285Z 01:52:39 INFO:68.7%, step:2799, perplexity: 17.745, speed: 14715 words
775 2018-11-30T05:52:46.426836725Z 01:52:46 INFO:71.1%, step:2899, perplexity: 17.738, speed: 14730 words
776 2018-11-30T05:52:53.397980668Z 01:52:53 INFO:73.6%, step:2999, perplexity: 17.730, speed: 14728 words
777 2018-11-30T05:53:01.05593674Z 01:53:01 INFO:76.0%, step:3099, perplexity: 17.722, speed: 14680 words
778 2018-11-30T05:53:08.456067763Z 01:53:08 INFO:78.5%, step:3199, perplexity: 17.714, speed: 14653 words
779 2018-11-30T05:53:14.990415891Z 01:53:14 INFO:80.9%, step:3299, perplexity: 17.705, speed: 14682 words
780 2018-11-30T05:53:21.587876639Z 01:53:21 INFO:83.4%, step:3399, perplexity: 17.695, speed: 14705 words
781 2018-11-30T05:53:28.908931843Z 01:53:28 INFO:85.8%, step:3499, perplexity: 17.685, speed: 14683 words
782 2018-11-30T05:53:35.9018097Z 01:53:35 INFO:88.3%, step:3599, perplexity: 17.676, speed: 14683 words
783 2018-11-30T05:53:42.882645701Z 01:53:42 INFO:90.7%, step:3699, perplexity: 17.668, speed: 14682 words
784 2018-11-30T05:53:49.488042978Z 01:53:49 INFO:93.2%, step:3799, perplexity: 17.660, speed: 14702 words
785 2018-11-30T05:53:56.648925358Z 01:53:56 INFO:95.6%, step:3899, perplexity: 17.653, speed: 14692 words
786 2018-11-30T05:54:04.269983637Z 01:54:04 INFO:98.1%, step:3999, perplexity: 17.646, speed: 14658 words
787 2018-11-30T05:54:10.268892466Z 01:54:10 INFO:Perplexity: 17.430, speed: 14629 words per sec
788 2018-11-30T05:54:12.343667598Z 01:54:12 INFO:Latest model saved in ./results/output_poem/save_model/model-57092
789 2018-11-30T05:54:12.343714908Z
790 2018-11-30T05:54:12.343722023Z 01:54:12 INFO:Evaluate on validation set
791 2018-11-30T05:54:17.895933125Z 01:54:17 INFO:41.7%, step:99, perplexity: 20.558, speed: 20826 words
792 2018-11-30T05:54:20.527601023Z 01:54:20 INFO:83.3%, step:199, perplexity: 20.380, speed: 27120 words
793 2018-11-30T05:54:21.870072112Z 01:54:21 INFO:Perplexity: 21.123, speed: 27628 words per sec
794 2018-11-30T05:54:21.870120896Z 01:54:21 INFO:Decay the learning rate: 7.8125e-05
795 2018-11-30T05:54:21.877164248Z 01:54:21 INFO:Best model is saved in ./results/output_poem/best_model/model-53014
796 2018-11-30T05:54:21.877182752Z 01:54:21 INFO:Best validation ppl is 21.121807
797 2018-11-30T05:54:21.877189362Z
798 2018-11-30T05:54:21.893970311Z 01:54:21 INFO:=================== Epoch 14 ===================
799 2018-11-30T05:54:21.894011948Z
800 2018-11-30T05:54:21.894019497Z 01:54:21 INFO:Training on training set
801 2018-11-30T05:54:29.139841124Z 01:54:29 INFO:2.5%, step:99, perplexity: 19.249, speed: 14148 words
802 2018-11-30T05:54:36.177542153Z 01:54:36 INFO:4.9%, step:199, perplexity: 18.654, speed: 14343 words
803 2018-11-30T05:54:43.087928377Z 01:54:43 INFO:7.4%, step:299, perplexity: 18.460, speed: 14496 words
804 2018-11-30T05:54:49.945881477Z 01:54:49 INFO:9.8%, step:399, perplexity: 18.408, speed: 14603 words
805 2018-11-30T05:54:57.43789082Z 01:54:57 INFO:12.3%, step:499, perplexity: 18.387, speed: 14408 words
806 2018-11-30T05:55:05.152225308Z 01:55:05 INFO:14.7%, step:599, perplexity: 18.349, speed: 14204 words
807 2018-11-30T05:55:12.859935605Z 01:55:12 INFO:17.2%, step:699, perplexity: 18.270, speed: 14067 words
808 2018-11-30T05:55:19.645987024Z 01:55:19 INFO:19.6%, step:799, perplexity: 18.153, speed: 14186 words
809 2018-11-30T05:55:26.598990807Z 01:55:26 INFO:22.1%, step:899, perplexity: 18.051, speed: 14244 words
810 2018-11-30T05:55:34.232959354Z 01:55:34 INFO:24.5%, step:999, perplexity: 17.982, speed: 14157 words
811 2018-11-30T05:55:41.102945118Z 01:55:41 INFO:27.0%, step:1099, perplexity: 17.933, speed: 14221 words
812 2018-11-30T05:55:48.310916368Z 01:55:48 INFO:29.4%, step:1199, perplexity: 17.881, speed: 14220 words
813 2018-11-30T05:55:55.232687858Z 01:55:55 INFO:31.9%, step:1299, perplexity: 17.838, speed: 14263 words
814 2018-11-30T05:56:02.317602256Z 01:56:02 INFO:34.3%, step:1399, perplexity: 17.800, speed: 14277 words
815 2018-11-30T05:56:10.160981518Z 01:56:10 INFO:36.8%, step:1499, perplexity: 17.769, speed: 14188 words
816 2018-11-30T05:56:16.65565804Z 01:56:16 INFO:39.2%, step:1599, perplexity: 17.748, speed: 14277 words
817 2018-11-30T05:56:23.464776092Z 01:56:23 INFO:41.7%, step:1699, perplexity: 17.734, speed: 14320 words
818 2018-11-30T05:56:30.305564655Z 01:56:30 INFO:44.1%, step:1799, perplexity: 17.725, speed: 14355 words
819 2018-11-30T05:56:37.297787066Z 01:56:37 INFO:46.6%, step:1899, perplexity: 17.716, speed: 14370 words
820 2018-11-30T05:56:44.36981556Z 01:56:44 INFO:49.0%, step:1999, perplexity: 17.705, speed: 14375 words
821 2018-11-30T05:56:51.286841324Z 01:56:51 INFO:51.5%, step:2099, perplexity: 17.702, speed: 14395 words
822 2018-11-30T05:56:58.099048746Z 01:56:58 INFO:53.9%, step:2199, perplexity: 17.704, speed: 14422 words
823 2018-11-30T05:57:05.552601209Z 01:57:05 INFO:56.4%, step:2299, perplexity: 17.709, speed: 14391 words
824 2018-11-30T05:57:13.317532578Z 01:57:13 INFO:58.9%, step:2399, perplexity: 17.713, speed: 14337 words
825 2018-11-30T05:57:20.125476866Z 01:57:20 INFO:61.3%, step:2499, perplexity: 17.718, speed: 14364 words
826 2018-11-30T05:57:26.948062616Z 01:57:26 INFO:63.8%, step:2599, perplexity: 17.719, speed: 14387 words
827 2018-11-30T05:57:33.532441607Z 01:57:33 INFO:66.2%, step:2699, perplexity: 17.716, speed: 14428 words
828 2018-11-30T05:57:41.025446676Z 01:57:41 INFO:68.7%, step:2799, perplexity: 17.711, speed: 14399 words
829 2018-11-30T05:57:47.846370476Z 01:57:47 INFO:71.1%, step:2899, perplexity: 17.705, speed: 14419 words
830 2018-11-30T05:57:54.821523949Z 01:57:54 INFO:73.6%, step:2999, perplexity: 17.696, speed: 14428 words
831 2018-11-30T05:58:02.551156139Z 01:58:02 INFO:76.0%, step:3099, perplexity: 17.688, speed: 14387 words
832 2018-11-30T05:58:09.896704979Z 01:58:09 INFO:78.5%, step:3199, perplexity: 17.680, speed: 14372 words
833 2018-11-30T05:58:16.837813154Z 01:58:16 INFO:80.9%, step:3299, perplexity: 17.671, speed: 14383 words
834 2018-11-30T05:58:23.506878166Z 01:58:23 INFO:83.4%, step:3399, perplexity: 17.661, speed: 14410 words
835 2018-11-30T05:58:30.281169545Z 01:58:30 INFO:85.8%, step:3499, perplexity: 17.651, speed: 14429 words
836 2018-11-30T05:58:37.123887501Z 01:58:37 INFO:88.3%, step:3599, perplexity: 17.642, speed: 14444 words
837 2018-11-30T05:58:44.411975762Z 01:58:44 INFO:90.7%, step:3699, perplexity: 17.634, speed: 14433 words
838 2018-11-30T05:58:51.239047311Z 01:58:51 INFO:93.2%, step:3799, perplexity: 17.626, speed: 14447 words
839 2018-11-30T05:58:58.024469854Z 01:58:58 INFO:95.6%, step:3899, perplexity: 17.618, speed: 14463 words
840 2018-11-30T05:59:05.415685079Z 01:59:05 INFO:98.1%, step:3999, perplexity: 17.611, speed: 14447 words
841 2018-11-30T05:59:11.287461032Z 01:59:11 INFO:Perplexity: 17.389, speed: 14430 words per sec
842 2018-11-30T05:59:13.441923968Z 01:59:13 INFO:Latest model saved in ./results/output_poem/save_model/model-61170
843 2018-11-30T05:59:13.441971371Z
844 2018-11-30T05:59:13.441978401Z 01:59:13 INFO:Evaluate on validation set
845 2018-11-30T05:59:18.343972222Z 01:59:18 INFO:41.7%, step:99, perplexity: 20.498, speed: 23868 words
846 2018-11-30T05:59:21.037656064Z 01:59:21 INFO:83.3%, step:199, perplexity: 20.321, speed: 29302 words
847 2018-11-30T05:59:22.281067922Z 01:59:22 INFO:Perplexity: 21.065, speed: 29848 words per sec
848 2018-11-30T05:59:24.478555798Z 01:59:24 INFO:Best model is saved in ./results/output_poem/best_model/model-61170
849 2018-11-30T05:59:24.478597514Z 01:59:24 INFO:Best validation ppl is 21.064644
850 2018-11-30T05:59:24.478605366Z
851 2018-11-30T05:59:24.482407082Z 01:59:24 INFO:Latest model is saved in ./results/output_poem/save_model/model-61170
852 2018-11-30T05:59:24.482423821Z 01:59:24 INFO:Best model is saved in ./results/output_poem/best_model/model-61170
853 2018-11-30T05:59:24.482428095Z 01:59:24 INFO:Best validation ppl is 21.064644
854 2018-11-30T05:59:24.482431436Z
855 2018-11-30T05:59:24.482434291Z 01:59:24 INFO:Evaluate the best model on test set
856 2018-11-30T05:59:29.589702083Z 01:59:29 INFO:20.8%, step:99, perplexity: 24.128, speed: 24662 words
857 2018-11-30T05:59:32.439987017Z 01:59:32 INFO:41.7%, step:199, perplexity: 25.266, speed: 29250 words
858 2018-11-30T05:59:35.450174869Z 01:59:35 INFO:62.5%, step:299, perplexity: 25.428, speed: 30700 words
859 2018-11-30T05:59:38.355844634Z 01:59:38 INFO:83.3%, step:399, perplexity: 25.498, speed: 31709 words
860 2018-11-30T05:59:40.519902775Z 01:59:40 INFO:Perplexity: 25.508, speed: 32599 words per sec
861 2018-11-30T05:59:42.862057327Z SYSTEM: Finishing...
+6
-0
results/output_poem/best_model/checkpoint less more
0 model_checkpoint_path: "model-61170"
1 all_model_checkpoint_paths: "model-24468"
2 all_model_checkpoint_paths: "model-32624"
3 all_model_checkpoint_paths: "model-36702"
4 all_model_checkpoint_paths: "model-53014"
5 all_model_checkpoint_paths: "model-61170"
results/output_poem/best_model/model-24468.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/best_model/model-24468.index less more
Binary diff not shown
results/output_poem/best_model/model-24468.meta less more
Binary diff not shown
results/output_poem/best_model/model-32624.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/best_model/model-32624.index less more
Binary diff not shown
results/output_poem/best_model/model-32624.meta less more
Binary diff not shown
results/output_poem/best_model/model-36702.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/best_model/model-36702.index less more
Binary diff not shown
results/output_poem/best_model/model-36702.meta less more
Binary diff not shown
results/output_poem/best_model/model-53014.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/best_model/model-53014.index less more
Binary diff not shown
results/output_poem/best_model/model-53014.meta less more
Binary diff not shown
results/output_poem/best_model/model-61170.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/best_model/model-61170.index less more
Binary diff not shown
results/output_poem/best_model/model-61170.meta less more
Binary diff not shown
+18
-0
results/output_poem/result.json less more
0 {
1 "best_model": "./results/output_poem/best_model/model-61170",
2 "best_valid_ppl": 21.06464385986328,
3 "latest_model": "./results/output_poem/save_model/model-61170",
4 "params": {
5 "batch_size": 16,
6 "cell_type": "lstm",
7 "dropout": 0.0,
8 "embedding_size": 128,
9 "hidden_size": 128,
10 "input_dropout": 0.0,
11 "learning_rate": 0.005,
12 "max_grad_norm": 5.0,
13 "num_layers": 2,
14 "num_unrollings": 64
15 },
16 "test_ppl": 25.508337020874023
17 }⏎
+6
-0
results/output_poem/save_model/checkpoint less more
0 model_checkpoint_path: "model-61170"
1 all_model_checkpoint_paths: "model-44858"
2 all_model_checkpoint_paths: "model-48936"
3 all_model_checkpoint_paths: "model-53014"
4 all_model_checkpoint_paths: "model-57092"
5 all_model_checkpoint_paths: "model-61170"
results/output_poem/save_model/model-44858.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/save_model/model-44858.index less more
Binary diff not shown
results/output_poem/save_model/model-44858.meta less more
Binary diff not shown
results/output_poem/save_model/model-48936.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/save_model/model-48936.index less more
Binary diff not shown
results/output_poem/save_model/model-48936.meta less more
Binary diff not shown
results/output_poem/save_model/model-53014.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/save_model/model-53014.index less more
Binary diff not shown
results/output_poem/save_model/model-53014.meta less more
Binary diff not shown
results/output_poem/save_model/model-57092.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/save_model/model-57092.index less more
Binary diff not shown
results/output_poem/save_model/model-57092.meta less more
Binary diff not shown
results/output_poem/save_model/model-61170.data-00000-of-00001 less more
Binary diff not shown
results/output_poem/save_model/model-61170.index less more
Binary diff not shown
results/output_poem/save_model/model-61170.meta less more
Binary diff not shown
results/tb_results/tensorboard_log/1543547440/summaries/train/.nfs00000000011509b40000001c less more
Binary diff not shown
results/tb_results/tensorboard_log/1543547440/summaries/train/events.out.tfevents.1543547464.job-gpu-5c00aa091afd942d0d0ee1d3-zpf8z less more
Binary diff not shown
results/tb_results/tensorboard_log/1543547440/summaries/valid/.nfs00000000011509b60000001d less more
Binary diff not shown
results/tb_results/tensorboard_log/1543547440/summaries/valid/events.out.tfevents.1543547467.job-gpu-5c00aa091afd942d0d0ee1d3-zpf8z less more
Binary diff not shown
results/tb_results/tensorboard_log/1543552764/summaries/train/events.out.tfevents.1543552788.job-gpu-5c00bed61afd943016b45263-6w7d6 less more
Binary diff not shown
results/tb_results/tensorboard_log/1543552764/summaries/valid/events.out.tfevents.1543552793.job-gpu-5c00bed61afd943016b45263-6w7d6 less more
Binary diff not shown
results/tb_results/tensorboard_log/1543553075/summaries/train/events.out.tfevents.1543553098.job-gpu-5c00c00d1afd942b54c88504-f2s8v less more
Binary diff not shown
results/tb_results/tensorboard_log/1543553075/summaries/valid/events.out.tfevents.1543553101.job-gpu-5c00c00d1afd942b54c88504-f2s8v less more
Binary diff not shown
+6
-5
train.py less more
2929 # Create necessary directories.
3030 if len(args.init_dir) != 0:
3131 args.output_dir = args.init_dir
32 else:
33 if os.path.exists(args.output_dir):
34 shutil.rmtree(args.output_dir)
35 for paths in [args.save_model, args.save_best_model, args.tb_log_dir]:
36 os.makedirs(os.path.dirname(paths))
32
33 # else:
34 # if os.path.exists(args.output_dir):
35 # shutil.rmtree(args.output_dir)
36 # for paths in [args.save_model, args.save_best_model, args.tb_log_dir]:
37 # os.makedirs(os.path.dirname(paths))
3738
3839 logging.basicConfig(stream=sys.stdout,
3940 format='%(asctime)s %(levelname)s:%(message)s',