yangsaisai
7 years ago
| 0 | import argparse | |
| 1 | import numpy as np | |
| 2 | ||
| 3 | def config_poem_train(args=''): | |
| 4 | parser = argparse.ArgumentParser() | |
| 5 | ||
| 6 | # Data and vocabulary file | |
| 7 | # parser.add_argument('--data_file', type=str, | |
| 8 | # default='../data/poem/poems_space.txt', | |
| 9 | # help='data file') | |
| 10 | ||
| 11 | parser.add_argument('--data_path', type=str, | |
| 12 | default='./data/poem/', | |
| 13 | help='data path') | |
| 14 | ||
| 15 | ||
| 16 | parser.add_argument('--encoding', type=str, | |
| 17 | default='utf-8', | |
| 18 | help='the encoding of the data file.') | |
| 19 | ||
| 20 | # Parameters for saving models. | |
| 21 | parser.add_argument('--output_dir', type=str, default='output_model', | |
| 22 | help=('directory to store final and' | |
| 23 | ' intermediate results and models.')) | |
| 24 | # Parameters for using saved best models. | |
| 25 | parser.add_argument('--init_dir', type=str, default='', | |
| 26 | help='continue from the outputs in the given directory') | |
| 27 | ||
| 28 | # Parameters to configure the neural network. | |
| 29 | parser.add_argument('--hidden_size', type=int, default=128,#128, | |
| 30 | help='size of RNN hidden state vector') | |
| 31 | parser.add_argument('--embedding_size', type=int, default=128,#0, | |
| 32 | help='size of character embeddings, 0 for one-hot') | |
| 33 | parser.add_argument('--num_layers', type=int, default=2, | |
| 34 | help='number of layers in the RNN') | |
| 35 | parser.add_argument('--num_unrollings', type=int, default=64,#10, | |
| 36 | help='number of unrolling steps.') | |
| 37 | parser.add_argument('--cell_type', type=str, default='lstm', | |
| 38 | help='which model to use (rnn, lstm or gru).') | |
| 39 | ||
| 40 | # Parameters to control the training. | |
| 41 | parser.add_argument('--num_epochs', type=int, default=8, | |
| 42 | help='number of epochs') | |
| 43 | parser.add_argument('--batch_size', type=int, default=16, | |
| 44 | help='minibatch size') | |
| 45 | parser.add_argument('--train_frac', type=float, default=0.9, | |
| 46 | help='fraction of data used for training.') | |
| 47 | parser.add_argument('--valid_frac', type=float, default=0.05, | |
| 48 | help='fraction of data used for validation.') | |
| 49 | # test_frac is computed as (1 - train_frac - valid_frac). | |
| 50 | parser.add_argument('--dropout', type=float, default=0.0, | |
| 51 | help='dropout rate, default to 0 (no dropout).') | |
| 52 | ||
| 53 | parser.add_argument('--input_dropout', type=float, default=0.0, | |
| 54 | help=('dropout rate on input layer, default to 0 (no dropout),' | |
| 55 | 'and no dropout if using one-hot representation.')) | |
| 56 | ||
| 57 | # Parameters for gradient descent. | |
| 58 | parser.add_argument('--max_grad_norm', type=float, default=5., | |
| 59 | help='clip global grad norm') | |
| 60 | parser.add_argument('--learning_rate', type=float, default=5e-3, | |
| 61 | help='initial learning rate') | |
| 62 | ||
| 63 | # Parameters for logging. | |
| 64 | parser.add_argument('--progress_freq', type=int, default=100, | |
| 65 | help=('frequency for progress report in training and evalution.')) | |
| 66 | parser.add_argument('--verbose', type=int, default=0, | |
| 67 | help=('whether to show progress report in training and evalution.')) | |
| 68 | ||
| 69 | # Parameters to feed in the initial model and current best model. | |
| 70 | parser.add_argument('--init_model', type=str, | |
| 71 | default='', help=('initial model')) | |
| 72 | parser.add_argument('--best_model', type=str, | |
| 73 | default='', help=('current best model')) | |
| 74 | parser.add_argument('--best_valid_ppl', type=float, | |
| 75 | default=np.Inf, help=('current valid perplexity')) | |
| 76 | ||
| 77 | # # Parameters for using saved best models. | |
| 78 | # parser.add_argument('--model_dir', type=str, default='', | |
| 79 | # help='continue from the outputs in the given directory') | |
| 80 | ||
| 81 | # Parameters for debugging. | |
| 82 | parser.add_argument('--debug', dest='debug', action='store_true', | |
| 83 | help='show debug information') | |
| 84 | parser.set_defaults(debug=False) | |
| 85 | ||
| 86 | # Parameters for unittesting the implementation. | |
| 87 | parser.add_argument('--test', dest='test', action='store_true', | |
| 88 | help=('use the first 1000 character to as data to test the implementation')) | |
| 89 | parser.set_defaults(test=False) | |
| 90 | ||
| 91 | # input_args = '--data_path ./data/poem --output_dir output_poem --hidden_size 256 --embedding_size 128 --num_unrollings 128 --debug --encoding utf-8' | |
| 92 | args = parser.parse_args(args.split()) | |
| 93 | ||
| 94 | return args | |
| 95 | ||
| 96 | ||
| 97 | ||
| 98 | def config_sample(args=''): | |
| 99 | parser = argparse.ArgumentParser() | |
| 100 | ||
| 101 | # hyper-parameters for using saved best models. | |
| 102 | # 学习日志和结果相关的超参数 | |
| 103 | logging_args = parser.add_argument_group('Logging_Options') | |
| 104 | logging_args.add_argument('--model_dir', type=str, | |
| 105 | default='demo_model/', | |
| 106 | help='continue from the outputs in the given directory') | |
| 107 | ||
| 108 | logging_args.add_argument('--data_dir', type=str, | |
| 109 | default='./data/poem', | |
| 110 | help='data file path') | |
| 111 | ||
| 112 | logging_args.add_argument('--best_model', type=str, | |
| 113 | default='', help=('current best model')) | |
| 114 | ||
| 115 | # hyper-parameters for sampling. | |
| 116 | # 设置sampling相关的超参数 | |
| 117 | testing_args = parser.add_argument_group('Sampling Options') | |
| 118 | testing_args.add_argument('--max_prob', dest='max_prob', action='store_true', | |
| 119 | help='always pick the most probable next character in sampling') | |
| 120 | testing_args.set_defaults(max_prob=False) | |
| 121 | ||
| 122 | testing_args.add_argument('--start_text', type=str, | |
| 123 | default='The meaning of life is ', | |
| 124 | help='the text to start with') | |
| 125 | ||
| 126 | testing_args.add_argument('--length', type=int, | |
| 127 | default=100, | |
| 128 | help='length of sampled sequence') | |
| 129 | ||
| 130 | testing_args.add_argument('--seed', type=int, | |
| 131 | default=-1, | |
| 132 | help=('seed for sampling to replicate results, ' | |
| 133 | 'an integer between 0 and 4294967295.')) | |
| 134 | ||
| 135 | args = parser.parse_args(args.split()) | |
| 136 | ||
| 137 | return args⏎ |
| 78 | 78 | w2v_file = os.path.join(base_path, "vectors_poem.bin") |
| 79 | 79 | w2v = Word2Vec(w2v_file) |
| 80 | 80 | |
| 81 | train_data_loader = DataLoader(base_path,batch_size,num_unrollings,w2v.model,'train') | |
| 82 | test1_data_loader = DataLoader(base_path,batch_size,num_unrollings,w2v.model,'test') | |
| 83 | valid_data_loader = DataLoader(base_path,batch_size,num_unrollings,w2v.model,'valid') | |
| 81 | train_data_loader = DataLoader(base_path, batch_size, num_unrollings, w2v.model, 'train') | |
| 82 | test1_data_loader = DataLoader(base_path, batch_size, num_unrollings, w2v.model, 'test') | |
| 83 | valid_data_loader = DataLoader(base_path, batch_size, num_unrollings, w2v.model, 'valid') | |
| 84 | 84 | |
| 85 | 85 | # Create graphs |
| 86 | 86 | logging.info('Creating graph') |
| 88 | 88 | with graph.as_default(): |
| 89 | 89 | w2v_vocab_size = len(w2v.model.vocab) |
| 90 | 90 | with tf.name_scope('training'): |
| 91 | train_model = CharRNNLM(is_training=True,w2v_model = w2v.model,vocab_size=w2v_vocab_size, infer=False, **params) | |
| 91 | train_model = CharRNNLM(is_training=True, w2v_model = w2v.model, vocab_size=w2v_vocab_size, infer=False, **params) | |
| 92 | 92 | tf.get_variable_scope().reuse_variables() |
| 93 | 93 | |
| 94 | 94 | with tf.name_scope('validation'): |
| 95 | valid_model = CharRNNLM(is_training=False,w2v_model = w2v.model, vocab_size=w2v_vocab_size, infer=False, **params) | |
| 95 | valid_model = CharRNNLM(is_training=False, w2v_model = w2v.model, vocab_size=w2v_vocab_size, infer=False, **params) | |
| 96 | 96 | |
| 97 | 97 | with tf.name_scope('evaluation'): |
| 98 | test_model = CharRNNLM(is_training=False,w2v_model = w2v.model,vocab_size=w2v_vocab_size, infer=False, **params) | |
| 98 | test_model = CharRNNLM(is_training=False, w2v_model = w2v.model,vocab_size=w2v_vocab_size, infer=False, **params) | |
| 99 | 99 | saver = tf.train.Saver(name='model_saver') |
| 100 | 100 | best_model_saver = tf.train.Saver(name='best_model_saver') |
| 101 | 101 | |
| 103 | 103 | |
| 104 | 104 | result = {} |
| 105 | 105 | result['params'] = params |
| 106 | ||
| 107 | 106 | |
| 108 | 107 | try: |
| 109 | 108 | with tf.Session(graph=graph) as session: |
| 186 | 185 | result_path = os.path.join(args.output_dir, 'result.json') |
| 187 | 186 | if os.path.exists(result_path): |
| 188 | 187 | os.remove(result_path) |
| 189 | with open(result_path, 'w',encoding='utf-8',errors='ignore') as f: | |
| 188 | with open(result_path, 'w', encoding='utf-8', errors='ignore') as f: | |
| 190 | 189 | json.dump(result, f, indent=2, sort_keys=True) |
| 191 | 190 | |
| 192 | 191 | |
| 193 | 192 | if __name__ == '__main__': |
| 194 | args = '--output_dir /home/jovyan/work/results/output_poem --data_path /home/jovyan/work/data/poem/ --hidden_size 128 --embedding_size 128 --cell_type lstm' | |
| 193 | args = '--output_dir ./results/output_poem --data_path ./data/poem/ --hidden_size 128 --embedding_size 128 --cell_type lstm' | |
| 195 | 194 | main(args) |
| 38 | 38 | help='which model to use (rnn, lstm or gru).') |
| 39 | 39 | |
| 40 | 40 | # Parameters to control the training. |
| 41 | parser.add_argument('--num_epochs', type=int, default=5, | |
| 41 | parser.add_argument('--num_epochs', type=int, default=8, | |
| 42 | 42 | help='number of epochs') |
| 43 | 43 | parser.add_argument('--batch_size', type=int, default=16, |
| 44 | 44 | help='minibatch size') |
| 0 | 2018-11-28T05:15:14.320735257Z SYSTEM: Preparing env... | |
| 1 | 2018-11-28T05:15:14.647877554Z SYSTEM: Running... | |
| 2 | 2018-11-28T05:15:18.106402357Z Writing to /home/jovyan/work/results/output_poem/tensorboard_log/1543382118 | |
| 3 | 2018-11-28T05:15:18.106430315Z | |
| 4 | 2018-11-28T05:15:18.241492422Z ============================================================ | |
| 5 | 2018-11-28T05:15:18.241525934Z All final and intermediate outputs will be stored in ./results/output_poem/ | |
| 6 | 2018-11-28T05:15:18.241530809Z ============================================================ | |
| 7 | 2018-11-28T05:15:18.241534122Z | |
| 8 | 2018-11-28T05:15:18.241537089Z 01:15:18 INFO:args are: | |
| 9 | 2018-11-28T05:15:18.241542025Z Namespace(batch_size=16, best_model='', best_valid_ppl=inf, cell_type='lstm', data_path='./data/poem/', debug=False, dropout=0.0, embedding_size=128, encoding='utf-8', hidden_size=128, init_dir='', init_model='', input_dropout=0.0, learning_rate=0.005, max_grad_norm=5.0, num_epochs=8, num_layers=2, num_unrollings=64, output_dir='./results/output_poem', progress_freq=100, save_best_model='./results/output_poem/best_model/model', save_model='./results/output_poem/save_model/model', tb_log_dir='/home/jovyan/work/results/output_poem/tensorboard_log/1543382118', test=False, train_frac=0.9, valid_frac=0.05, verbose=0) | |
| 10 | 2018-11-28T05:15:18.241548415Z 01:15:18 INFO:Parameters are: | |
| 11 | 2018-11-28T05:15:18.241551292Z { | |
| 12 | 2018-11-28T05:15:18.241555549Z "batch_size": 16, | |
| 13 | 2018-11-28T05:15:18.241560357Z "cell_type": "lstm", | |
| 14 | 2018-11-28T05:15:18.241565107Z "dropout": 0.0, | |
| 15 | 2018-11-28T05:15:18.241569874Z "embedding_size": 128, | |
| 16 | 2018-11-28T05:15:18.241574722Z "hidden_size": 128, | |
| 17 | 2018-11-28T05:15:18.241577874Z "input_dropout": 0.0, | |
| 18 | 2018-11-28T05:15:18.241581158Z "learning_rate": 0.005, | |
| 19 | 2018-11-28T05:15:18.241584298Z "max_grad_norm": 5.0, | |
| 20 | 2018-11-28T05:15:18.241587178Z "num_layers": 2, | |
| 21 | 2018-11-28T05:15:18.241590185Z "num_unrollings": 64 | |
| 22 | 2018-11-28T05:15:18.241593164Z } | |
| 23 | 2018-11-28T05:15:18.241595822Z | |
| 24 | 2018-11-28T05:15:18.396889511Z tensor_file:./data/poem/poem_ids.txt | |
| 25 | 2018-11-28T05:15:18.397244689Z Loading dataset from ./data/poem/poem_ids.txt | |
| 26 | 2018-11-28T05:15:18.711302229Z file maxSeqLen = 64 | |
| 27 | 2018-11-28T05:15:18.712849098Z Loaded ./data/poem/: training samples:65235 ,validationSamples:3837,testingSamples:7676 | |
| 28 | 2018-11-28T05:15:19.225663457Z tensor_file:./data/poem/poem_ids.txt | |
| 29 | 2018-11-28T05:15:19.225695279Z Loading dataset from ./data/poem/poem_ids.txt | |
| 30 | 2018-11-28T05:15:19.634889498Z file maxSeqLen = 64 | |
| 31 | 2018-11-28T05:15:19.634937044Z Loaded ./data/poem/: training samples:65235 ,validationSamples:3837,testingSamples:7676 | |
| 32 | 2018-11-28T05:15:19.677671888Z tensor_file:./data/poem/poem_ids.txt | |
| 33 | 2018-11-28T05:15:19.677708851Z Loading dataset from ./data/poem/poem_ids.txt | |
| 34 | 2018-11-28T05:15:20.182924225Z file maxSeqLen = 64 | |
| 35 | 2018-11-28T05:15:20.182974095Z Loaded ./data/poem/: training samples:65235 ,validationSamples:3837,testingSamples:7676 | |
| 36 | 2018-11-28T05:15:20.215050752Z 01:15:20 INFO:Creating graph | |
| 37 | 2018-11-28T05:15:33.324106085Z 01:15:33 INFO:Start training | |
| 38 | 2018-11-28T05:15:33.324155825Z | |
| 39 | 2018-11-28T05:15:33.32475036Z 2018-11-28 13:15:33.324366: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA | |
| 40 | 2018-11-28T05:15:33.50406859Z 2018-11-28 13:15:33.503475: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:964] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero | |
| 41 | 2018-11-28T05:15:33.504319293Z 2018-11-28 13:15:33.504140: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1432] Found device 0 with properties: | |
| 42 | 2018-11-28T05:15:33.504328004Z name: Tesla P100-PCIE-16GB major: 6 minor: 0 memoryClockRate(GHz): 1.3285 | |
| 43 | 2018-11-28T05:15:33.504331566Z pciBusID: 0000:00:07.0 | |
| 44 | 2018-11-28T05:15:33.504334469Z totalMemory: 15.90GiB freeMemory: 15.61GiB | |
| 45 | 2018-11-28T05:15:33.504337518Z 2018-11-28 13:15:33.504183: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1511] Adding visible gpu devices: 0 | |
| 46 | 2018-11-28T05:15:34.733181051Z 2018-11-28 13:15:34.732672: I tensorflow/core/common_runtime/gpu/gpu_device.cc:982] Device interconnect StreamExecutor with strength 1 edge matrix: | |
| 47 | 2018-11-28T05:15:34.733221672Z 2018-11-28 13:15:34.732718: I tensorflow/core/common_runtime/gpu/gpu_device.cc:988] 0 | |
| 48 | 2018-11-28T05:15:34.733230214Z 2018-11-28 13:15:34.732726: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1001] 0: N | |
| 49 | 2018-11-28T05:15:34.733512258Z 2018-11-28 13:15:34.733245: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 15129 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:07.0, compute capability: 6.0) | |
| 50 | 2018-11-28T05:15:41.290521983Z 01:15:41 INFO:=================== Epoch 0 =================== | |
| 51 | 2018-11-28T05:15:41.290566673Z | |
| 52 | 2018-11-28T05:15:41.290573607Z 01:15:41 INFO:Training on training set | |
| 53 | 2018-11-28T05:15:52.151909785Z 01:15:52 INFO:2.5%, step:99, perplexity: 578.325, speed: 9591 words | |
| 54 | 2018-11-28T05:15:57.430336017Z 01:15:57 INFO:4.9%, step:199, perplexity: 357.929, speed: 12836 words | |
| 55 | 2018-11-28T05:16:02.116662673Z 01:16:02 INFO:7.4%, step:299, perplexity: 273.130, speed: 14883 words | |
| 56 | 2018-11-28T05:16:07.253916222Z 01:16:07 INFO:9.8%, step:399, perplexity: 227.320, speed: 15890 words | |
| 57 | 2018-11-28T05:16:11.835896741Z 01:16:11 INFO:12.3%, step:499, perplexity: 198.077, speed: 16864 words | |
| 58 | 2018-11-28T05:16:18.358938679Z 01:16:18 INFO:14.7%, step:599, perplexity: 177.427, speed: 16659 words | |
| 59 | 2018-11-28T05:16:24.523903582Z 01:16:24 INFO:17.2%, step:699, perplexity: 161.822, speed: 16651 words | |
| 60 | 2018-11-28T05:16:30.173918306Z 01:16:30 INFO:19.6%, step:799, perplexity: 149.461, speed: 16823 words | |
| 61 | 2018-11-28T05:16:34.752631718Z 01:16:34 INFO:22.1%, step:899, perplexity: 139.500, speed: 17299 words | |
| 62 | 2018-11-28T05:16:39.546106241Z 01:16:39 INFO:24.5%, step:999, perplexity: 131.369, speed: 17634 words | |
| 63 | 2018-11-28T05:16:44.397107699Z 01:16:44 INFO:27.0%, step:1099, perplexity: 124.590, speed: 17902 words | |
| 64 | 2018-11-28T05:16:49.074953597Z 01:16:49 INFO:29.4%, step:1199, perplexity: 118.776, speed: 18180 words | |
| 65 | 2018-11-28T05:16:54.465967056Z 01:16:54 INFO:31.9%, step:1299, perplexity: 113.747, speed: 18239 words | |
| 66 | 2018-11-28T05:16:59.400682889Z 01:16:59 INFO:34.3%, step:1399, perplexity: 109.345, speed: 18397 words | |
| 67 | 2018-11-28T05:17:04.438936976Z 01:17:04 INFO:36.8%, step:1499, perplexity: 105.459, speed: 18514 words | |
| 68 | 2018-11-28T05:17:09.406938999Z 01:17:09 INFO:39.2%, step:1599, perplexity: 102.013, speed: 18633 words | |
| 69 | 2018-11-28T05:17:14.492306748Z 01:17:14 INFO:41.7%, step:1699, perplexity: 98.932, speed: 18715 words | |
| 70 | 2018-11-28T05:17:18.955922673Z 01:17:18 INFO:44.1%, step:1799, perplexity: 96.158, speed: 18909 words | |
| 71 | 2018-11-28T05:17:23.83692923Z 01:17:23 INFO:46.6%, step:1899, perplexity: 93.630, speed: 19007 words | |
| 72 | 2018-11-28T05:17:28.314893125Z 01:17:28 INFO:49.0%, step:1999, perplexity: 91.307, speed: 19169 words | |
| 73 | 2018-11-28T05:17:32.944925873Z 01:17:32 INFO:51.5%, step:2099, perplexity: 89.191, speed: 19291 words | |
| 74 | 2018-11-28T05:17:37.499867891Z 01:17:37 INFO:53.9%, step:2199, perplexity: 87.256, speed: 19417 words | |
| 75 | 2018-11-28T05:17:42.102061433Z 01:17:42 INFO:56.4%, step:2299, perplexity: 85.471, speed: 19525 words | |
| 76 | 2018-11-28T05:17:46.701617682Z 01:17:46 INFO:58.9%, step:2399, perplexity: 83.806, speed: 19625 words | |
| 77 | 2018-11-28T05:17:51.156559266Z 01:17:51 INFO:61.3%, step:2499, perplexity: 82.256, speed: 19741 words | |
| 78 | 2018-11-28T05:17:55.621960785Z 01:17:55 INFO:63.8%, step:2599, perplexity: 80.791, speed: 19847 words | |
| 79 | 2018-11-28T05:18:00.095881941Z 01:18:00 INFO:66.2%, step:2699, perplexity: 79.407, speed: 19945 words | |
| 80 | 2018-11-28T05:18:04.564095005Z 01:18:04 INFO:68.7%, step:2799, perplexity: 78.098, speed: 20038 words | |
| 81 | 2018-11-28T05:18:09.212007978Z 01:18:09 INFO:71.1%, step:2899, perplexity: 76.856, speed: 20101 words | |
| 82 | 2018-11-28T05:18:13.827493369Z 01:18:13 INFO:73.6%, step:2999, perplexity: 75.676, speed: 20164 words | |
| 83 | 2018-11-28T05:18:18.407461963Z 01:18:18 INFO:76.0%, step:3099, perplexity: 74.560, speed: 20228 words | |
| 84 | 2018-11-28T05:18:22.885785963Z 01:18:22 INFO:78.5%, step:3199, perplexity: 73.501, speed: 20301 words | |
| 85 | 2018-11-28T05:18:27.241545233Z 01:18:27 INFO:80.9%, step:3299, perplexity: 72.491, speed: 20385 words | |
| 86 | 2018-11-28T05:18:31.824936735Z 01:18:31 INFO:83.4%, step:3399, perplexity: 71.525, speed: 20438 words | |
| 87 | 2018-11-28T05:18:36.274325471Z 01:18:36 INFO:85.8%, step:3499, perplexity: 70.604, speed: 20504 words | |
| 88 | 2018-11-28T05:18:42.083556975Z 01:18:42 INFO:88.3%, step:3599, perplexity: 69.727, speed: 20412 words | |
| 89 | 2018-11-28T05:18:46.920702765Z 01:18:46 INFO:90.7%, step:3699, perplexity: 68.891, speed: 20431 words | |
| 90 | 2018-11-28T05:18:51.64409895Z 01:18:51 INFO:93.2%, step:3799, perplexity: 68.090, speed: 20462 words | |
| 91 | 2018-11-28T05:18:56.179571666Z 01:18:56 INFO:95.6%, step:3899, perplexity: 67.321, speed: 20511 words | |
| 92 | 2018-11-28T05:19:00.463665777Z 01:19:00 INFO:98.1%, step:3999, perplexity: 66.586, speed: 20585 words | |
| 93 | 2018-11-28T05:19:04.217818589Z 01:19:04 INFO:Perplexity: 37.638, speed: 20597 words per sec | |
| 94 | 2018-11-28T05:19:05.623957366Z 01:19:05 INFO:Latest model saved in ./results/output_poem/save_model/model-4078 | |
| 95 | 2018-11-28T05:19:05.623995605Z | |
| 96 | 2018-11-28T05:19:05.624001752Z 01:19:05 INFO:Evaluate on validation set | |
| 97 | 2018-11-28T05:19:09.109920512Z 01:19:09 INFO:41.7%, step:99, perplexity: 25.095, speed: 34222 words | |
| 98 | 2018-11-28T05:19:11.062413884Z 01:19:11 INFO:83.3%, step:199, perplexity: 24.852, speed: 41418 words | |
| 99 | 2018-11-28T05:19:11.95742685Z 01:19:11 INFO:Perplexity: 25.451, speed: 42076 words per sec | |
| 100 | 2018-11-28T05:19:13.797936688Z 01:19:13 INFO:Best model is saved in ./results/output_poem/best_model/model-4078 | |
| 101 | 2018-11-28T05:19:13.797975448Z 01:19:13 INFO:Best validation ppl is 25.451208 | |
| 102 | 2018-11-28T05:19:13.797982653Z | |
| 103 | 2018-11-28T05:19:13.800808043Z 01:19:13 INFO:=================== Epoch 1 =================== | |
| 104 | 2018-11-28T05:19:13.80083244Z | |
| 105 | 2018-11-28T05:19:13.80083835Z 01:19:13 INFO:Training on training set | |
| 106 | 2018-11-28T05:19:18.300777807Z 01:19:18 INFO:2.5%, step:99, perplexity: 29.545, speed: 22765 words | |
| 107 | 2018-11-28T05:19:22.940339608Z 01:19:22 INFO:4.9%, step:199, perplexity: 28.671, speed: 22412 words | |
| 108 | 2018-11-28T05:19:27.479189102Z 01:19:27 INFO:7.4%, step:299, perplexity: 28.460, speed: 22461 words | |
| 109 | 2018-11-28T05:19:32.002925368Z 01:19:31 INFO:9.8%, step:399, perplexity: 28.393, speed: 22507 words | |
| 110 | 2018-11-28T05:19:36.56603061Z 01:19:36 INFO:12.3%, step:499, perplexity: 28.350, speed: 22492 words | |
| 111 | 2018-11-28T05:19:41.013002929Z 01:19:41 INFO:14.7%, step:599, perplexity: 28.298, speed: 22580 words | |
| 112 | 2018-11-28T05:19:45.691995675Z 01:19:45 INFO:17.2%, step:699, perplexity: 28.192, speed: 22478 words | |
| 113 | 2018-11-28T05:19:50.735863468Z 01:19:50 INFO:19.6%, step:799, perplexity: 28.013, speed: 22181 words | |
| 114 | 2018-11-28T05:19:56.363837035Z 01:19:56 INFO:22.1%, step:899, perplexity: 27.850, speed: 21654 words | |
| 115 | 2018-11-28T05:20:00.652844593Z 01:20:00 INFO:24.5%, step:999, perplexity: 27.736, speed: 21859 words | |
| 116 | 2018-11-28T05:20:05.169376631Z 01:20:05 INFO:27.0%, step:1099, perplexity: 27.648, speed: 21929 words | |
| 117 | 2018-11-28T05:20:09.963226341Z 01:20:09 INFO:29.4%, step:1199, perplexity: 27.551, speed: 21880 words | |
| 118 | 2018-11-28T05:20:14.317996577Z 01:20:14 INFO:31.9%, step:1299, perplexity: 27.464, speed: 21998 words | |
| 119 | 2018-11-28T05:20:19.020423739Z 01:20:19 INFO:34.3%, step:1399, perplexity: 27.384, speed: 21982 words | |
| 120 | 2018-11-28T05:20:23.436615332Z 01:20:23 INFO:36.8%, step:1499, perplexity: 27.314, speed: 22058 words | |
| 121 | 2018-11-28T05:20:27.965321026Z 01:20:27 INFO:39.2%, step:1599, perplexity: 27.259, speed: 22092 words | |
| 122 | 2018-11-28T05:20:32.405933788Z 01:20:32 INFO:41.7%, step:1699, perplexity: 27.214, speed: 22147 words | |
| 123 | 2018-11-28T05:20:36.840325902Z 01:20:36 INFO:44.1%, step:1799, perplexity: 27.178, speed: 22197 words | |
| 124 | 2018-11-28T05:20:41.284337972Z 01:20:41 INFO:46.6%, step:1899, perplexity: 27.140, speed: 22240 words | |
| 125 | 2018-11-28T05:20:45.853882344Z 01:20:45 INFO:49.0%, step:1999, perplexity: 27.099, speed: 22249 words | |
| 126 | 2018-11-28T05:20:50.403481951Z 01:20:50 INFO:51.5%, step:2099, perplexity: 27.069, speed: 22261 words | |
| 127 | 2018-11-28T05:20:55.0149652Z 01:20:55 INFO:53.9%, step:2199, perplexity: 27.051, speed: 22258 words | |
| 128 | 2018-11-28T05:20:59.368873395Z 01:20:59 INFO:56.4%, step:2299, perplexity: 27.037, speed: 22311 words | |
| 129 | 2018-11-28T05:21:03.839913475Z 01:21:03 INFO:58.9%, step:2399, perplexity: 27.021, speed: 22334 words | |
| 130 | 2018-11-28T05:21:08.292414533Z 01:21:08 INFO:61.3%, step:2499, perplexity: 27.006, speed: 22360 words | |
| 131 | 2018-11-28T05:21:13.881829178Z 01:21:13 INFO:63.8%, step:2599, perplexity: 26.983, speed: 22172 words | |
| 132 | 2018-11-28T05:21:19.282779709Z 01:21:19 INFO:66.2%, step:2699, perplexity: 26.955, speed: 22034 words | |
| 133 | 2018-11-28T05:21:24.088115395Z 01:21:24 INFO:68.7%, step:2799, perplexity: 26.923, speed: 22007 words | |
| 134 | 2018-11-28T05:21:28.646995184Z 01:21:28 INFO:71.1%, step:2899, perplexity: 26.888, speed: 22023 words | |
| 135 | 2018-11-28T05:21:33.112124137Z 01:21:33 INFO:73.6%, step:2999, perplexity: 26.850, speed: 22052 words | |
| 136 | 2018-11-28T05:21:37.642299653Z 01:21:37 INFO:76.0%, step:3099, perplexity: 26.813, speed: 22069 words | |
| 137 | 2018-11-28T05:21:42.211277597Z 01:21:42 INFO:78.5%, step:3199, perplexity: 26.777, speed: 22080 words | |
| 138 | 2018-11-28T05:21:46.714947184Z 01:21:46 INFO:80.9%, step:3299, perplexity: 26.739, speed: 22099 words | |
| 139 | 2018-11-28T05:21:51.196736419Z 01:21:51 INFO:83.4%, step:3399, perplexity: 26.700, speed: 22120 words | |
| 140 | 2018-11-28T05:21:56.150497463Z 01:21:56 INFO:85.8%, step:3499, perplexity: 26.661, speed: 22076 words | |
| 141 | 2018-11-28T05:22:00.636990661Z 01:22:00 INFO:88.3%, step:3599, perplexity: 26.624, speed: 22096 words | |
| 142 | 2018-11-28T05:22:05.157937912Z 01:22:05 INFO:90.7%, step:3699, perplexity: 26.590, speed: 22111 words | |
| 143 | 2018-11-28T05:22:09.823902429Z 01:22:09 INFO:93.2%, step:3799, perplexity: 26.555, speed: 22106 words | |
| 144 | 2018-11-28T05:22:14.373324064Z 01:22:14 INFO:95.6%, step:3899, perplexity: 26.522, speed: 22117 words | |
| 145 | 2018-11-28T05:22:19.15143595Z 01:22:19 INFO:98.1%, step:3999, perplexity: 26.490, speed: 22099 words | |
| 146 | 2018-11-28T05:22:23.31504694Z 01:22:23 INFO:Perplexity: 25.254, speed: 22035 words per sec | |
| 147 | 2018-11-28T05:22:24.656984837Z 01:22:24 INFO:Latest model saved in ./results/output_poem/save_model/model-8156 | |
| 148 | 2018-11-28T05:22:24.657026417Z | |
| 149 | 2018-11-28T05:22:24.657041095Z 01:22:24 INFO:Evaluate on validation set | |
| 150 | 2018-11-28T05:22:28.268539683Z 01:22:28 INFO:41.7%, step:99, perplexity: 22.229, speed: 32722 words | |
| 151 | 2018-11-28T05:22:30.121644601Z 01:22:30 INFO:83.3%, step:199, perplexity: 22.053, speed: 41101 words | |
| 152 | 2018-11-28T05:22:30.872943005Z 01:22:30 INFO:Perplexity: 22.685, speed: 42857 words per sec | |
| 153 | 2018-11-28T05:22:31.994848413Z 01:22:31 INFO:Best model is saved in ./results/output_poem/best_model/model-8156 | |
| 154 | 2018-11-28T05:22:31.994899778Z 01:22:31 INFO:Best validation ppl is 22.685474 | |
| 155 | 2018-11-28T05:22:31.994908703Z | |
| 156 | 2018-11-28T05:22:32.000031225Z 01:22:31 INFO:=================== Epoch 2 =================== | |
| 157 | 2018-11-28T05:22:32.000063957Z | |
| 158 | 2018-11-28T05:22:32.000067847Z 01:22:31 INFO:Training on training set | |
| 159 | 2018-11-28T05:22:37.01322994Z 01:22:36 INFO:2.5%, step:99, perplexity: 25.183, speed: 20578 words | |
| 160 | 2018-11-28T05:22:42.345905405Z 01:22:42 INFO:4.9%, step:199, perplexity: 24.417, speed: 19797 words | |
| 161 | 2018-11-28T05:22:46.811464779Z 01:22:46 INFO:7.4%, step:299, perplexity: 24.233, speed: 20745 words | |
| 162 | 2018-11-28T05:22:51.339805925Z 01:22:51 INFO:9.8%, step:399, perplexity: 24.185, speed: 21183 words | |
| 163 | 2018-11-28T05:22:55.75755766Z 01:22:55 INFO:12.3%, step:499, perplexity: 24.163, speed: 21552 words | |
| 164 | 2018-11-28T05:23:00.223069181Z 01:23:00 INFO:14.7%, step:599, perplexity: 24.106, speed: 21771 words | |
| 165 | 2018-11-28T05:23:04.757544648Z 01:23:04 INFO:17.2%, step:699, perplexity: 23.988, speed: 21884 words | |
| 166 | 2018-11-28T05:23:09.657961466Z 01:23:09 INFO:19.6%, step:799, perplexity: 23.818, speed: 21755 words | |
| 167 | 2018-11-28T05:23:14.221313031Z 01:23:14 INFO:22.1%, step:899, perplexity: 23.671, speed: 21829 words | |
| 168 | 2018-11-28T05:23:18.816911047Z 01:23:18 INFO:24.5%, step:999, perplexity: 23.571, speed: 21873 words | |
| 169 | 2018-11-28T05:23:23.491947631Z 01:23:23 INFO:27.0%, step:1099, perplexity: 23.497, speed: 21876 words | |
| 170 | 2018-11-28T05:23:27.968567177Z 01:23:27 INFO:29.4%, step:1199, perplexity: 23.420, speed: 21956 words | |
| 171 | 2018-11-28T05:23:32.372116584Z 01:23:32 INFO:31.9%, step:1299, perplexity: 23.355, speed: 22051 words | |
| 172 | 2018-11-28T05:23:36.837942718Z 01:23:36 INFO:34.3%, step:1399, perplexity: 23.298, speed: 22111 words | |
| 173 | 2018-11-28T05:23:41.530493226Z 01:23:41 INFO:36.8%, step:1499, perplexity: 23.251, speed: 22092 words | |
| 174 | 2018-11-28T05:23:46.171053889Z 01:23:46 INFO:39.2%, step:1599, perplexity: 23.218, speed: 22090 words | |
| 175 | 2018-11-28T05:23:51.156219491Z 01:23:51 INFO:41.7%, step:1699, perplexity: 23.195, speed: 21995 words | |
| 176 | 2018-11-28T05:23:57.037946213Z 01:23:57 INFO:44.1%, step:1799, perplexity: 23.180, speed: 21676 words | |
| 177 | 2018-11-28T05:24:01.733920546Z 01:24:01 INFO:46.6%, step:1899, perplexity: 23.164, speed: 21682 words | |
| 178 | 2018-11-28T05:24:06.543857715Z 01:24:06 INFO:49.0%, step:1999, perplexity: 23.145, speed: 21662 words | |
| 179 | 2018-11-28T05:24:11.1022342Z 01:24:11 INFO:51.5%, step:2099, perplexity: 23.137, speed: 21699 words | |
| 180 | 2018-11-28T05:24:15.555619529Z 01:24:15 INFO:53.9%, step:2199, perplexity: 23.138, speed: 21755 words | |
| 181 | 2018-11-28T05:24:20.324677068Z 01:24:20 INFO:56.4%, step:2299, perplexity: 23.142, speed: 21742 words | |
| 182 | 2018-11-28T05:24:24.767991934Z 01:24:24 INFO:58.9%, step:2399, perplexity: 23.145, speed: 21794 words | |
| 183 | 2018-11-28T05:24:29.263320179Z 01:24:29 INFO:61.3%, step:2499, perplexity: 23.148, speed: 21831 words | |
| 184 | 2018-11-28T05:24:33.837418532Z 01:24:33 INFO:63.8%, step:2599, perplexity: 23.146, speed: 21852 words | |
| 185 | 2018-11-28T05:24:38.245001975Z 01:24:38 INFO:66.2%, step:2699, perplexity: 23.138, speed: 21901 words | |
| 186 | 2018-11-28T05:24:43.067659081Z 01:24:43 INFO:68.7%, step:2799, perplexity: 23.127, speed: 21877 words | |
| 187 | 2018-11-28T05:24:47.570327429Z 01:24:47 INFO:71.1%, step:2899, perplexity: 23.113, speed: 21905 words | |
| 188 | 2018-11-28T05:24:52.106962793Z 01:24:52 INFO:73.6%, step:2999, perplexity: 23.097, speed: 21926 words | |
| 189 | 2018-11-28T05:24:56.733937331Z 01:24:56 INFO:76.0%, step:3099, perplexity: 23.081, speed: 21933 words | |
| 190 | 2018-11-28T05:25:01.3840531Z 01:25:01 INFO:78.5%, step:3199, perplexity: 23.066, speed: 21936 words | |
| 191 | 2018-11-28T05:25:05.850832622Z 01:25:05 INFO:80.9%, step:3299, perplexity: 23.049, speed: 21965 words | |
| 192 | 2018-11-28T05:25:11.302970236Z 01:25:11 INFO:83.4%, step:3399, perplexity: 23.030, speed: 21855 words | |
| 193 | 2018-11-28T05:25:16.443282651Z 01:25:16 INFO:85.8%, step:3499, perplexity: 23.012, speed: 21795 words | |
| 194 | 2018-11-28T05:25:20.904907735Z 01:25:20 INFO:88.3%, step:3599, perplexity: 22.996, speed: 21825 words | |
| 195 | 2018-11-28T05:25:25.99788701Z 01:25:25 INFO:90.7%, step:3699, perplexity: 22.981, speed: 21775 words | |
| 196 | 2018-11-28T05:25:30.435583999Z 01:25:30 INFO:93.2%, step:3799, perplexity: 22.966, speed: 21807 words | |
| 197 | 2018-11-28T05:25:34.962901956Z 01:25:34 INFO:95.6%, step:3899, perplexity: 22.951, speed: 21827 words | |
| 198 | 2018-11-28T05:25:39.461122588Z 01:25:39 INFO:98.1%, step:3999, perplexity: 22.938, speed: 21850 words | |
| 199 | 2018-11-28T05:25:43.11879421Z 01:25:43 INFO:Perplexity: 22.465, speed: 21850 words per sec | |
| 200 | 2018-11-28T05:25:44.32372101Z 01:25:44 INFO:Latest model saved in ./results/output_poem/save_model/model-12234 | |
| 201 | 2018-11-28T05:25:44.323789691Z | |
| 202 | 2018-11-28T05:25:44.323799938Z 01:25:44 INFO:Evaluate on validation set | |
| 203 | 2018-11-28T05:25:47.654595159Z 01:25:47 INFO:41.7%, step:99, perplexity: 21.375, speed: 35303 words | |
| 204 | 2018-11-28T05:25:49.458907733Z 01:25:49 INFO:83.3%, step:199, perplexity: 21.171, speed: 43524 words | |
| 205 | 2018-11-28T05:25:50.267339Z 01:25:50 INFO:Perplexity: 21.812, speed: 44671 words per sec | |
| 206 | 2018-11-28T05:25:51.437721053Z 01:25:51 INFO:Best model is saved in ./results/output_poem/best_model/model-12234 | |
| 207 | 2018-11-28T05:25:51.437796304Z 01:25:51 INFO:Best validation ppl is 21.812408 | |
| 208 | 2018-11-28T05:25:51.437805478Z | |
| 209 | 2018-11-28T05:25:51.447694929Z 01:25:51 INFO:=================== Epoch 3 =================== | |
| 210 | 2018-11-28T05:25:51.447752254Z | |
| 211 | 2018-11-28T05:25:51.447770475Z 01:25:51 INFO:Training on training set | |
| 212 | 2018-11-28T05:25:56.163104052Z 01:25:56 INFO:2.5%, step:99, perplexity: 23.369, speed: 21685 words | |
| 213 | 2018-11-28T05:26:00.771960189Z 01:26:00 INFO:4.9%, step:199, perplexity: 22.668, speed: 21949 words | |
| 214 | 2018-11-28T05:26:05.264627514Z 01:26:05 INFO:7.4%, step:299, perplexity: 22.480, speed: 22224 words | |
| 215 | 2018-11-28T05:26:09.737915227Z 01:26:09 INFO:9.8%, step:399, perplexity: 22.428, speed: 22386 words | |
| 216 | 2018-11-28T05:26:14.149234808Z 01:26:14 INFO:12.3%, step:499, perplexity: 22.404, speed: 22547 words | |
| 217 | 2018-11-28T05:26:18.631519097Z 01:26:18 INFO:14.7%, step:599, perplexity: 22.353, speed: 22597 words | |
| 218 | 2018-11-28T05:26:23.328940848Z 01:26:23 INFO:17.2%, step:699, perplexity: 22.249, speed: 22479 words | |
| 219 | 2018-11-28T05:26:28.011841335Z 01:26:28 INFO:19.6%, step:799, perplexity: 22.095, speed: 22401 words | |
| 220 | 2018-11-28T05:26:34.160596502Z 01:26:34 INFO:22.1%, step:899, perplexity: 21.962, speed: 21573 words | |
| 221 | 2018-11-28T05:26:38.691265843Z 01:26:38 INFO:24.5%, step:999, perplexity: 21.873, speed: 21672 words | |
| 222 | 2018-11-28T05:26:43.331959385Z 01:26:43 INFO:27.0%, step:1099, perplexity: 21.809, speed: 21707 words | |
| 223 | 2018-11-28T05:26:47.838880244Z 01:26:47 INFO:29.4%, step:1199, perplexity: 21.746, speed: 21788 words | |
| 224 | 2018-11-28T05:26:52.309418041Z 01:26:52 INFO:31.9%, step:1299, perplexity: 21.696, speed: 21870 words | |
| 225 | 2018-11-28T05:26:56.978889617Z 01:26:56 INFO:34.3%, step:1399, perplexity: 21.653, speed: 21877 words | |
| 226 | 2018-11-28T05:27:01.786059562Z 01:27:01 INFO:36.8%, step:1499, perplexity: 21.619, speed: 21835 words | |
| 227 | 2018-11-28T05:27:06.341602616Z 01:27:06 INFO:39.2%, step:1599, perplexity: 21.597, speed: 21874 words | |
| 228 | 2018-11-28T05:27:10.622703722Z 01:27:10 INFO:41.7%, step:1699, perplexity: 21.583, speed: 21985 words | |
| 229 | 2018-11-28T05:27:15.287529709Z 01:27:15 INFO:44.1%, step:1799, perplexity: 21.576, speed: 21984 words | |
| 230 | 2018-11-28T05:27:19.604181296Z 01:27:19 INFO:46.6%, step:1899, perplexity: 21.568, speed: 22068 words | |
| 231 | 2018-11-28T05:27:24.268040948Z 01:27:24 INFO:49.0%, step:1999, perplexity: 21.557, speed: 22064 words | |
| 232 | 2018-11-28T05:27:28.700909801Z 01:27:28 INFO:51.5%, step:2099, perplexity: 21.555, speed: 22110 words | |
| 233 | 2018-11-28T05:27:33.128729474Z 01:27:33 INFO:53.9%, step:2199, perplexity: 21.561, speed: 22154 words | |
| 234 | 2018-11-28T05:27:37.712923749Z 01:27:37 INFO:56.4%, step:2299, perplexity: 21.571, speed: 22162 words | |
| 235 | 2018-11-28T05:27:43.721394242Z 01:27:43 INFO:58.9%, step:2399, perplexity: 21.578, speed: 21888 words | |
| 236 | 2018-11-28T05:27:48.394192028Z 01:27:48 INFO:61.3%, step:2499, perplexity: 21.587, speed: 21889 words | |
| 237 | 2018-11-28T05:27:52.671101475Z 01:27:52 INFO:63.8%, step:2599, perplexity: 21.589, speed: 21961 words | |
| 238 | 2018-11-28T05:27:57.456770277Z 01:27:57 INFO:66.2%, step:2699, perplexity: 21.587, speed: 21940 words | |
| 239 | 2018-11-28T05:28:01.925033947Z 01:28:01 INFO:68.7%, step:2799, perplexity: 21.581, speed: 21974 words | |
| 240 | 2018-11-28T05:28:06.179046633Z 01:28:06 INFO:71.1%, step:2899, perplexity: 21.573, speed: 22040 words | |
| 241 | 2018-11-28T05:28:11.579914925Z 01:28:11 INFO:73.6%, step:2999, perplexity: 21.562, speed: 21921 words | |
| 242 | 2018-11-28T05:28:16.073542252Z 01:28:16 INFO:76.0%, step:3099, perplexity: 21.552, speed: 21948 words | |
| 243 | 2018-11-28T05:28:20.491060097Z 01:28:20 INFO:78.5%, step:3199, perplexity: 21.543, speed: 21985 words | |
| 244 | 2018-11-28T05:28:25.036105231Z 01:28:25 INFO:80.9%, step:3299, perplexity: 21.532, speed: 22001 words | |
| 245 | 2018-11-28T05:28:29.718805904Z 01:28:29 INFO:83.4%, step:3399, perplexity: 21.519, speed: 21997 words | |
| 246 | 2018-11-28T05:28:34.330994683Z 01:28:34 INFO:85.8%, step:3499, perplexity: 21.506, speed: 22003 words | |
| 247 | 2018-11-28T05:28:38.797031311Z 01:28:38 INFO:88.3%, step:3599, perplexity: 21.495, speed: 22027 words | |
| 248 | 2018-11-28T05:28:43.538902828Z 01:28:43 INFO:90.7%, step:3699, perplexity: 21.486, speed: 22015 words | |
| 249 | 2018-11-28T05:28:47.968136508Z 01:28:47 INFO:93.2%, step:3799, perplexity: 21.476, speed: 22044 words | |
| 250 | 2018-11-28T05:28:53.702905292Z 01:28:53 INFO:95.6%, step:3899, perplexity: 21.467, speed: 21911 words | |
| 251 | 2018-11-28T05:28:58.871337987Z 01:28:58 INFO:98.1%, step:3999, perplexity: 21.459, speed: 21854 words | |
| 252 | 2018-11-28T05:29:02.574677671Z 01:29:02 INFO:Perplexity: 21.200, speed: 21848 words per sec | |
| 253 | 2018-11-28T05:29:03.895102225Z 01:29:03 INFO:Latest model saved in ./results/output_poem/save_model/model-16312 | |
| 254 | 2018-11-28T05:29:03.895152581Z | |
| 255 | 2018-11-28T05:29:03.89515908Z 01:29:03 INFO:Evaluate on validation set | |
| 256 | 2018-11-28T05:29:07.220458535Z 01:29:07 INFO:41.7%, step:99, perplexity: 21.318, speed: 35732 words | |
| 257 | 2018-11-28T05:29:09.166104371Z 01:29:09 INFO:83.3%, step:199, perplexity: 21.100, speed: 42574 words | |
| 258 | 2018-11-28T05:29:09.890979Z 01:29:09 INFO:Perplexity: 21.767, speed: 44400 words per sec | |
| 259 | 2018-11-28T05:29:11.103512427Z 01:29:11 INFO:Best model is saved in ./results/output_poem/best_model/model-16312 | |
| 260 | 2018-11-28T05:29:11.103560344Z 01:29:11 INFO:Best validation ppl is 21.767382 | |
| 261 | 2018-11-28T05:29:11.103568424Z | |
| 262 | 2018-11-28T05:29:11.106738122Z 01:29:11 INFO:=================== Epoch 4 =================== | |
| 263 | 2018-11-28T05:29:11.106777023Z | |
| 264 | 2018-11-28T05:29:11.106784816Z 01:29:11 INFO:Training on training set | |
| 265 | 2018-11-28T05:29:15.646892058Z 01:29:15 INFO:2.5%, step:99, perplexity: 22.391, speed: 22570 words | |
| 266 | 2018-11-28T05:29:20.135912922Z 01:29:20 INFO:4.9%, step:199, perplexity: 21.689, speed: 22683 words | |
| 267 | 2018-11-28T05:29:24.884797113Z 01:29:24 INFO:7.4%, step:299, perplexity: 21.493, speed: 22303 words | |
| 268 | 2018-11-28T05:29:29.413523238Z 01:29:29 INFO:9.8%, step:399, perplexity: 21.438, speed: 22376 words | |
| 269 | 2018-11-28T05:29:34.206511815Z 01:29:34 INFO:12.3%, step:499, perplexity: 21.414, speed: 22168 words | |
| 270 | 2018-11-28T05:29:38.726895264Z 01:29:38 INFO:14.7%, step:599, perplexity: 21.365, speed: 22247 words | |
| 271 | 2018-11-28T05:29:43.21747835Z 01:29:43 INFO:17.2%, step:699, perplexity: 21.265, speed: 22324 words | |
| 272 | 2018-11-28T05:29:47.892395135Z 01:29:47 INFO:19.6%, step:799, perplexity: 21.121, speed: 22270 words | |
| 273 | 2018-11-28T05:29:52.372947152Z 01:29:52 INFO:22.1%, step:899, perplexity: 20.998, speed: 22334 words | |
| 274 | 2018-11-28T05:29:57.16613738Z 01:29:57 INFO:24.5%, step:999, perplexity: 20.916, speed: 22233 words | |
| 275 | 2018-11-28T05:30:02.127932718Z 01:30:02 INFO:27.0%, step:1099, perplexity: 20.858, speed: 22077 words | |
| 276 | 2018-11-28T05:30:07.958247984Z 01:30:07 INFO:29.4%, step:1199, perplexity: 20.797, speed: 21616 words | |
| 277 | 2018-11-28T05:30:12.738110322Z 01:30:12 INFO:31.9%, step:1299, perplexity: 20.746, speed: 21603 words | |
| 278 | 2018-11-28T05:30:17.363699049Z 01:30:17 INFO:34.3%, step:1399, perplexity: 20.704, speed: 21638 words | |
| 279 | 2018-11-28T05:30:22.224928344Z 01:30:22 INFO:36.8%, step:1499, perplexity: 20.669, speed: 21598 words | |
| 280 | 2018-11-28T05:30:26.785686985Z 01:30:26 INFO:39.2%, step:1599, perplexity: 20.647, speed: 21650 words | |
| 281 | 2018-11-28T05:30:31.437942052Z 01:30:31 INFO:41.7%, step:1699, perplexity: 20.635, speed: 21671 words | |
| 282 | 2018-11-28T05:30:36.095579361Z 01:30:36 INFO:44.1%, step:1799, perplexity: 20.629, speed: 21688 words | |
| 283 | 2018-11-28T05:30:40.825893933Z 01:30:40 INFO:46.6%, step:1899, perplexity: 20.623, speed: 21686 words | |
| 284 | 2018-11-28T05:30:45.456927806Z 01:30:45 INFO:49.0%, step:1999, perplexity: 20.614, speed: 21707 words | |
| 285 | 2018-11-28T05:30:50.003129325Z 01:30:49 INFO:51.5%, step:2099, perplexity: 20.614, speed: 21745 words | |
| 286 | 2018-11-28T05:30:54.843951861Z 01:30:54 INFO:53.9%, step:2199, perplexity: 20.623, speed: 21717 words | |
| 287 | 2018-11-28T05:30:59.415949138Z 01:30:59 INFO:56.4%, step:2299, perplexity: 20.634, speed: 21746 words | |
| 288 | 2018-11-28T05:31:04.189452019Z 01:31:04 INFO:58.9%, step:2399, perplexity: 20.644, speed: 21733 words | |
| 289 | 2018-11-28T05:31:08.846121082Z 01:31:08 INFO:61.3%, step:2499, perplexity: 20.654, speed: 21743 words | |
| 290 | 2018-11-28T05:31:14.405105746Z 01:31:14 INFO:63.8%, step:2599, perplexity: 20.659, speed: 21593 words | |
| 291 | 2018-11-28T05:31:19.732932121Z 01:31:19 INFO:66.2%, step:2699, perplexity: 20.659, speed: 21495 words | |
| 292 | 2018-11-28T05:31:24.400819388Z 01:31:24 INFO:68.7%, step:2799, perplexity: 20.656, speed: 21511 words | |
| 293 | 2018-11-28T05:31:29.124944257Z 01:31:29 INFO:71.1%, step:2899, perplexity: 20.651, speed: 21516 words | |
| 294 | 2018-11-28T05:31:33.809891023Z 01:31:33 INFO:73.6%, step:2999, perplexity: 20.643, speed: 21528 words | |
| 295 | 2018-11-28T05:31:38.513603607Z 01:31:38 INFO:76.0%, step:3099, perplexity: 20.636, speed: 21535 words | |
| 296 | 2018-11-28T05:31:43.406432591Z 01:31:43 INFO:78.5%, step:3199, perplexity: 20.629, speed: 21516 words | |
| 297 | 2018-11-28T05:31:48.119764689Z 01:31:48 INFO:80.9%, step:3299, perplexity: 20.621, speed: 21522 words | |
| 298 | 2018-11-28T05:31:52.634808232Z 01:31:52 INFO:83.4%, step:3399, perplexity: 20.611, speed: 21554 words | |
| 299 | 2018-11-28T05:31:57.196914874Z 01:31:57 INFO:85.8%, step:3499, perplexity: 20.601, speed: 21579 words | |
| 300 | 2018-11-28T05:32:02.05182765Z 01:32:02 INFO:88.3%, step:3599, perplexity: 20.593, speed: 21565 words | |
| 301 | 2018-11-28T05:32:06.629024079Z 01:32:06 INFO:90.7%, step:3699, perplexity: 20.586, speed: 21586 words | |
| 302 | 2018-11-28T05:32:11.305463028Z 01:32:11 INFO:93.2%, step:3799, perplexity: 20.578, speed: 21594 words | |
| 303 | 2018-11-28T05:32:15.893979365Z 01:32:15 INFO:95.6%, step:3899, perplexity: 20.572, speed: 21612 words | |
| 304 | 2018-11-28T05:32:20.332299705Z 01:32:20 INFO:98.1%, step:3999, perplexity: 20.566, speed: 21646 words | |
| 305 | 2018-11-28T05:32:24.438717422Z 01:32:24 INFO:Perplexity: 20.395, speed: 21600 words per sec | |
| 306 | 2018-11-28T05:32:25.971929217Z 01:32:25 INFO:Latest model saved in ./results/output_poem/save_model/model-20390 | |
| 307 | 2018-11-28T05:32:25.971965872Z | |
| 308 | 2018-11-28T05:32:25.97197032Z 01:32:25 INFO:Evaluate on validation set | |
| 309 | 2018-11-28T05:32:29.830859497Z 01:32:29 INFO:41.7%, step:99, perplexity: 21.123, speed: 29554 words | |
| 310 | 2018-11-28T05:32:31.739870111Z 01:32:31 INFO:83.3%, step:199, perplexity: 20.887, speed: 38112 words | |
| 311 | 2018-11-28T05:32:32.424183727Z 01:32:32 INFO:Perplexity: 21.564, speed: 40562 words per sec | |
| 312 | 2018-11-28T05:32:33.689601041Z 01:32:33 INFO:Best model is saved in ./results/output_poem/best_model/model-20390 | |
| 313 | 2018-11-28T05:32:33.689635576Z 01:32:33 INFO:Best validation ppl is 21.564394 | |
| 314 | 2018-11-28T05:32:33.689639886Z | |
| 315 | 2018-11-28T05:32:33.697379503Z 01:32:33 INFO:=================== Epoch 5 =================== | |
| 316 | 2018-11-28T05:32:33.69739793Z | |
| 317 | 2018-11-28T05:32:33.697405143Z 01:32:33 INFO:Training on training set | |
| 318 | 2018-11-28T05:32:37.985787519Z 01:32:37 INFO:2.5%, step:99, perplexity: 21.658, speed: 23874 words | |
| 319 | 2018-11-28T05:32:42.621622433Z 01:32:42 INFO:4.9%, step:199, perplexity: 20.995, speed: 22947 words | |
| 320 | 2018-11-28T05:32:47.172662816Z 01:32:47 INFO:7.4%, step:299, perplexity: 20.824, speed: 22796 words | |
| 321 | 2018-11-28T05:32:51.600789975Z 01:32:51 INFO:9.8%, step:399, perplexity: 20.790, speed: 22880 words | |
| 322 | 2018-11-28T05:32:56.243515346Z 01:32:56 INFO:12.3%, step:499, perplexity: 20.779, speed: 22709 words | |
| 323 | 2018-11-28T05:33:00.502435021Z 01:33:00 INFO:14.7%, step:599, perplexity: 20.741, speed: 22920 words | |
| 324 | 2018-11-28T05:33:05.238875614Z 01:33:05 INFO:17.2%, step:699, perplexity: 20.652, speed: 22725 words | |
| 325 | 2018-11-28T05:33:10.197519026Z 01:33:10 INFO:19.6%, step:799, perplexity: 20.519, speed: 22443 words | |
| 326 | 2018-11-28T05:33:14.916545325Z 01:33:14 INFO:22.1%, step:899, perplexity: 20.402, speed: 22358 words | |
| 327 | 2018-11-28T05:33:19.135109223Z 01:33:19 INFO:24.5%, step:999, perplexity: 20.324, speed: 22537 words | |
| 328 | 2018-11-28T05:33:23.791288483Z 01:33:23 INFO:27.0%, step:1099, perplexity: 20.269, speed: 22486 words | |
| 329 | 2018-11-28T05:33:28.16992119Z 01:33:28 INFO:29.4%, step:1199, perplexity: 20.210, speed: 22558 words | |
| 330 | 2018-11-28T05:33:32.519097677Z 01:33:32 INFO:31.9%, step:1299, perplexity: 20.162, speed: 22631 words | |
| 331 | 2018-11-28T05:33:38.263877109Z 01:33:38 INFO:34.3%, step:1399, perplexity: 20.121, speed: 22203 words | |
| 332 | 2018-11-28T05:33:43.08828305Z 01:33:43 INFO:36.8%, step:1499, perplexity: 20.088, speed: 22135 words | |
| 333 | 2018-11-28T05:33:47.553131028Z 01:33:47 INFO:39.2%, step:1599, perplexity: 20.067, speed: 22184 words | |
| 334 | 2018-11-28T05:33:51.986490924Z 01:33:51 INFO:41.7%, step:1699, perplexity: 20.055, speed: 22236 words | |
| 335 | 2018-11-28T05:33:57.332936597Z 01:33:57 INFO:44.1%, step:1799, perplexity: 20.050, speed: 22038 words | |
| 336 | 2018-11-28T05:34:01.8038692Z 01:34:01 INFO:46.6%, step:1899, perplexity: 20.044, speed: 22082 words | |
| 337 | 2018-11-28T05:34:07.354678573Z 01:34:07 INFO:49.0%, step:1999, perplexity: 20.036, speed: 21867 words | |
| 338 | 2018-11-28T05:34:13.649983064Z 01:34:13 INFO:51.5%, step:2099, perplexity: 20.037, speed: 21514 words | |
| 339 | 2018-11-28T05:34:18.732371673Z 01:34:18 INFO:53.9%, step:2199, perplexity: 20.046, speed: 21448 words | |
| 340 | 2018-11-28T05:34:24.546469012Z 01:34:24 INFO:56.4%, step:2299, perplexity: 20.057, speed: 21247 words | |
| 341 | 2018-11-28T05:34:30.177949572Z 01:34:30 INFO:58.9%, step:2399, perplexity: 20.067, speed: 21099 words | |
| 342 | 2018-11-28T05:34:35.152075087Z 01:34:35 INFO:61.3%, step:2499, perplexity: 20.077, speed: 21078 words | |
| 343 | 2018-11-28T05:34:39.761042024Z 01:34:39 INFO:63.8%, step:2599, perplexity: 20.083, speed: 21119 words | |
| 344 | 2018-11-28T05:34:44.479961469Z 01:34:44 INFO:66.2%, step:2699, perplexity: 20.084, speed: 21140 words | |
| 345 | 2018-11-28T05:34:50.139953305Z 01:34:50 INFO:68.7%, step:2799, perplexity: 20.082, speed: 21015 words | |
| 346 | 2018-11-28T05:34:55.311821519Z 01:34:55 INFO:71.1%, step:2899, perplexity: 20.078, speed: 20970 words | |
| 347 | 2018-11-28T05:35:00.411536286Z 01:35:00 INFO:73.6%, step:2999, perplexity: 20.071, speed: 20939 words | |
| 348 | 2018-11-28T05:35:05.860039108Z 01:35:05 INFO:76.0%, step:3099, perplexity: 20.065, speed: 20862 words | |
| 349 | 2018-11-28T05:35:10.528738136Z 01:35:10 INFO:78.5%, step:3199, perplexity: 20.059, speed: 20894 words | |
| 350 | 2018-11-28T05:35:15.445247176Z 01:35:15 INFO:80.9%, step:3299, perplexity: 20.052, speed: 20892 words | |
| 351 | 2018-11-28T05:35:20.020995604Z 01:35:20 INFO:83.4%, step:3399, perplexity: 20.043, speed: 20933 words | |
| 352 | 2018-11-28T05:35:24.680512371Z 01:35:24 INFO:85.8%, step:3499, perplexity: 20.034, speed: 20961 words | |
| 353 | 2018-11-28T05:35:29.256086196Z 01:35:29 INFO:88.3%, step:3599, perplexity: 20.028, speed: 20998 words | |
| 354 | 2018-11-28T05:35:33.786591338Z 01:35:33 INFO:90.7%, step:3699, perplexity: 20.023, speed: 21039 words | |
| 355 | 2018-11-28T05:35:38.648986985Z 01:35:38 INFO:93.2%, step:3799, perplexity: 20.018, speed: 21039 words | |
| 356 | 2018-11-28T05:35:44.669905213Z 01:35:44 INFO:95.6%, step:3899, perplexity: 20.013, speed: 20912 words | |
| 357 | 2018-11-28T05:35:50.732917319Z 01:35:50 INFO:98.1%, step:3999, perplexity: 20.009, speed: 20788 words | |
| 358 | 2018-11-28T05:35:54.785723813Z 01:35:54 INFO:Perplexity: 19.914, speed: 20766 words per sec | |
| 359 | 2018-11-28T05:35:56.479199974Z 01:35:56 INFO:Latest model saved in ./results/output_poem/save_model/model-24468 | |
| 360 | 2018-11-28T05:35:56.479246472Z | |
| 361 | 2018-11-28T05:35:56.479253591Z 01:35:56 INFO:Evaluate on validation set | |
| 362 | 2018-11-28T05:36:00.140294992Z 01:36:00 INFO:41.7%, step:99, perplexity: 21.246, speed: 32185 words | |
| 363 | 2018-11-28T05:36:02.062325051Z 01:36:02 INFO:83.3%, step:199, perplexity: 21.008, speed: 40125 words | |
| 364 | 2018-11-28T05:36:02.811858743Z 01:36:02 INFO:Perplexity: 21.731, speed: 41982 words per sec | |
| 365 | 2018-11-28T05:36:02.811906452Z 01:36:02 INFO:Decay the learning rate: 0.0025 | |
| 366 | 2018-11-28T05:36:02.815668365Z 01:36:02 INFO:Best model is saved in ./results/output_poem/best_model/model-20390 | |
| 367 | 2018-11-28T05:36:02.815690115Z 01:36:02 INFO:Best validation ppl is 21.564394 | |
| 368 | 2018-11-28T05:36:02.815696861Z | |
| 369 | 2018-11-28T05:36:02.822729485Z 01:36:02 INFO:=================== Epoch 6 =================== | |
| 370 | 2018-11-28T05:36:02.822800258Z | |
| 371 | 2018-11-28T05:36:02.822806952Z 01:36:02 INFO:Training on training set | |
| 372 | 2018-11-28T05:36:08.616841996Z 01:36:08 INFO:2.5%, step:99, perplexity: 21.108, speed: 17683 words | |
| 373 | 2018-11-28T05:36:14.432072783Z 01:36:14 INFO:4.9%, step:199, perplexity: 20.455, speed: 17646 words | |
| 374 | 2018-11-28T05:36:19.02813022Z 01:36:19 INFO:7.4%, step:299, perplexity: 20.259, speed: 18969 words | |
| 375 | 2018-11-28T05:36:23.749238827Z 01:36:23 INFO:9.8%, step:399, perplexity: 20.197, speed: 19576 words | |
| 376 | 2018-11-28T05:36:28.212223508Z 01:36:28 INFO:12.3%, step:499, perplexity: 20.163, speed: 20167 words | |
| 377 | 2018-11-28T05:36:32.820523302Z 01:36:32 INFO:14.7%, step:599, perplexity: 20.107, speed: 20484 words | |
| 378 | 2018-11-28T05:36:37.291275716Z 01:36:37 INFO:17.2%, step:699, perplexity: 20.005, speed: 20797 words | |
| 379 | 2018-11-28T05:36:42.189822946Z 01:36:42 INFO:19.6%, step:799, perplexity: 19.862, speed: 20810 words | |
| 380 | 2018-11-28T05:36:46.790198097Z 01:36:46 INFO:22.1%, step:899, perplexity: 19.738, speed: 20962 words | |
| 381 | 2018-11-28T05:36:51.508890275Z 01:36:51 INFO:24.5%, step:999, perplexity: 19.652, speed: 21035 words | |
| 382 | 2018-11-28T05:36:56.684992781Z 01:36:56 INFO:27.0%, step:1099, perplexity: 19.587, speed: 20915 words | |
| 383 | 2018-11-28T05:37:01.584967426Z 01:37:01 INFO:29.4%, step:1199, perplexity: 19.521, speed: 20912 words | |
| 384 | 2018-11-28T05:37:06.282888328Z 01:37:06 INFO:31.9%, step:1299, perplexity: 19.464, speed: 20978 words | |
| 385 | 2018-11-28T05:37:10.907942308Z 01:37:10 INFO:34.3%, step:1399, perplexity: 19.416, speed: 21057 words | |
| 386 | 2018-11-28T05:37:16.317963599Z 01:37:16 INFO:36.8%, step:1499, perplexity: 19.375, speed: 20900 words | |
| 387 | 2018-11-28T05:37:23.358881306Z 01:37:23 INFO:39.2%, step:1599, perplexity: 19.346, speed: 20347 words | |
| 388 | 2018-11-28T05:37:29.990216183Z 01:37:29 INFO:41.7%, step:1699, perplexity: 19.325, speed: 19971 words | |
| 389 | 2018-11-28T05:37:36.187724632Z 01:37:36 INFO:44.1%, step:1799, perplexity: 19.311, speed: 19742 words | |
| 390 | 2018-11-28T05:37:40.68887598Z 01:37:40 INFO:46.6%, step:1899, perplexity: 19.296, speed: 19881 words | |
| 391 | 2018-11-28T05:37:45.673687131Z 01:37:45 INFO:49.0%, step:1999, perplexity: 19.280, speed: 19913 words | |
| 392 | 2018-11-28T05:37:50.439706829Z 01:37:50 INFO:51.5%, step:2099, perplexity: 19.272, speed: 19983 words | |
| 393 | 2018-11-28T05:37:55.224934462Z 01:37:55 INFO:53.9%, step:2199, perplexity: 19.272, speed: 20043 words | |
| 394 | 2018-11-28T05:37:59.886703294Z 01:37:59 INFO:56.4%, step:2299, perplexity: 19.274, speed: 20119 words | |
| 395 | 2018-11-28T05:38:04.457938224Z 01:38:04 INFO:58.9%, step:2399, perplexity: 19.275, speed: 20205 words | |
| 396 | 2018-11-28T05:38:09.552948444Z 01:38:09 INFO:61.3%, step:2499, perplexity: 19.277, speed: 20200 words | |
| 397 | 2018-11-28T05:38:14.408207141Z 01:38:14 INFO:63.8%, step:2599, perplexity: 19.275, speed: 20233 words | |
| 398 | 2018-11-28T05:38:19.083949352Z 01:38:19 INFO:66.2%, step:2699, perplexity: 19.269, speed: 20292 words | |
| 399 | 2018-11-28T05:38:23.726901821Z 01:38:23 INFO:68.7%, step:2799, perplexity: 19.260, speed: 20349 words | |
| 400 | 2018-11-28T05:38:28.409923335Z 01:38:28 INFO:71.1%, step:2899, perplexity: 19.250, speed: 20398 words | |
| 401 | 2018-11-28T05:38:33.303088534Z 01:38:33 INFO:73.6%, step:2999, perplexity: 19.236, speed: 20415 words | |
| 402 | 2018-11-28T05:38:37.825371941Z 01:38:37 INFO:76.0%, step:3099, perplexity: 19.224, speed: 20480 words | |
| 403 | 2018-11-28T05:38:43.614140756Z 01:38:43 INFO:78.5%, step:3199, perplexity: 19.213, speed: 20380 words | |
| 404 | 2018-11-28T05:38:49.909926786Z 01:38:49 INFO:80.9%, step:3299, perplexity: 19.200, speed: 20224 words | |
| 405 | 2018-11-28T05:38:56.832879889Z 01:38:56 INFO:83.4%, step:3399, perplexity: 19.185, speed: 20009 words | |
| 406 | 2018-11-28T05:39:01.8463188Z 01:39:01 INFO:85.8%, step:3499, perplexity: 19.171, speed: 20020 words | |
| 407 | 2018-11-28T05:39:06.62809651Z 01:39:06 INFO:88.3%, step:3599, perplexity: 19.159, speed: 20056 words | |
| 408 | 2018-11-28T05:39:11.254979442Z 01:39:11 INFO:90.7%, step:3699, perplexity: 19.148, speed: 20107 words | |
| 409 | 2018-11-28T05:39:16.042919407Z 01:39:16 INFO:93.2%, step:3799, perplexity: 19.136, speed: 20139 words | |
| 410 | 2018-11-28T05:39:20.729901429Z 01:39:20 INFO:95.6%, step:3899, perplexity: 19.126, speed: 20180 words | |
| 411 | 2018-11-28T05:39:25.524913007Z 01:39:25 INFO:98.1%, step:3999, perplexity: 19.116, speed: 20207 words | |
| 412 | 2018-11-28T05:39:29.19798531Z 01:39:29 INFO:Perplexity: 18.780, speed: 20235 words per sec | |
| 413 | 2018-11-28T05:39:30.488379394Z 01:39:30 INFO:Latest model saved in ./results/output_poem/save_model/model-28546 | |
| 414 | 2018-11-28T05:39:30.488427349Z | |
| 415 | 2018-11-28T05:39:30.490843202Z 01:39:30 INFO:Evaluate on validation set | |
| 416 | 2018-11-28T05:39:33.853919169Z 01:39:33 INFO:41.7%, step:99, perplexity: 20.626, speed: 35171 words | |
| 417 | 2018-11-28T05:39:35.859929388Z 01:39:35 INFO:83.3%, step:199, perplexity: 20.405, speed: 41713 words | |
| 418 | 2018-11-28T05:39:36.65815797Z 01:39:36 INFO:Perplexity: 21.129, speed: 43047 words per sec | |
| 419 | 2018-11-28T05:39:37.924819697Z 01:39:37 INFO:Best model is saved in ./results/output_poem/best_model/model-28546 | |
| 420 | 2018-11-28T05:39:37.924862968Z 01:39:37 INFO:Best validation ppl is 21.129450 | |
| 421 | 2018-11-28T05:39:37.924869814Z | |
| 422 | 2018-11-28T05:39:37.926842322Z 01:39:37 INFO:=================== Epoch 7 =================== | |
| 423 | 2018-11-28T05:39:37.926857488Z | |
| 424 | 2018-11-28T05:39:37.926861075Z 01:39:37 INFO:Training on training set | |
| 425 | 2018-11-28T05:39:42.545986885Z 01:39:42 INFO:2.5%, step:99, perplexity: 20.231, speed: 22175 words | |
| 426 | 2018-11-28T05:39:47.219042928Z 01:39:47 INFO:4.9%, step:199, perplexity: 19.613, speed: 22050 words | |
| 427 | 2018-11-28T05:39:51.713899439Z 01:39:51 INFO:7.4%, step:299, perplexity: 19.427, speed: 22285 words | |
| 428 | 2018-11-28T05:39:56.4159164Z 01:39:56 INFO:9.8%, step:399, perplexity: 19.375, speed: 22157 words | |
| 429 | 2018-11-28T05:40:01.056732197Z 01:40:01 INFO:12.3%, step:499, perplexity: 19.349, speed: 22146 words | |
| 430 | 2018-11-28T05:40:06.150904102Z 01:40:06 INFO:14.7%, step:599, perplexity: 19.303, speed: 21769 words | |
| 431 | 2018-11-28T05:40:12.303871251Z 01:40:12 INFO:17.2%, step:699, perplexity: 19.213, speed: 20853 words | |
| 432 | 2018-11-28T05:40:17.468218711Z 01:40:17 INFO:19.6%, step:799, perplexity: 19.084, speed: 20719 words | |
| 433 | 2018-11-28T05:40:22.685622701Z 01:40:22 INFO:22.1%, step:899, perplexity: 18.972, speed: 20591 words | |
| 434 | 2018-11-28T05:40:27.995858362Z 01:40:27 INFO:24.5%, step:999, perplexity: 18.896, speed: 20453 words | |
| 435 | 2018-11-28T05:40:33.625996882Z 01:40:33 INFO:27.0%, step:1099, perplexity: 18.841, speed: 20223 words | |
| 436 | 2018-11-28T05:40:38.237403159Z 01:40:38 INFO:29.4%, step:1199, perplexity: 18.784, speed: 20375 words | |
| 437 | 2018-11-28T05:40:43.130527899Z 01:40:43 INFO:31.9%, step:1299, perplexity: 18.736, speed: 20417 words | |
| 438 | 2018-11-28T05:40:47.688159913Z 01:40:47 INFO:34.3%, step:1399, perplexity: 18.696, speed: 20551 words | |
| 439 | 2018-11-28T05:40:52.295745027Z 01:40:52 INFO:36.8%, step:1499, perplexity: 18.663, speed: 20655 words | |
| 440 | 2018-11-28T05:40:57.021959616Z 01:40:57 INFO:39.2%, step:1599, perplexity: 18.640, speed: 20717 words | |
| 441 | 2018-11-28T05:41:01.533930833Z 01:41:01 INFO:41.7%, step:1699, perplexity: 18.626, speed: 20822 words | |
| 442 | 2018-11-28T05:41:06.280907452Z 01:41:06 INFO:44.1%, step:1799, perplexity: 18.618, speed: 20862 words | |
| 443 | 2018-11-28T05:41:11.105830858Z 01:41:11 INFO:46.6%, step:1899, perplexity: 18.609, speed: 20881 words | |
| 444 | 2018-11-28T05:41:16.045993013Z 01:41:16 INFO:49.0%, step:1999, perplexity: 18.599, speed: 20873 words | |
| 445 | 2018-11-28T05:41:20.645861233Z 01:41:20 INFO:51.5%, step:2099, perplexity: 18.596, speed: 20935 words | |
| 446 | 2018-11-28T05:41:25.464343549Z 01:41:25 INFO:53.9%, step:2199, perplexity: 18.601, speed: 20950 words | |
| 447 | 2018-11-28T05:41:30.177886259Z 01:41:30 INFO:56.4%, step:2299, perplexity: 18.608, speed: 20982 words | |
| 448 | 2018-11-28T05:41:34.694890957Z 01:41:34 INFO:58.9%, step:2399, perplexity: 18.614, speed: 21047 words | |
| 449 | 2018-11-28T05:41:39.471856164Z 01:41:39 INFO:61.3%, step:2499, perplexity: 18.621, speed: 21063 words | |
| 450 | 2018-11-28T05:41:44.007353311Z 01:41:44 INFO:63.8%, step:2599, perplexity: 18.624, speed: 21117 words | |
| 451 | 2018-11-28T05:41:48.659023016Z 01:41:48 INFO:66.2%, step:2699, perplexity: 18.622, speed: 21149 words | |
| 452 | 2018-11-28T05:41:53.105647699Z 01:41:53 INFO:68.7%, step:2799, perplexity: 18.619, speed: 21211 words | |
| 453 | 2018-11-28T05:41:58.023522994Z 01:41:58 INFO:71.1%, step:2899, perplexity: 18.613, speed: 21197 words | |
| 454 | 2018-11-28T05:42:02.908491615Z 01:42:02 INFO:73.6%, step:2999, perplexity: 18.605, speed: 21189 words | |
| 455 | 2018-11-28T05:42:07.454416166Z 01:42:07 INFO:76.0%, step:3099, perplexity: 18.598, speed: 21230 words | |
| 456 | 2018-11-28T05:42:12.203017455Z 01:42:12 INFO:78.5%, step:3199, perplexity: 18.591, speed: 21240 words | |
| 457 | 2018-11-28T05:42:16.695884994Z 01:42:16 INFO:80.9%, step:3299, perplexity: 18.582, speed: 21284 words | |
| 458 | 2018-11-28T05:42:21.331409954Z 01:42:21 INFO:83.4%, step:3399, perplexity: 18.573, speed: 21307 words | |
| 459 | 2018-11-28T05:42:26.091776613Z 01:42:26 INFO:85.8%, step:3499, perplexity: 18.563, speed: 21313 words | |
| 460 | 2018-11-28T05:42:30.853723817Z 01:42:30 INFO:88.3%, step:3599, perplexity: 18.555, speed: 21318 words | |
| 461 | 2018-11-28T05:42:35.424723952Z 01:42:35 INFO:90.7%, step:3699, perplexity: 18.548, speed: 21346 words | |
| 462 | 2018-11-28T05:42:39.898239337Z 01:42:39 INFO:93.2%, step:3799, perplexity: 18.541, speed: 21384 words | |
| 463 | 2018-11-28T05:42:44.620485928Z 01:42:44 INFO:95.6%, step:3899, perplexity: 18.534, speed: 21392 words | |
| 464 | 2018-11-28T05:42:49.182772303Z 01:42:49 INFO:98.1%, step:3999, perplexity: 18.529, speed: 21417 words | |
| 465 | 2018-11-28T05:42:52.808659933Z 01:42:52 INFO:Perplexity: 18.360, speed: 21428 words per sec | |
| 466 | 2018-11-28T05:42:53.997961882Z 01:42:53 INFO:Latest model saved in ./results/output_poem/save_model/model-32624 | |
| 467 | 2018-11-28T05:42:53.998007082Z | |
| 468 | 2018-11-28T05:42:53.998014156Z 01:42:53 INFO:Evaluate on validation set | |
| 469 | 2018-11-28T05:42:57.319734688Z 01:42:57 INFO:41.7%, step:99, perplexity: 20.762, speed: 34947 words | |
| 470 | 2018-11-28T05:42:59.213694127Z 01:42:59 INFO:83.3%, step:199, perplexity: 20.515, speed: 42447 words | |
| 471 | 2018-11-28T05:42:59.979589189Z 01:42:59 INFO:Perplexity: 21.241, speed: 43960 words per sec | |
| 472 | 2018-11-28T05:42:59.979632036Z 01:42:59 INFO:Decay the learning rate: 0.00125 | |
| 473 | 2018-11-28T05:42:59.980369065Z 01:42:59 INFO:Best model is saved in ./results/output_poem/best_model/model-28546 | |
| 474 | 2018-11-28T05:42:59.980384934Z 01:42:59 INFO:Best validation ppl is 21.129450 | |
| 475 | 2018-11-28T05:42:59.980389656Z | |
| 476 | 2018-11-28T05:42:59.981776916Z 01:42:59 INFO:Latest model is saved in ./results/output_poem/save_model/model-32624 | |
| 477 | 2018-11-28T05:42:59.981790536Z 01:42:59 INFO:Best model is saved in ./results/output_poem/best_model/model-28546 | |
| 478 | 2018-11-28T05:42:59.981794359Z 01:42:59 INFO:Best validation ppl is 21.129450 | |
| 479 | 2018-11-28T05:42:59.981810163Z | |
| 480 | 2018-11-28T05:42:59.981813401Z 01:42:59 INFO:Evaluate the best model on test set | |
| 481 | 2018-11-28T05:43:03.798086219Z 01:43:03 INFO:20.8%, step:99, perplexity: 24.469, speed: 32869 words | |
| 482 | 2018-11-28T05:43:05.659887299Z 01:43:05 INFO:41.7%, step:199, perplexity: 25.556, speed: 41153 words | |
| 483 | 2018-11-28T05:43:07.551507217Z 01:43:07 INFO:62.5%, step:299, perplexity: 25.675, speed: 44730 words | |
| 484 | 2018-11-28T05:43:09.502860421Z 01:43:09 INFO:83.3%, step:399, perplexity: 25.702, speed: 46441 words | |
| 485 | 2018-11-28T05:43:11.366013365Z 01:43:11 INFO:Perplexity: 25.562, speed: 46007 words per sec | |
| 486 | 2018-11-28T05:43:12.680105735Z SYSTEM: Finishing... |
| 0 | { | |
| 1 | "best_model": "./results/output_poem/best_model/model-20390", | |
| 2 | "best_valid_ppl": 22.313692092895508, | |
| 3 | "latest_model": "./results/output_poem/save_model/model-20390", | |
| 4 | "params": { | |
| 5 | "batch_size": 16, | |
| 6 | "cell_type": "lstm", | |
| 7 | "dropout": 0.0, | |
| 8 | "embedding_size": 128, | |
| 9 | "hidden_size": 128, | |
| 10 | "input_dropout": 0.0, | |
| 11 | "learning_rate": 0.005, | |
| 12 | "max_grad_norm": 5.0, | |
| 13 | "num_layers": 2, | |
| 14 | "num_unrollings": 64 | |
| 15 | }, | |
| 16 | "test_ppl": 27.004379272460938 | |
| 17 | }⏎ |
| 0 | model_checkpoint_path: "/home/jovyan/work/results/output_poem/best_model/model-20390" | |
| 1 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/best_model/model-4078" | |
| 2 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/best_model/model-8156" | |
| 3 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/best_model/model-12234" | |
| 4 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/best_model/model-16312" | |
| 5 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/best_model/model-20390" | |
| 0 | model_checkpoint_path: "model-28546" | |
| 1 | all_model_checkpoint_paths: "model-8156" | |
| 2 | all_model_checkpoint_paths: "model-12234" | |
| 3 | all_model_checkpoint_paths: "model-16312" | |
| 4 | all_model_checkpoint_paths: "model-20390" | |
| 5 | all_model_checkpoint_paths: "model-28546" |
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
| 0 | 0 | { |
| 1 | "best_model": "./results/output_poem/best_model/model-20390", | |
| 2 | "best_valid_ppl": 22.313692092895508, | |
| 3 | "latest_model": "./results/output_poem/save_model/model-20390", | |
| 1 | "best_model": "./results/output_poem/best_model/model-28546", | |
| 2 | "best_valid_ppl": 21.12944984436035, | |
| 3 | "latest_model": "./results/output_poem/save_model/model-32624", | |
| 4 | 4 | "params": { |
| 5 | 5 | "batch_size": 16, |
| 6 | 6 | "cell_type": "lstm", |
| 13 | 13 | "num_layers": 2, |
| 14 | 14 | "num_unrollings": 64 |
| 15 | 15 | }, |
| 16 | "test_ppl": 27.004379272460938 | |
| 16 | "test_ppl": 25.561819076538086 | |
| 17 | 17 | }⏎ |
| 0 | model_checkpoint_path: "/home/jovyan/work/results/output_poem/save_model/model-20390" | |
| 1 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/save_model/model-4078" | |
| 2 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/save_model/model-8156" | |
| 3 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/save_model/model-12234" | |
| 4 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/save_model/model-16312" | |
| 5 | all_model_checkpoint_paths: "/home/jovyan/work/results/output_poem/save_model/model-20390" | |
| 0 | model_checkpoint_path: "model-32624" | |
| 1 | all_model_checkpoint_paths: "model-16312" | |
| 2 | all_model_checkpoint_paths: "model-20390" | |
| 3 | all_model_checkpoint_paths: "model-24468" | |
| 4 | all_model_checkpoint_paths: "model-28546" | |
| 5 | all_model_checkpoint_paths: "model-32624" |
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
Binary diff not shown
results/output_poem/tensorboard_log/1543382118/summaries/train/events.out.tfevents.1543382136.job-gpu-5bfe24501afd942eee5c5020-nqf5m
less
more
Binary diff not shown
results/output_poem/tensorboard_log/1543382118/summaries/valid/events.out.tfevents.1543382138.job-gpu-5bfe24501afd942eee5c5020-nqf5m
less
more
Binary diff not shown
| 78 | 78 | w2v_file = os.path.join(base_path, "vectors_poem.bin") |
| 79 | 79 | w2v = Word2Vec(w2v_file) |
| 80 | 80 | |
| 81 | train_data_loader = DataLoader(base_path,batch_size,num_unrollings,w2v.model,'train') | |
| 82 | test1_data_loader = DataLoader(base_path,batch_size,num_unrollings,w2v.model,'test') | |
| 83 | valid_data_loader = DataLoader(base_path,batch_size,num_unrollings,w2v.model,'valid') | |
| 81 | train_data_loader = DataLoader(base_path, batch_size, num_unrollings, w2v.model, 'train') | |
| 82 | test1_data_loader = DataLoader(base_path, batch_size, num_unrollings, w2v.model, 'test') | |
| 83 | valid_data_loader = DataLoader(base_path, batch_size, num_unrollings, w2v.model, 'valid') | |
| 84 | 84 | |
| 85 | 85 | # Create graphs |
| 86 | 86 | logging.info('Creating graph') |
| 88 | 88 | with graph.as_default(): |
| 89 | 89 | w2v_vocab_size = len(w2v.model.vocab) |
| 90 | 90 | with tf.name_scope('training'): |
| 91 | train_model = CharRNNLM(is_training=True,w2v_model = w2v.model,vocab_size=w2v_vocab_size, infer=False, **params) | |
| 91 | train_model = CharRNNLM(is_training=True, w2v_model = w2v.model, vocab_size=w2v_vocab_size, infer=False, **params) | |
| 92 | 92 | tf.get_variable_scope().reuse_variables() |
| 93 | 93 | |
| 94 | 94 | with tf.name_scope('validation'): |
| 95 | valid_model = CharRNNLM(is_training=False,w2v_model = w2v.model, vocab_size=w2v_vocab_size, infer=False, **params) | |
| 95 | valid_model = CharRNNLM(is_training=False, w2v_model = w2v.model, vocab_size=w2v_vocab_size, infer=False, **params) | |
| 96 | 96 | |
| 97 | 97 | with tf.name_scope('evaluation'): |
| 98 | test_model = CharRNNLM(is_training=False,w2v_model = w2v.model,vocab_size=w2v_vocab_size, infer=False, **params) | |
| 98 | test_model = CharRNNLM(is_training=False, w2v_model = w2v.model,vocab_size=w2v_vocab_size, infer=False, **params) | |
| 99 | 99 | saver = tf.train.Saver(name='model_saver') |
| 100 | 100 | best_model_saver = tf.train.Saver(name='best_model_saver') |
| 101 | 101 | |
| 103 | 103 | |
| 104 | 104 | result = {} |
| 105 | 105 | result['params'] = params |
| 106 | ||
| 107 | 106 | |
| 108 | 107 | try: |
| 109 | 108 | with tf.Session(graph=graph) as session: |
| 186 | 185 | result_path = os.path.join(args.output_dir, 'result.json') |
| 187 | 186 | if os.path.exists(result_path): |
| 188 | 187 | os.remove(result_path) |
| 189 | with open(result_path, 'w',encoding='utf-8',errors='ignore') as f: | |
| 188 | with open(result_path, 'w', encoding='utf-8', errors='ignore') as f: | |
| 190 | 189 | json.dump(result, f, indent=2, sort_keys=True) |
| 191 | 190 | |
| 192 | 191 | |
| 193 | 192 | if __name__ == '__main__': |
| 194 | args = '--output_dir /home/jovyan/work/results/output_poem --data_path /home/jovyan/work/data/poem/ --hidden_size 128 --embedding_size 128 --cell_type lstm' | |
| 193 | args = '--output_dir ./results/output_poem --data_path ./data/poem/ --hidden_size 128 --embedding_size 128 --cell_type lstm' | |
| 195 | 194 | main(args) |