master
/ 10_introduction_to_artificial_neural_networks.ipynb

10_introduction_to_artificial_neural_networks.ipynb @master — view markup · raw · history · blame

Notebook

Chapter 10 – Introduction to Artificial Neural Networks

This notebook contains all the sample code and solutions to the exercises in chapter 10.

Setup

First, let's make sure this notebook works well in both python 2 and 3, import a few common modules, ensure MatplotLib plots figures inline and prepare a function to save the figures:

In [1]:
# To support both python 2 and python 3
from __future__ import division, print_function, unicode_literals

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
def reset_graph(seed=42):
    tf.reset_default_graph()
    tf.set_random_seed(seed)
    np.random.seed(seed)

# To plot pretty figures
%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

# Where to save the figures
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "ann"

def save_fig(fig_id, tight_layout=True):
    path = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID, fig_id + ".png")
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format='png', dpi=300)

Perceptrons

In [2]:
import numpy as np
from sklearn.datasets import load_iris
from sklearn.linear_model import Perceptron

iris = load_iris()
X = iris.data[:, (2, 3)]  # petal length, petal width
y = (iris.target == 0).astype(np.int)

per_clf = Perceptron(max_iter=100, random_state=42)
per_clf.fit(X, y)

y_pred = per_clf.predict([[2, 0.5]])
In [3]:
y_pred
Out[3]:
array([1])
In [4]:
a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]
b = -per_clf.intercept_ / per_clf.coef_[0][1]

axes = [0, 5, 0, 2]

x0, x1 = np.meshgrid(
        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),
        np.linspace(axes[2], axes[3], 200).reshape(-1, 1),
    )
X_new = np.c_[x0.ravel(), x1.ravel()]
y_predict = per_clf.predict(X_new)
zz = y_predict.reshape(x0.shape)

plt.figure(figsize=(10, 4))
plt.plot(X[y==0, 0], X[y==0, 1], "bs", label="Not Iris-Setosa")
plt.plot(X[y==1, 0], X[y==1, 1], "yo", label="Iris-Setosa")

plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], "k-", linewidth=3)
from matplotlib.colors import ListedColormap
custom_cmap = ListedColormap(['#9898ff', '#fafab0'])

plt.contourf(x0, x1, zz, cmap=custom_cmap)
plt.xlabel("Petal length", fontsize=14)
plt.ylabel("Petal width", fontsize=14)
plt.legend(loc="lower right", fontsize=14)
plt.axis(axes)

save_fig("perceptron_iris_plot")
plt.show()

Saving figure perceptron_iris_plot

Activation functions

In [5]:
def logit(z):
    return 1 / (1 + np.exp(-z))

def relu(z):
    return np.maximum(0, z)

def derivative(f, z, eps=0.000001):
    return (f(z + eps) - f(z - eps))/(2 * eps)
In [6]:
z = np.linspace(-5, 5, 200)

plt.figure(figsize=(11,4))

plt.subplot(121)
plt.plot(z, np.sign(z), "r-", linewidth=2, label="Step")
plt.plot(z, logit(z), "g--", linewidth=2, label="Logit")
plt.plot(z, np.tanh(z), "b-", linewidth=2, label="Tanh")
plt.plot(z, relu(z), "m-.", linewidth=2, label="ReLU")
plt.grid(True)
plt.legend(loc="center right", fontsize=14)
plt.title("Activation functions", fontsize=14)
plt.axis([-5, 5, -1.2, 1.2])

plt.subplot(122)
plt.plot(z, derivative(np.sign, z), "r-", linewidth=2, label="Step")
plt.plot(0, 0, "ro", markersize=5)
plt.plot(0, 0, "rx", markersize=10)
plt.plot(z, derivative(logit, z), "g--", linewidth=2, label="Logit")
plt.plot(z, derivative(np.tanh, z), "b-", linewidth=2, label="Tanh")
plt.plot(z, derivative(relu, z), "m-.", linewidth=2, label="ReLU")
plt.grid(True)
#plt.legend(loc="center right", fontsize=14)
plt.title("Derivatives", fontsize=14)
plt.axis([-5, 5, -0.2, 1.2])

save_fig("activation_functions_plot")
plt.show()

Saving figure activation_functions_plot
In [7]:
def heaviside(z):
    return (z >= 0).astype(z.dtype)

def sigmoid(z):
    return 1/(1+np.exp(-z))

def mlp_xor(x1, x2, activation=heaviside):
    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)
In [8]:
x1s = np.linspace(-0.2, 1.2, 100)
x2s = np.linspace(-0.2, 1.2, 100)
x1, x2 = np.meshgrid(x1s, x2s)

z1 = mlp_xor(x1, x2, activation=heaviside)
z2 = mlp_xor(x1, x2, activation=sigmoid)

plt.figure(figsize=(10,4))

plt.subplot(121)
plt.contourf(x1, x2, z1)
plt.plot([0, 1], [0, 1], "gs", markersize=20)
plt.plot([0, 1], [1, 0], "y^", markersize=20)
plt.title("Activation function: heaviside", fontsize=14)
plt.grid(True)

plt.subplot(122)
plt.contourf(x1, x2, z2)
plt.plot([0, 1], [0, 1], "gs", markersize=20)
plt.plot([0, 1], [1, 0], "y^", markersize=20)
plt.title("Activation function: sigmoid", fontsize=14)
plt.grid(True)

FNN for MNIST

Using the Estimator API (formerly tf.contrib.learn)

In [9]:
import tensorflow as tf

Warning: tf.examples.tutorials.mnist is deprecated. We will use tf.keras.datasets.mnist instead. Moreover, the tf.contrib.learn API was promoted to tf.estimators and tf.feature_columns, and it has changed considerably. In particular, there is no infer_real_valued_columns_from_input() function or SKCompat class.

In [10]:
(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()
X_train = X_train.astype(np.float32).reshape(-1, 28*28) / 255.0
X_test = X_test.astype(np.float32).reshape(-1, 28*28) / 255.0
y_train = y_train.astype(np.int32)
y_test = y_test.astype(np.int32)
X_valid, X_train = X_train[:5000], X_train[5000:]
y_valid, y_train = y_train[:5000], y_train[5000:]
In [11]:
feature_cols = [tf.feature_column.numeric_column("X", shape=[28 * 28])]
dnn_clf = tf.estimator.DNNClassifier(hidden_units=[300,100], n_classes=10,
                                     feature_columns=feature_cols)

input_fn = tf.estimator.inputs.numpy_input_fn(
    x={"X": X_train}, y=y_train, num_epochs=40, batch_size=50, shuffle=True)
dnn_clf.train(input_fn=input_fn)

INFO:tensorflow:Using default config.
WARNING:tensorflow:Using temporary folder as model directory: /tmp/tmpuflzeb_h
INFO:tensorflow:Using config: {'_evaluation_master': '', '_session_config': None, '_model_dir': '/tmp/tmpuflzeb_h', '_task_type': 'worker', '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4fcb4e15c0>, '_save_summary_steps': 100, '_is_chief': True, '_save_checkpoints_steps': None, '_log_step_count_steps': 100, '_master': '', '_service': None, '_keep_checkpoint_every_n_hours': 10000, '_task_id': 0, '_tf_random_seed': None, '_num_ps_replicas': 0, '_global_id_in_cluster': 0, '_train_distribute': None, '_num_worker_replicas': 1, '_save_checkpoints_secs': 600, '_keep_checkpoint_max': 5}
INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Create CheckpointSaverHook.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Saving checkpoints for 1 into /tmp/tmpuflzeb_h/model.ckpt.
INFO:tensorflow:loss = 122.883514, step = 0
INFO:tensorflow:global_step/sec: 480.267
INFO:tensorflow:loss = 9.599711, step = 100 (0.209 sec)
INFO:tensorflow:global_step/sec: 599.191
INFO:tensorflow:loss = 19.580772, step = 200 (0.167 sec)
INFO:tensorflow:global_step/sec: 640.184
INFO:tensorflow:loss = 2.1866307, step = 300 (0.157 sec)
INFO:tensorflow:global_step/sec: 716.395
INFO:tensorflow:loss = 11.493204, step = 400 (0.138 sec)
INFO:tensorflow:global_step/sec: 713.653
INFO:tensorflow:loss = 4.0078278, step = 500 (0.140 sec)
INFO:tensorflow:global_step/sec: 722.021
INFO:tensorflow:loss = 10.612131, step = 600 (0.139 sec)
INFO:tensorflow:global_step/sec: 669.446
INFO:tensorflow:loss = 6.692636, step = 700 (0.149 sec)
INFO:tensorflow:global_step/sec: 720.49
INFO:tensorflow:loss = 4.2058306, step = 800 (0.139 sec)
INFO:tensorflow:global_step/sec: 766.548
INFO:tensorflow:loss = 9.13055, step = 900 (0.130 sec)
INFO:tensorflow:global_step/sec: 773.506
INFO:tensorflow:loss = 4.1445055, step = 1000 (0.129 sec)
INFO:tensorflow:global_step/sec: 755.713
INFO:tensorflow:loss = 8.442559, step = 1100 (0.132 sec)
INFO:tensorflow:global_step/sec: 762.721
INFO:tensorflow:loss = 1.4401194, step = 1200 (0.131 sec)
INFO:tensorflow:global_step/sec: 659.992
INFO:tensorflow:loss = 13.526959, step = 1300 (0.152 sec)
INFO:tensorflow:global_step/sec: 701.683
INFO:tensorflow:loss = 5.039109, step = 1400 (0.143 sec)
INFO:tensorflow:global_step/sec: 751.167
INFO:tensorflow:loss = 1.8074234, step = 1500 (0.133 sec)
INFO:tensorflow:global_step/sec: 700.915
INFO:tensorflow:loss = 6.4867635, step = 1600 (0.142 sec)
INFO:tensorflow:global_step/sec: 733.041
INFO:tensorflow:loss = 0.5804969, step = 1700 (0.136 sec)
INFO:tensorflow:global_step/sec: 764.306
INFO:tensorflow:loss = 1.5091155, step = 1800 (0.131 sec)
INFO:tensorflow:global_step/sec: 664.096
INFO:tensorflow:loss = 3.6764488, step = 1900 (0.151 sec)
INFO:tensorflow:global_step/sec: 687.762
INFO:tensorflow:loss = 1.4820085, step = 2000 (0.145 sec)
INFO:tensorflow:global_step/sec: 667.013
INFO:tensorflow:loss = 1.2964534, step = 2100 (0.150 sec)
INFO:tensorflow:global_step/sec: 704.008
INFO:tensorflow:loss = 1.0711427, step = 2200 (0.142 sec)
INFO:tensorflow:global_step/sec: 664.444
INFO:tensorflow:loss = 2.6691673, step = 2300 (0.151 sec)
INFO:tensorflow:global_step/sec: 691.387
INFO:tensorflow:loss = 0.9668397, step = 2400 (0.144 sec)
INFO:tensorflow:global_step/sec: 749.867
INFO:tensorflow:loss = 1.4468323, step = 2500 (0.133 sec)
INFO:tensorflow:global_step/sec: 725.923
INFO:tensorflow:loss = 1.8073778, step = 2600 (0.138 sec)
INFO:tensorflow:global_step/sec: 732.432
INFO:tensorflow:loss = 3.8904514, step = 2700 (0.136 sec)
INFO:tensorflow:global_step/sec: 710.886
INFO:tensorflow:loss = 2.3015192, step = 2800 (0.141 sec)
INFO:tensorflow:global_step/sec: 713.338
INFO:tensorflow:loss = 4.671579, step = 2900 (0.140 sec)
INFO:tensorflow:global_step/sec: 708.305
INFO:tensorflow:loss = 1.3551085, step = 3000 (0.141 sec)
INFO:tensorflow:global_step/sec: 647.185
INFO:tensorflow:loss = 1.9145899, step = 3100 (0.155 sec)
INFO:tensorflow:global_step/sec: 673.601
INFO:tensorflow:loss = 1.0741266, step = 3200 (0.148 sec)
INFO:tensorflow:global_step/sec: 721.531
INFO:tensorflow:loss = 1.0713774, step = 3300 (0.138 sec)
INFO:tensorflow:global_step/sec: 713.421
INFO:tensorflow:loss = 1.3074391, step = 3400 (0.140 sec)
INFO:tensorflow:global_step/sec: 633.857
INFO:tensorflow:loss = 2.0073137, step = 3500 (0.158 sec)
INFO:tensorflow:global_step/sec: 672.287
INFO:tensorflow:loss = 13.952677, step = 3600 (0.149 sec)
INFO:tensorflow:global_step/sec: 639.094
INFO:tensorflow:loss = 1.6767453, step = 3700 (0.157 sec)
INFO:tensorflow:global_step/sec: 731.891
INFO:tensorflow:loss = 0.27798674, step = 3800 (0.137 sec)
INFO:tensorflow:global_step/sec: 728.154
INFO:tensorflow:loss = 3.5524733, step = 3900 (0.137 sec)
INFO:tensorflow:global_step/sec: 714.217
INFO:tensorflow:loss = 0.6761815, step = 4000 (0.140 sec)
INFO:tensorflow:global_step/sec: 744.721
INFO:tensorflow:loss = 0.79083383, step = 4100 (0.134 sec)
INFO:tensorflow:global_step/sec: 685.419
INFO:tensorflow:loss = 1.3305103, step = 4200 (0.146 sec)
INFO:tensorflow:global_step/sec: 632.625
INFO:tensorflow:loss = 0.14447726, step = 4300 (0.158 sec)
INFO:tensorflow:global_step/sec: 679.755
INFO:tensorflow:loss = 1.8386902, step = 4400 (0.148 sec)
INFO:tensorflow:global_step/sec: 751.428
INFO:tensorflow:loss = 0.94889283, step = 4500 (0.132 sec)
INFO:tensorflow:global_step/sec: 750.376
INFO:tensorflow:loss = 0.28424773, step = 4600 (0.134 sec)
INFO:tensorflow:global_step/sec: 735.964
INFO:tensorflow:loss = 3.266353, step = 4700 (0.136 sec)
INFO:tensorflow:global_step/sec: 742.054
INFO:tensorflow:loss = 3.171119, step = 4800 (0.134 sec)
INFO:tensorflow:global_step/sec: 779.657
INFO:tensorflow:loss = 1.12006, step = 4900 (0.128 sec)
INFO:tensorflow:global_step/sec: 730.952
INFO:tensorflow:loss = 0.5669488, step = 5000 (0.137 sec)
INFO:tensorflow:global_step/sec: 715.897
INFO:tensorflow:loss = 0.3067366, step = 5100 (0.140 sec)
INFO:tensorflow:global_step/sec: 631.438
INFO:tensorflow:loss = 0.5437011, step = 5200 (0.159 sec)
INFO:tensorflow:global_step/sec: 654.717
INFO:tensorflow:loss = 0.25085437, step = 5300 (0.153 sec)
INFO:tensorflow:global_step/sec: 658.829
INFO:tensorflow:loss = 0.30891788, step = 5400 (0.151 sec)
INFO:tensorflow:global_step/sec: 672.908
INFO:tensorflow:loss = 0.8258436, step = 5500 (0.149 sec)
INFO:tensorflow:global_step/sec: 574.211
INFO:tensorflow:loss = 0.19280735, step = 5600 (0.173 sec)
INFO:tensorflow:global_step/sec: 653.783
INFO:tensorflow:loss = 0.17635345, step = 5700 (0.155 sec)
INFO:tensorflow:global_step/sec: 635.343
INFO:tensorflow:loss = 2.1531484, step = 5800 (0.155 sec)
INFO:tensorflow:global_step/sec: 628.505
INFO:tensorflow:loss = 1.3385924, step = 5900 (0.159 sec)
INFO:tensorflow:global_step/sec: 743.577
INFO:tensorflow:loss = 1.7493693, step = 6000 (0.134 sec)
INFO:tensorflow:global_step/sec: 759.487
INFO:tensorflow:loss = 1.3990214, step = 6100 (0.132 sec)
INFO:tensorflow:global_step/sec: 715.001
INFO:tensorflow:loss = 0.08636901, step = 6200 (0.140 sec)
INFO:tensorflow:global_step/sec: 682.812
INFO:tensorflow:loss = 1.2878852, step = 6300 (0.147 sec)
INFO:tensorflow:global_step/sec: 668.255
INFO:tensorflow:loss = 2.8647041, step = 6400 (0.149 sec)
INFO:tensorflow:global_step/sec: 702.609
INFO:tensorflow:loss = 0.38349468, step = 6500 (0.143 sec)
INFO:tensorflow:global_step/sec: 717.91
INFO:tensorflow:loss = 0.71950877, step = 6600 (0.139 sec)
INFO:tensorflow:global_step/sec: 719.603
INFO:tensorflow:loss = 0.8812942, step = 6700 (0.139 sec)
INFO:tensorflow:global_step/sec: 697.269
INFO:tensorflow:loss = 0.5575855, step = 6800 (0.143 sec)
INFO:tensorflow:global_step/sec: 733.281
INFO:tensorflow:loss = 0.97547567, step = 6900 (0.136 sec)
INFO:tensorflow:global_step/sec: 731.343
INFO:tensorflow:loss = 0.10276175, step = 7000 (0.137 sec)
INFO:tensorflow:global_step/sec: 739.462
INFO:tensorflow:loss = 1.1644993, step = 7100 (0.135 sec)
INFO:tensorflow:global_step/sec: 674.078
INFO:tensorflow:loss = 0.87335706, step = 7200 (0.149 sec)
INFO:tensorflow:global_step/sec: 754.457
INFO:tensorflow:loss = 0.02294194, step = 7300 (0.132 sec)
INFO:tensorflow:global_step/sec: 665.539
INFO:tensorflow:loss = 0.03988394, step = 7400 (0.151 sec)
INFO:tensorflow:global_step/sec: 664.489
INFO:tensorflow:loss = 0.35611078, step = 7500 (0.150 sec)
INFO:tensorflow:global_step/sec: 642.162
INFO:tensorflow:loss = 0.37163323, step = 7600 (0.156 sec)
INFO:tensorflow:global_step/sec: 710.797
INFO:tensorflow:loss = 0.2848217, step = 7700 (0.141 sec)
INFO:tensorflow:global_step/sec: 761.344
INFO:tensorflow:loss = 0.7845428, step = 7800 (0.131 sec)
INFO:tensorflow:global_step/sec: 765.77
INFO:tensorflow:loss = 0.10574758, step = 7900 (0.130 sec)
INFO:tensorflow:global_step/sec: 723.478
INFO:tensorflow:loss = 0.69526076, step = 8000 (0.138 sec)
INFO:tensorflow:global_step/sec: 750.877
INFO:tensorflow:loss = 0.58130324, step = 8100 (0.133 sec)
INFO:tensorflow:global_step/sec: 730.965
INFO:tensorflow:loss = 0.31362545, step = 8200 (0.137 sec)
INFO:tensorflow:global_step/sec: 717.223
INFO:tensorflow:loss = 0.8284373, step = 8300 (0.139 sec)
INFO:tensorflow:global_step/sec: 660.215
INFO:tensorflow:loss = 0.82293755, step = 8400 (0.151 sec)
INFO:tensorflow:global_step/sec: 650.197
INFO:tensorflow:loss = 0.08349186, step = 8500 (0.154 sec)
INFO:tensorflow:global_step/sec: 686.024
INFO:tensorflow:loss = 0.48411164, step = 8600 (0.146 sec)
INFO:tensorflow:global_step/sec: 775.917
INFO:tensorflow:loss = 0.35096797, step = 8700 (0.129 sec)
INFO:tensorflow:global_step/sec: 773.152
INFO:tensorflow:loss = 0.22725578, step = 8800 (0.130 sec)
INFO:tensorflow:global_step/sec: 691.659
INFO:tensorflow:loss = 0.0485073, step = 8900 (0.144 sec)
INFO:tensorflow:global_step/sec: 773.385
INFO:tensorflow:loss = 0.07985984, step = 9000 (0.129 sec)
INFO:tensorflow:global_step/sec: 741.822
INFO:tensorflow:loss = 0.0542172, step = 9100 (0.135 sec)
INFO:tensorflow:global_step/sec: 749.122
INFO:tensorflow:loss = 0.50927365, step = 9200 (0.133 sec)
INFO:tensorflow:global_step/sec: 681.55
INFO:tensorflow:loss = 0.9489171, step = 9300 (0.147 sec)
INFO:tensorflow:global_step/sec: 680.382
INFO:tensorflow:loss = 0.060666546, step = 9400 (0.147 sec)
INFO:tensorflow:global_step/sec: 764.193
INFO:tensorflow:loss = 0.28517923, step = 9500 (0.131 sec)
INFO:tensorflow:global_step/sec: 702.206
INFO:tensorflow:loss = 0.20915146, step = 9600 (0.142 sec)
INFO:tensorflow:global_step/sec: 678.51
INFO:tensorflow:loss = 0.579557, step = 9700 (0.148 sec)
INFO:tensorflow:global_step/sec: 593.939
INFO:tensorflow:loss = 0.93460476, step = 9800 (0.168 sec)
INFO:tensorflow:global_step/sec: 665.897
INFO:tensorflow:loss = 0.047924682, step = 9900 (0.150 sec)
INFO:tensorflow:global_step/sec: 711.483
INFO:tensorflow:loss = 0.4922812, step = 10000 (0.141 sec)
INFO:tensorflow:global_step/sec: 712.135
INFO:tensorflow:loss = 0.047895633, step = 10100 (0.140 sec)
INFO:tensorflow:global_step/sec: 645.55
INFO:tensorflow:loss = 0.5398847, step = 10200 (0.155 sec)
INFO:tensorflow:global_step/sec: 638.582
INFO:tensorflow:loss = 0.15529627, step = 10300 (0.157 sec)
INFO:tensorflow:global_step/sec: 751.801
INFO:tensorflow:loss = 0.9468999, step = 10400 (0.133 sec)
INFO:tensorflow:global_step/sec: 699.759
INFO:tensorflow:loss = 0.117633805, step = 10500 (0.143 sec)
INFO:tensorflow:global_step/sec: 681.536
INFO:tensorflow:loss = 2.7077622, step = 10600 (0.147 sec)
INFO:tensorflow:global_step/sec: 639.102
INFO:tensorflow:loss = 0.4701725, step = 10700 (0.158 sec)
INFO:tensorflow:global_step/sec: 657.787
INFO:tensorflow:loss = 0.036744572, step = 10800 (0.150 sec)
INFO:tensorflow:global_step/sec: 610.198
INFO:tensorflow:loss = 0.5130552, step = 10900 (0.164 sec)
INFO:tensorflow:global_step/sec: 641.731
INFO:tensorflow:loss = 0.10233489, step = 11000 (0.156 sec)
INFO:tensorflow:global_step/sec: 676.149
INFO:tensorflow:loss = 0.27252752, step = 11100 (0.148 sec)
INFO:tensorflow:global_step/sec: 685.492
INFO:tensorflow:loss = 0.03905679, step = 11200 (0.145 sec)
INFO:tensorflow:global_step/sec: 739.538
INFO:tensorflow:loss = 0.4037779, step = 11300 (0.135 sec)
INFO:tensorflow:global_step/sec: 761.957
INFO:tensorflow:loss = 0.07931179, step = 11400 (0.131 sec)
INFO:tensorflow:global_step/sec: 690.435
INFO:tensorflow:loss = 0.25471577, step = 11500 (0.145 sec)
INFO:tensorflow:global_step/sec: 731.688
INFO:tensorflow:loss = 0.095315784, step = 11600 (0.137 sec)
INFO:tensorflow:global_step/sec: 737.292
INFO:tensorflow:loss = 0.26888248, step = 11700 (0.136 sec)
INFO:tensorflow:global_step/sec: 717.634
INFO:tensorflow:loss = 0.0791325, step = 11800 (0.139 sec)
INFO:tensorflow:global_step/sec: 612.395
INFO:tensorflow:loss = 0.22025564, step = 11900 (0.164 sec)
INFO:tensorflow:global_step/sec: 674.538
INFO:tensorflow:loss = 0.23279706, step = 12000 (0.148 sec)
INFO:tensorflow:global_step/sec: 694.35
INFO:tensorflow:loss = 0.12199997, step = 12100 (0.144 sec)
INFO:tensorflow:global_step/sec: 666.45
INFO:tensorflow:loss = 0.28451073, step = 12200 (0.150 sec)
INFO:tensorflow:global_step/sec: 714.277
INFO:tensorflow:loss = 0.118836306, step = 12300 (0.140 sec)
INFO:tensorflow:global_step/sec: 697.121
INFO:tensorflow:loss = 0.040629867, step = 12400 (0.143 sec)
INFO:tensorflow:global_step/sec: 695.514
INFO:tensorflow:loss = 0.12426446, step = 12500 (0.144 sec)
INFO:tensorflow:global_step/sec: 748.65
INFO:tensorflow:loss = 0.123113535, step = 12600 (0.134 sec)
INFO:tensorflow:global_step/sec: 705.816
INFO:tensorflow:loss = 0.08067431, step = 12700 (0.142 sec)
INFO:tensorflow:global_step/sec: 677.035
INFO:tensorflow:loss = 0.12166683, step = 12800 (0.148 sec)
INFO:tensorflow:global_step/sec: 715.319
INFO:tensorflow:loss = 0.44292378, step = 12900 (0.140 sec)
INFO:tensorflow:global_step/sec: 663.768
INFO:tensorflow:loss = 0.115894824, step = 13000 (0.151 sec)
INFO:tensorflow:global_step/sec: 730.272
INFO:tensorflow:loss = 0.48085117, step = 13100 (0.137 sec)
INFO:tensorflow:global_step/sec: 715.079
INFO:tensorflow:loss = 0.04950906, step = 13200 (0.140 sec)
INFO:tensorflow:global_step/sec: 728.079
INFO:tensorflow:loss = 0.22041032, step = 13300 (0.137 sec)
INFO:tensorflow:global_step/sec: 743.21
INFO:tensorflow:loss = 0.96188104, step = 13400 (0.135 sec)
INFO:tensorflow:global_step/sec: 736.865
INFO:tensorflow:loss = 0.48876682, step = 13500 (0.136 sec)
INFO:tensorflow:global_step/sec: 753.917
INFO:tensorflow:loss = 0.016977577, step = 13600 (0.133 sec)
INFO:tensorflow:global_step/sec: 723.362
INFO:tensorflow:loss = 0.21165565, step = 13700 (0.138 sec)
INFO:tensorflow:global_step/sec: 717.43
INFO:tensorflow:loss = 0.07759167, step = 13800 (0.140 sec)
INFO:tensorflow:global_step/sec: 720.41
INFO:tensorflow:loss = 0.30665547, step = 13900 (0.138 sec)
INFO:tensorflow:global_step/sec: 628.631
INFO:tensorflow:loss = 0.061501402, step = 14000 (0.159 sec)
INFO:tensorflow:global_step/sec: 652.142
INFO:tensorflow:loss = 0.22736073, step = 14100 (0.156 sec)
INFO:tensorflow:global_step/sec: 615.068
INFO:tensorflow:loss = 0.07389079, step = 14200 (0.160 sec)
INFO:tensorflow:global_step/sec: 688.411
INFO:tensorflow:loss = 0.29698288, step = 14300 (0.145 sec)
INFO:tensorflow:global_step/sec: 711.249
INFO:tensorflow:loss = 0.14422297, step = 14400 (0.142 sec)
INFO:tensorflow:global_step/sec: 719.837
INFO:tensorflow:loss = 0.014385125, step = 14500 (0.137 sec)
INFO:tensorflow:global_step/sec: 728.481
INFO:tensorflow:loss = 0.072385326, step = 14600 (0.137 sec)
INFO:tensorflow:global_step/sec: 633.549
INFO:tensorflow:loss = 0.2490507, step = 14700 (0.158 sec)
INFO:tensorflow:global_step/sec: 655.833
INFO:tensorflow:loss = 0.13730074, step = 14800 (0.152 sec)
INFO:tensorflow:global_step/sec: 669.018
INFO:tensorflow:loss = 0.094497725, step = 14900 (0.150 sec)
INFO:tensorflow:global_step/sec: 648.228
INFO:tensorflow:loss = 0.02053002, step = 15000 (0.154 sec)
INFO:tensorflow:global_step/sec: 705.368
INFO:tensorflow:loss = 0.07519817, step = 15100 (0.142 sec)
INFO:tensorflow:global_step/sec: 716.654
INFO:tensorflow:loss = 5.0296702, step = 15200 (0.139 sec)
INFO:tensorflow:global_step/sec: 694.54
INFO:tensorflow:loss = 0.0641305, step = 15300 (0.144 sec)
INFO:tensorflow:global_step/sec: 706.583
INFO:tensorflow:loss = 0.046975322, step = 15400 (0.142 sec)
INFO:tensorflow:global_step/sec: 760.143
INFO:tensorflow:loss = 0.19096217, step = 15500 (0.131 sec)
INFO:tensorflow:global_step/sec: 729.355
INFO:tensorflow:loss = 0.07447162, step = 15600 (0.137 sec)
INFO:tensorflow:global_step/sec: 721.879
INFO:tensorflow:loss = 0.008595724, step = 15700 (0.138 sec)
INFO:tensorflow:global_step/sec: 725.15
INFO:tensorflow:loss = 0.02350374, step = 15800 (0.138 sec)
INFO:tensorflow:global_step/sec: 750.054
INFO:tensorflow:loss = 0.23393407, step = 15900 (0.135 sec)
INFO:tensorflow:global_step/sec: 686.044
INFO:tensorflow:loss = 0.052663296, step = 16000 (0.145 sec)
INFO:tensorflow:global_step/sec: 571.582
INFO:tensorflow:loss = 0.07737052, step = 16100 (0.175 sec)
INFO:tensorflow:global_step/sec: 621.862
INFO:tensorflow:loss = 0.059392054, step = 16200 (0.161 sec)
INFO:tensorflow:global_step/sec: 680.005
INFO:tensorflow:loss = 0.112042084, step = 16300 (0.147 sec)
INFO:tensorflow:global_step/sec: 733.16
INFO:tensorflow:loss = 0.019262556, step = 16400 (0.136 sec)
INFO:tensorflow:global_step/sec: 747.072
INFO:tensorflow:loss = 0.11752561, step = 16500 (0.134 sec)
INFO:tensorflow:global_step/sec: 734.132
INFO:tensorflow:loss = 0.23796175, step = 16600 (0.136 sec)
INFO:tensorflow:global_step/sec: 737.207
INFO:tensorflow:loss = 0.2813179, step = 16700 (0.136 sec)
INFO:tensorflow:global_step/sec: 735.658
INFO:tensorflow:loss = 0.08527176, step = 16800 (0.136 sec)
INFO:tensorflow:global_step/sec: 760.729
INFO:tensorflow:loss = 0.0708789, step = 16900 (0.132 sec)
INFO:tensorflow:global_step/sec: 780.434
INFO:tensorflow:loss = 0.10494808, step = 17000 (0.128 sec)
INFO:tensorflow:global_step/sec: 802.2
INFO:tensorflow:loss = 0.053045884, step = 17100 (0.124 sec)
INFO:tensorflow:global_step/sec: 758.93
INFO:tensorflow:loss = 0.019496916, step = 17200 (0.132 sec)
INFO:tensorflow:global_step/sec: 777.573
INFO:tensorflow:loss = 0.2686581, step = 17300 (0.128 sec)
INFO:tensorflow:global_step/sec: 798.333
INFO:tensorflow:loss = 0.0589393, step = 17400 (0.125 sec)
INFO:tensorflow:global_step/sec: 779.037
INFO:tensorflow:loss = 0.04246951, step = 17500 (0.128 sec)
INFO:tensorflow:global_step/sec: 781.962
INFO:tensorflow:loss = 0.1414603, step = 17600 (0.128 sec)
INFO:tensorflow:global_step/sec: 637.081
INFO:tensorflow:loss = 0.03246697, step = 17700 (0.157 sec)
INFO:tensorflow:global_step/sec: 617.478
INFO:tensorflow:loss = 0.1726821, step = 17800 (0.162 sec)
INFO:tensorflow:global_step/sec: 670.541
INFO:tensorflow:loss = 0.16460724, step = 17900 (0.148 sec)
INFO:tensorflow:global_step/sec: 775.135
INFO:tensorflow:loss = 0.2628479, step = 18000 (0.129 sec)
INFO:tensorflow:global_step/sec: 732.683
INFO:tensorflow:loss = 0.092953764, step = 18100 (0.136 sec)
INFO:tensorflow:global_step/sec: 766.433
INFO:tensorflow:loss = 0.04855925, step = 18200 (0.130 sec)
INFO:tensorflow:global_step/sec: 724.314
INFO:tensorflow:loss = 0.1251901, step = 18300 (0.138 sec)
INFO:tensorflow:global_step/sec: 617.447
INFO:tensorflow:loss = 0.07948378, step = 18400 (0.162 sec)
INFO:tensorflow:global_step/sec: 605.474
INFO:tensorflow:loss = 0.0709505, step = 18500 (0.165 sec)
INFO:tensorflow:global_step/sec: 681.306
INFO:tensorflow:loss = 0.047453385, step = 18600 (0.147 sec)
INFO:tensorflow:global_step/sec: 763.621
INFO:tensorflow:loss = 0.25552556, step = 18700 (0.131 sec)
INFO:tensorflow:global_step/sec: 782.048
INFO:tensorflow:loss = 0.09249738, step = 18800 (0.128 sec)
INFO:tensorflow:global_step/sec: 592.506
INFO:tensorflow:loss = 0.013389738, step = 18900 (0.171 sec)
INFO:tensorflow:global_step/sec: 673.123
INFO:tensorflow:loss = 0.13284124, step = 19000 (0.147 sec)
INFO:tensorflow:global_step/sec: 734.839
INFO:tensorflow:loss = 0.1085002, step = 19100 (0.136 sec)
INFO:tensorflow:global_step/sec: 679.734
INFO:tensorflow:loss = 0.2795027, step = 19200 (0.147 sec)
INFO:tensorflow:global_step/sec: 630.326
INFO:tensorflow:loss = 0.04235497, step = 19300 (0.159 sec)
INFO:tensorflow:global_step/sec: 744.772
INFO:tensorflow:loss = 0.184058, step = 19400 (0.134 sec)
INFO:tensorflow:global_step/sec: 720.66
INFO:tensorflow:loss = 0.10944146, step = 19500 (0.139 sec)
INFO:tensorflow:global_step/sec: 682.055
INFO:tensorflow:loss = 0.61868834, step = 19600 (0.147 sec)
INFO:tensorflow:global_step/sec: 669.81
INFO:tensorflow:loss = 0.013134284, step = 19700 (0.149 sec)
INFO:tensorflow:global_step/sec: 728.217
INFO:tensorflow:loss = 0.07665045, step = 19800 (0.137 sec)
INFO:tensorflow:global_step/sec: 765.268
INFO:tensorflow:loss = 0.13330126, step = 19900 (0.131 sec)
INFO:tensorflow:global_step/sec: 754.119
INFO:tensorflow:loss = 0.1017472, step = 20000 (0.133 sec)
INFO:tensorflow:global_step/sec: 766.664
INFO:tensorflow:loss = 0.32613552, step = 20100 (0.130 sec)
INFO:tensorflow:global_step/sec: 755.936
INFO:tensorflow:loss = 0.08524483, step = 20200 (0.133 sec)
INFO:tensorflow:global_step/sec: 750.554
INFO:tensorflow:loss = 0.10158778, step = 20300 (0.133 sec)
INFO:tensorflow:global_step/sec: 776.959
INFO:tensorflow:loss = 0.029059863, step = 20400 (0.129 sec)
INFO:tensorflow:global_step/sec: 788.694
INFO:tensorflow:loss = 0.012308217, step = 20500 (0.127 sec)
INFO:tensorflow:global_step/sec: 782.133
INFO:tensorflow:loss = 0.059527285, step = 20600 (0.128 sec)
INFO:tensorflow:global_step/sec: 751.991
INFO:tensorflow:loss = 0.008613001, step = 20700 (0.133 sec)
INFO:tensorflow:global_step/sec: 600.703
INFO:tensorflow:loss = 0.006468304, step = 20800 (0.167 sec)
INFO:tensorflow:global_step/sec: 555.84
INFO:tensorflow:loss = 0.041916292, step = 20900 (0.180 sec)
INFO:tensorflow:global_step/sec: 636.831
INFO:tensorflow:loss = 0.004384146, step = 21000 (0.157 sec)
INFO:tensorflow:global_step/sec: 683.558
INFO:tensorflow:loss = 0.024349477, step = 21100 (0.150 sec)
INFO:tensorflow:global_step/sec: 640.381
INFO:tensorflow:loss = 0.04808281, step = 21200 (0.152 sec)
INFO:tensorflow:global_step/sec: 679.647
INFO:tensorflow:loss = 0.12497167, step = 21300 (0.147 sec)
INFO:tensorflow:global_step/sec: 746.897
INFO:tensorflow:loss = 0.037052717, step = 21400 (0.134 sec)
INFO:tensorflow:global_step/sec: 755.997
INFO:tensorflow:loss = 0.006926307, step = 21500 (0.132 sec)
INFO:tensorflow:global_step/sec: 773.32
INFO:tensorflow:loss = 0.074007586, step = 21600 (0.129 sec)
INFO:tensorflow:global_step/sec: 718.055
INFO:tensorflow:loss = 0.10291884, step = 21700 (0.139 sec)
INFO:tensorflow:global_step/sec: 750.981
INFO:tensorflow:loss = 0.012438581, step = 21800 (0.133 sec)
INFO:tensorflow:global_step/sec: 765.381
INFO:tensorflow:loss = 0.0038113636, step = 21900 (0.131 sec)
INFO:tensorflow:global_step/sec: 748.057
INFO:tensorflow:loss = 0.03924905, step = 22000 (0.133 sec)
INFO:tensorflow:global_step/sec: 758.258
INFO:tensorflow:loss = 0.0941266, step = 22100 (0.132 sec)
INFO:tensorflow:global_step/sec: 748.376
INFO:tensorflow:loss = 0.0047066705, step = 22200 (0.134 sec)
INFO:tensorflow:global_step/sec: 717.068
INFO:tensorflow:loss = 0.049610026, step = 22300 (0.139 sec)
INFO:tensorflow:global_step/sec: 764.656
INFO:tensorflow:loss = 0.026602667, step = 22400 (0.131 sec)
INFO:tensorflow:global_step/sec: 719.336
INFO:tensorflow:loss = 0.037756026, step = 22500 (0.139 sec)
INFO:tensorflow:global_step/sec: 708.671
INFO:tensorflow:loss = 0.009034704, step = 22600 (0.141 sec)
INFO:tensorflow:global_step/sec: 696.592
INFO:tensorflow:loss = 0.121186815, step = 22700 (0.144 sec)
INFO:tensorflow:global_step/sec: 719.805
INFO:tensorflow:loss = 0.018444436, step = 22800 (0.139 sec)
INFO:tensorflow:global_step/sec: 750.769
INFO:tensorflow:loss = 0.15817365, step = 22900 (0.133 sec)
INFO:tensorflow:global_step/sec: 735.057
INFO:tensorflow:loss = 0.07102643, step = 23000 (0.136 sec)
INFO:tensorflow:global_step/sec: 723.982
INFO:tensorflow:loss = 0.053571053, step = 23100 (0.138 sec)
INFO:tensorflow:global_step/sec: 728.232
INFO:tensorflow:loss = 0.0041952957, step = 23200 (0.137 sec)
INFO:tensorflow:global_step/sec: 746.255
INFO:tensorflow:loss = 0.07324758, step = 23300 (0.134 sec)
INFO:tensorflow:global_step/sec: 698.442
INFO:tensorflow:loss = 0.013589903, step = 23400 (0.143 sec)
INFO:tensorflow:global_step/sec: 672.646
INFO:tensorflow:loss = 0.10314279, step = 23500 (0.149 sec)
INFO:tensorflow:global_step/sec: 720.016
INFO:tensorflow:loss = 0.05611958, step = 23600 (0.139 sec)
INFO:tensorflow:global_step/sec: 670.816
INFO:tensorflow:loss = 0.040629435, step = 23700 (0.149 sec)
INFO:tensorflow:global_step/sec: 679.358
INFO:tensorflow:loss = 0.039468687, step = 23800 (0.147 sec)
INFO:tensorflow:global_step/sec: 591.633
INFO:tensorflow:loss = 0.07725768, step = 23900 (0.169 sec)
INFO:tensorflow:global_step/sec: 626.386
INFO:tensorflow:loss = 0.07133426, step = 24000 (0.160 sec)
INFO:tensorflow:global_step/sec: 622.443
INFO:tensorflow:loss = 0.01620267, step = 24100 (0.160 sec)
INFO:tensorflow:global_step/sec: 757.569
INFO:tensorflow:loss = 0.033674255, step = 24200 (0.132 sec)
INFO:tensorflow:global_step/sec: 759.175
INFO:tensorflow:loss = 0.02904886, step = 24300 (0.132 sec)
INFO:tensorflow:global_step/sec: 643.196
INFO:tensorflow:loss = 0.021125063, step = 24400 (0.156 sec)
INFO:tensorflow:global_step/sec: 764.936
INFO:tensorflow:loss = 0.010064006, step = 24500 (0.131 sec)
INFO:tensorflow:global_step/sec: 623.595
INFO:tensorflow:loss = 0.22093584, step = 24600 (0.160 sec)
INFO:tensorflow:global_step/sec: 642.635
INFO:tensorflow:loss = 0.04337728, step = 24700 (0.156 sec)
INFO:tensorflow:global_step/sec: 642.833
INFO:tensorflow:loss = 0.017090034, step = 24800 (0.156 sec)
INFO:tensorflow:global_step/sec: 648.077
INFO:tensorflow:loss = 0.058798455, step = 24900 (0.154 sec)
INFO:tensorflow:global_step/sec: 611.414
INFO:tensorflow:loss = 0.026018398, step = 25000 (0.164 sec)
INFO:tensorflow:global_step/sec: 637.37
INFO:tensorflow:loss = 0.05266387, step = 25100 (0.157 sec)
INFO:tensorflow:global_step/sec: 662.643
INFO:tensorflow:loss = 0.02904391, step = 25200 (0.151 sec)
INFO:tensorflow:global_step/sec: 614.739
INFO:tensorflow:loss = 0.03309081, step = 25300 (0.163 sec)
INFO:tensorflow:global_step/sec: 598.224
INFO:tensorflow:loss = 0.114326894, step = 25400 (0.167 sec)
INFO:tensorflow:global_step/sec: 663.694
INFO:tensorflow:loss = 0.038087483, step = 25500 (0.151 sec)
INFO:tensorflow:global_step/sec: 636.034
INFO:tensorflow:loss = 0.042078175, step = 25600 (0.158 sec)
INFO:tensorflow:global_step/sec: 587.527
INFO:tensorflow:loss = 0.021071143, step = 25700 (0.170 sec)
INFO:tensorflow:global_step/sec: 664.395
INFO:tensorflow:loss = 0.039268643, step = 25800 (0.151 sec)
INFO:tensorflow:global_step/sec: 705.829
INFO:tensorflow:loss = 0.00580811, step = 25900 (0.142 sec)
INFO:tensorflow:global_step/sec: 636.116
INFO:tensorflow:loss = 0.06418324, step = 26000 (0.157 sec)
INFO:tensorflow:global_step/sec: 619.763
INFO:tensorflow:loss = 0.030529939, step = 26100 (0.161 sec)
INFO:tensorflow:global_step/sec: 592.966
INFO:tensorflow:loss = 0.019166447, step = 26200 (0.169 sec)
INFO:tensorflow:global_step/sec: 589.27
INFO:tensorflow:loss = 0.0116247, step = 26300 (0.169 sec)
INFO:tensorflow:global_step/sec: 695.168
INFO:tensorflow:loss = 0.030719474, step = 26400 (0.144 sec)
INFO:tensorflow:global_step/sec: 732.142
INFO:tensorflow:loss = 0.11685369, step = 26500 (0.137 sec)
INFO:tensorflow:global_step/sec: 760.473
INFO:tensorflow:loss = 0.01657794, step = 26600 (0.131 sec)
INFO:tensorflow:global_step/sec: 671.327
INFO:tensorflow:loss = 0.02757436, step = 26700 (0.149 sec)
INFO:tensorflow:global_step/sec: 668.14
INFO:tensorflow:loss = 0.013070018, step = 26800 (0.149 sec)
INFO:tensorflow:global_step/sec: 771.784
INFO:tensorflow:loss = 0.16727, step = 26900 (0.131 sec)
INFO:tensorflow:global_step/sec: 768.783
INFO:tensorflow:loss = 0.026858501, step = 27000 (0.129 sec)
INFO:tensorflow:global_step/sec: 806.951
INFO:tensorflow:loss = 0.020017473, step = 27100 (0.124 sec)
INFO:tensorflow:global_step/sec: 792.151
INFO:tensorflow:loss = 0.0063375104, step = 27200 (0.126 sec)
INFO:tensorflow:global_step/sec: 743.85
INFO:tensorflow:loss = 0.038745653, step = 27300 (0.134 sec)
INFO:tensorflow:global_step/sec: 763.696
INFO:tensorflow:loss = 0.07117924, step = 27400 (0.131 sec)
INFO:tensorflow:global_step/sec: 718.526
INFO:tensorflow:loss = 0.23104118, step = 27500 (0.139 sec)
INFO:tensorflow:global_step/sec: 691.104
INFO:tensorflow:loss = 0.045554783, step = 27600 (0.145 sec)
INFO:tensorflow:global_step/sec: 708.787
INFO:tensorflow:loss = 0.054858394, step = 27700 (0.141 sec)
INFO:tensorflow:global_step/sec: 749.613
INFO:tensorflow:loss = 0.1919387, step = 27800 (0.133 sec)
INFO:tensorflow:global_step/sec: 676.335
INFO:tensorflow:loss = 0.035167128, step = 27900 (0.148 sec)
INFO:tensorflow:global_step/sec: 756.045
INFO:tensorflow:loss = 0.06809587, step = 28000 (0.132 sec)
INFO:tensorflow:global_step/sec: 745.169
INFO:tensorflow:loss = 0.020983905, step = 28100 (0.134 sec)
INFO:tensorflow:global_step/sec: 731.403
INFO:tensorflow:loss = 0.009154754, step = 28200 (0.137 sec)
INFO:tensorflow:global_step/sec: 761.517
INFO:tensorflow:loss = 0.013444103, step = 28300 (0.132 sec)
INFO:tensorflow:global_step/sec: 735.488
INFO:tensorflow:loss = 0.019219223, step = 28400 (0.136 sec)
INFO:tensorflow:global_step/sec: 701.509
INFO:tensorflow:loss = 0.0007980311, step = 28500 (0.143 sec)
INFO:tensorflow:global_step/sec: 672.906
INFO:tensorflow:loss = 0.037370995, step = 28600 (0.148 sec)
INFO:tensorflow:global_step/sec: 693.067
INFO:tensorflow:loss = 0.034441676, step = 28700 (0.146 sec)
INFO:tensorflow:global_step/sec: 715.073
INFO:tensorflow:loss = 0.019205665, step = 28800 (0.139 sec)
INFO:tensorflow:global_step/sec: 754.271
INFO:tensorflow:loss = 0.04330594, step = 28900 (0.133 sec)
INFO:tensorflow:global_step/sec: 717.943
INFO:tensorflow:loss = 0.02258328, step = 29000 (0.139 sec)
INFO:tensorflow:global_step/sec: 778.761
INFO:tensorflow:loss = 0.033814937, step = 29100 (0.128 sec)
INFO:tensorflow:global_step/sec: 736.08
INFO:tensorflow:loss = 0.06819173, step = 29200 (0.136 sec)
INFO:tensorflow:global_step/sec: 694.266
INFO:tensorflow:loss = 0.03306775, step = 29300 (0.144 sec)
INFO:tensorflow:global_step/sec: 717.808
INFO:tensorflow:loss = 0.044940256, step = 29400 (0.139 sec)
INFO:tensorflow:global_step/sec: 754.514
INFO:tensorflow:loss = 0.041515682, step = 29500 (0.132 sec)
INFO:tensorflow:global_step/sec: 741.502
INFO:tensorflow:loss = 0.0137351295, step = 29600 (0.135 sec)
INFO:tensorflow:global_step/sec: 726.773
INFO:tensorflow:loss = 0.058922045, step = 29700 (0.137 sec)
INFO:tensorflow:global_step/sec: 772.537
INFO:tensorflow:loss = 0.004406413, step = 29800 (0.129 sec)
INFO:tensorflow:global_step/sec: 719.1
INFO:tensorflow:loss = 0.02618828, step = 29900 (0.140 sec)
INFO:tensorflow:global_step/sec: 723.709
INFO:tensorflow:loss = 0.035493106, step = 30000 (0.138 sec)
INFO:tensorflow:global_step/sec: 762.042
INFO:tensorflow:loss = 0.032594115, step = 30100 (0.131 sec)
INFO:tensorflow:global_step/sec: 774.994
INFO:tensorflow:loss = 0.012513466, step = 30200 (0.129 sec)
INFO:tensorflow:global_step/sec: 727.626
INFO:tensorflow:loss = 0.030434133, step = 30300 (0.138 sec)
INFO:tensorflow:global_step/sec: 751.067
INFO:tensorflow:loss = 0.009930114, step = 30400 (0.133 sec)
INFO:tensorflow:global_step/sec: 740.545
INFO:tensorflow:loss = 0.012347832, step = 30500 (0.135 sec)
INFO:tensorflow:global_step/sec: 768.107
INFO:tensorflow:loss = 0.008713552, step = 30600 (0.131 sec)
INFO:tensorflow:global_step/sec: 737.775
INFO:tensorflow:loss = 0.01328897, step = 30700 (0.135 sec)
INFO:tensorflow:global_step/sec: 700.529
INFO:tensorflow:loss = 0.005648759, step = 30800 (0.143 sec)
INFO:tensorflow:global_step/sec: 741.498
INFO:tensorflow:loss = 0.045800988, step = 30900 (0.135 sec)
INFO:tensorflow:global_step/sec: 759.22
INFO:tensorflow:loss = 0.0049998416, step = 31000 (0.132 sec)
INFO:tensorflow:global_step/sec: 778.944
INFO:tensorflow:loss = 0.025347317, step = 31100 (0.128 sec)
INFO:tensorflow:global_step/sec: 756.877
INFO:tensorflow:loss = 0.02644661, step = 31200 (0.132 sec)
INFO:tensorflow:global_step/sec: 722.356
INFO:tensorflow:loss = 0.012870766, step = 31300 (0.138 sec)
INFO:tensorflow:global_step/sec: 659.329
INFO:tensorflow:loss = 0.030599056, step = 31400 (0.152 sec)
INFO:tensorflow:global_step/sec: 724.764
INFO:tensorflow:loss = 0.09482564, step = 31500 (0.139 sec)
INFO:tensorflow:global_step/sec: 738.252
INFO:tensorflow:loss = 0.019195255, step = 31600 (0.135 sec)
INFO:tensorflow:global_step/sec: 724.457
INFO:tensorflow:loss = 0.011149004, step = 31700 (0.138 sec)
INFO:tensorflow:global_step/sec: 749.343
INFO:tensorflow:loss = 0.011274818, step = 31800 (0.134 sec)
INFO:tensorflow:global_step/sec: 717.722
INFO:tensorflow:loss = 0.016880032, step = 31900 (0.139 sec)
INFO:tensorflow:global_step/sec: 686.074
INFO:tensorflow:loss = 0.034965005, step = 32000 (0.146 sec)
INFO:tensorflow:global_step/sec: 747.228
INFO:tensorflow:loss = 0.0711512, step = 32100 (0.134 sec)
INFO:tensorflow:global_step/sec: 758.061
INFO:tensorflow:loss = 0.020935431, step = 32200 (0.132 sec)
INFO:tensorflow:global_step/sec: 754.156
INFO:tensorflow:loss = 0.05625681, step = 32300 (0.133 sec)
INFO:tensorflow:global_step/sec: 752.545
INFO:tensorflow:loss = 0.008945845, step = 32400 (0.133 sec)
INFO:tensorflow:global_step/sec: 774.541
INFO:tensorflow:loss = 0.0025368293, step = 32500 (0.129 sec)
INFO:tensorflow:global_step/sec: 778.982
INFO:tensorflow:loss = 0.025532966, step = 32600 (0.128 sec)
INFO:tensorflow:global_step/sec: 749.828
INFO:tensorflow:loss = 0.024839351, step = 32700 (0.133 sec)
INFO:tensorflow:global_step/sec: 740.441
INFO:tensorflow:loss = 0.05547411, step = 32800 (0.135 sec)
INFO:tensorflow:global_step/sec: 721.51
INFO:tensorflow:loss = 0.09110242, step = 32900 (0.139 sec)
INFO:tensorflow:global_step/sec: 763.617
INFO:tensorflow:loss = 0.027063178, step = 33000 (0.132 sec)
INFO:tensorflow:global_step/sec: 704.674
INFO:tensorflow:loss = 0.14633262, step = 33100 (0.141 sec)
INFO:tensorflow:global_step/sec: 754.149
INFO:tensorflow:loss = 0.007330508, step = 33200 (0.133 sec)
INFO:tensorflow:global_step/sec: 733.544
INFO:tensorflow:loss = 0.008549547, step = 33300 (0.136 sec)
INFO:tensorflow:global_step/sec: 750.367
INFO:tensorflow:loss = 0.024134047, step = 33400 (0.133 sec)
INFO:tensorflow:global_step/sec: 756.464
INFO:tensorflow:loss = 0.0074961022, step = 33500 (0.132 sec)
INFO:tensorflow:global_step/sec: 778.015
INFO:tensorflow:loss = 0.018352449, step = 33600 (0.128 sec)
INFO:tensorflow:global_step/sec: 790.108
INFO:tensorflow:loss = 0.016640741, step = 33700 (0.127 sec)
INFO:tensorflow:global_step/sec: 781.526
INFO:tensorflow:loss = 0.0017197947, step = 33800 (0.128 sec)
INFO:tensorflow:global_step/sec: 785.147
INFO:tensorflow:loss = 0.0111332275, step = 33900 (0.127 sec)
INFO:tensorflow:global_step/sec: 760.909
INFO:tensorflow:loss = 0.040406324, step = 34000 (0.132 sec)
INFO:tensorflow:global_step/sec: 772.981
INFO:tensorflow:loss = 0.012423429, step = 34100 (0.129 sec)
INFO:tensorflow:global_step/sec: 781.531
INFO:tensorflow:loss = 0.015810683, step = 34200 (0.128 sec)
INFO:tensorflow:global_step/sec: 720.78
INFO:tensorflow:loss = 0.0105865635, step = 34300 (0.139 sec)
INFO:tensorflow:global_step/sec: 734.963
INFO:tensorflow:loss = 0.022520129, step = 34400 (0.136 sec)
INFO:tensorflow:global_step/sec: 687.737
INFO:tensorflow:loss = 0.007853265, step = 34500 (0.145 sec)
INFO:tensorflow:global_step/sec: 755.826
INFO:tensorflow:loss = 0.017153783, step = 34600 (0.133 sec)
INFO:tensorflow:global_step/sec: 733.16
INFO:tensorflow:loss = 0.04800478, step = 34700 (0.136 sec)
INFO:tensorflow:global_step/sec: 687.501
INFO:tensorflow:loss = 0.022419866, step = 34800 (0.145 sec)
INFO:tensorflow:global_step/sec: 614.493
INFO:tensorflow:loss = 0.0017927951, step = 34900 (0.163 sec)
INFO:tensorflow:global_step/sec: 745.438
INFO:tensorflow:loss = 0.04672495, step = 35000 (0.134 sec)
INFO:tensorflow:global_step/sec: 757.189
INFO:tensorflow:loss = 0.0890024, step = 35100 (0.132 sec)
INFO:tensorflow:global_step/sec: 751.348
INFO:tensorflow:loss = 0.0034388623, step = 35200 (0.133 sec)
INFO:tensorflow:global_step/sec: 737.144
INFO:tensorflow:loss = 0.033513878, step = 35300 (0.136 sec)
INFO:tensorflow:global_step/sec: 730.169
INFO:tensorflow:loss = 0.0026553052, step = 35400 (0.137 sec)
INFO:tensorflow:global_step/sec: 716.332
INFO:tensorflow:loss = 0.040838942, step = 35500 (0.140 sec)
INFO:tensorflow:global_step/sec: 693.586
INFO:tensorflow:loss = 0.023297546, step = 35600 (0.144 sec)
INFO:tensorflow:global_step/sec: 766.726
INFO:tensorflow:loss = 0.039266217, step = 35700 (0.130 sec)
INFO:tensorflow:global_step/sec: 800.56
INFO:tensorflow:loss = 0.006094549, step = 35800 (0.125 sec)
INFO:tensorflow:global_step/sec: 779.842
INFO:tensorflow:loss = 0.024597563, step = 35900 (0.128 sec)
INFO:tensorflow:global_step/sec: 791.645
INFO:tensorflow:loss = 0.0005974211, step = 36000 (0.127 sec)
INFO:tensorflow:global_step/sec: 760.571
INFO:tensorflow:loss = 0.008686724, step = 36100 (0.131 sec)
INFO:tensorflow:global_step/sec: 779.966
INFO:tensorflow:loss = 0.003942257, step = 36200 (0.128 sec)
INFO:tensorflow:global_step/sec: 786.328
INFO:tensorflow:loss = 0.012746293, step = 36300 (0.127 sec)
INFO:tensorflow:global_step/sec: 609.898
INFO:tensorflow:loss = 0.03244071, step = 36400 (0.164 sec)
INFO:tensorflow:global_step/sec: 701.438
INFO:tensorflow:loss = 0.0060920594, step = 36500 (0.142 sec)
INFO:tensorflow:global_step/sec: 789.399
INFO:tensorflow:loss = 0.02364352, step = 36600 (0.127 sec)
INFO:tensorflow:global_step/sec: 798.938
INFO:tensorflow:loss = 0.03801505, step = 36700 (0.125 sec)
INFO:tensorflow:global_step/sec: 643.811
INFO:tensorflow:loss = 0.016948957, step = 36800 (0.155 sec)
INFO:tensorflow:global_step/sec: 664.83
INFO:tensorflow:loss = 0.017786317, step = 36900 (0.151 sec)
INFO:tensorflow:global_step/sec: 543.587
INFO:tensorflow:loss = 0.058662392, step = 37000 (0.183 sec)
INFO:tensorflow:global_step/sec: 632.46
INFO:tensorflow:loss = 0.014718251, step = 37100 (0.158 sec)
INFO:tensorflow:global_step/sec: 775.433
INFO:tensorflow:loss = 0.047604885, step = 37200 (0.129 sec)
INFO:tensorflow:global_step/sec: 786.68
INFO:tensorflow:loss = 0.002843613, step = 37300 (0.127 sec)
INFO:tensorflow:global_step/sec: 799.111
INFO:tensorflow:loss = 0.023884777, step = 37400 (0.125 sec)
INFO:tensorflow:global_step/sec: 756.131
INFO:tensorflow:loss = 0.017163115, step = 37500 (0.133 sec)
INFO:tensorflow:global_step/sec: 633.787
INFO:tensorflow:loss = 0.026272407, step = 37600 (0.157 sec)
INFO:tensorflow:global_step/sec: 723.891
INFO:tensorflow:loss = 0.0028121169, step = 37700 (0.138 sec)
INFO:tensorflow:global_step/sec: 754.086
INFO:tensorflow:loss = 0.016791945, step = 37800 (0.133 sec)
INFO:tensorflow:global_step/sec: 701.247
INFO:tensorflow:loss = 0.047742378, step = 37900 (0.143 sec)
INFO:tensorflow:global_step/sec: 682.181
INFO:tensorflow:loss = 0.04361719, step = 38000 (0.146 sec)
INFO:tensorflow:global_step/sec: 786.376
INFO:tensorflow:loss = 0.038325347, step = 38100 (0.127 sec)
INFO:tensorflow:global_step/sec: 804.283
INFO:tensorflow:loss = 0.010711223, step = 38200 (0.124 sec)
INFO:tensorflow:global_step/sec: 770.032
INFO:tensorflow:loss = 0.11263705, step = 38300 (0.130 sec)
INFO:tensorflow:global_step/sec: 759.344
INFO:tensorflow:loss = 0.06331622, step = 38400 (0.132 sec)
INFO:tensorflow:global_step/sec: 793.153
INFO:tensorflow:loss = 0.03488349, step = 38500 (0.126 sec)
INFO:tensorflow:global_step/sec: 748.129
INFO:tensorflow:loss = 0.024869196, step = 38600 (0.133 sec)
INFO:tensorflow:global_step/sec: 760.015
INFO:tensorflow:loss = 0.049668543, step = 38700 (0.131 sec)
INFO:tensorflow:global_step/sec: 795.537
INFO:tensorflow:loss = 0.024653997, step = 38800 (0.126 sec)
INFO:tensorflow:global_step/sec: 754.715
INFO:tensorflow:loss = 0.0064418544, step = 38900 (0.132 sec)
INFO:tensorflow:global_step/sec: 805.211
INFO:tensorflow:loss = 0.025671637, step = 39000 (0.124 sec)
INFO:tensorflow:global_step/sec: 763.917
INFO:tensorflow:loss = 0.007240239, step = 39100 (0.131 sec)
INFO:tensorflow:global_step/sec: 756.067
INFO:tensorflow:loss = 0.0044421586, step = 39200 (0.132 sec)
INFO:tensorflow:global_step/sec: 778.837
INFO:tensorflow:loss = 0.024619173, step = 39300 (0.128 sec)
INFO:tensorflow:global_step/sec: 754.662
INFO:tensorflow:loss = 0.012689214, step = 39400 (0.132 sec)
INFO:tensorflow:global_step/sec: 746.614
INFO:tensorflow:loss = 0.018057879, step = 39500 (0.134 sec)
INFO:tensorflow:global_step/sec: 694.068
INFO:tensorflow:loss = 0.02785282, step = 39600 (0.145 sec)
INFO:tensorflow:global_step/sec: 654.979
INFO:tensorflow:loss = 0.012543851, step = 39700 (0.152 sec)
INFO:tensorflow:global_step/sec: 731.693
INFO:tensorflow:loss = 0.034165256, step = 39800 (0.137 sec)
INFO:tensorflow:global_step/sec: 712.66
INFO:tensorflow:loss = 0.018817844, step = 39900 (0.140 sec)
INFO:tensorflow:global_step/sec: 767.888
INFO:tensorflow:loss = 0.064586915, step = 40000 (0.130 sec)
INFO:tensorflow:global_step/sec: 682.678
INFO:tensorflow:loss = 0.039944634, step = 40100 (0.147 sec)
INFO:tensorflow:global_step/sec: 637.928
INFO:tensorflow:loss = 0.006592812, step = 40200 (0.158 sec)
INFO:tensorflow:global_step/sec: 654.42
INFO:tensorflow:loss = 0.0012000705, step = 40300 (0.152 sec)
INFO:tensorflow:global_step/sec: 735.869
INFO:tensorflow:loss = 0.0021813547, step = 40400 (0.136 sec)
INFO:tensorflow:global_step/sec: 739.684
INFO:tensorflow:loss = 0.020016534, step = 40500 (0.135 sec)
INFO:tensorflow:global_step/sec: 722.897
INFO:tensorflow:loss = 0.010720066, step = 40600 (0.138 sec)
INFO:tensorflow:global_step/sec: 750.822
INFO:tensorflow:loss = 0.05658368, step = 40700 (0.133 sec)
INFO:tensorflow:global_step/sec: 761.754
INFO:tensorflow:loss = 0.013007379, step = 40800 (0.131 sec)
INFO:tensorflow:global_step/sec: 721.558
INFO:tensorflow:loss = 0.0246561, step = 40900 (0.139 sec)
INFO:tensorflow:global_step/sec: 658.625
INFO:tensorflow:loss = 0.01263381, step = 41000 (0.152 sec)
INFO:tensorflow:global_step/sec: 691.157
INFO:tensorflow:loss = 0.004255068, step = 41100 (0.144 sec)
INFO:tensorflow:global_step/sec: 710.883
INFO:tensorflow:loss = 0.015441139, step = 41200 (0.141 sec)
INFO:tensorflow:global_step/sec: 797.458
INFO:tensorflow:loss = 0.010338774, step = 41300 (0.125 sec)
INFO:tensorflow:global_step/sec: 747.132
INFO:tensorflow:loss = 0.021103038, step = 41400 (0.134 sec)
INFO:tensorflow:global_step/sec: 750.845
INFO:tensorflow:loss = 0.025617313, step = 41500 (0.133 sec)
INFO:tensorflow:global_step/sec: 754.583
INFO:tensorflow:loss = 0.013572226, step = 41600 (0.133 sec)
INFO:tensorflow:global_step/sec: 756.386
INFO:tensorflow:loss = 0.0033336552, step = 41700 (0.132 sec)
INFO:tensorflow:global_step/sec: 782.906
INFO:tensorflow:loss = 0.053168457, step = 41800 (0.128 sec)
INFO:tensorflow:global_step/sec: 719.6
INFO:tensorflow:loss = 0.029688315, step = 41900 (0.139 sec)
INFO:tensorflow:global_step/sec: 788.136
INFO:tensorflow:loss = 0.01654516, step = 42000 (0.127 sec)
INFO:tensorflow:global_step/sec: 738.526
INFO:tensorflow:loss = 0.015659308, step = 42100 (0.135 sec)
INFO:tensorflow:global_step/sec: 777.578
INFO:tensorflow:loss = 0.01840079, step = 42200 (0.129 sec)
INFO:tensorflow:global_step/sec: 786.416
INFO:tensorflow:loss = 0.021663003, step = 42300 (0.127 sec)
INFO:tensorflow:global_step/sec: 763.347
INFO:tensorflow:loss = 0.011599571, step = 42400 (0.131 sec)
INFO:tensorflow:global_step/sec: 762.321
INFO:tensorflow:loss = 0.0044469903, step = 42500 (0.131 sec)
INFO:tensorflow:global_step/sec: 768.549
INFO:tensorflow:loss = 0.0019147585, step = 42600 (0.130 sec)
INFO:tensorflow:global_step/sec: 771.429
INFO:tensorflow:loss = 0.0054854164, step = 42700 (0.130 sec)
INFO:tensorflow:global_step/sec: 793.871
INFO:tensorflow:loss = 0.0017117725, step = 42800 (0.126 sec)
INFO:tensorflow:global_step/sec: 770.1
INFO:tensorflow:loss = 0.012048513, step = 42900 (0.130 sec)
INFO:tensorflow:global_step/sec: 744.636
INFO:tensorflow:loss = 0.06634566, step = 43000 (0.134 sec)
INFO:tensorflow:global_step/sec: 696.882
INFO:tensorflow:loss = 0.0003919307, step = 43100 (0.144 sec)
INFO:tensorflow:global_step/sec: 705.516
INFO:tensorflow:loss = 0.06582007, step = 43200 (0.141 sec)
INFO:tensorflow:global_step/sec: 699.244
INFO:tensorflow:loss = 0.0038124803, step = 43300 (0.143 sec)
INFO:tensorflow:global_step/sec: 792.079
INFO:tensorflow:loss = 0.003364585, step = 43400 (0.126 sec)
INFO:tensorflow:global_step/sec: 753.586
INFO:tensorflow:loss = 0.00725976, step = 43500 (0.133 sec)
INFO:tensorflow:global_step/sec: 720.951
INFO:tensorflow:loss = 0.024148291, step = 43600 (0.139 sec)
INFO:tensorflow:global_step/sec: 770.384
INFO:tensorflow:loss = 0.013779048, step = 43700 (0.130 sec)
INFO:tensorflow:global_step/sec: 799.363
INFO:tensorflow:loss = 0.014951154, step = 43800 (0.125 sec)
INFO:tensorflow:global_step/sec: 791.774
INFO:tensorflow:loss = 0.0015594304, step = 43900 (0.126 sec)
INFO:tensorflow:Saving checkpoints for 44000 into /tmp/tmpuflzeb_h/model.ckpt.
INFO:tensorflow:Loss for final step: 0.0012097486.
Out[11]:
<tensorflow.python.estimator.canned.dnn.DNNClassifier at 0x7f4f62b23be0>
In [12]:
test_input_fn = tf.estimator.inputs.numpy_input_fn(
    x={"X": X_test}, y=y_test, shuffle=False)
eval_results = dnn_clf.evaluate(input_fn=test_input_fn)

INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Starting evaluation at 2018-05-18-19:12:49
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /tmp/tmpuflzeb_h/model.ckpt-44000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
INFO:tensorflow:Finished evaluation at 2018-05-18-19:12:50
INFO:tensorflow:Saving dict for global step 44000: accuracy = 0.9798, average_loss = 0.10096103, global_step = 44000, loss = 12.779877
In [13]:
eval_results
Out[13]:
{'accuracy': 0.9798,
 'average_loss': 0.10096103,
 'global_step': 44000,
 'loss': 12.779877}
In [14]:
y_pred_iter = dnn_clf.predict(input_fn=test_input_fn)
y_pred = list(y_pred_iter)
y_pred[0]

INFO:tensorflow:Calling model_fn.
INFO:tensorflow:Done calling model_fn.
INFO:tensorflow:Graph was finalized.
INFO:tensorflow:Restoring parameters from /tmp/tmpuflzeb_h/model.ckpt-44000
INFO:tensorflow:Running local_init_op.
INFO:tensorflow:Done running local_init_op.
Out[14]:
{'class_ids': array([7]),
 'classes': array([b'7'], dtype=object),
 'logits': array([ -3.809414 ,  -4.1564407,  -0.426081 ,   3.2636993, -11.065331 ,
         -8.790985 , -10.436305 ,  19.935707 ,  -6.9282775,   2.2807484],
       dtype=float32),
 'probabilities': array([4.8710768e-11, 3.4428106e-11, 1.4354495e-09, 5.7469666e-08,
        3.4389070e-14, 3.3431518e-13, 6.4506329e-14, 1.0000000e+00,
        2.1533745e-12, 2.1505466e-08], dtype=float32)}

Using plain TensorFlow

In [15]:
import tensorflow as tf

n_inputs = 28*28  # MNIST
n_hidden1 = 300
n_hidden2 = 100
n_outputs = 10
In [16]:
reset_graph()

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int32, shape=(None), name="y")
In [17]:
def neuron_layer(X, n_neurons, name, activation=None):
    with tf.name_scope(name):
        n_inputs = int(X.get_shape()[1])
        stddev = 2 / np.sqrt(n_inputs)
        init = tf.truncated_normal((n_inputs, n_neurons), stddev=stddev)
        W = tf.Variable(init, name="kernel")
        b = tf.Variable(tf.zeros([n_neurons]), name="bias")
        Z = tf.matmul(X, W) + b
        if activation is not None:
            return activation(Z)
        else:
            return Z
In [18]:
with tf.name_scope("dnn"):
    hidden1 = neuron_layer(X, n_hidden1, name="hidden1",
                           activation=tf.nn.relu)
    hidden2 = neuron_layer(hidden1, n_hidden2, name="hidden2",
                           activation=tf.nn.relu)
    logits = neuron_layer(hidden2, n_outputs, name="outputs")
In [19]:
with tf.name_scope("loss"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y,
                                                              logits=logits)
    loss = tf.reduce_mean(xentropy, name="loss")
In [20]:
learning_rate = 0.01

with tf.name_scope("train"):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    training_op = optimizer.minimize(loss)
In [21]:
with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
In [22]:
init = tf.global_variables_initializer()
saver = tf.train.Saver()
In [23]:
n_epochs = 40
batch_size = 50
In [24]:
def shuffle_batch(X, y, batch_size):
    rnd_idx = np.random.permutation(len(X))
    n_batches = len(X) // batch_size
    for batch_idx in np.array_split(rnd_idx, n_batches):
        X_batch, y_batch = X[batch_idx], y[batch_idx]
        yield X_batch, y_batch
In [25]:
with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        acc_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid})
        print(epoch, "Batch accuracy:", acc_batch, "Val accuracy:", acc_val)

    save_path = saver.save(sess, "./my_model_final.ckpt")

0 Batch accuracy: 0.9 Val accuracy: 0.9146
1 Batch accuracy: 0.92 Val accuracy: 0.936
2 Batch accuracy: 0.96 Val accuracy: 0.945
3 Batch accuracy: 0.92 Val accuracy: 0.9512
4 Batch accuracy: 0.98 Val accuracy: 0.9558
5 Batch accuracy: 0.96 Val accuracy: 0.9566
6 Batch accuracy: 1.0 Val accuracy: 0.9612
7 Batch accuracy: 0.94 Val accuracy: 0.963
8 Batch accuracy: 0.98 Val accuracy: 0.9652
9 Batch accuracy: 0.96 Val accuracy: 0.966
10 Batch accuracy: 0.92 Val accuracy: 0.9688
11 Batch accuracy: 0.98 Val accuracy: 0.969
12 Batch accuracy: 0.98 Val accuracy: 0.967
13 Batch accuracy: 0.98 Val accuracy: 0.9706
14 Batch accuracy: 1.0 Val accuracy: 0.9714
15 Batch accuracy: 0.94 Val accuracy: 0.9732
16 Batch accuracy: 1.0 Val accuracy: 0.9736
17 Batch accuracy: 1.0 Val accuracy: 0.9742
18 Batch accuracy: 1.0 Val accuracy: 0.9746
19 Batch accuracy: 0.98 Val accuracy: 0.9748
20 Batch accuracy: 1.0 Val accuracy: 0.9752
21 Batch accuracy: 1.0 Val accuracy: 0.9752
22 Batch accuracy: 0.98 Val accuracy: 0.9764
23 Batch accuracy: 0.98 Val accuracy: 0.9752
24 Batch accuracy: 0.98 Val accuracy: 0.9772
25 Batch accuracy: 1.0 Val accuracy: 0.977
26 Batch accuracy: 0.98 Val accuracy: 0.9778
27 Batch accuracy: 1.0 Val accuracy: 0.9774
28 Batch accuracy: 0.96 Val accuracy: 0.9754
29 Batch accuracy: 0.98 Val accuracy: 0.9776
30 Batch accuracy: 1.0 Val accuracy: 0.9756
31 Batch accuracy: 0.98 Val accuracy: 0.9772
32 Batch accuracy: 0.98 Val accuracy: 0.9772
33 Batch accuracy: 0.98 Val accuracy: 0.979
34 Batch accuracy: 1.0 Val accuracy: 0.9784
35 Batch accuracy: 1.0 Val accuracy: 0.9778
36 Batch accuracy: 0.98 Val accuracy: 0.978
37 Batch accuracy: 1.0 Val accuracy: 0.9776
38 Batch accuracy: 1.0 Val accuracy: 0.9792
39 Batch accuracy: 1.0 Val accuracy: 0.9776
In [26]:
with tf.Session() as sess:
    saver.restore(sess, "./my_model_final.ckpt") # or better, use save_path
    X_new_scaled = X_test[:20]
    Z = logits.eval(feed_dict={X: X_new_scaled})
    y_pred = np.argmax(Z, axis=1)

INFO:tensorflow:Restoring parameters from ./my_model_final.ckpt
In [27]:
print("Predicted classes:", y_pred)
print("Actual classes:   ", y_test[:20])

Predicted classes: [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]
Actual classes:    [7 2 1 0 4 1 4 9 5 9 0 6 9 0 1 5 9 7 3 4]
In [28]:
from tensorflow_graph_in_jupyter import show_graph
In [29]:
show_graph(tf.get_default_graph())

Using dense() instead of neuron_layer()

Note: previous releases of the book used tensorflow.contrib.layers.fully_connected() rather than tf.layers.dense() (which did not exist when this chapter was written). It is now preferable to use tf.layers.dense(), because anything in the contrib module may change or be deleted without notice. The dense() function is almost identical to the fully_connected() function, except for a few minor differences:

  • several parameters are renamed: scope becomes name, activation_fn becomes activation (and similarly the _fn suffix is removed from other parameters such as normalizer_fn), weights_initializer becomes kernel_initializer, etc.
  • the default activation is now None rather than tf.nn.relu.
  • a few more differences are presented in chapter 11.
In [30]:
n_inputs = 28*28  # MNIST
n_hidden1 = 300
n_hidden2 = 100
n_outputs = 10
In [31]:
reset_graph()

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int32, shape=(None), name="y")
In [32]:
with tf.name_scope("dnn"):
    hidden1 = tf.layers.dense(X, n_hidden1, name="hidden1",
                              activation=tf.nn.relu)
    hidden2 = tf.layers.dense(hidden1, n_hidden2, name="hidden2",
                              activation=tf.nn.relu)
    logits = tf.layers.dense(hidden2, n_outputs, name="outputs")
    y_proba = tf.nn.softmax(logits)
In [33]:
with tf.name_scope("loss"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
    loss = tf.reduce_mean(xentropy, name="loss")
In [34]:
learning_rate = 0.01

with tf.name_scope("train"):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    training_op = optimizer.minimize(loss)
In [35]:
with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
In [36]:
init = tf.global_variables_initializer()
saver = tf.train.Saver()
In [37]:
n_epochs = 20
n_batches = 50

with tf.Session() as sess:
    init.run()
    for epoch in range(n_epochs):
        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
        acc_batch = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
        acc_valid = accuracy.eval(feed_dict={X: X_valid, y: y_valid})
        print(epoch, "Batch accuracy:", acc_batch, "Validation accuracy:", acc_valid)

    save_path = saver.save(sess, "./my_model_final.ckpt")

0 Batch accuracy: 0.9 Validation accuracy: 0.9024
1 Batch accuracy: 0.92 Validation accuracy: 0.9254
2 Batch accuracy: 0.94 Validation accuracy: 0.9372
3 Batch accuracy: 0.9 Validation accuracy: 0.9416
4 Batch accuracy: 0.94 Validation accuracy: 0.9472
5 Batch accuracy: 0.94 Validation accuracy: 0.9512
6 Batch accuracy: 1.0 Validation accuracy: 0.9548
7 Batch accuracy: 0.94 Validation accuracy: 0.961
8 Batch accuracy: 0.96 Validation accuracy: 0.962
9 Batch accuracy: 0.94 Validation accuracy: 0.9648
10 Batch accuracy: 0.92 Validation accuracy: 0.9656
11 Batch accuracy: 0.98 Validation accuracy: 0.9668
12 Batch accuracy: 0.98 Validation accuracy: 0.9684
13 Batch accuracy: 0.98 Validation accuracy: 0.9702
14 Batch accuracy: 1.0 Validation accuracy: 0.9696
15 Batch accuracy: 0.94 Validation accuracy: 0.9718
16 Batch accuracy: 0.98 Validation accuracy: 0.9728
17 Batch accuracy: 1.0 Validation accuracy: 0.973
18 Batch accuracy: 0.98 Validation accuracy: 0.9748
19 Batch accuracy: 0.98 Validation accuracy: 0.9756
In [38]:
show_graph(tf.get_default_graph())

Exercise solutions

1. to 8.

See appendix A.

9.

Train a deep MLP on the MNIST dataset and see if you can get over 98% precision. Just like in the last exercise of chapter 9, try adding all the bells and whistles (i.e., save checkpoints, restore the last checkpoint in case of an interruption, add summaries, plot learning curves using TensorBoard, and so on).

First let's create the deep net. It's exactly the same as earlier, with just one addition: we add a tf.summary.scalar() to track the loss and the accuracy during training, so we can view nice learning curves using TensorBoard.

In [40]:
n_inputs = 28*28  # MNIST
n_hidden1 = 300
n_hidden2 = 100
n_outputs = 10
In [41]:
reset_graph()

X = tf.placeholder(tf.float32, shape=(None, n_inputs), name="X")
y = tf.placeholder(tf.int32, shape=(None), name="y")
In [42]:
with tf.name_scope("dnn"):
    hidden1 = tf.layers.dense(X, n_hidden1, name="hidden1",
                              activation=tf.nn.relu)
    hidden2 = tf.layers.dense(hidden1, n_hidden2, name="hidden2",
                              activation=tf.nn.relu)
    logits = tf.layers.dense(hidden2, n_outputs, name="outputs")
In [43]:
with tf.name_scope("loss"):
    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
    loss = tf.reduce_mean(xentropy, name="loss")
    loss_summary = tf.summary.scalar('log_loss', loss)
In [44]:
learning_rate = 0.01

with tf.name_scope("train"):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    training_op = optimizer.minimize(loss)
In [45]:
with tf.name_scope("eval"):
    correct = tf.nn.in_top_k(logits, y, 1)
    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))
    accuracy_summary = tf.summary.scalar('accuracy', accuracy)
In [46]:
init = tf.global_variables_initializer()
saver = tf.train.Saver()

Now we need to define the directory to write the TensorBoard logs to:

In [47]:
from datetime import datetime

def log_dir(prefix=""):
    now = datetime.utcnow().strftime("%Y%m%d%H%M%S")
    root_logdir = "tf_logs"
    if prefix:
        prefix += "-"
    name = prefix + "run-" + now
    return "{}/{}/".format(root_logdir, name)
In [48]:
logdir = log_dir("mnist_dnn")

Now we can create the FileWriter that we will use to write the TensorBoard logs:

In [49]:
file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())

Hey! Why don't we implement early stopping? For this, we are going to need to use the validation set.

In [50]:
m, n = X_train.shape
In [52]:
n_epochs = 10001
batch_size = 50
n_batches = int(np.ceil(m / batch_size))

checkpoint_path = "/tmp/my_deep_mnist_model.ckpt"
checkpoint_epoch_path = checkpoint_path + ".epoch"
final_model_path = "./my_deep_mnist_model"

best_loss = np.infty
epochs_without_progress = 0
max_epochs_without_progress = 50

with tf.Session() as sess:
    if os.path.isfile(checkpoint_epoch_path):
        # if the checkpoint file exists, restore the model and load the epoch number
        with open(checkpoint_epoch_path, "rb") as f:
            start_epoch = int(f.read())
        print("Training was interrupted. Continuing at epoch", start_epoch)
        saver.restore(sess, checkpoint_path)
    else:
        start_epoch = 0
        sess.run(init)

    for epoch in range(start_epoch, n_epochs):
        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):
            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})
        accuracy_val, loss_val, accuracy_summary_str, loss_summary_str = sess.run([accuracy, loss, accuracy_summary, loss_summary], feed_dict={X: X_valid, y: y_valid})
        file_writer.add_summary(accuracy_summary_str, epoch)
        file_writer.add_summary(loss_summary_str, epoch)
        if epoch % 5 == 0:
            print("Epoch:", epoch,
                  "\tValidation accuracy: {:.3f}%".format(accuracy_val * 100),
                  "\tLoss: {:.5f}".format(loss_val))
            saver.save(sess, checkpoint_path)
            with open(checkpoint_epoch_path, "wb") as f:
                f.write(b"%d" % (epoch + 1))
            if loss_val < best_loss:
                saver.save(sess, final_model_path)
                best_loss = loss_val
            else:
                epochs_without_progress += 5
                if epochs_without_progress > max_epochs_without_progress:
                    print("Early stopping")
                    break

Epoch: 0 	Validation accuracy: 92.180% 	Loss: 0.30208
Epoch: 5 	Validation accuracy: 95.980% 	Loss: 0.15037
Epoch: 10 	Validation accuracy: 97.100% 	Loss: 0.11160
Epoch: 15 	Validation accuracy: 97.700% 	Loss: 0.09562
Epoch: 20 	Validation accuracy: 97.840% 	Loss: 0.08309
Epoch: 25 	Validation accuracy: 98.040% 	Loss: 0.07706
Epoch: 30 	Validation accuracy: 98.140% 	Loss: 0.07287
Epoch: 35 	Validation accuracy: 98.280% 	Loss: 0.07133
Epoch: 40 	Validation accuracy: 98.220% 	Loss: 0.06968
Epoch: 45 	Validation accuracy: 98.220% 	Loss: 0.06993
Epoch: 50 	Validation accuracy: 98.160% 	Loss: 0.07093
Epoch: 55 	Validation accuracy: 98.280% 	Loss: 0.06994
Epoch: 60 	Validation accuracy: 98.200% 	Loss: 0.06894
Epoch: 65 	Validation accuracy: 98.260% 	Loss: 0.06906
Epoch: 70 	Validation accuracy: 98.220% 	Loss: 0.07057
Epoch: 75 	Validation accuracy: 98.280% 	Loss: 0.06963
Epoch: 80 	Validation accuracy: 98.320% 	Loss: 0.07264
Epoch: 85 	Validation accuracy: 98.200% 	Loss: 0.07403
Epoch: 90 	Validation accuracy: 98.300% 	Loss: 0.07332
Epoch: 95 	Validation accuracy: 98.180% 	Loss: 0.07535
Epoch: 100 	Validation accuracy: 98.260% 	Loss: 0.07542
Early stopping
In [53]:
os.remove(checkpoint_epoch_path)
In [54]:
with tf.Session() as sess:
    saver.restore(sess, final_model_path)
    accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test})

INFO:tensorflow:Restoring parameters from ./my_deep_mnist_model
In [55]:
accuracy_val
Out[55]:
0.9796
In [ ]: