diff --git a/.gitignore b/.gitignore index db4561e..ead0dec 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ docs/_build/ # PyBuilder target/ + +#ipython notebook +.ipynb_checkpoints/ diff --git a/README.md b/README.md index 0db9e9f..f879126 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,6 @@ Theano-Tutorials ================ Bare bones introduction to machine learning from linear regression to convolutional neural networks using Theano. + +See [ipython notebooks](http://nbviewer.ipython.org/github/udibr/Theano-Tutorials/blob/master/notebooks/index.ipynb) + diff --git a/load.py b/load.py index 25d931d..cae4cd5 100644 --- a/load.py +++ b/load.py @@ -13,19 +13,19 @@ def one_hot(x,n): def mnist(ntrain=60000,ntest=10000,onehot=True): data_dir = os.path.join(datasets_dir,'mnist/') - fd = open(os.path.join(data_dir,'train-images.idx3-ubyte')) + fd = open(os.path.join(data_dir,'train-images-idx3-ubyte')) loaded = np.fromfile(file=fd,dtype=np.uint8) trX = loaded[16:].reshape((60000,28*28)).astype(float) - fd = open(os.path.join(data_dir,'train-labels.idx1-ubyte')) + fd = open(os.path.join(data_dir,'train-labels-idx1-ubyte')) loaded = np.fromfile(file=fd,dtype=np.uint8) trY = loaded[8:].reshape((60000)) - fd = open(os.path.join(data_dir,'t10k-images.idx3-ubyte')) + fd = open(os.path.join(data_dir,'t10k-images-idx3-ubyte')) loaded = np.fromfile(file=fd,dtype=np.uint8) teX = loaded[16:].reshape((10000,28*28)).astype(float) - fd = open(os.path.join(data_dir,'t10k-labels.idx1-ubyte')) + fd = open(os.path.join(data_dir,'t10k-labels-idx1-ubyte')) loaded = np.fromfile(file=fd,dtype=np.uint8) teY = loaded[8:].reshape((10000)) @@ -45,4 +45,4 @@ def mnist(ntrain=60000,ntest=10000,onehot=True): trY = np.asarray(trY) teY = np.asarray(teY) - return trX,teX,trY,teY \ No newline at end of file + return trX,teX,trY,teY diff --git a/notebooks/0_multiply.ipynb b/notebooks/0_multiply.ipynb new file mode 100644 index 0000000..fb96d08 --- /dev/null +++ b/notebooks/0_multiply.ipynb @@ -0,0 +1,143 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:f13eb3b414e3e34aa7531d120d38fcd6602672de65630585a7952395a63c3f7e" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "import Theano" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "tensor is the section of theano that deals with data (like numpy)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "create Theano symbolic variables" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "a = T.scalar()\n", + "b = T.scalar()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "our model" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "y = a * b" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "compile to a python function (to CPU or GPU depending on configuration)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "multiply = theano.function(inputs=[a, b], outputs=y)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and use the function" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "multiply(1, 2)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 5, + "text": [ + "array(2.0)" + ] + } + ], + "prompt_number": 5 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "multiply(3, 3)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 6, + "text": [ + "array(9.0)" + ] + } + ], + "prompt_number": 6 + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/1_linear_regression.ipynb b/notebooks/1_linear_regression.ipynb new file mode 100644 index 0000000..2f8e70b --- /dev/null +++ b/notebooks/1_linear_regression.ipynb @@ -0,0 +1,240 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:65a56707d32e8e45ec327145d45b5a9a273bc13b6c18fc9b7e7bf8c84fdf53ba" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "imports" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "import numpy as np" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "generate data with noise" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "trX = np.linspace(-1, 1, 101)\n", + "trY = 2 * trX + np.random.randn(*trX.shape) * 0.33" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "symbolic vriable initalization" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "X = T.scalar()\n", + "Y = T.scalar()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "our model" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w):\n", + " return X * w" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "model parameter initalization. hyper variables, with real value" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w = theano.shared(np.asarray(0., dtype=theano.config.floatX))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "y = model(X, w)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "metric to be optimized by model" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "cost = T.mean(T.sqr(y - Y))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "learning signal for parameters. Computes the gradient symbolically" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "gradient = T.grad(cost=cost, wrt=w)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 9 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "how to update in each step. 0.01 is the learning rate" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "updates = [[w, w - gradient * 0.01]]" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 10 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "compile to a python function. `allow_input_downcast=True` is set to ignore typing issue with Theano on different platforms (GPU handle only 32bit)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 11 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "iterate 100 times over the entire data (epoch.) In each epoch iterate over all the data samples" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "for i in range(100):\n", + " for x, y in zip(trX, trY):\n", + " train(x, y)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 12 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we should get something close to the true weight of the data: 2" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w.get_value()" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 13, + "text": [ + "array(1.9382810308787022)" + ] + } + ], + "prompt_number": 13 + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/2_logistic_regression.ipynb b/notebooks/2_logistic_regression.ipynb new file mode 100644 index 0000000..568c985 --- /dev/null +++ b/notebooks/2_logistic_regression.ipynb @@ -0,0 +1,1095 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:8e7467687cabf437572893b54a343b6de5be3c2f06c6247823de13c0201bf14a" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os\n", + "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our input is now the handwritten digits, each is a vector" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "import numpy as np" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "Using gpu device 0: GeForce GT 650M\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Convert to correct dtype" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def floatX(X):\n", + " return np.asarray(X, dtype=theano.config.floatX)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "initalize model parameters (small random guassian distribution)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def init_weights(shape):\n", + " return theano.shared(floatX(np.random.randn(*shape) * 0.01))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In our model we will replace scalars with vectors" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "X = T.fmatrix()\n", + "Y = T.fmatrix()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and weights with matrices" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w = init_weights((784, 10))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and use softmax convert our weighted input to probability" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w):\n", + " return T.nnet.softmax(T.dot(X, w))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "probability output" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "py_x = model(X, w)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 9 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "maxima prediction" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "y_pred = T.argmax(py_x, axis=1)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 10 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "different cost function, maximize the value that is correct and minize the others" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))\n", + "gradient = T.grad(cost=cost, wrt=w)\n", + "update = [[w, w - gradient * 0.05]]\n", + "\n", + "train = theano.function(inputs=[X, Y], outputs=cost, updates=update, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 11 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "now we also have a predict function that will be used on the test data" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "predict = theano.function(inputs=[X], outputs=y_pred, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 12 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "train on mini-batches of 128 samples each" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%time\n", + "for i in range(100):\n", + " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", + " cost = train(trX[start:end], trY[start:end])\n", + " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " errs = []\n", + " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", + " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", + " print i, np.mean(np.concatenate(errs))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0 0.918269230769\n", + "1" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.918569711538\n", + "2" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.918970352564\n", + "3" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919070512821\n", + "4" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919170673077\n", + "5" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919671474359\n", + "6" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919871794872\n", + "7" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920372596154\n", + "8" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920572916667\n", + "9" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920372596154\n", + "10" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920572916667\n", + "11" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920673076923\n", + "12" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920572916667\n", + "13" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920673076923\n", + "14" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921274038462\n", + "15" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921474358974\n", + "16" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921674679487\n", + "17" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921875\n", + "18" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921674679487\n", + "19" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921474358974\n", + "20" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921474358974\n", + "21" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921774839744\n", + "22" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921975160256\n", + "23" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921875\n", + "24" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921574519231\n", + "25" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921875\n", + "26" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922075320513\n", + "27" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922275641026\n", + "28" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922275641026\n", + "29" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922375801282\n", + "30" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922576121795\n", + "31" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922375801282\n", + "32" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922576121795\n", + "33" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922576121795\n", + "34" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922676282051\n", + "35" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922976762821\n", + "36" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922776442308\n", + "37" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922776442308\n", + "38" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922876602564\n", + "39" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923076923077\n", + "40" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923076923077\n", + "41" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.92327724359\n", + "42" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923377403846\n", + "43" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923677884615\n", + "44" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923778044872\n", + "45" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "46" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "47" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "48" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "49" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923878205128\n", + "50" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "51" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924078525641\n", + "52" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923978365385\n", + "53" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924078525641\n", + "54" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924078525641\n", + "55" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924178685897\n", + "56" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924479166667\n", + "57" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "58" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "59" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "60" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "61" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "62" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "63" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "64" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "65" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "66" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "67" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "68" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "69" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "70" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "71" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "72" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "73" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "74" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "75" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "76" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "77" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "78" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "79" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "80" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "81" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "82" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "83" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "84" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924979967949\n", + "85" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "86" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "87" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "88" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "89" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "90" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924879807692\n", + "91" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "92" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924779647436\n", + "93" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "94" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "95" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "96" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "97" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "98" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "99" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924679487179\n", + "CPU times: user 1min 17s, sys: 4.96 s, total: 1min 22s\n", + "Wall time: 1min 23s\n" + ] + } + ], + "prompt_number": 14 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "prediction is not as good as kNN" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "28.5 sec for CPU. GPU was 1min23s or x3 slower :-(" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/3_net.ipynb b/notebooks/3_net.ipynb new file mode 100644 index 0000000..56f4443 --- /dev/null +++ b/notebooks/3_net.ipynb @@ -0,0 +1,1002 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:fc684fd1168901d7d18baf83d926aaf0042beea79127f8b118f4da677b8dc70b" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "An \"old\" network before 2006" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os\n", + "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "import numpy as np\n", + "\n", + "def floatX(X):\n", + " return np.asarray(X, dtype=theano.config.floatX)\n", + "\n", + "def init_weights(shape):\n", + " return theano.shared(floatX(np.random.randn(*shape) * 0.01))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "Using gpu device 0: GeForce GT 650M\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Generalize to compute stochastic gradient descent (SGD) on all model parametes. Note that SGD has a fixed learning rate which slow things down as you near the goal" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def sgd(cost, params, lr=0.05):\n", + " grads = T.grad(cost=cost, wrt=params)\n", + " updates = []\n", + " for p, g in zip(params, grads):\n", + " updates.append([p, p - g * lr])\n", + " return updates" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "logistic regression (LR) model is replaced with a neural network (NN) model with one hidden layer" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w_h, w_o):\n", + " h = T.nnet.sigmoid(T.dot(X, w_h))\n", + " pyx = T.nnet.softmax(T.dot(h, w_o))\n", + " return pyx" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "X = T.fmatrix()\n", + "Y = T.fmatrix()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have 625 hidden units" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w_h = init_weights((784, 625))\n", + "w_o = init_weights((625, 10))\n", + "\n", + "py_x = model(X, w_h, w_o)\n", + "y_x = T.argmax(py_x, axis=1)\n", + "\n", + "cost = T.mean(T.nnet.categorical_crossentropy(py_x, Y))\n", + "params = [w_h, w_o]\n", + "updates = sgd(cost, params)\n", + "\n", + "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n", + "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%time\n", + "for i in range(100):\n", + " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", + " cost = train(trX[start:end], trY[start:end])\n", + " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " errs = []\n", + " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", + " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", + " print i, np.mean(np.concatenate(errs))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0 0.708233173077\n", + "1" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.831129807692\n", + "2" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.868189102564\n", + "3" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.883413461538\n", + "4" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.889623397436\n", + "5" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.894931891026\n", + "6" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.8984375\n", + "7" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.901642628205\n", + "8" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.90484775641\n", + "9" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.907251602564\n", + "10" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.909154647436\n", + "11" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.911157852564\n", + "12" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.912960737179\n", + "13" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.913361378205\n", + "14" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.914763621795\n", + "15" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.915765224359\n", + "16" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.916466346154\n", + "17" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.916967147436\n", + "18" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.917367788462\n", + "19" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.918269230769\n", + "20" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919070512821\n", + "21" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919571314103\n", + "22" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919571314103\n", + "23" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.919571314103\n", + "24" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920272435897\n", + "25" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.920873397436\n", + "26" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921073717949\n", + "27" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921474358974\n", + "28" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.921774839744\n", + "29" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922475961538\n", + "30" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922475961538\n", + "31" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.922776442308\n", + "32" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.923677884615\n", + "33" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924579326923\n", + "34" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.924979967949\n", + "35" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.925280448718\n", + "36" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.926081730769\n", + "37" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.926181891026\n", + "38" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.926682692308\n", + "39" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.927483974359\n", + "40" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.927984775641\n", + "41" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.928485576923\n", + "42" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.928886217949\n", + "43" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.929487179487\n", + "44" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.929887820513\n", + "45" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.930588942308\n", + "46" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.931189903846\n", + "47" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.93108974359\n", + "48" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.931690705128\n", + "49" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.932391826923\n", + "50" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.933994391026\n", + "51" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.934495192308\n", + "52" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.93499599359\n", + "53" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.935697115385\n", + "54" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.936698717949\n", + "55" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.937600160256\n", + "56" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.938000801282\n", + "57" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.938401442308\n", + "58" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.939002403846\n", + "59" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.939503205128\n", + "60" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.939903846154\n", + "61" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.940604967949\n", + "62" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.941205929487\n", + "63" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.941706730769\n", + "64" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.941907051282\n", + "65" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.942608173077\n", + "66" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.943309294872\n", + "67" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.943709935897\n", + "68" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.944110576923\n", + "69" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.945212339744\n", + "70" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.945913461538\n", + "71" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.946314102564\n", + "72" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.946614583333\n", + "73" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.94671474359\n", + "74" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.946915064103\n", + "75" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.947215544872\n", + "76" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.947315705128\n", + "77" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.94781650641\n", + "78" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.948617788462\n", + "79" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.949018429487\n", + "80" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.949719551282\n", + "81" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.950721153846\n", + "82" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.951021634615\n", + "83" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.951522435897\n", + "84" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.951923076923\n", + "85" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.952223557692\n", + "86" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.952924679487\n", + "87" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.953525641026\n", + "88" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.953826121795\n", + "89" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.954427083333\n", + "90" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.955028044872\n", + "91" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.955228365385\n", + "92" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.955528846154\n", + "93" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.955729166667\n", + "94" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956029647436\n", + "95" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956330128205\n", + "96" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956330128205\n", + "97" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956730769231\n", + "98" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.956931089744\n", + "99" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.957331730769\n", + "CPU times: user 2min 9s, sys: 5.21 s, total: 2min 14s\n", + "Wall time: 2min 14s\n" + ] + } + ], + "prompt_number": 8 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "CPU 2min 56s, GPU 2min 14s. The result are 0.957 and not 0.98 as discussed in the video lecture" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/4_modern_net.ipynb b/notebooks/4_modern_net.ipynb new file mode 100644 index 0000000..acc84a2 --- /dev/null +++ b/notebooks/4_modern_net.ipynb @@ -0,0 +1,1075 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:a24946d3f359951c3345d365cd78ed271a7c2fe1ce2db20406363ca25c9b1806" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "two main changes: replace sigmoid with rectifier (LeRU). The problem of overfitting of old NN is solved by adding noise. This allows to add hidden layers (in the fast this caused overfitting.)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os\n", + "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n", + "import numpy as np\n", + "\n", + "srng = RandomStreams()\n", + "\n", + "def floatX(X):\n", + " return np.asarray(X, dtype=theano.config.floatX)\n", + "\n", + "def init_weights(shape):\n", + " return theano.shared(floatX(np.random.randn(*shape) * 0.01))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "replacement for sigmoid function" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def rectify(X):\n", + " return T.maximum(X, 0.)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "modify softmax to remove max value to avoid explosition of exp (numeric stability)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def softmax(X):\n", + " e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n", + " return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we use a running RMS average of the gradient to scale the gradient size" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):\n", + " grads = T.grad(cost=cost, wrt=params)\n", + " updates = []\n", + " for p, g in zip(params, grads):\n", + " acc = theano.shared(p.get_value() * 0.) # acc is allocated for each parameter (p) with 0 values with the shape of p\n", + " acc_new = rho * acc + (1 - rho) * g ** 2\n", + " gradient_scaling = T.sqrt(acc_new + epsilon)\n", + " g = g / gradient_scaling\n", + " updates.append((acc, acc_new))\n", + " updates.append((p, p - lr * g))\n", + " return updates" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "inject noise with dropout. randomly set to zero some (p) of the nodes. We have to compensate by scaling the nodes that were not dropped out." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def dropout(X, p=0.):\n", + " if p > 0:\n", + " retain_prob = 1 - p\n", + " X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX)\n", + " X /= retain_prob\n", + " return X" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "new model has two hidden layers of 625 nodes" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w_h, w_h2, w_o, p_drop_input, p_drop_hidden):\n", + " X = dropout(X, p_drop_input)\n", + " h = rectify(T.dot(X, w_h))\n", + "\n", + " h = dropout(h, p_drop_hidden)\n", + " h2 = rectify(T.dot(h, w_h2))\n", + "\n", + " h2 = dropout(h2, p_drop_hidden)\n", + " py_x = softmax(T.dot(h2, w_o))\n", + " return h, h2, py_x" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "X = T.fmatrix()\n", + "Y = T.fmatrix()\n", + "\n", + "w_h = init_weights((784, 625))\n", + "w_h2 = init_weights((625, 625))\n", + "w_o = init_weights((625, 10))\n", + "\n", + "noise_h, noise_h2, noise_py_x = model(X, w_h, w_h2, w_o, 0.2, 0.5)\n", + "h, h2, py_x = model(X, w_h, w_h2, w_o, 0., 0.)\n", + "y_x = T.argmax(py_x, axis=1)\n", + "\n", + "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n", + "params = [w_h, w_h2, w_o]\n", + "updates = RMSprop(cost, params, lr=0.001)\n", + "\n", + "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n", + "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "/Users/udi/anaconda/lib/python2.7/site-packages/theano/sandbox/rng_mrg.py:1195: UserWarning: MRG_RandomStreams Can't determine #streams from size (Shape.0), guessing 60*256\n", + " nstreams = self.n_streams(size)\n" + ] + } + ], + "prompt_number": 9 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%time\n", + "for i in range(100):\n", + " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", + " cost = train(trX[start:end], trY[start:end])\n", + " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " errs = []\n", + " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", + " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", + " print i, np.mean(np.concatenate(errs))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "0 0.940104166667\n", + "1" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.965344551282\n", + "2" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.972856570513\n", + "3" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.973858173077\n", + "4" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.975260416667\n", + "5" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.978665865385\n", + "6" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.978866185897\n", + "7" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.979166666667\n", + "8" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.982171474359\n", + "9" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.980268429487\n", + "10" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.982171474359\n", + "11" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98297275641\n", + "12" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983072916667\n", + "13" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983673878205\n", + "14" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984074519231\n", + "15" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983673878205\n", + "16" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983373397436\n", + "17" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984074519231\n", + "18" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984775641026\n", + "19" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.983874198718\n", + "20" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985977564103\n", + "21" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984174679487\n", + "22" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984675480769\n", + "23" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984274839744\n", + "24" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984775641026\n", + "25" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "26" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985176282051\n", + "27" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.984575320513\n", + "28" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985076121795\n", + "29" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985576923077\n", + "30" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98577724359\n", + "31" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98577724359\n", + "32" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98687900641\n", + "33" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985376602564\n", + "34" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986077724359\n", + "35" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985877403846\n", + "36" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985677083333\n", + "37" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98687900641\n", + "38" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986678685897\n", + "39" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98577724359\n", + "40" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985376602564\n", + "41" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986578525641\n", + "42" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986478365385\n", + "43" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985677083333\n", + "44" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986278044872\n", + "45" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "46" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "47" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.988581730769\n", + "48" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986378205128\n", + "49" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987479967949\n", + "50" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985977564103\n", + "51" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.98687900641\n", + "52" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987580128205\n", + "53" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "54" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985877403846\n", + "55" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986077724359\n", + "56" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "57" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986077724359\n", + "58" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "59" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985877403846\n", + "60" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "61" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "62" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "63" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "64" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987780448718\n", + "65" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986678685897\n", + "66" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987079326923\n", + "67" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987079326923\n", + "68" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986678685897\n", + "69" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "70" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986177884615\n", + "71" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986378205128\n", + "72" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "73" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "74" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987780448718\n", + "75" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986678685897\n", + "76" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987179487179\n", + "77" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987980769231\n", + "78" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.985977564103\n", + "79" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986578525641\n", + "80" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "81" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.988782051282\n", + "82" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987379807692\n", + "83" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "84" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987479967949\n", + "85" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987980769231\n", + "86" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987479967949\n", + "87" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987580128205\n", + "88" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986578525641\n", + "89" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987680288462\n", + "90" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987880608974\n", + "91" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "92" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986979166667\n", + "93" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "94" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.986778846154\n", + "95" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987780448718\n", + "96" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987580128205\n", + "97" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987980769231\n", + "98" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987279647436\n", + "99" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.987079326923\n", + "CPU times: user 4min 53s, sys: 30.4 s, total: 5min 23s\n", + "Wall time: 5min 23s\n" + ] + } + ], + "prompt_number": 10 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "CPU 17min 4s, GPU 5min 23s" + ], + "language": "python", + "metadata": {}, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "98.8% not 99% in video" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/5_convolutional_net.ipynb b/notebooks/5_convolutional_net.ipynb new file mode 100644 index 0000000..9503c00 --- /dev/null +++ b/notebooks/5_convolutional_net.ipynb @@ -0,0 +1,1109 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:5d9f76b1d1dc426c1b07e4480adfabee73388a181a66dd7bbdf1e9c1e0896efe" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Use 2D convolution - respect the 2D structure of the data by running the same network (same weights) on small 2D patches of the image." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "import os\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On OSX this example is really slow and the GPU cause the Mac to shutdown! So run it on AWS with GPU (e.g. g2.2xlarge )" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import theano\n", + "from theano import tensor as T\n", + "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams\n", + "import numpy as np\n", + "from theano.tensor.nnet.conv import conv2d\n", + "from theano.tensor.signal.downsample import max_pool_2d\n", + "\n", + "srng = RandomStreams()\n", + "\n", + "def floatX(X):\n", + " return np.asarray(X, dtype=theano.config.floatX)\n", + "\n", + "def init_weights(shape):\n", + " return theano.shared(floatX(np.random.randn(*shape) * 0.01))\n", + "\n", + "def rectify(X):\n", + " return T.maximum(X, 0.)\n", + "\n", + "def softmax(X):\n", + " e_x = T.exp(X - X.max(axis=1).dimshuffle(0, 'x'))\n", + " return e_x / e_x.sum(axis=1).dimshuffle(0, 'x')\n", + "\n", + "def dropout(X, p=0.):\n", + " if p > 0:\n", + " retain_prob = 1 - p\n", + " X *= srng.binomial(X.shape, p=retain_prob, dtype=theano.config.floatX, nstreams=60*256)\n", + " X /= retain_prob\n", + " return X\n", + "\n", + "def RMSprop(cost, params, lr=0.001, rho=0.9, epsilon=1e-6):\n", + " grads = T.grad(cost=cost, wrt=params)\n", + " updates = []\n", + " for p, g in zip(params, grads):\n", + " acc = theano.shared(p.get_value() * 0.)\n", + " acc_new = rho * acc + (1 - rho) * g ** 2\n", + " gradient_scaling = T.sqrt(acc_new + epsilon)\n", + " g = g / gradient_scaling\n", + " updates.append((acc, acc_new))\n", + " updates.append((p, p - lr * g))\n", + " return updates" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "Using gpu device 0: GRID K520\n" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "block to compute conv->activate->pool->noise" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def model(X, w, w2, w3, w4, w_o, p_drop_conv, p_drop_hidden):\n", + " # block to compute conv->activate->pool->noise\n", + " l1a = rectify(conv2d(X, w, border_mode='full'))\n", + " l1 = max_pool_2d(l1a, (2, 2))\n", + " l1 = dropout(l1, p_drop_conv)\n", + "\n", + " l2a = rectify(conv2d(l1, w2))\n", + " l2 = max_pool_2d(l2a, (2, 2))\n", + " l2 = dropout(l2, p_drop_conv)\n", + "\n", + " l3a = rectify(conv2d(l2, w3))\n", + " l3b = max_pool_2d(l3a, (2, 2))\n", + " # return back to original vector represnation\n", + " l3 = T.flatten(l3b, outdim=2)\n", + " l3 = dropout(l3, p_drop_conv)\n", + "\n", + " l4 = rectify(T.dot(l3, w4))\n", + " l4 = dropout(l4, p_drop_hidden)\n", + "\n", + " pyx = softmax(T.dot(l4, w_o))\n", + " return l1, l2, l3, l4, pyx" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "convert data from vector to color images, but we have just one color channel " + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "trX = trX.reshape(-1, 1, 28, 28)\n", + "teX = teX.reshape(-1, 1, 28, 28)\n", + "\n", + "X = T.ftensor4()\n", + "Y = T.fmatrix()" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "covolution weights (n_kernels, n_channels, kernel_w, kernel_h). We will use 3x3 kernel. Smallest possible. Instead of bigger kernels use more layers. It looks as if each convolution layer is twice as big as the previous one, but keep in mind that we perform maxpool which will reduce the dimension by factor of two" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w = init_weights((32, 1, 3, 3))\n", + "w2 = init_weights((64, 32, 3, 3))\n", + "w3 = init_weights((128, 64, 3, 3))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "last layer is flat but its input size must match the size of the previous convolution layer" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "w4 = init_weights((128 * 3 * 3, 625))\n", + "w_o = init_weights((625, 10))" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "noise during training" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "noise_l1, noise_l2, noise_l3, noise_l4, noise_py_x = model(X, w, w2, w3, w4, w_o, 0.2, 0.5)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 8 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "noiseless model for testing" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "l1, l2, l3, l4, py_x = model(X, w, w2, w3, w4, w_o, 0., 0.)\n", + "y_x = T.argmax(py_x, axis=1)\n", + "\n", + "\n", + "cost = T.mean(T.nnet.categorical_crossentropy(noise_py_x, Y))\n", + "params = [w, w2, w3, w4, w_o]\n", + "updates = RMSprop(cost, params, lr=0.001)\n", + "\n", + "train = theano.function(inputs=[X, Y], outputs=cost, updates=updates, allow_input_downcast=True)\n", + "predict = theano.function(inputs=[X], outputs=y_x, allow_input_downcast=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "/usr/local/lib/python2.7/dist-packages/theano/tensor/subtensor.py:110: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.\n", + " start in [None, 0] or\n", + "/usr/local/lib/python2.7/dist-packages/theano/tensor/subtensor.py:114: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.\n", + " stop in [None, length, maxsize] or\n" + ] + } + ], + "prompt_number": 9 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%time\n", + "for i in range(100):\n", + " for start, end in zip(range(0, len(trX), 128), range(128, len(trX), 128)):\n", + " cost = train(trX[start:end], trY[start:end])\n", + " #print '%d:%.4f'%(start/128,cost),\n", + " #print\n", + " # prediction is also done in minibatches because the entire test data does not fit the Mac GPU :-(\n", + " errs = []\n", + " for start, end in zip(range(0, len(teX), 128), range(128, len(teX), 128)):\n", + " errs.append(np.argmax(teY[start:end], axis=1) == predict(teX[start:end]))\n", + " print '###',i, np.mean(np.concatenate(errs))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "### 0 0.934795673077\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 1 0.973657852564\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 2 0.982872596154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 3 0.987179487179\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 4 0.987479967949\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 5 0.989783653846\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 6 0.987379807692\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 7 0.991987179487\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 8 0.991786858974\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 9 0.991786858974\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 10 0.992287660256\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 11 0.992487980769\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 12 0.991786858974\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 13 0.9921875\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 14 0.992588141026\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 15 0.993689903846\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 16 0.992788461538\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 17 0.993289262821\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 18 0.992287660256\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 19 0.992588141026\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 20 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 21 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 22 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 23 0.993289262821\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 24 0.993189102564\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 25 0.993489583333\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 26 0.993389423077\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 27 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 28 0.993790064103\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 29 0.993489583333\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 30 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 31 0.993289262821\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 32 0.993189102564\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 33 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 34 0.994190705128\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 35 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 36 0.993489583333\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 37 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 38 0.993790064103\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 39 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 40 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 41 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 42 0.993790064103\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 43 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 44 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 45 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 46 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 47 0.993389423077\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 48 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 49 0.993389423077\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 50 0.994290865385\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 51 0.994290865385\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 52 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 53 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 54 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 55 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 56 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 57 0.994090544872\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 58 0.99469150641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 59 0.993990384615\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 60 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 61 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 62 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 63 0.994190705128\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 64 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 65 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 66 0.994491185897\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 67 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 68 0.995092147436\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 69 0.995192307692\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 70 0.99469150641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 71 0.994491185897\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 72 0.995092147436\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 73 0.994891826923\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 74 0.993489583333\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 75 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 76 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 77 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 78 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 79 0.993790064103\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 80 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 81 0.994491185897\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 82 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 83 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 84 0.994290865385\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 85 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 86 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 87 0.994891826923\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 88 0.994991987179\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 89 0.99358974359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 90 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 91 0.995092147436\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 92 0.994290865385\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 93 0.994791666667\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 94 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 95 0.994391025641\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 96 0.994591346154\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 97 0.994190705128\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 98 0.993890224359\n", + "###" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 99 0.994391025641\n", + "CPU times: user 1h 34min 43s, sys: 45min 13s, total: 2h 19min 57s\n", + "Wall time: 2h 19min 56s\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "GPU 2h 19min 56s 0.994-0.995" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/KNN.ipynb b/notebooks/KNN.ipynb new file mode 100644 index 0000000..6fbe425 --- /dev/null +++ b/notebooks/KNN.ipynb @@ -0,0 +1,102 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:2a506df8c7b7baaab1282610827bb2333815c5150a1098d957f2d4d75d49ef8b" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Simplest way to perform classification of the handwritten digits is to use nearest neighbor" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "import os\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trLabel, teLabel = load.mnist(onehot=False)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import sklearn.neighbors\n", + "clf = sklearn.neighbors.KNeighborsClassifier(n_neighbors=1)\n", + "clf.fit(trX,trLabel)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 3, + "text": [ + "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n", + " metric_params=None, n_neighbors=1, p=2, weights='uniform')" + ] + } + ], + "prompt_number": 3 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "tePredict = clf.predict(teX)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "np.mean(tePredict == teLabel)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 5, + "text": [ + "0.96909999999999996" + ] + } + ], + "prompt_number": 5 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "which works very well but not good enough (0.995 or more)" + ] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/notebooks/index.ipynb b/notebooks/index.ipynb new file mode 100644 index 0000000..0a74ee8 --- /dev/null +++ b/notebooks/index.ipynb @@ -0,0 +1,877 @@ +{ + "metadata": { + "name": "", + "signature": "sha256:cb17786a324ccc4066b6446428b8d06831376bd8ca1f416866ee0d68af35f208" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Follow the lecture [Introduction to Deep Learning with Python](https://www.youtube.com/watch?v=S75EdAcXHKk)" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from IPython.display import YouTubeVideo\n", + "YouTubeVideo('S75EdAcXHKk')" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "html": [ + "\n", + " \n", + " " + ], + "metadata": {}, + "output_type": "pyout", + "prompt_number": 1, + "text": [ + "" + ] + } + ], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "after performing the set described below you can walk through the notebooks:\n", + " \n", + "* [KNN](./KNN.ipynb)\n", + "* [Mulitply two numbers with Theano](./0_multiply.ipynb)\n", + "* [Linear regression](./1_linear_regression.ipynb)\n", + "* [Logistic Regression](./2_logistic_regression.ipynb)\n", + "* [Neural Network](./3_net.ipynb)\n", + "* [Modern Neural Network](./4_modern_net.ipynb)\n", + "* [Convolution Neural Network](./5_convolutional_net.ipynb)" + ] + }, + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "Get Data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "download mnist files from http://yann.lecun.com/exdb/mnist/ and open the zipped file into `~/Downloads/lisa/data/mnist/`" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "%%bash\n", + "mkdir -p ~/Downloads/lisa/data/mnist/\n", + "cd ~/Downloads/lisa/data/mnist/\n", + "#wget http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n", + "#wget http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n", + "#wget http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n", + "#wget http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n", + "gunzip *.gz" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "!ls ~/Downloads/lisa/data/mnist/" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "t10k-images-idx3-ubyte\ttrain-images-idx3-ubyte\r\n", + "t10k-labels-idx1-ubyte\ttrain-labels-idx1-ubyte\r\n" + ] + } + ], + "prompt_number": 5 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os,sys,inspect\n", + "currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))\n", + "parentdir = os.path.dirname(currentdir)\n", + "sys.path.insert(0,parentdir) \n", + "import load\n", + "\n", + "load.datasets_dir = os.path.expanduser(\"~/Downloads/lisa/data/\")\n", + "\n", + "trX, teX, trY, teY = load.mnist(onehot=True)" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "we have 60K images of handwritten numbers for training and 10K for testing. Each image has 28*28 gray-level pixels" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "trX.shape, teX.shape, trY.shape, teY.shape" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 7, + "text": [ + "((60000, 784), (10000, 784), (60000, 10), (10000, 10))" + ] + } + ], + "prompt_number": 7 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import numpy as np\n", + "np.min(trX),np.max(trX),np.min(teX),np.max(teX)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 9, + "text": [ + "(0.0, 1.0, 0.0, 1.0)" + ] + } + ], + "prompt_number": 9 + }, + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "Installation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Install Theano. On OS X it is highly recommended to [install using anaconda](http://deeplearning.net/software/theano/install.html#anaconda-1-5). If you also [installed MKL](http://continuum.io/blog/mkl-optimizations), which is also highly recommended, you will need to export the following environment variables before running `ipython notebook`\n", + "\n", + " export DYLD_FALLBACK_LIBRARY_PATH=$HOME/anaconda/lib:$DYLD_FALLBACK_LIBRARY_PATH" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On AWS I started from the following community image \"Ubuntu-14.04-Caffe-GPU - ami-588d0030\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "install Theano" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "!sudo pip install theano" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Downloading/unpacking theano\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " Downloading Theano-0.6.0.tar.gz (1.8MB): \r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 0% 4.1kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 0% 8.2kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 0% 12kB \r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 0% 16kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1% 20kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1% 24kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1% 28kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1% 32kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 2% 36kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 2% 40kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 2% 45kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 2% 49kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 53kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 57kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 61kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 65kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 3% 69kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 4% 73kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 4% 77kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 4% 81kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 4% 86kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 5% 90kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 5% 94kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 5% 98kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 5% 102kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 106kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 110kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 114kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 118kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 6% 122kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 7% 126kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 7% 131kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 7% 135kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 7% 139kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 8% 143kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 8% 147kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 8% 151kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 8% 155kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 159kB" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 163kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 167kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 172kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 9% 176kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 10% 180kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 10% 184kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 10% 188kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 10% 192kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 11% 196kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 11% 200kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 11% 204kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 11% 208kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 12% 212kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 12% 217kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 12% 221kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 12% 225kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 229kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 233kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 237kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 241kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 13% 245kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 14% 249kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 14% 253kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 14% 258kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 14% 262kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 15% 266kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 15% 270kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 15% 274kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 15% 278kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 282kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 286kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 290kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 294kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 16% 299kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 17% 303kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 17% 307kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 17% 311kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 17% 315kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 18% 319kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 18% 323kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 18% 327kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 18% 331kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 335kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 339kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 344kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 348kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 19% 352kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 20% 356kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 20% 360kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 20% 364kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 20% 368kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 21% 372kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 21% 376kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 21% 380kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 21% 385kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 389kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 393kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 397kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 401kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 22% 405kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 23% 409kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 23% 413kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 23% 417kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 23% 421kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 24% 425kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 24% 430kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 24% 434kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 24% 438kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 25% 442kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 25% 446kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 25% 450kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 25% 454kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 458kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 462kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 466kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 471kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 26% 475kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 27% 479kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 27% 483kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 27% 487kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 27% 491kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 28% 495kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 28% 499kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 28% 503kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 28% 507kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 512kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 516kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 520kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 524kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 29% 528kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 30% 532kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 30% 536kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 30% 540kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 30% 544kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 31% 548kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 31% 552kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 31% 557kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 31% 561kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 565kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 569kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 573kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 577kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 32% 581kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 33% 585kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 33% 589kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 33% 593kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 33% 598kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 34% 602kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 34% 606kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 34% 610kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 34% 614kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 618kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 622kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 626kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 630kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 35% 634kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 36% 638kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 36% 643kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 36% 647kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 36% 651kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 37% 655kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 37% 659kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 37% 663kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 37% 667kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 38% 671kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 38% 675kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 38% 679kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 38% 684kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 688kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 692kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 696kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 700kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 39% 704kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 40% 708kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 40% 712kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 40% 716kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 40% 720kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 41% 724kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 41% 729kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 41% 733kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 41% 737kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 741kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 745kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 749kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 753kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 42% 757kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 43% 761kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 43% 765kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 43% 770kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 43% 774kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 44% 778kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 44% 782kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 44% 786kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 44% 790kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 794kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 798kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 802kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 806kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 45% 811kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 46% 815kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 46% 819kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 46% 823kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 46% 827kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 47% 831kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 47% 835kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 47% 839kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 47% 843kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 48% 847kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 48% 851kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 48% 856kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 48% 860kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 864kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 868kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 872kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 876kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 49% 880kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 50% 884kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 50% 888kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 50% 892kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 50% 897kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 51% 901kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 51% 905kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 51% 909kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 51% 913kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 917kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 921kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 925kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 929kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 52% 933kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 53% 937kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 53% 942kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 53% 946kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 53% 950kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 54% 954kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 54% 958kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 54% 962kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 54% 966kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 970kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 974kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 978kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 983kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 55% 987kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 56% 991kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 56% 995kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 56% 999kB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 56% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 57% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 57% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 57% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 57% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 58% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 59% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 59% 1.0MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 59% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 59% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 60% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 60% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 60% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 60% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 61% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 61% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 61% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 61% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 62% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 63% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 63% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 63% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 63% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 64% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 64% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 64% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 64% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.1MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 65% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 66% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 66% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 66% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 66% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 67% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 67% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 67% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 67% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 68% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 69% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 69% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 69% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 69% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 70% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 70% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 70% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 70% 1.2MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 71% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 72% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 72% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 72% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 72% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 73% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 73% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 73% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 73% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 74% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 74% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 74% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 74% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 75% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 76% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 76% 1.3MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 76% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 76% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 77% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 77% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 77% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 77% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 78% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 79% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 79% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 79% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 79% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 80% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 80% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 80% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 80% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 81% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 82% 1.4MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 82% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 82% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 82% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 83% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 83% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 83% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 83% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 84% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 85% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 85% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 85% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 85% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 86% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 86% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 86% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 86% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 87% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 87% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 87% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 87% 1.5MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 88% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 89% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 89% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 89% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 89% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 90% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 90% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 90% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 90% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 91% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 92% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 92% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 92% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 92% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 93% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 93% 1.6MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 93% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 93% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 94% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 95% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 95% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 95% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 95% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 96% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 96% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 96% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 96% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 97% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 97% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 97% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 97% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 98% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 99% 1.7MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 99% 1.8MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 99% 1.8MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 99% 1.8MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 100% 1.8MB\r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): \r", + " Downloading Theano-0.6.0.tar.gz (1.8MB): 1.8MB downloaded\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " Running setup.py (path:/tmp/pip_build_root/theano/setup.py) egg_info for package theano\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " \r\n", + " warning: manifest_maker: MANIFEST.in, line 7: 'recursive-include' expects ...\r\n", + " \r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "Requirement already satisfied (use --upgrade to upgrade): numpy>=1.5.0 in /usr/local/lib/python2.7/dist-packages/numpy-1.9.0-py2.7-linux-x86_64.egg (from theano)\r\n", + "Requirement already satisfied (use --upgrade to upgrade): scipy>=0.7.2 in /usr/local/lib/python2.7/dist-packages (from theano)\r\n", + "Installing collected packages: theano\r\n", + " Running setup.py install for theano\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " changing mode of build/scripts-2.7/theano-cache from 644 to 755\r\n", + " changing mode of build/scripts-2.7/theano-nose from 644 to 755\r\n", + " changing mode of build/scripts-2.7/theano-test from 644 to 755\r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " \r\n", + " warning: manifest_maker: MANIFEST.in, line 7: 'recursive-include' expects ...\r\n", + " \r\n" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " changing mode of /usr/local/bin/theano-nose to 755\r\n", + " changing mode of /usr/local/bin/theano-cache to 755\r\n", + " changing mode of /usr/local/bin/theano-test to 755\r\n", + "Successfully installed theano\r\n", + "Cleaning up...\r\n" + ] + } + ], + "prompt_number": 14 + }, + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "Using GPU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You will need to install CUDA and export the following environment variables before running `ipython notebook`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On OSX\n", + "```bash\n", + "export DYLD_FALLBACK_LIBRARY_PATH=/Developer/NVIDIA/CUDA-6.5/lib:$DYLD_FALLBACK_LIBRARY_PATH\n", + "export PATH=/Developer/NVIDIA/CUDA-6.5/bin:$PATH\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On AWS\n", + "```bash\n", + "export PATH=/usr/local/cuda-6.5/bin/:$PATH\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "you can control if CPU or GPU is used inside the notebook, however, you can only set it once. You need to restart the notebook if you want to change it" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import os\n", + "#os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=cpu,floatX=float32'\n", + "os.environ['THEANO_FLAGS'] = 'mode=FAST_RUN,device=gpu,floatX=float32'" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and check if you are using CPU or GPU" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from theano import function, config, shared, sandbox\n", + "import theano.tensor as T\n", + "import numpy\n", + "import time\n", + "\n", + "vlen = 10 * 30 * 768 # 10 x #cores x # threads per core\n", + "iters = 1000\n", + "\n", + "rng = numpy.random.RandomState(22)\n", + "x = shared(numpy.asarray(rng.rand(vlen), config.floatX))\n", + "f = function([], T.exp(x))\n", + "print f.maker.fgraph.toposort()\n", + "t0 = time.time()\n", + "for i in xrange(iters):\n", + " r = f()\n", + "t1 = time.time()\n", + "print 'Looping %d times took' % iters, t1 - t0, 'seconds'\n", + "print 'Result is', r\n", + "if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]):\n", + " print 'Used the cpu'\n", + "else:\n", + " print 'Used the gpu'" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "[GpuElemwise{exp,no_inplace}(), HostFromGpu(GpuElemwise{exp,no_inplace}.0)]\n", + "Looping 1000 times took" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + " 0.738588094711 seconds\n", + "Result is [ 1.23178029 1.61879349 1.52278066 ..., 2.20771813 2.29967761\n", + " 1.62323296]\n", + "Used the gpu\n" + ] + }, + { + "output_type": "stream", + "stream": "stderr", + "text": [ + "Using gpu device 0: GRID K520\n" + ] + } + ], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "On OSX, for this example, the GPU is x4-x5 faster than CPU" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [], + "language": "python", + "metadata": {}, + "outputs": [] + } + ], + "metadata": {} + } + ] +} \ No newline at end of file