From dfa204818eebb0a462a0f65dd35545854207d611 Mon Sep 17 00:00:00 2001 From: spike Date: Tue, 28 Mar 2017 20:08:55 +0200 Subject: [PATCH] wip project 3, build_nn --- ...dlnd_tv_script_generation-checkpoint.ipynb | 137 +++++++++++++++--- .../dlnd_tv_script_generation.ipynb | 137 +++++++++++++++--- 2 files changed, 234 insertions(+), 40 deletions(-) diff --git a/tv-script-generation/.ipynb_checkpoints/dlnd_tv_script_generation-checkpoint.ipynb b/tv-script-generation/.ipynb_checkpoints/dlnd_tv_script_generation-checkpoint.ipynb index ac7b12e..4351eab 100644 --- a/tv-script-generation/.ipynb_checkpoints/dlnd_tv_script_generation-checkpoint.ipynb +++ b/tv-script-generation/.ipynb_checkpoints/dlnd_tv_script_generation-checkpoint.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 21, "metadata": { "collapsed": false, "deletable": true, @@ -292,6 +292,35 @@ "int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Extra hyper parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from collections import namedtuple\n", + "\n", + "hyper_params = (('embedding_size', 128),\n", + " \n", + " )\n", + "\n", + "\n", + "\n", + "\n", + "Hyper = namedtuple('Hyper', map(lambda x: x[0], hyper_params))\n", + "HYPER = Hyper(*list(map(lambda x: x[1], hyper_params)))\n", + "\n" + ] + }, { "cell_type": "markdown", "metadata": { @@ -367,21 +396,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tests Passed\n" + ] + } + ], "source": [ "def get_inputs():\n", " \"\"\"\n", " Create TF Placeholders for input, targets, and learning rate.\n", " :return: Tuple (input, targets, learning rate)\n", " \"\"\"\n", - " # TODO: Implement Function\n", - " return None, None, None\n", + " \n", + " # We use shape [None, None] to feed any batch size and any sequence length\n", + " input_placeholder = tf.placeholder(tf.int32, [None, None],name='input')\n", + " \n", + " # Targets are [batch_size, seq_length]\n", + " targets_placeholder = tf.placeholder(tf.int32, [None, None]) \n", + " \n", + " \n", + " learning_rate_placeholder = tf.placeholder(tf.float32)\n", + " return input_placeholder, targets_placeholder, learning_rate_placeholder\n", "\n", "\n", "\"\"\"\n", @@ -408,13 +453,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tests Passed\n" + ] + } + ], "source": [ "def get_init_cell(batch_size, rnn_size):\n", " \"\"\"\n", @@ -423,8 +476,14 @@ " :param rnn_size: Size of RNNs\n", " :return: Tuple (cell, initialize state)\n", " \"\"\"\n", - " # TODO: Implement Function\n", - " return None, None\n", + " lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n", + " \n", + " cell = tf.contrib.rnn.MultiRNNCell([lstm] * 2)\n", + " \n", + " initial_state = cell.zero_state(batch_size, tf.float32)\n", + " initial_state = tf.identity(initial_state, name='initial_state')\n", + " \n", + " return cell, initial_state\n", "\n", "\n", "\"\"\"\n", @@ -446,13 +505,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tests Passed\n" + ] + } + ], "source": [ "def get_embed(input_data, vocab_size, embed_dim):\n", " \"\"\"\n", @@ -462,8 +529,13 @@ " :param embed_dim: Number of embedding dimensions\n", " :return: Embedded input.\n", " \"\"\"\n", - " # TODO: Implement Function\n", - " return None\n", + " embeddings = tf.Variable(\n", + " tf.random_uniform([vocab_size, embed_dim], -1.0, 1.0)\n", + " )\n", + " \n", + " embed = tf.nn.embedding_lookup(embeddings, input_data)\n", + " \n", + " return embed\n", "\n", "\n", "\"\"\"\n", @@ -489,13 +561,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tests Passed\n" + ] + } + ], "source": [ "def build_rnn(cell, inputs):\n", " \"\"\"\n", @@ -504,8 +584,11 @@ " :param inputs: Input text data\n", " :return: Tuple (Outputs, Final State)\n", " \"\"\"\n", - " # TODO: Implement Function\n", - " return None, None\n", + " outputs, final_state = tf.nn.dynamic_rnn(cell, inputs, dtype=tf.float32)\n", + " final_state = tf.identity(final_state, name='final_state')\n", + " \n", + " \n", + " return outputs, final_state\n", "\n", "\n", "\"\"\"\n", @@ -532,13 +615,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "ename": "AssertionError", + "evalue": "Final state doesn't have the \"name\" attribute. Are you using build_rnn?", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0mDON\u001b[0m\u001b[0;31m'\u001b[0m\u001b[0mT\u001b[0m \u001b[0mMODIFY\u001b[0m \u001b[0mANYTHING\u001b[0m \u001b[0mIN\u001b[0m \u001b[0mTHIS\u001b[0m \u001b[0mCELL\u001b[0m \u001b[0mTHAT\u001b[0m \u001b[0mIS\u001b[0m \u001b[0mBELOW\u001b[0m \u001b[0mTHIS\u001b[0m \u001b[0mLINE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \"\"\"\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mtests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_build_nn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuild_nn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/home/spike/ml/udacity/nd101/deep-learning-modified/tv-script-generation/problem_unittests.py\u001b[0m in \u001b[0;36mtest_build_nn\u001b[0;34m(build_nn)\u001b[0m\n\u001b[1;32m 242\u001b[0m \u001b[0;31m# Check name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 243\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfinal_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'name'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 244\u001b[0;31m \u001b[0;34m'Final state doesn\\'t have the \"name\" attribute. Are you using build_rnn?'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 245\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mfinal_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'final_state:0'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 246\u001b[0m \u001b[0;34m'Final state doesn\\'t have the correct name. Found the name {}. Are you using build_rnn?'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfinal_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mAssertionError\u001b[0m: Final state doesn't have the \"name\" attribute. Are you using build_rnn?" + ] + } + ], "source": [ "def build_nn(cell, rnn_size, input_data, vocab_size):\n", " \"\"\"\n", @@ -549,7 +645,8 @@ " :param vocab_size: Vocabulary size\n", " :return: Tuple (Logits, FinalState)\n", " \"\"\"\n", - " # TODO: Implement Function\n", + " embed = get_embed(input_data, vocab_size, HYPER.embedding_size)\n", + " \n", " return None, None\n", "\n", "\n", diff --git a/tv-script-generation/dlnd_tv_script_generation.ipynb b/tv-script-generation/dlnd_tv_script_generation.ipynb index ac7b12e..4351eab 100644 --- a/tv-script-generation/dlnd_tv_script_generation.ipynb +++ b/tv-script-generation/dlnd_tv_script_generation.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 21, "metadata": { "collapsed": false, "deletable": true, @@ -292,6 +292,35 @@ "int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Extra hyper parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from collections import namedtuple\n", + "\n", + "hyper_params = (('embedding_size', 128),\n", + " \n", + " )\n", + "\n", + "\n", + "\n", + "\n", + "Hyper = namedtuple('Hyper', map(lambda x: x[0], hyper_params))\n", + "HYPER = Hyper(*list(map(lambda x: x[1], hyper_params)))\n", + "\n" + ] + }, { "cell_type": "markdown", "metadata": { @@ -367,21 +396,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tests Passed\n" + ] + } + ], "source": [ "def get_inputs():\n", " \"\"\"\n", " Create TF Placeholders for input, targets, and learning rate.\n", " :return: Tuple (input, targets, learning rate)\n", " \"\"\"\n", - " # TODO: Implement Function\n", - " return None, None, None\n", + " \n", + " # We use shape [None, None] to feed any batch size and any sequence length\n", + " input_placeholder = tf.placeholder(tf.int32, [None, None],name='input')\n", + " \n", + " # Targets are [batch_size, seq_length]\n", + " targets_placeholder = tf.placeholder(tf.int32, [None, None]) \n", + " \n", + " \n", + " learning_rate_placeholder = tf.placeholder(tf.float32)\n", + " return input_placeholder, targets_placeholder, learning_rate_placeholder\n", "\n", "\n", "\"\"\"\n", @@ -408,13 +453,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tests Passed\n" + ] + } + ], "source": [ "def get_init_cell(batch_size, rnn_size):\n", " \"\"\"\n", @@ -423,8 +476,14 @@ " :param rnn_size: Size of RNNs\n", " :return: Tuple (cell, initialize state)\n", " \"\"\"\n", - " # TODO: Implement Function\n", - " return None, None\n", + " lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)\n", + " \n", + " cell = tf.contrib.rnn.MultiRNNCell([lstm] * 2)\n", + " \n", + " initial_state = cell.zero_state(batch_size, tf.float32)\n", + " initial_state = tf.identity(initial_state, name='initial_state')\n", + " \n", + " return cell, initial_state\n", "\n", "\n", "\"\"\"\n", @@ -446,13 +505,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tests Passed\n" + ] + } + ], "source": [ "def get_embed(input_data, vocab_size, embed_dim):\n", " \"\"\"\n", @@ -462,8 +529,13 @@ " :param embed_dim: Number of embedding dimensions\n", " :return: Embedded input.\n", " \"\"\"\n", - " # TODO: Implement Function\n", - " return None\n", + " embeddings = tf.Variable(\n", + " tf.random_uniform([vocab_size, embed_dim], -1.0, 1.0)\n", + " )\n", + " \n", + " embed = tf.nn.embedding_lookup(embeddings, input_data)\n", + " \n", + " return embed\n", "\n", "\n", "\"\"\"\n", @@ -489,13 +561,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tests Passed\n" + ] + } + ], "source": [ "def build_rnn(cell, inputs):\n", " \"\"\"\n", @@ -504,8 +584,11 @@ " :param inputs: Input text data\n", " :return: Tuple (Outputs, Final State)\n", " \"\"\"\n", - " # TODO: Implement Function\n", - " return None, None\n", + " outputs, final_state = tf.nn.dynamic_rnn(cell, inputs, dtype=tf.float32)\n", + " final_state = tf.identity(final_state, name='final_state')\n", + " \n", + " \n", + " return outputs, final_state\n", "\n", "\n", "\"\"\"\n", @@ -532,13 +615,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 37, "metadata": { "collapsed": false, "deletable": true, "editable": true }, - "outputs": [], + "outputs": [ + { + "ename": "AssertionError", + "evalue": "Final state doesn't have the \"name\" attribute. Are you using build_rnn?", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0mDON\u001b[0m\u001b[0;31m'\u001b[0m\u001b[0mT\u001b[0m \u001b[0mMODIFY\u001b[0m \u001b[0mANYTHING\u001b[0m \u001b[0mIN\u001b[0m \u001b[0mTHIS\u001b[0m \u001b[0mCELL\u001b[0m \u001b[0mTHAT\u001b[0m \u001b[0mIS\u001b[0m \u001b[0mBELOW\u001b[0m \u001b[0mTHIS\u001b[0m \u001b[0mLINE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \"\"\"\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0mtests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_build_nn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuild_nn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/home/spike/ml/udacity/nd101/deep-learning-modified/tv-script-generation/problem_unittests.py\u001b[0m in \u001b[0;36mtest_build_nn\u001b[0;34m(build_nn)\u001b[0m\n\u001b[1;32m 242\u001b[0m \u001b[0;31m# Check name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 243\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfinal_state\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'name'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 244\u001b[0;31m \u001b[0;34m'Final state doesn\\'t have the \"name\" attribute. Are you using build_rnn?'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 245\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mfinal_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'final_state:0'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 246\u001b[0m \u001b[0;34m'Final state doesn\\'t have the correct name. Found the name {}. Are you using build_rnn?'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfinal_state\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mAssertionError\u001b[0m: Final state doesn't have the \"name\" attribute. Are you using build_rnn?" + ] + } + ], "source": [ "def build_nn(cell, rnn_size, input_data, vocab_size):\n", " \"\"\"\n", @@ -549,7 +645,8 @@ " :param vocab_size: Vocabulary size\n", " :return: Tuple (Logits, FinalState)\n", " \"\"\"\n", - " # TODO: Implement Function\n", + " embed = get_embed(input_data, vocab_size, HYPER.embedding_size)\n", + " \n", " return None, None\n", "\n", "\n",