sequence_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Example dataset : SumTask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SumTaskDataset\n",
      "Training example (sorted):\n",
      "tensor([1, 3, 3, 3, 5, 5, 5, 5, 7, 8]) tensor(45)\n",
      "Test example (extrapolated):\n",
      "tensor([6, 3, 9, 5, 9, 2, 2, 2, 3, 6]) tensor(47)\n"
     ]
    }
   ],
   "source": [
    "%run -i Sequences/datasets.py"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run CG-invariant Feedforward network on this dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: Using cuda\n",
      "[INFO]: ================================================================================\n",
      "[INFO]: Configuration: \n",
      "[INFO]: {'activation': 'relu',\n",
      "[INFO]:  'batchSize': 64,\n",
      "[INFO]:  'dataDir': 'data',\n",
      "[INFO]:  'dataset': 'SumTask',\n",
      "[INFO]:  'dropout': 0.0,\n",
      "[INFO]:  'embeddingSize': 128,\n",
      "[INFO]:  'hiddenDimension': 128,\n",
      "[INFO]:  'logLevel': 'info',\n",
      "[INFO]:  'lr': 0.001,\n",
      "[INFO]:  'model': 'cgff',\n",
      "[INFO]:  'nSamples': 10000,\n",
      "[INFO]:  'numEpochs': 100,\n",
      "[INFO]:  'patience': 10,\n",
      "[INFO]:  'penaltyAlpha': 10,\n",
      "[INFO]:  'penaltyMode': 'simple',\n",
      "[INFO]:  'penaltyT': 1,\n",
      "[INFO]:  'phiLayerDims': [128],\n",
      "[INFO]:  'rhoLayerDims': [128, 1],\n",
      "[INFO]:  'seed': 314271,\n",
      "[INFO]:  'sequenceLength': 10,\n",
      "[INFO]:  'tqdmDisable': False,\n",
      "[INFO]:  'vocabularyRange': [1, 100],\n",
      "[INFO]:  'vocabularySize': 100,\n",
      "[INFO]:  'weightDecay': 0,\n",
      "[INFO]:  'weightsAcrossDims': False}\n",
      "[INFO]: ================================================================================\n",
      "[INFO]: Train Dataset: SumTaskDataset of shape torch.Size([8000, 10])\n",
      "[INFO]: Test Dataset: SumTaskDataset of shape torch.Size([2000, 10])\n",
      "[INFO]: Test Dataset: SumTaskDataset of shape torch.Size([2000, 10])\n",
      "[INFO]: Begin Run : cgff => Sequences.models.CGFF\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epochs', style=ProgressStyle(description_width='initial')), H…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 1 out of 10 | Train Loss: 252861.5781 (Penalty: 66.8520) | Valid Loss: 252525.8594 (Penalty: 66.8520) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 2 out of 10 | Train Loss: 42578.0689 (Penalty: 66.9865) | Valid Loss: 2839.4849 (Penalty: 67.1600) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 3 out of 10 | Train Loss: 1410.7454 (Penalty: 67.4058) | Valid Loss: 908.6765 (Penalty: 67.5799) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 4 out of 10 | Train Loss: 747.1261 (Penalty: 67.5897) | Valid Loss: 705.3759 (Penalty: 67.5821) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 5 out of 10 | Train Loss: 607.6743 (Penalty: 67.5427) | Valid Loss: 568.1722 (Penalty: 67.5079) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 6 out of 12 | Train Loss: 484.5359 (Penalty: 67.4576) | Valid Loss: 423.0894 (Penalty: 67.4159) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 7 out of 14 | Train Loss: 362.1086 (Penalty: 67.3595) | Valid Loss: 325.0298 (Penalty: 67.3007) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 8 out of 16 | Train Loss: 303.7137 (Penalty: 67.2272) | Valid Loss: 281.8472 (Penalty: 67.1582) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 9 out of 18 | Train Loss: 267.5329 (Penalty: 67.0858) | Valid Loss: 244.8448 (Penalty: 67.0154) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 10 out of 20 | Train Loss: 230.3128 (Penalty: 66.9463) | Valid Loss: 208.4343 (Penalty: 66.8754) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 11 out of 22 | Train Loss: 194.7115 (Penalty: 66.8122) | Valid Loss: 176.6043 (Penalty: 66.7446) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 12 out of 24 | Train Loss: 165.0804 (Penalty: 66.6888) | Valid Loss: 152.2957 (Penalty: 66.6260) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 13 out of 26 | Train Loss: 143.3505 (Penalty: 66.5747) | Valid Loss: 136.4982 (Penalty: 66.5146) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 14 out of 28 | Train Loss: 126.1098 (Penalty: 66.4589) | Valid Loss: 117.1147 (Penalty: 66.3935) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 15 out of 30 | Train Loss: 104.5273 (Penalty: 66.3258) | Valid Loss: 91.4435 (Penalty: 66.2507) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 16 out of 32 | Train Loss: 80.4586 (Penalty: 66.1786) | Valid Loss: 72.7040 (Penalty: 66.1082) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 17 out of 34 | Train Loss: 69.3366 (Penalty: 66.0446) | Valid Loss: 67.3956 (Penalty: 65.9816) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 18 out of 36 | Train Loss: 66.6571 (Penalty: 65.9198) | Valid Loss: 66.2176 (Penalty: 65.8563) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 19 out of 38 | Train Loss: 66.0052 (Penalty: 65.7918) | Valid Loss: 65.8656 (Penalty: 65.7247) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 20 out of 38 | Train Loss: 65.7531 (Penalty: 65.6557) | Valid Loss: 65.6678 (Penalty: 65.5837) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 21 out of 38 | Train Loss: 65.5739 (Penalty: 65.5096) | Valid Loss: 65.4934 (Penalty: 65.4321) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 22 out of 38 | Train Loss: 65.4007 (Penalty: 65.3523) | Valid Loss: 65.3161 (Penalty: 65.2688) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 23 out of 38 | Train Loss: 65.2200 (Penalty: 65.1827) | Valid Loss: 65.1294 (Penalty: 65.0925) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 24 out of 38 | Train Loss: 65.0283 (Penalty: 64.9995) | Valid Loss: 64.9306 (Penalty: 64.9022) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 25 out of 38 | Train Loss: 64.8235 (Penalty: 64.8016) | Valid Loss: 64.7176 (Penalty: 64.6963) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 26 out of 38 | Train Loss: 64.6039 (Penalty: 64.5875) | Valid Loss: 64.4888 (Penalty: 64.4735) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 27 out of 38 | Train Loss: 64.3675 (Penalty: 64.3555) | Valid Loss: 64.2429 (Penalty: 64.2318) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 28 out of 38 | Train Loss: 64.1124 (Penalty: 64.1036) | Valid Loss: 63.9773 (Penalty: 63.9692) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 29 out of 38 | Train Loss: 63.8361 (Penalty: 63.8298) | Valid Loss: 63.6892 (Penalty: 63.6834) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 30 out of 38 | Train Loss: 63.5357 (Penalty: 63.5312) | Valid Loss: 63.3756 (Penalty: 63.3714) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 31 out of 62 | Train Loss: 63.2082 (Penalty: 63.2050) | Valid Loss: 63.0328 (Penalty: 63.0300) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 32 out of 64 | Train Loss: 62.8496 (Penalty: 62.8475) | Valid Loss: 62.6571 (Penalty: 62.6552) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 33 out of 66 | Train Loss: 62.4556 (Penalty: 62.4542) | Valid Loss: 62.2434 (Penalty: 62.2422) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 34 out of 68 | Train Loss: 62.0209 (Penalty: 62.0199) | Valid Loss: 61.7859 (Penalty: 61.7851) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 35 out of 70 | Train Loss: 61.5386 (Penalty: 61.5380) | Valid Loss: 61.2771 (Penalty: 61.2766) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 36 out of 72 | Train Loss: 61.0008 (Penalty: 61.0003) | Valid Loss: 60.7077 (Penalty: 60.7074) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 37 out of 74 | Train Loss: 60.3966 (Penalty: 60.3964) | Valid Loss: 60.0658 (Penalty: 60.0656) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 38 out of 76 | Train Loss: 59.7125 (Penalty: 59.7124) | Valid Loss: 59.3356 (Penalty: 59.3354) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 39 out of 78 | Train Loss: 58.9299 (Penalty: 58.9298) | Valid Loss: 58.4953 (Penalty: 58.4951) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 40 out of 80 | Train Loss: 58.0227 (Penalty: 58.0226) | Valid Loss: 57.5137 (Penalty: 57.5135) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 41 out of 82 | Train Loss: 56.9528 (Penalty: 56.9527) | Valid Loss: 56.3437 (Penalty: 56.3436) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 42 out of 84 | Train Loss: 55.6598 (Penalty: 55.6597) | Valid Loss: 54.9084 (Penalty: 54.9083) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 43 out of 86 | Train Loss: 54.0393 (Penalty: 54.0392) | Valid Loss: 53.0664 (Penalty: 53.0662) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 44 out of 88 | Train Loss: 51.8828 (Penalty: 51.8827) | Valid Loss: 50.5134 (Penalty: 50.5132) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 45 out of 90 | Train Loss: 48.8670 (Penalty: 48.8669) | Valid Loss: 47.4888 (Penalty: 47.4886) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 46 out of 92 | Train Loss: 46.4562 (Penalty: 46.4561) | Valid Loss: 45.2845 (Penalty: 45.2843) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 47 out of 94 | Train Loss: 43.7686 (Penalty: 43.7684) | Valid Loss: 42.1394 (Penalty: 42.1392) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 48 out of 96 | Train Loss: 41.3075 (Penalty: 41.3073) | Valid Loss: 40.4416 (Penalty: 40.4414) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 49 out of 98 | Train Loss: 39.4550 (Penalty: 39.4547) | Valid Loss: 38.3548 (Penalty: 38.3546) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 50 out of 100 | Train Loss: 37.0471 (Penalty: 37.0467) | Valid Loss: 35.5577 (Penalty: 35.5574) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 51 out of 100 | Train Loss: 33.6851 (Penalty: 33.6844) | Valid Loss: 31.4768 (Penalty: 31.4763) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 52 out of 100 | Train Loss: 28.9595 (Penalty: 28.9585) | Valid Loss: 26.6909 (Penalty: 26.6893) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 53 out of 100 | Train Loss: 24.3533 (Penalty: 24.3508) | Valid Loss: 22.3124 (Penalty: 22.3052) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 54 out of 100 | Train Loss: 20.1758 (Penalty: 20.1731) | Valid Loss: 17.6528 (Penalty: 17.6461) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 55 out of 100 | Train Loss: 14.4090 (Penalty: 14.3919) | Valid Loss: 11.2899 (Penalty: 11.1187) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 56 out of 100 | Train Loss: 9.1630 (Penalty: 8.8743) | Valid Loss: 7.5699 (Penalty: 7.1026) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 57 out of 100 | Train Loss: 8.3259 (Penalty: 6.3549) | Valid Loss: 16.4342 (Penalty: 6.0399) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 58 out of 100 | Train Loss: 11.1281 (Penalty: 5.6699) | Valid Loss: 9.5203 (Penalty: 5.3265) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 59 out of 100 | Train Loss: 8.8088 (Penalty: 5.1064) | Valid Loss: 7.8483 (Penalty: 4.8414) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 60 out of 100 | Train Loss: 10.2283 (Penalty: 4.6472) | Valid Loss: 6.2007 (Penalty: 4.3261) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 61 out of 100 | Train Loss: 4.6541 (Penalty: 4.0664) | Valid Loss: 4.1440 (Penalty: 3.8073) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 62 out of 100 | Train Loss: 4.0867 (Penalty: 3.5427) | Valid Loss: 3.7157 (Penalty: 3.2600) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 63 out of 100 | Train Loss: 3.7705 (Penalty: 2.9595) | Valid Loss: 3.0554 (Penalty: 2.6466) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 64 out of 100 | Train Loss: 3.0113 (Penalty: 2.3115) | Valid Loss: 2.0562 (Penalty: 1.9458) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 65 out of 100 | Train Loss: 1.6791 (Penalty: 1.5807) | Valid Loss: 2.0284 (Penalty: 1.2050) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 66 out of 100 | Train Loss: 1.1341 (Penalty: 0.8148) | Valid Loss: 1.1823 (Penalty: 0.4201) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 67 out of 100 | Train Loss: 1.5760 (Penalty: 0.1560) | Valid Loss: 1.8820 (Penalty: 0.0571) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 68 out of 100 | Train Loss: 13.6803 (Penalty: 0.1683) | Valid Loss: 22.4016 (Penalty: 0.3238) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 69 out of 100 | Train Loss: 20.1600 (Penalty: 0.2884) | Valid Loss: 1.1878 (Penalty: 0.0506) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 70 out of 100 | Train Loss: 0.8029 (Penalty: 0.0378) | Valid Loss: 0.0969 (Penalty: 0.0170) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 71 out of 100 | Train Loss: 0.0475 (Penalty: 0.0200) | Valid Loss: 0.0292 (Penalty: 0.0178) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 72 out of 100 | Train Loss: 0.0243 (Penalty: 0.0180) | Valid Loss: 0.0194 (Penalty: 0.0154) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 73 out of 100 | Train Loss: 0.0215 (Penalty: 0.0165) | Valid Loss: 0.0200 (Penalty: 0.0164) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 74 out of 100 | Train Loss: 0.0222 (Penalty: 0.0181) | Valid Loss: 0.0175 (Penalty: 0.0151) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 75 out of 100 | Train Loss: 0.0179 (Penalty: 0.0159) | Valid Loss: 0.0168 (Penalty: 0.0152) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 76 out of 100 | Train Loss: 0.0185 (Penalty: 0.0164) | Valid Loss: 0.0204 (Penalty: 0.0150) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 77 out of 100 | Train Loss: 0.0192 (Penalty: 0.0161) | Valid Loss: 0.0280 (Penalty: 0.0178) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 78 out of 100 | Train Loss: 0.0838 (Penalty: 0.0177) | Valid Loss: 1.1236 (Penalty: 0.0342) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 79 out of 100 | Train Loss: 4.6443 (Penalty: 0.1048) | Valid Loss: 4.6945 (Penalty: 0.2981) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 80 out of 100 | Train Loss: 18.4326 (Penalty: 0.1870) | Valid Loss: 84.4478 (Penalty: 0.4286) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 81 out of 100 | Train Loss: 20.2103 (Penalty: 0.2216) | Valid Loss: 1.3020 (Penalty: 0.0370) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 82 out of 100 | Train Loss: 0.4125 (Penalty: 0.0249) | Valid Loss: 0.3979 (Penalty: 0.0246) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 83 out of 100 | Train Loss: 0.1237 (Penalty: 0.0182) | Valid Loss: 0.0416 (Penalty: 0.0144) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 84 out of 100 | Train Loss: 0.0218 (Penalty: 0.0152) | Valid Loss: 0.0152 (Penalty: 0.0136) | (Saved model)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 85 out of 100 | Train Loss: 0.0195 (Penalty: 0.0157) | Valid Loss: 0.0180 (Penalty: 0.0131) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 86 out of 100 | Train Loss: 0.0242 (Penalty: 0.0156) | Valid Loss: 0.0232 (Penalty: 0.0165) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 87 out of 100 | Train Loss: 0.0214 (Penalty: 0.0158) | Valid Loss: 0.0214 (Penalty: 0.0172) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 88 out of 100 | Train Loss: 0.0188 (Penalty: 0.0163) | Valid Loss: 0.0188 (Penalty: 0.0155) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 89 out of 100 | Train Loss: 0.0178 (Penalty: 0.0149) | Valid Loss: 0.0191 (Penalty: 0.0134) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 90 out of 100 | Train Loss: 0.0202 (Penalty: 0.0147) | Valid Loss: 0.0444 (Penalty: 0.0152) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 91 out of 100 | Train Loss: 0.2467 (Penalty: 0.0199) | Valid Loss: 3.3027 (Penalty: 0.0452) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 92 out of 100 | Train Loss: 11.5080 (Penalty: 0.1380) | Valid Loss: 17.2846 (Penalty: 0.2759) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 93 out of 100 | Train Loss: 15.4778 (Penalty: 0.1495) | Valid Loss: 4.5375 (Penalty: 0.0468) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 94 out of 100 | Train Loss: 5.9918 (Penalty: 0.1103) | Valid Loss: 3.0672 (Penalty: 0.0418) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 95 out of 100 | Train Loss: 2.4832 (Penalty: 0.0578) | Valid Loss: 1.6375 (Penalty: 0.0349) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 96 out of 100 | Train Loss: 2.0523 (Penalty: 0.0486) | Valid Loss: 2.4306 (Penalty: 0.0327) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 97 out of 100 | Train Loss: 5.4794 (Penalty: 0.0993) | Valid Loss: 4.3073 (Penalty: 0.0456) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 98 out of 100 | Train Loss: 3.2798 (Penalty: 0.0669) | Valid Loss: 2.1149 (Penalty: 0.0401) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 99 out of 100 | Train Loss: 2.2441 (Penalty: 0.0507) | Valid Loss: 2.5389 (Penalty: 0.0352) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: 100 out of 100 | Train Loss: 5.3753 (Penalty: 0.1051) | Valid Loss: 3.9172 (Penalty: 0.0546) | \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Minibatches', style=ProgressStyle(description_width='initial'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO]: ================================================================================\n",
      "[INFO]: Train Results:\n",
      "[INFO]: {'accuracy': 100.0,\n",
      "[INFO]:  'loss': 0.0015051240846514701,\n",
      "[INFO]:  'lossAndPenalty': 0.015121068507432937,\n",
      "[INFO]:  'penalty': 0.013615944422781467}\n",
      "[INFO]: ================================================================================\n",
      "[INFO]: ================================================================================\n",
      "[INFO]: Validation Results:\n",
      "[INFO]: {'accuracy': 100.0,\n",
      "[INFO]:  'loss': 0.00157875781878829,\n",
      "[INFO]:  'lossAndPenalty': 0.015194702241569758,\n",
      "[INFO]:  'penalty': 0.013615944422781467}\n",
      "[INFO]: ================================================================================\n",
      "[INFO]: ================================================================================\n",
      "[INFO]: Test Results:\n",
      "[INFO]: {'in': {'accuracy': 100.0,\n",
      "[INFO]:         'loss': 0.0014593727327883244,\n",
      "[INFO]:         'lossAndPenalty': 0.015075317155569791,\n",
      "[INFO]:         'penalty': 0.013615944422781467},\n",
      "[INFO]:  'ood': {'accuracy': 100.0,\n",
      "[INFO]:          'loss': 0.001409832153469324,\n",
      "[INFO]:          'lossAndPenalty': 0.015025776576250791,\n",
      "[INFO]:          'penalty': 0.013615944422781467}}\n",
      "[INFO]: ================================================================================\n"
     ]
    }
   ],
   "source": [
    "%run -i Sequences/main.py --dataset=SumTask --model=cgff --numEpochs=100"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}