{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Text Embedding - IMDB dataset\n",
"=============================\n",
"---\n",
"Introduction au Deep Learning (IDLE) - S. Arias, E. Maldonado, JL. Parouty - CNRS/SARI/DEVLOG - 2020 \n",
"\n",
"## Reviews analysis :\n",
"\n",
"The objective is to guess whether our new and personals films reviews are **positive or negative** . \n",
"For this, we will use our previously saved model.\n",
"\n",
"What we're going to do:\n",
"\n",
" - Preparing the data\n",
" - Retrieve our saved model\n",
" - Evaluate the result\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 1 - Init python stuff"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'seaborn'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-1-94e372328354>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 7\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyplot\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmatplotlib\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mseaborn\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msns\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 10\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'seaborn'"
]
}
],
"source": [
"import numpy as np\n",
"\n",
"import tensorflow as tf\n",
"import tensorflow.keras as keras\n",
"import tensorflow.keras.datasets.imdb as imdb\n",
"\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib\n",
"import seaborn as sns\n",
"import pandas as pd\n",
"\n",
"import os,sys,h5py,json,re\n",
"\n",
"from importlib import reload\n",
"\n",
"sys.path.append('..')\n",
"import fidle.pwk as ooo\n",
"\n",
"ooo.init()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2 : Preparing the data\n",
"### 2.1 - Our reviews :"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"reviews = [ \"This film is particularly nice, a must see.\",\n",
" \"Some films are classics and cannot be ignored.\",\n",
" \"This movie is just abominable and doesn't deserve to be seen!\"]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 2.2 - Retrieve dictionaries"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"with open('./data/word_index.json', 'r') as fp:\n",
" word_index = json.load(fp)\n",
" index_word = {index:word for word,index in word_index.items()} "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 2.3 - Clean, index and padd"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"max_len = 256\n",
"vocab_size = 10000\n",
"\n",
"\n",
"nb_reviews = len(reviews)\n",
"x_data = []\n",
"\n",
"# ---- For all reviews\n",
"for review in reviews:\n",
" # ---- First index must be <start>\n",
" index_review=[1]\n",
" # ---- For all words\n",
" for w in review.split(' '):\n",
" # ---- Clean it\n",
" w_clean = re.sub(r\"[^a-zA-Z0-9]\", \"\", w)\n",
" # ---- Not empty ?\n",
" if len(w_clean)>0:\n",
" # ---- Get the index\n",
" w_index = word_index.get(w,2)\n",
" if w_index>vocab_size : w_index=2\n",
" # ---- Add the index if < vocab_size\n",
" index_review.append(w_index)\n",
" # ---- Add the indexed review\n",
" x_data.append(index_review) \n",
"\n",
"# ---- Padding\n",
"x_data = keras.preprocessing.sequence.pad_sequences(x_data, value = 0, padding = 'post', maxlen = max_len)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### 2.4 - Have a look"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def translate(x):\n",
" return ' '.join( [index_word.get(i,'?') for i in x] )\n",
"\n",
"for i in range(nb_reviews):\n",
" imax=np.where(x_data[i]==0)[0][0]+5\n",
" print(f'\\nText review :', reviews[i])\n",
" print( f'x_train[{i:}] :', list(x_data[i][:imax]), '(...)')\n",
" print( 'Translation :', translate(x_data[i][:imax]), '(...)')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 2 - Bring back the model"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"model = keras.models.load_model('./run/models/best_model.h5')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Step 4 - Predict"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"y_pred = model.predict(x_data)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"#### And the winner is :"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for i in range(nb_reviews):\n",
" print(f'\\n{reviews[i]:<70} =>',('NEGATIVE' if y_pred[i][0]<0.5 else 'POSITIVE'),f'({y_pred[i][0]:.2f})')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"a=[1]+[i for i in range(3)]\n",
"a"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.5"
}
},
"nbformat": 4,
"nbformat_minor": 4
}