电影评论情感分析：RNN循环网络原理及自然语言处理NLP应用

import os
import tarfile
import urllib.request

import tensorflow as tf
import numpy as np

import re
import string
from random import randint
#数据地址
url="http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz"
#数据存放路径
filepath="data/aclImdb_vl.tar.gz"

#如果当前目录下不存在data文件夹，则建立
if not os.path.exists("data"):
    os. makedirs("data")
#下载数据，80兆左右
if not os.path.isfile(filepath):
    print ('downloading...')
    result=urllib.request.urlretrieve(url,filepath)
    print (' downloaded:' ,result)
else:
    print(filepath,'is existed ! ')

#解压数据
if not os.path.exists("data/aclImdb") :
    tfile=tarfile. open(filepath,"r:gz")
    print (' extracting.. .')
    result=tfile.extractall("data/")
    print (' extraction completed')
else:
    print("data/aclImdb is existed!")

#将文本中不需要的字符清除，如html标签<br />
def remove_tags(text):
    re_tag = re.compile(r'<[^>]+>')
    return re_tag.sub('', text)

#读取文件
def read_files(filetype):
    path = "data/aclImdb/"
    file_list=[]
#读取正面评价的文件的路径，存到file_list列表里
    positive_path=path + filetype+"/pos/"
    for f in os.listdir(positive_path):
        file_list+=[positive_path+f]
    pos_files_num=len(file_list)
    #读取负面评价的文件的路径，存到file_list列表里
    negative_path=path + filetype+"/neg/"
    for f in os.listdir(negative_path):
        file_list+=[negative_path+f]
    neg_files_num=len(file_list)-pos_files_num
    print('read',filetype,'files:' , len(file_list))
    print(pos_files_num,'pos files in',filetype,'files')
    print (neg_files_num, 'neg files in',filetype, 'files')
    #得到所有标签。标签用one-hot编码表示，正面评价标签为[1 0]，负面评价标签为[0 1]
    all_labels = ([[1,0]] * pos_files_num +[[0,1]] *neg_files_num)
    #得到所有文本。
    all_texts= []
    for fi in file_list:
        with open(fi, encoding='utf8') as file_input:
            #文本中有<br/>这类html标签，将文本传入remove_tags函数
            #函数里使用正则表达式可以将这样的标签清除掉。
            all_texts +=[remove_tags(" ".join(file_input.readlines()))]
    return all_labels, all_texts

#得到训练与测试用的标签和文本
train_labels,train_texts=read_files("train")
test_labels,test_texts=read_files("test")

#查看数据、标签
print("训练数据")
print("正面评价:")
print(train_texts[0])
print(train_labels[0])
print("负面评价:")
print(train_texts[12500])
print(train_labels[12500])
print("测试数据")
print("正面评价:")
print(test_texts[0])
print(test_labels[0])
print("负面评价:")
print(test_texts[12500])
print(test_labels[12500])

import tensorflow.keras as keras
#建立词汇词典Token
token = keras.preprocessing.text.Tokenizer(num_words=4000)
token.fit_on_texts(train_texts)
#查看读取文档数量
token.document_count
#将单词映射为他们的排名或索引
print(token.word_index)
#将单词映射为它们在训练期间所出现的文档或文本数量
token.word_docs
#查看词汇出现的频次排名
print(token.word_counts)
#文字转数字列表
train_sequences = token.texts_to_sequences(train_texts)
test_sequences = token.texts_to_sequences(test_texts)

print(train_texts[0])

print(train_sequences[0])

x_train = keras.preprocessing.sequence.pad_sequences(train_sequences,
padding='post',truncating='post',maxlen=400)
x_test = keras.preprocessing.sequence.pad_sequences(test_sequences,
padding='post',truncating='post',maxlen=400)

x_train.shape
print(x_train[0])
#让转换后的数字列表长度相同
x_train = keras.preprocessing. sequence.pad_sequences(train_sequences,
                                                    padding='post',
                                                      truncating='post',
                                                      maxlen=400)
x_test = keras.preprocessing. sequence.pad_sequences(test_sequences,
                                                    padding='post',
                                                     truncating='post',
                                                     maxlen=400)
y_train=np.array(train_labels)
#要把标签改为ndarray格式
y_test=np.array(test_labels)

model = keras.models.Sequential()
model.add (keras.layers.Embedding(output_dim=32,
                                  input_dim=4000,
                                  input_length=400))
model.add(keras.layers.Flatten())
model.add(keras.layers.Dense(units=256,activation='relu'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(units=2,activation='softmax'))
model.summary()
#模型设置与训练
model.compile(optimizer='adam',
             loss='categorical_crossentropy',
             metrics=['accuracy'])
history = model.fit(x_train, y_train,
                   validation_split=0.2,
                   epochs=10,
                   batch_size=128,
                   verbose=1)
#训练过程可视化
import matplotlib.pyplot as plt
acc=history.history['accuracy']
val_acc=history.history['val_accuracy']
loss=history.history['loss']
val_loss=history.history['val_loss']
epochs=range(1,len(acc)+1)
plt.plot(epochs,loss,'r',label='Training loss')
plt.plot(epochs,val_loss,'b',label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
plt.show()
plt.clf()# clear figure
acc_values=history.history['accuracy']
val_acc_values=history.history['val_accuracy']
plt.plot(epochs,acc,'r',label='Training acc')
plt.plot(epochs,val_acc,'b',label='Validation acc')
plt.title('Training and validation accutacy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
plt.show()
#评估模型准确率
test_loss,test_acc=model.evaluate(x_test,y_test,verbose=1)
print('Test Accuracy',test_acc)
#执行模型预测
predictions=model.predict(x_test)
predictions[0]
#定义预测结果显示函数
sentiment_dict={0:'pos',1:'neg'}
def display_test_sentiment(i):
    print(test_texts[i])
    print('label values:',sentiment_dict[np.argmax(y_test[i])],'predict value:',sentiment_dict[np.argmax(predictions[i])])
#查看预测结果
display_test_sentiment(0)

tf.keras.layers.Embedding(input_dim, # 输入词汇表的长度，最大词汇数+1
output_dim, # 输出词向量的维度
input_length=None # 输入Tensor的长度

#建立模型
model = keras.models.Sequential()
model.add (keras.layers.Embedding(output_dim=32,
                                  input_dim=4000,
                                  input_length=400))
model.add(keras.layers.Bidirectional(keras.layers.LSTM(units=8)))
model.add(keras.layers.Dense(units=32,activation='relu'))
model.add(keras.layers.Dropout(0.3))
model.add(keras.layers.Dense(units=2,activation='softmax'))
model.summary()
#模型设置与训练
model.compile(optimizer='adam',
             loss='categorical_crossentropy',
             metrics=['accuracy'])
history = model.fit(x_train, y_train,
                   validation_split=0.2,
                   epochs=6,
                   batch_size=128,
                   verbose=1)