import urllib.request
import os
import tarfile
url = 'http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz'
filepath = './data/aclImdb_v1.tar.gz'
if not os.path.isfile(filepath):
result = urllib.request.urlretrieve(url, filepath)
print('download:', result)
if not os.path.exists('./data/aclImdb'):
tfile = tarfile.open('./data/aclImdb_v1.tar.gz', 'r:gz')
result = tfile.extractall('./data/')
import tensorflow as tf
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.preprocessing.text import Tokenizer
Using TensorFlow backend.
E:\Anaconda3\envs\ml\lib\site-packages\tensorflow\python\framework\dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_qint8 = np.dtype([("qint8", np.int8, 1)])
E:\Anaconda3\envs\ml\lib\site-packages\tensorflow\python\framework\dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_quint8 = np.dtype([("quint8", np.uint8, 1)])
E:\Anaconda3\envs\ml\lib\site-packages\tensorflow\python\framework\dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_qint16 = np.dtype([("qint16", np.int16, 1)])
E:\Anaconda3\envs\ml\lib\site-packages\tensorflow\python\framework\dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_quint16 = np.dtype([("quint16", np.uint16, 1)])
E:\Anaconda3\envs\ml\lib\site-packages\tensorflow\python\framework\dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
_np_qint32 = np.dtype([("qint32", np.int32, 1)])
E:\Anaconda3\envs\ml\lib\site-packages\tensorflow\python\framework\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
np_resource = np.dtype([("resource", np.ubyte, 1)])
WARNING: Logging before flag parsing goes to stderr.
W0819 14:47:47.112198 9352 __init__.py:308] Limited tf.compat.v2.summary API due to missing TensorBoard installation.
import re
def rm_tags(text):
re_tag = re.compile('<[^>]+>')
return re_tag.sub('', text)
import os
def read_files(filetype):
path = './data/aclImdb/'
file_list = []
positive_path = path + filetype + '/pos/'
for f in os.listdir(positive_path):
file_list += [positive_path + f]
negative_path = path + filetype + '/neg/'
for f in os.listdir(negative_path):
file_list += [negative_path + f]
print('read', filetype, 'files:', len(file_list))
all_labels = ([1] * 12500 + [0] * 12500)
all_texts = []
for fi in file_list:
with open(fi, encoding='utf8') as file_input:
all_texts += [rm_tags(''.join(file_input.readlines()))]
return all_labels, all_texts
y_train, train_text = read_files('train')
read train files: 25000
y_test, test_text = read_files('test')
read test files: 25000
train_text[0]
'Bromwell High is a cartoon comedy. It ran at the same time as some other programs about school life, such as "Teachers". My 35 years in the teaching profession lead me to believe that Bromwell High\'s satire is much closer to reality than is "Teachers". The scramble to survive financially, the insightful students who can see right through their pathetic teachers\' pomp, the pettiness of the whole situation, all remind me of the schools I knew and their students. When I saw the episode in which a student repeatedly tried to burn down the school, I immediately recalled ......... at .......... High. A classic line: INSPECTOR: I\'m here to sack one of your teachers. STUDENT: Welcome to Bromwell High. I expect that many adults of my age think that Bromwell High is far fetched. What a pity that it isn\'t!'
y_train[0]
1
train_text[12501]
"Airport '77 starts as a brand new luxury 747 plane is loaded up with valuable paintings & such belonging to rich businessman Philip Stevens (James Stewart) who is flying them & a bunch of VIP's to his estate in preparation of it being opened to the public as a museum, also on board is Stevens daughter Julie (Kathleen Quinlan) & her son. The luxury jetliner takes off as planned but mid-air the plane is hi-jacked by the co-pilot Chambers (Robert Foxworth) & his two accomplice's Banker (Monte Markham) & Wilson (Michael Pataki) who knock the passengers & crew out with sleeping gas, they plan to steal the valuable cargo & land on a disused plane strip on an isolated island but while making his descent Chambers almost hits an oil rig in the Ocean & loses control of the plane sending it crashing into the sea where it sinks to the bottom right bang in the middle of the Bermuda Triangle. With air in short supply, water leaking in & having flown over 200 miles off course the problems mount for the survivor's as they await help with time fast running out...Also known under the slightly different tile Airport 1977 this second sequel to the smash-hit disaster thriller Airport (1970) was directed by Jerry Jameson & while once again like it's predecessors I can't say Airport '77 is any sort of forgotten classic it is entertaining although not necessarily for the right reasons. Out of the three Airport films I have seen so far I actually liked this one the best, just. It has my favourite plot of the three with a nice mid-air hi-jacking & then the crashing (didn't he see the oil rig?) & sinking of the 747 (maybe the makers were trying to cross the original Airport with another popular disaster flick of the period The Poseidon Adventure (1972)) & submerged is where it stays until the end with a stark dilemma facing those trapped inside, either suffocate when the air runs out or drown as the 747 floods or if any of the doors are opened & it's a decent idea that could have made for a great little disaster flick but bad unsympathetic character's, dull dialogue, lethargic set-pieces & a real lack of danger or suspense or tension means this is a missed opportunity. While the rather sluggish plot keeps one entertained for 108 odd minutes not that much happens after the plane sinks & there's not as much urgency as I thought there should have been. Even when the Navy become involved things don't pick up that much with a few shots of huge ships & helicopters flying about but there's just something lacking here. George Kennedy as the jinxed airline worker Joe Patroni is back but only gets a couple of scenes & barely even says anything preferring to just look worried in the background.The home video & theatrical version of Airport '77 run 108 minutes while the US TV versions add an extra hour of footage including a new opening credits sequence, many more scenes with George Kennedy as Patroni, flashbacks to flesh out character's, longer rescue scenes & the discovery or another couple of dead bodies including the navigator. While I would like to see this extra footage I am not sure I could sit through a near three hour cut of Airport '77. As expected the film has dated badly with horrible fashions & interior design choices, I will say no more other than the toy plane model effects aren't great either. Along with the other two Airport sequels this takes pride of place in the Razzie Award's Hall of Shame although I can think of lots of worse films than this so I reckon that's a little harsh. The action scenes are a little dull unfortunately, the pace is slow & not much excitement or tension is generated which is a shame as I reckon this could have been a pretty good film if made properly.The production values are alright if nothing spectacular. The acting isn't great, two time Oscar winner Jack Lemmon has said since it was a mistake to star in this, one time Oscar winner James Stewart looks old & frail, also one time Oscar winner Lee Grant looks drunk while Sir Christopher Lee is given little to do & there are plenty of other familiar faces to look out for too.Airport '77 is the most disaster orientated of the three Airport films so far & I liked the ideas behind it even if they were a bit silly, the production & bland direction doesn't help though & a film about a sunken plane just shouldn't be this boring or lethargic. Followed by The Concorde ... Airport '79 (1979)."
token = Tokenizer(num_words=2000)
token.fit_on_texts(train_text)
x_train_seq = token.texts_to_sequences(train_text)
x_test_seq = token.texts_to_sequences(test_text)
x_train = sequence.pad_sequences(x_train_seq, maxlen=100)
x_test = sequence.pad_sequences(x_test_seq, maxlen=100)
from tensorflow.keras.datasets import imdb
from tensorflow.keras.preprocessing import sequence
from tensorflow.keras.preprocessing.text import Tokenizer
y_train, train_text = read_files('train')
read train files: 25000
y_test, train_text =read_files('test')
read test files: 25000
token = Tokenizer(num_words=2000)
token.fit_on_texts(train_text)
x_train_seq = token.texts_to_sequences(train_text)
x_test_seq = token.texts_to_sequences(test_text)
x_train = sequence.pad_sequences(x_train_seq, maxlen=100)
x_test = sequence.pad_sequences(x_test_seq, maxlen=100)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers.core import Dense, Dropout, Activation, Flatten
from tensorflow.keras.layers.embeddings import Embedding
model = Sequential()
model.add(Embedding(output_dim=32, input_dim=2000, input_length=100))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(units=256,
activation='relu'))
model.add(Dropout(0.35))
model.add(Dense(units=1,
activation='sigmoid'))
W0819 14:48:52.688199 9352 deprecation_wrapper.py:119] From E:\Anaconda3\envs\ml\lib\site-packages\keras\backend\tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.
W0819 14:48:52.700166 9352 deprecation_wrapper.py:119] From E:\Anaconda3\envs\ml\lib\site-packages\keras\backend\tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.
W0819 14:48:52.705153 9352 deprecation_wrapper.py:119] From E:\Anaconda3\envs\ml\lib\site-packages\keras\backend\tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.
W0819 14:48:52.720114 9352 deprecation_wrapper.py:119] From E:\Anaconda3\envs\ml\lib\site-packages\keras\backend\tensorflow_backend.py:133: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.
W0819 14:48:52.730087 9352 deprecation.py:506] From E:\Anaconda3\envs\ml\lib\site-packages\keras\backend\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding_1 (Embedding) (None, 100, 32) 64000
_________________________________________________________________
dropout_1 (Dropout) (None, 100, 32) 0
_________________________________________________________________
flatten_1 (Flatten) (None, 3200) 0
_________________________________________________________________
dense_1 (Dense) (None, 256) 819456
_________________________________________________________________
dropout_2 (Dropout) (None, 256) 0
_________________________________________________________________
dense_2 (Dense) (None, 1) 257
=================================================================
Total params: 883,713
Trainable params: 883,713
Non-trainable params: 0
_________________________________________________________________
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
W0819 14:48:52.872310 9352 deprecation_wrapper.py:119] From E:\Anaconda3\envs\ml\lib\site-packages\keras\optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.
W0819 14:48:52.896245 9352 deprecation_wrapper.py:119] From E:\Anaconda3\envs\ml\lib\site-packages\keras\backend\tensorflow_backend.py:3376: The name tf.log is deprecated. Please use tf.math.log instead.
W0819 14:48:52.902229 9352 deprecation.py:323] From E:\Anaconda3\envs\ml\lib\site-packages\tensorflow\python\ops\nn_impl.py:180: add_dispatch_support..wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.where in 2.0, which has the same broadcast rule as np.where
train_history = model.fit(x_train,
y_train,
batch_size=100,
epochs=10,
verbose=2,
validation_split=0.2)
Train on 20000 samples, validate on 5000 samples
Epoch 1/10
- 4s - loss: 0.4585 - acc: 0.7695 - val_loss: 0.5365 - val_acc: 0.7516
Epoch 2/10
- 4s - loss: 0.2481 - acc: 0.8981 - val_loss: 0.5019 - val_acc: 0.7854
Epoch 3/10
- 4s - loss: 0.1478 - acc: 0.9465 - val_loss: 0.5992 - val_acc: 0.7796
Epoch 4/10
- 4s - loss: 0.0739 - acc: 0.9754 - val_loss: 0.9505 - val_acc: 0.7286
Epoch 5/10
- 4s - loss: 0.0425 - acc: 0.9866 - val_loss: 0.9713 - val_acc: 0.7640
Epoch 6/10
- 4s - loss: 0.0321 - acc: 0.9888 - val_loss: 1.1840 - val_acc: 0.7470
Epoch 7/10
- 4s - loss: 0.0246 - acc: 0.9914 - val_loss: 1.4026 - val_acc: 0.7220
Epoch 8/10
- 4s - loss: 0.0247 - acc: 0.9914 - val_loss: 1.4726 - val_acc: 0.7214
Epoch 9/10
- 4s - loss: 0.0243 - acc: 0.9907 - val_loss: 1.6475 - val_acc: 0.7066
Epoch 10/10
- 4s - loss: 0.0216 - acc: 0.9920 - val_loss: 1.9692 - val_acc: 0.6810
scores = model.evaluate(x_test, y_test, verbose=1)
25000/25000 [==============================] - 1s 36us/step
scores[1]
0.93452
predict = model.predict_classes(x_test)
predict[:10]
array([[1],
[1],
[1],
[1],
[1],
[1],
[1],
[1],
[1],
[1]])
predict_classes = predict.reshape(-1)
predict_classes[:10]
array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1])
SentimentDict = {1: '正面的', 2: '负面的'}
def display_test_SentimentDisct(i):
print(test_text[i])
print('label真实值：', SentimentDict[y_test[1]], '预测结果:',
SentimentDict[predict_classes[i]])
display_test_SentimentDisct(2)
As a recreational golfer with some knowledge of the sport's history, I was pleased with Disney's sensitivity to the issues of class in golf in the early twentieth century. The movie depicted well the psychological battles that Harry Vardon fought within himself, from his childhood trauma of being evicted to his own inability to break that glass ceiling that prevents him from being accepted as an equal in English golf society. Likewise, the young Ouimet goes through his own class struggles, being a mere caddie in the eyes of the upper crust Americans who scoff at his attempts to rise above his standing. What I loved best, however, is how this theme of class is manifested in the characters of Ouimet's parents. His father is a working-class drone who sees the value of hard work but is intimidated by the upper class; his mother, however, recognizes her son's talent and desire and encourages him to pursue his dream of competing against those who think he is inferior.Finally, the golf scenes are well photographed. Although the course used in the movie was not the actual site of the historical tournament, the little liberties taken by Disney do not detract from the beauty of the film. There's one little Disney moment at the pool table; otherwise, the viewer does not really think Disney. The ending, as in "Miracle," is not some Disney creation, but one that only human history could have written.
label真实值： 正面的 预测结果: 正面的
display_test_SentimentDisct(1250)
This movie was well done but it also made me feel very down at times as well. For anyone that is considering show business this is a must see as it shows the raw deal in what goes on for these struggling workers. The soundtrack was definitely cool and the acting and dancing complimented it nicely. Some of the student's attitudes might have been a little far-fetched like Leroy's especially because I'm sure someone like that would've been kicked out immediately for refusing to read and such if this was the real High School For Performing Arts. The Coco screen test is hard to watch for any people out there with weak stomachs, please heed my warning. While it's very gritty I know it's the truth on what happens so in this respect the movie is right on. Overall it's entertaining and even though some parts drag on the majority goes by really quickly.Final Grouping:Movies: Probably would've skipped this one.DVD Purchase: Not something I'd need to see again and again.Rental: Worth renting at least once in your life!
label真实值： 正面的 预测结果: 正面的
token = Tokenizer(num_words=2800)
token.fit_on_texts(train_text)
x_train_seq = token.texts_to_sequences(train_text)
x_test_seq = token.texts_to_sequences(test_text)
x_train = sequence.pad_sequences(x_train_seq, maxlen=380)
x_test = sequence.pad_sequences(x_test_seq, maxlen=380)
model = Sequential()
model.add(Embedding(output_dim=32,
input_dim=3800,
input_length=380))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(units=256,
activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(units=1,
activation='sigmoid'))
model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding_2 (Embedding) (None, 380, 32) 121600
_________________________________________________________________
dropout_3 (Dropout) (None, 380, 32) 0
_________________________________________________________________
flatten_2 (Flatten) (None, 12160) 0
_________________________________________________________________
dense_3 (Dense) (None, 256) 3113216
_________________________________________________________________
dropout_4 (Dropout) (None, 256) 0
_________________________________________________________________
dense_4 (Dense) (None, 1) 257
=================================================================
Total params: 3,235,073
Trainable params: 3,235,073
Non-trainable params: 0
_________________________________________________________________
