2020中国华录杯·数据湖算法大赛—定向算法赛（吸烟打电话检测）
!pip install paddlex
!pip install imgaug
import os
os.chdir('/home/aistudio/work/')
os.mkdir('./raw_data')
os.mkdir('./new_data')
os.mkdir('./new_data/train/')
os.mkdir('./new_data/test/')
!unzip /home/aistudio/data/data49342/data.zip -d /home/aistudio/work/raw_data/
import cv2
import numpy as np
count = 0
def PaddlingImage(path):
global count
img = cv2.imread(path)
h, w = img.shape[:2]
if h > w * 1.5:
f = img[:int(w*1.2), :]
count += 1
else:
f = img
return f
labels = sorted(os.listdir('raw_data/train'))
with open('labels.txt', 'w') as f:
for v in labels:
f.write(v.split('_')[0]+'\n')
print(labels)
['calling_images', 'normal_images', 'smoking_images']
from tqdm import tqdm
base = 'raw_data/train'
datas = []
for i, cls_fold in enumerate(labels):
cls_base = os.path.join(base, cls_fold)
files = os.listdir(cls_base)
for pt in tqdm(files):
img = os.path.join(cls_base, pt)
pad = PaddlingImage(img)
new_pt = os.path.join('train/', pt)
cv2.imwrite('new_data/'+new_pt, pad)
info = new_pt + ' ' + str(i) + '\n'
datas.append(info)
print(info)
print('{}/{}'.format(count, len(datas)))
split = 0.9
train_num = int(len(datas) * split)
eval_num = len(datas) - train_num
print('train:', train_num)
print('eval:', eval_num)
np.random.seed(120)
np.random.shuffle(datas)
with open('train_list.txt', 'w') as f:
for v in datas[:train_num]:
f.write(v)
with open('eval_list.txt', 'w') as f:
for v in datas[train_num:]:
f.write(v)
100%|██████████| 1227/1227 [00:04<00:00, 268.45it/s]
100%|██████████| 166/166 [00:00<00:00, 861.11it/s]
100%|██████████| 2168/2168 [00:12<00:00, 168.20it/s]
train/2141.jpg 2
622/3561
train: 3204
eval: 357
from paddlex.cls import transforms
import paddlex as pdx
import imgaug.augmenters as iaa
train_transforms = transforms.Compose([
transforms.ResizeByShort(short_size=230),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.RandomDistort(),
transforms.Normalize()
])
eval_transforms = transforms.Compose([
transforms.ResizeByShort(short_size=224),
transforms.CenterCrop(224),
transforms.Normalize()
])
base = 'new_data'
train_dataset = pdx.datasets.ImageNet(
data_dir=base,
file_list='train_list.txt',
label_list='labels.txt',
transforms=train_transforms,
shuffle=True)
eval_dataset = pdx.datasets.ImageNet(
data_dir=base,
file_list='eval_list.txt',
label_list='labels.txt',
transforms=eval_transforms,
shuffle=False)
2020-08-14 15:11:32 [INFO] Starting to read file list from dataset...
2020-08-14 15:11:32 [INFO] 3204 samples in file train_list.txt
2020-08-14 15:11:32 [INFO] Starting to read file list from dataset...
2020-08-14 15:11:32 [INFO] 357 samples in file eval_list.txt
model_name = 'ResNet101_vd'
num_classes = len(train_dataset.labels)
model = pdx.cls.ResNet101_vd(num_classes=num_classes)
model.train(num_epochs=100,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
train_batch_size=64,
lr_decay_epochs=[40, 80],
learning_rate=2e-4,
save_dir='model',
log_interval_steps=100,
save_interval_epochs=20)
2020-08-14 15:11:42 [INFO] Connecting PaddleHub server to get pretrain weights...
2020-08-14 15:11:45 [INFO] Load pretrain weights from model/pretrain/ResNet101_vd.
2020-08-14 15:11:45 [WARNING] [SKIP] Shape of pretrained weight model/pretrain/ResNet101_vd/fc_0.w_0 doesn't match.(Pretrained: (2048, 1000), Actual: (2048, 3))
2020-08-14 15:11:45 [WARNING] [SKIP] Shape of pretrained weight model/pretrain/ResNet101_vd/fc_0.b_0 doesn't match.(Pretrained: (1000,), Actual: (3,))
2020-08-14 15:11:45,378-WARNING: model/pretrain/ResNet101_vd.pdparams not found, try to load model file saved with [ save_params, save_persistables, save_vars ]
2020-08-14 15:11:46 [INFO] There are 530 varaibles in model/pretrain/ResNet101_vd are loaded.
2020-08-14 15:12:10 [INFO] [TRAIN] Epoch 1 finished, loss=0.872781, acc1=0.593437, acc3=1.0, lr=0.0002 .
2020-08-14 15:12:32 [INFO] [TRAIN] Epoch=2/100, Step=50/50, loss=0.775406, acc1=0.484375, acc3=1.0, lr=0.0002, time_each_step=0.37s, eta=0:39:16
2020-08-14 15:12:32 [INFO] [TRAIN] Epoch 2 finished, loss=0.798228, acc1=0.609062, acc3=1.0, lr=0.0002 .
2020-08-14 15:13:01 [INFO] [TRAIN] Epoch 3 finished, loss=0.787145, acc1=0.613125, acc3=1.0, lr=0.0002 .
2020-08-14 15:13:25 [INFO] [TRAIN] Epoch=4/100, Step=50/50, loss=0.639058, acc1=0.65625, acc3=1.0, lr=0.0002, time_each_step=0.37s, eta=0:47:15
2020-08-14 15:13:25 [INFO] [TRAIN] Epoch 4 finished, loss=0.774129, acc1=0.614688, acc3=1.0, lr=0.0002 .
2020-08-14 15:13:49 [INFO] [TRAIN] Epoch 5 finished, loss=0.77001, acc1=0.618438, acc3=1.0, lr=0.0002 .
2020-08-14 15:14:12 [INFO] [TRAIN] Epoch=6/100, Step=50/50, loss=0.684268, acc1=0.671875, acc3=1.0, lr=0.0002, time_each_step=0.39s, eta=0:37:45
2020-08-14 15:14:12 [INFO] [TRAIN] Epoch 6 finished, loss=0.76205, acc1=0.619375, acc3=1.0, lr=0.0002 .
2020-08-14 15:14:36 [INFO] [TRAIN] Epoch 7 finished, loss=0.762282, acc1=0.61125, acc3=1.0, lr=0.0002 .
2020-08-14 15:15:01 [INFO] [TRAIN] Epoch=8/100, Step=50/50, loss=0.655588, acc1=0.609375, acc3=1.0, lr=0.0002, time_each_step=0.4s, eta=0:36:54
2020-08-14 15:15:01 [INFO] [TRAIN] Epoch 8 finished, loss=0.750704, acc1=0.6275, acc3=1.0, lr=0.0002 .
2020-08-14 15:15:25 [INFO] [TRAIN] Epoch 9 finished, loss=0.753532, acc1=0.624375, acc3=1.0, lr=0.0002 .
2020-08-14 15:15:51 [INFO] [TRAIN] Epoch=10/100, Step=50/50, loss=0.71359, acc1=0.640625, acc3=1.0, lr=0.0002, time_each_step=0.38s, eta=0:36:28
2020-08-14 15:15:51 [INFO] [TRAIN] Epoch 10 finished, loss=0.747947, acc1=0.629062, acc3=1.0, lr=0.0002 .
2020-08-14 15:16:15 [INFO] [TRAIN] Epoch 11 finished, loss=0.749648, acc1=0.626562, acc3=1.0, lr=0.0002 .
2020-08-14 15:16:39 [INFO] [TRAIN] Epoch=12/100, Step=50/50, loss=0.771269, acc1=0.65625, acc3=1.0, lr=0.0002, time_each_step=0.38s, eta=0:35:38
2020-08-14 15:16:39 [INFO] [TRAIN] Epoch 12 finished, loss=0.742428, acc1=0.6325, acc3=1.0, lr=0.0002 .
2020-08-14 15:17:02 [INFO] [TRAIN] Epoch 13 finished, loss=0.741229, acc1=0.627813, acc3=1.0, lr=0.0002 .
2020-08-14 15:17:25 [INFO] [TRAIN] Epoch=14/100, Step=50/50, loss=0.783685, acc1=0.59375, acc3=1.0, lr=0.0002, time_each_step=0.39s, eta=0:33:40
2020-08-14 15:17:25 [INFO] [TRAIN] Epoch 14 finished, loss=0.738079, acc1=0.644687, acc3=1.0, lr=0.0002 .
2020-08-14 15:17:53 [INFO] [TRAIN] Epoch 15 finished, loss=0.739008, acc1=0.626562, acc3=1.0, lr=0.0002 .
2020-08-14 15:18:19 [INFO] [TRAIN] Epoch=16/100, Step=50/50, loss=0.648112, acc1=0.640625, acc3=1.0, lr=0.0002, time_each_step=0.4s, eta=0:39:21
2020-08-14 15:18:19 [INFO] [TRAIN] Epoch 16 finished, loss=0.730248, acc1=0.640312, acc3=1.0, lr=0.0002 .
2020-08-14 15:18:44 [INFO] [TRAIN] Epoch 17 finished, loss=0.725165, acc1=0.6475, acc3=1.0, lr=0.0002 .
2020-08-14 15:19:10 [INFO] [TRAIN] Epoch=18/100, Step=50/50, loss=0.647713, acc1=0.65625, acc3=1.0, lr=0.0002, time_each_step=0.39s, eta=0:35:14
2020-08-14 15:19:10 [INFO] [TRAIN] Epoch 18 finished, loss=0.720243, acc1=0.653437, acc3=1.0, lr=0.0002 .
2020-08-14 15:19:34 [INFO] [TRAIN] Epoch 19 finished, loss=0.727277, acc1=0.6375, acc3=1.0, lr=0.0002 .
2020-08-14 15:19:58 [INFO] [TRAIN] Epoch=20/100, Step=50/50, loss=0.719195, acc1=0.65625, acc3=1.0, lr=0.0002, time_each_step=0.38s, eta=0:32:53
2020-08-14 15:19:58 [INFO] [TRAIN] Epoch 20 finished, loss=0.721341, acc1=0.63125, acc3=1.0, lr=0.0002 .
2020-08-14 15:19:58 [INFO] Start to evaluating(total_samples=357, total_steps=6)...
0%| | 0/6 [00:00
model.evaluate(eval_dataset, batch_size=1, epoch_id=None, return_details=False)
2020-08-14 15:58:07 [INFO] Start to evaluating(total_samples=357, total_steps=357)...
100%|██████████| 357/357 [00:07<00:00, 47.98it/s]
OrderedDict([('acc1', 0.5518207282913166), ('acc3', 1.0)])
test_base = 'raw_data/test'
test_data = []
for img in os.listdir(test_base):
pt = os.path.join(test_base, img)
pad = PaddlingImage(pt)
new_pt = os.path.join('new_data/test/', img)
cv2.imwrite(new_pt, pad)
test_data.append(new_pt)
print(new_pt)
print('test_num:', len(test_data))
new_data/test/1444.jpg
test_num: 1752
import numpy as np
import paddlex as pdx
from tqdm import tqdm
import json
submission = []
labels = sorted(os.listdir('raw_data/train'))
for im_pt in tqdm(test_data):
value = {"image_name":None, "category":None, "score":None}
result = model.predict(im_pt, topk=1, transforms=eval_transforms)
value['category'] = result[0]['category']
value['score'] = str(result[0]['score'])
value['image_name'] = im_pt.split('/')[-1]
submission.append(value)
with open('result.json', 'w') as f:
json.dump(submission, f)
print('done')
