Merge pull request #1111 from Laicheng0830/added_app

added yolov4

Author: Laicheng0830

.travis.yml

@@ -64,6 +64,7 @@ install:
   - |
     if [[ -v _DOC_AND_YAPF_TEST ]]; then
         pip install tensorflow==2.0.0-rc1
+        pip install opencv-python
         pip install yapf
         pip install -e .[doc]
     else

tensorlayer/__init__.py

@@ -44,6 +44,7 @@
     from tensorlayer import optimizers
     from tensorlayer import rein
     from tensorlayer import utils
+    from tensorlayer import app
 
     from tensorlayer.lazy_imports import LazyImport
 

tensorlayer/app/__init__.py

@@ -0,0 +1,5 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from .computer_vision_object_detection import *
+from .computer_vision import *

tensorlayer/app/computer_vision.py

@@ -0,0 +1,88 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from tensorlayer.app import YOLOv4, get_anchors, decode, filter_boxes
+import numpy as np
+import tensorflow as tf
+from tensorlayer import logging
+import cv2
+
+
+class object_detection(object):
+
+    def __init__(self, model_name='yolo4-mscoco'):
+        self.model_name = model_name
+        if self.model_name == 'yolo4-mscoco':
+            self.model = YOLOv4(NUM_CLASS=80, pretrained=True)
+        else:
+            raise ("The model does not support.")
+
+    def __call__(self, input_data):
+        if self.model_name == 'yolo4-mscoco':
+            batch_data = yolo4_input_processing(input_data)
+            feature_maps = self.model(batch_data, is_train=False)
+            output = yolo4_output_processing(feature_maps)
+        else:
+            raise NotImplementedError
+
+        return output
+
+    def __repr__(self):
+        s = ('(model_name={model_name}, model_structure={model}')
+        s += ')'
+        return s.format(classname=self.__class__.__name__, **self.__dict__)
+
+    @property
+    def list(self):
+        logging.info("The model name list: yolov4-mscoco")
+
+
+def yolo4_input_processing(original_image):
+    image_data = cv2.resize(original_image, (416, 416))
+    image_data = image_data / 255.
+    images_data = []
+    for i in range(1):
+        images_data.append(image_data)
+    images_data = np.asarray(images_data).astype(np.float32)
+    batch_data = tf.constant(images_data)
+    return batch_data
+
+
+def yolo4_output_processing(feature_maps):
+    STRIDES = [8, 16, 32]
+    ANCHORS = get_anchors([12, 16, 19, 36, 40, 28, 36, 75, 76, 55, 72, 146, 142, 110, 192, 243, 459, 401])
+    NUM_CLASS = 80
+    XYSCALE = [1.2, 1.1, 1.05]
+    iou_threshold = 0.45
+    score_threshold = 0.25
+
+    bbox_tensors = []
+    prob_tensors = []
+    score_thres = 0.2
+    for i, fm in enumerate(feature_maps):
+        if i == 0:
+            output_tensors = decode(fm, 416 // 8, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE)
+        elif i == 1:
+            output_tensors = decode(fm, 416 // 16, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE)
+        else:
+            output_tensors = decode(fm, 416 // 32, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE)
+        bbox_tensors.append(output_tensors[0])
+        prob_tensors.append(output_tensors[1])
+    pred_bbox = tf.concat(bbox_tensors, axis=1)
+    pred_prob = tf.concat(prob_tensors, axis=1)
+    boxes, pred_conf = filter_boxes(
+        pred_bbox, pred_prob, score_threshold=score_thres, input_shape=tf.constant([416, 416])
+    )
+    pred = {'concat': tf.concat([boxes, pred_conf], axis=-1)}
+
+    for key, value in pred.items():
+        boxes = value[:, :, 0:4]
+        pred_conf = value[:, :, 4:]
+
+    boxes, scores, classes, valid_detections = tf.image.combined_non_max_suppression(
+        boxes=tf.reshape(boxes, (tf.shape(boxes)[0], -1, 1, 4)),
+        scores=tf.reshape(pred_conf, (tf.shape(pred_conf)[0], -1, tf.shape(pred_conf)[-1])),
+        max_output_size_per_class=50, max_total_size=50, iou_threshold=iou_threshold, score_threshold=score_threshold
+    )
+    output = [boxes.numpy(), scores.numpy(), classes.numpy(), valid_detections.numpy()]
+    return output

tensorlayer/app/computer_vision_object_detection/__init__.py

@@ -0,0 +1,5 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+from .yolov4 import YOLOv4
+from .common import *

tensorlayer/app/computer_vision_object_detection/common.py

@@ -0,0 +1,173 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import tensorflow as tf
+import colorsys, random, cv2
+import numpy as np
+
+
+def decode_tf(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=0, XYSCALE=[1, 1, 1]):
+    batch_size = tf.shape(conv_output)[0]
+    conv_output = tf.reshape(conv_output, (batch_size, output_size, output_size, 3, 5 + NUM_CLASS))
+
+    conv_raw_dxdy, conv_raw_dwdh, conv_raw_conf, conv_raw_prob = tf.split(conv_output, (2, 2, 1, NUM_CLASS), axis=-1)
+
+    xy_grid = tf.meshgrid(tf.range(output_size), tf.range(output_size))
+    xy_grid = tf.expand_dims(tf.stack(xy_grid, axis=-1), axis=2)  # [gx, gy, 1, 2]
+    xy_grid = tf.tile(tf.expand_dims(xy_grid, axis=0), [batch_size, 1, 1, 3, 1])
+
+    xy_grid = tf.cast(xy_grid, tf.float32)
+
+    pred_xy = ((tf.sigmoid(conv_raw_dxdy) * XYSCALE[i]) - 0.5 * (XYSCALE[i] - 1) + xy_grid) * \
+              STRIDES[i]
+    pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i])
+    pred_xywh = tf.concat([pred_xy, pred_wh], axis=-1)
+
+    pred_conf = tf.sigmoid(conv_raw_conf)
+    pred_prob = tf.sigmoid(conv_raw_prob)
+
+    pred_prob = pred_conf * pred_prob
+    pred_prob = tf.reshape(pred_prob, (batch_size, -1, NUM_CLASS))
+    pred_xywh = tf.reshape(pred_xywh, (batch_size, -1, 4))
+
+    return pred_xywh, pred_prob
+
+
+def decode(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i, XYSCALE=[1, 1, 1]):
+    return decode_tf(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=i, XYSCALE=XYSCALE)
+
+
+def filter_boxes(box_xywh, scores, score_threshold=0.4, input_shape=tf.constant([416, 416])):
+    scores_max = tf.math.reduce_max(scores, axis=-1)
+
+    mask = scores_max >= score_threshold
+    class_boxes = tf.boolean_mask(box_xywh, mask)
+    pred_conf = tf.boolean_mask(scores, mask)
+    class_boxes = tf.reshape(class_boxes, [tf.shape(scores)[0], -1, tf.shape(class_boxes)[-1]])
+    pred_conf = tf.reshape(pred_conf, [tf.shape(scores)[0], -1, tf.shape(pred_conf)[-1]])
+
+    box_xy, box_wh = tf.split(class_boxes, (2, 2), axis=-1)
+
+    input_shape = tf.cast(input_shape, dtype=tf.float32)
+    box_yx = box_xy[..., ::-1]
+    box_hw = box_wh[..., ::-1]
+
+    box_mins = (box_yx - (box_hw / 2.)) / input_shape
+    box_maxes = (box_yx + (box_hw / 2.)) / input_shape
+    boxes = tf.concat(
+        [
+            box_mins[..., 0:1],  # y_min
+            box_mins[..., 1:2],  # x_min
+            box_maxes[..., 0:1],  # y_max
+            box_maxes[..., 1:2]  # x_max
+        ],
+        axis=-1
+    )
+    # return tf.concat([boxes, pred_conf], axis=-1)
+    return (boxes, pred_conf)
+
+
+def read_class_names(class_file_name):
+    names = {}
+    with open(class_file_name, 'r') as data:
+        for ID, name in enumerate(data):
+            names[ID] = name.strip('\n')
+    return names
+
+
+def draw_bbox(image, bboxes, show_label=True):
+    classes = read_class_names('model/coco.names')
+    num_classes = len(classes)
+    image_h, image_w, _ = image.shape
+    hsv_tuples = [(1.0 * x / num_classes, 1., 1.) for x in range(num_classes)]
+    colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
+    colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), colors))
+
+    random.seed(0)
+    random.shuffle(colors)
+    random.seed(None)
+
+    out_boxes, out_scores, out_classes, num_boxes = bboxes
+    for i in range(num_boxes[0]):
+        if int(out_classes[0][i]) < 0 or int(out_classes[0][i]) > num_classes: continue
+        coor = out_boxes[0][i]
+        coor[0] = int(coor[0] * image_h)
+        coor[2] = int(coor[2] * image_h)
+        coor[1] = int(coor[1] * image_w)
+        coor[3] = int(coor[3] * image_w)
+
+        fontScale = 0.5
+        score = out_scores[0][i]
+        class_ind = int(out_classes[0][i])
+        bbox_color = colors[class_ind]
+        bbox_thick = int(0.6 * (image_h + image_w) / 600)
+        c1, c2 = (coor[1], coor[0]), (coor[3], coor[2])
+        cv2.rectangle(image, c1, c2, bbox_color, bbox_thick)
+
+        if show_label:
+            bbox_mess = '%s: %.2f' % (classes[class_ind], score)
+            t_size = cv2.getTextSize(bbox_mess, 0, fontScale, thickness=bbox_thick // 2)[0]
+            c3 = (c1[0] + t_size[0], c1[1] - t_size[1] - 3)
+            cv2.rectangle(image, c1, (np.float32(c3[0]), np.float32(c3[1])), bbox_color, -1)  #filled
+
+            cv2.putText(
+                image, bbox_mess, (c1[0], np.float32(c1[1] - 2)), cv2.FONT_HERSHEY_SIMPLEX, fontScale, (0, 0, 0),
+                bbox_thick // 2, lineType=cv2.LINE_AA
+            )
+    return image
+
+
+def get_anchors(anchors_path, tiny=False):
+    anchors = np.array(anchors_path)
+    if tiny:
+        return anchors.reshape(2, 3, 2)
+    else:
+        return anchors.reshape(3, 3, 2)
+
+
+def decode_train(conv_output, output_size, NUM_CLASS, STRIDES, ANCHORS, i=0, XYSCALE=[1, 1, 1]):
+    conv_output = tf.reshape(conv_output, (tf.shape(conv_output)[0], output_size, output_size, 3, 5 + NUM_CLASS))
+
+    conv_raw_dxdy, conv_raw_dwdh, conv_raw_conf, conv_raw_prob = tf.split(conv_output, (2, 2, 1, NUM_CLASS), axis=-1)
+
+    xy_grid = tf.meshgrid(tf.range(output_size), tf.range(output_size))
+    xy_grid = tf.expand_dims(tf.stack(xy_grid, axis=-1), axis=2)  # [gx, gy, 1, 2]
+    xy_grid = tf.tile(tf.expand_dims(xy_grid, axis=0), [tf.shape(conv_output)[0], 1, 1, 3, 1])
+
+    xy_grid = tf.cast(xy_grid, tf.float32)
+
+    pred_xy = ((tf.sigmoid(conv_raw_dxdy) * XYSCALE[i]) - 0.5 * (XYSCALE[i] - 1) + xy_grid) * \
+              STRIDES[i]
+    pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i])
+    pred_xywh = tf.concat([pred_xy, pred_wh], axis=-1)
+
+    pred_conf = tf.sigmoid(conv_raw_conf)
+    pred_prob = tf.sigmoid(conv_raw_prob)
+
+    return tf.concat([pred_xywh, pred_conf, pred_prob], axis=-1)
+
+
+# def weights_sorted():
+#     # download weights
+#     maybe_download_and_extract(
+#         'yolov4.npz',
+#         model_path,
+#         'https://github.com/',
+#     )  # ls -al
+#     weights = []
+#     track_weights = []
+#     weights_dict = {}
+#
+#     npz = np.load(model_path, allow_pickle=True)
+#     # get weight list
+#     for val in sorted(npz.items()):
+#         logging.info("  Loading weights %s in %s" % (str(val[1].shape), val[0]))
+#         try:
+#             weights.append(int(val[0].split('/')[0].split('-')[-1]))
+#             track_weights.append(val[0])
+#         except:
+#             pass
+#     zip_weights = zip(weights, track_weights)
+#     zip_weights = sorted(zip_weights)
+#     for value, key in zip_weights:
+#         print(key)