what.examples.yolov3_tog_attack_demo

  1import cv2
  2import numpy as np
  3
  4from what.models.detection.datasets.coco import COCO_CLASS_NAMES
  5from what.models.detection.utils.box_utils import draw_bounding_boxes
  6from what.models.detection.yolo.yolov3 import YOLOV3
  7from what.models.detection.yolo.utils.yolo_utils import yolo_process_output, yolov3_anchors, yolov3_tiny_anchors
  8
  9from what.attacks.detection.yolo.TOG import TOGAttack
 10from what.utils.resize import bilinear_resize
 11
 12from what.cli.model import *
 13from what.utils.file import get_file
 14
 15# Target Model
 16what_yolov3_model_list = what_model_list[0:4]
 17
 18def yolov3_tog_attack_demo():
 19
 20    classes = COCO_CLASS_NAMES
 21
 22    colors = np.random.uniform(0, 255, size=(len(classes), 3))
 23
 24    # Check what_model_list for all supported models
 25    index = 3
 26
 27    # Download the model first if not exists
 28    if not os.path.isfile(os.path.join(WHAT_MODEL_PATH, what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX])):
 29        get_file(what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX],
 30                    WHAT_MODEL_PATH,
 31                    what_yolov3_model_list[index][WHAT_MODEL_URL_INDEX],
 32                    what_yolov3_model_list[index][WHAT_MODEL_HASH_INDEX])
 33
 34    attack = TOGAttack(os.path.join(WHAT_MODEL_PATH, what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX]), "multi_untargeted", classes)
 35    attack.fixed = False
 36
 37    last_outs = None
 38    last_boxes = None
 39    last_probs = None
 40
 41    # Initialize the camera
 42    video = input(f"Please input the OpenCV capture device (e.g. 0, 1, 2): ")
 43
 44    while not video.isdigit():
 45        video = input(f"Please input the OpenCV capture device (e.g. 0, 1, 2): ")
 46
 47    # Capture from camera
 48    cap = cv2.VideoCapture(int(video))
 49    #cap.set(3, 1920)
 50    #cap.set(4, 1080)
 51
 52    n = 0
 53
 54    while(True): 
 55        # Capture the video frame
 56        success, origin_cv_image = cap.read()  # read the camera frame
 57        if not success:
 58            break
 59
 60        # For YOLO, the input pixel values are normalized to [0, 1]
 61        input_cv_image = cv2.resize(origin_cv_image, (416, 416))
 62        input_cv_image = np.array(input_cv_image).astype(np.float32) / 255.0
 63
 64        # Yolo inference
 65        input_cv_image, outs = attack.attack(input_cv_image)
 66
 67        if last_outs is not None:
 68            res_list = []
 69            for out, last_out in zip(outs, last_outs):
 70                out = out.reshape((-1, 5 + len(classes)))
 71                last_out = last_out.reshape((-1, 5 + len(classes)))
 72
 73                res = np.mean(out[:, 4] - last_out[:, 4])
 74                res_list.append(res)
 75        else:
 76            last_outs = outs
 77
 78        boxes, labels, probs = yolo_process_output(outs, yolov3_tiny_anchors, len(classes))
 79
 80        if last_boxes is not None:
 81            # Eliminate the boxes with low confidence and overlaped boxes
 82            if last_boxes.size > 0 and boxes.size > 0:
 83                indexes = cv2.dnn.NMSBoxes(np.vstack((boxes, last_boxes)).tolist(), np.hstack((np.array(probs), np.array(last_probs))), 0.5, 0.4)
 84                if len(indexes) > 0:
 85                    indexes = indexes.flatten()
 86
 87        last_boxes = np.copy(boxes)
 88        last_probs = np.copy(probs)
 89
 90        # Draw bounding boxes
 91        out_img = cv2.cvtColor(origin_cv_image, cv2.COLOR_RGB2BGR)
 92        out_img = out_img.astype(np.float32) / 255.0
 93        height, width, _ = out_img.shape
 94        noise = attack.noise
 95
 96        # Resize the noise to the same shape as the input image
 97        # noise_r = bilinear_resize(noise[:, :, 0], height, width)
 98        # noise_g = bilinear_resize(noise[:, :, 1], height, width)
 99        # noise_b = bilinear_resize(noise[:, :, 2], height, width)
100        # noise = np.dstack((noise_r, noise_g, noise_b))
101
102        # out_img = out_img + noise
103        out_img = np.clip(out_img, 0, 1)
104
105        out_img = (out_img * 255.0).astype(np.uint8)
106
107        out_img = draw_bounding_boxes(out_img, boxes, labels, classes, probs);
108
109        cv2.namedWindow("result", cv2.WINDOW_NORMAL)
110        cv2.imshow("result", out_img)
111
112        if (cv2.waitKey(1) & 0xFF == ord('q')):
113            break
def yolov3_tog_attack_demo():
 19def yolov3_tog_attack_demo():
 20
 21    classes = COCO_CLASS_NAMES
 22
 23    colors = np.random.uniform(0, 255, size=(len(classes), 3))
 24
 25    # Check what_model_list for all supported models
 26    index = 3
 27
 28    # Download the model first if not exists
 29    if not os.path.isfile(os.path.join(WHAT_MODEL_PATH, what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX])):
 30        get_file(what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX],
 31                    WHAT_MODEL_PATH,
 32                    what_yolov3_model_list[index][WHAT_MODEL_URL_INDEX],
 33                    what_yolov3_model_list[index][WHAT_MODEL_HASH_INDEX])
 34
 35    attack = TOGAttack(os.path.join(WHAT_MODEL_PATH, what_yolov3_model_list[index][WHAT_MODEL_FILE_INDEX]), "multi_untargeted", classes)
 36    attack.fixed = False
 37
 38    last_outs = None
 39    last_boxes = None
 40    last_probs = None
 41
 42    # Initialize the camera
 43    video = input(f"Please input the OpenCV capture device (e.g. 0, 1, 2): ")
 44
 45    while not video.isdigit():
 46        video = input(f"Please input the OpenCV capture device (e.g. 0, 1, 2): ")
 47
 48    # Capture from camera
 49    cap = cv2.VideoCapture(int(video))
 50    #cap.set(3, 1920)
 51    #cap.set(4, 1080)
 52
 53    n = 0
 54
 55    while(True): 
 56        # Capture the video frame
 57        success, origin_cv_image = cap.read()  # read the camera frame
 58        if not success:
 59            break
 60
 61        # For YOLO, the input pixel values are normalized to [0, 1]
 62        input_cv_image = cv2.resize(origin_cv_image, (416, 416))
 63        input_cv_image = np.array(input_cv_image).astype(np.float32) / 255.0
 64
 65        # Yolo inference
 66        input_cv_image, outs = attack.attack(input_cv_image)
 67
 68        if last_outs is not None:
 69            res_list = []
 70            for out, last_out in zip(outs, last_outs):
 71                out = out.reshape((-1, 5 + len(classes)))
 72                last_out = last_out.reshape((-1, 5 + len(classes)))
 73
 74                res = np.mean(out[:, 4] - last_out[:, 4])
 75                res_list.append(res)
 76        else:
 77            last_outs = outs
 78
 79        boxes, labels, probs = yolo_process_output(outs, yolov3_tiny_anchors, len(classes))
 80
 81        if last_boxes is not None:
 82            # Eliminate the boxes with low confidence and overlaped boxes
 83            if last_boxes.size > 0 and boxes.size > 0:
 84                indexes = cv2.dnn.NMSBoxes(np.vstack((boxes, last_boxes)).tolist(), np.hstack((np.array(probs), np.array(last_probs))), 0.5, 0.4)
 85                if len(indexes) > 0:
 86                    indexes = indexes.flatten()
 87
 88        last_boxes = np.copy(boxes)
 89        last_probs = np.copy(probs)
 90
 91        # Draw bounding boxes
 92        out_img = cv2.cvtColor(origin_cv_image, cv2.COLOR_RGB2BGR)
 93        out_img = out_img.astype(np.float32) / 255.0
 94        height, width, _ = out_img.shape
 95        noise = attack.noise
 96
 97        # Resize the noise to the same shape as the input image
 98        # noise_r = bilinear_resize(noise[:, :, 0], height, width)
 99        # noise_g = bilinear_resize(noise[:, :, 1], height, width)
100        # noise_b = bilinear_resize(noise[:, :, 2], height, width)
101        # noise = np.dstack((noise_r, noise_g, noise_b))
102
103        # out_img = out_img + noise
104        out_img = np.clip(out_img, 0, 1)
105
106        out_img = (out_img * 255.0).astype(np.uint8)
107
108        out_img = draw_bounding_boxes(out_img, boxes, labels, classes, probs);
109
110        cv2.namedWindow("result", cv2.WINDOW_NORMAL)
111        cv2.imshow("result", out_img)
112
113        if (cv2.waitKey(1) & 0xFF == ord('q')):
114            break