本次實驗對100張飛機圖片組成的數據集，分別使用pgd攻擊和fgsm攻擊，達到對每張圖片飛機區域的攻擊，并使用getmap程序對攻擊的效果進行評估。

文章目錄

• 1、運行1.py程序和auto.py程序對飛機數據集的所有圖片進行獲取掩碼操作
• • （1）1.py程序
  • （2）auto.py程序
  • （3）運行后得到自動生成的掩碼圖像
• 2、使用pgd對數據集生成對抗樣本
• 3、使用fgsm方法生成對抗樣本
• 4、使用map方法進行評估
• • （1）使用get_mappgd對生成的對抗樣本進行評估
  • （2）使用get_mapfgsm對生成的對抗樣本進行評估

1、運行1.py程序和auto.py程序對飛機數據集的所有圖片進行獲取掩碼操作

（1）1.py程序

import os from tqdm import tqdm from PIL import Imagefrom torchvision import transforms from auto import YOLOif __name__ == "__main__":yolo = YOLO()# -------------------------------------------------------------------------## dir_origin_path指定了用于檢測的圖片的文件夾路徑# dir_save_path指定了檢測完圖片的保存路徑# dir_origin_path和dir_save_path僅在mode='dir_predict'時有效dir_origin_path = "/home/yolo程序/yolov4/VOCdevkit/VOC2007/JPEGImages"dir_save_path = "/home/yolo程序/yolov4/patch/mask"img_names = os.listdir(dir_origin_path)for img_name in tqdm(img_names):if img_name.lower().endswith(('.bmp', '.dib', '.png', '.jpg', '.jpeg', '.pbm', '.pgm', '.ppm', '.tif', '.tiff')):image_path = os.path.join(dir_origin_path, img_name)image = Image.open(image_path)mask = yolo.auto_mask(image)toPIL = transforms.ToPILImage() # 這個函數可以將張量轉為PIL圖片，由小數轉為0-255之間的像素值mask = toPIL(mask)if not os.path.exists(dir_save_path):os.makedirs(dir_save_path)mask.save(os.path.join(dir_save_path, img_name))

（2）auto.py程序

import colorsys import os import time import cv2 import numpy as np import torch.nn as nn from PIL import ImageDraw, ImageFontfrom nets.yolo import YoloBody from utils.utils import (cvtColor, get_anchors, get_classes, preprocess_input,resize_image) from utils.utils_bbox import DecodeBoximport torch from torchvision import transforms ''' 訓練自己的數據集必看注釋！ '''class YOLO(object):_defaults = {# --------------------------------------------------------------------------## 使用自己訓練好的模型進行預測一定要修改model_path和classes_path！# model_path指向logs文件夾下的權值文件，classes_path指向model_data下的txt## 訓練好后logs文件夾下存在多個權值文件，選擇驗證集損失較低的即可。# 驗證集損失較低不代表mAP較高，僅代表該權值在驗證集上泛化性能較好。# 如果出現shape不匹配，同時要注意訓練時的model_path和classes_path參數的修改# --------------------------------------------------------------------------#"model_path": '/home/yolo程序/yolov4/model_data/ep098-loss0.443-val_loss1.670.pth',"classes_path": '/home/yolo程序/yolov4/model_data/dota.txt',# ---------------------------------------------------------------------## anchors_path代表先驗框對應的txt文件，一般不修改。# anchors_mask用于幫助代碼找到對應的先驗框，一般不修改。# ---------------------------------------------------------------------#"anchors_path": '/home/yolo程序/yolov4/model_data/yolo_anchors.txt',"anchors_mask": [[6, 7, 8], [3, 4, 5], [0, 1, 2]],# ---------------------------------------------------------------------## 輸入圖片的大小，必須為32的倍數。# ---------------------------------------------------------------------#"input_shape": [1024, 1024],# ---------------------------------------------------------------------## 只有得分大于置信度的預測框會被保留下來# ---------------------------------------------------------------------#"confidence": 0.3,# ---------------------------------------------------------------------## 非極大抑制所用到的nms_iou大小# ---------------------------------------------------------------------#"nms_iou": 0.3,# ---------------------------------------------------------------------## 該變量用于控制是否使用letterbox_image對輸入圖像進行不失真的resize，# 在多次測試后，發現關閉letterbox_image直接resize的效果更好# ---------------------------------------------------------------------#"letterbox_image": False,# -------------------------------## 是否使用Cuda# 沒有GPU可以設置成False# -------------------------------#"cuda": True,}@classmethoddef get_defaults(cls, n):if n in cls._defaults:return cls._defaults[n]else:return "Unrecognized attribute name '" + n + "'"# ---------------------------------------------------## 初始化YOLO# ---------------------------------------------------#def __init__(self, **kwargs):self.__dict__.update(self._defaults)for name, value in kwargs.items():setattr(self, name, value)# ---------------------------------------------------## 獲得種類和先驗框的數量# ---------------------------------------------------#self.class_names, self.num_classes = get_classes(self.classes_path)self.anchors, self.num_anchors = get_anchors(self.anchors_path)self.bbox_util = DecodeBox(self.anchors, self.num_classes, (self.input_shape[0], self.input_shape[1]),self.anchors_mask)# ---------------------------------------------------## 畫框設置不同的顏色# ---------------------------------------------------#hsv_tuples = [(x / self.num_classes, 1., 1.) for x in range(self.num_classes)]self.colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))self.colors = list(map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)), self.colors))self.generate()# ---------------------------------------------------## 生成模型# ---------------------------------------------------#def generate(self):# ---------------------------------------------------## 建立yolo模型，載入yolo模型的權重# ---------------------------------------------------#self.net = YoloBody(self.anchors_mask, self.num_classes)device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')# device = torch.device('cpu')self.net.load_state_dict(torch.load(self.model_path, map_location=device))self.net = self.net.eval()print('{} model, anchors, and classes loaded.'.format(self.model_path))if self.cuda:self.net = nn.DataParallel(self.net)self.net = self.net.cuda()# ---------------------------------------------------## 制作mask# ---------------------------------------------------#def auto_mask(self,image):# mask = np.ones((1024, 1024, 3), dtype=float)# mask = mask.transpose(2, 0, 1)# ---------------------------------------------------## 計算輸入圖片的高和寬# ---------------------------------------------------#image_shape = np.array(np.shape(image)[0:2])# ---------------------------------------------------------## 在這里將圖像轉換成RGB圖像，防止灰度圖在預測時報錯。# 代碼僅僅支持RGB圖像的預測，所有其它類型的圖像都會轉化成RGB# ---------------------------------------------------------#image = cvtColor(image)# ---------------------------------------------------------## 給圖像增加灰條，實現不失真的resize# 也可以直接resize進行識別# ---------------------------------------------------------#image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)# ---------------------------------------------------------## 添加上batch_size維度# ---------------------------------------------------------#image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)with torch.no_grad():images = torch.from_numpy(image_data)if self.cuda:images = images.cuda()# ---------------------------------------------------------## 將圖像輸入網絡當中進行預測！# ---------------------------------------------------------#outputs = self.net(images)outputs = self.bbox_util.decode_box(outputs)# ---------------------------------------------------------## 將預測框進行堆疊，然后進行非極大抑制# ---------------------------------------------------------#results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,image_shape, self.letterbox_image, conf_thres=self.confidence,nms_thres=self.nms_iou)if results[0] is None:mask = np.ones((3,1024, 1024))mask = torch.from_numpy(mask) # ndarray轉換為tensorreturn maskmask = np.zeros((1024, 1024, 3), dtype=np.uint8)top_label = np.array(results[0][:, 6], dtype='int32')top_conf = results[0][:, 4] * results[0][:, 5]top_boxes = results[0][:, :4]# ---------------------------------------------------------## 圖像繪制# ---------------------------------------------------------#for i, c in list(enumerate(top_label)):box = top_boxes[i]top, left, bottom, right = boxtop = max(0, np.floor(top).astype('int32'))left = max(0, np.floor(left).astype('int32'))bottom = min(image.size[1], np.floor(bottom).astype('int32'))right = min(image.size[0], np.floor(right).astype('int32'))polygon = np.array([[left, top], [right, top], [right, bottom], [left, bottom]], np.int32) # 坐標為順時針方向cv2.fillConvexPoly(mask, polygon, (255, 255, 255))mask = torch.from_numpy(mask.transpose(2, 0, 1)) # ndarray轉換為tensorreturn mask# ---------------------------------------------------## 檢測圖片# ---------------------------------------------------#def detect_image(self, image):# ---------------------------------------------------## 計算輸入圖片的高和寬# ---------------------------------------------------#image_shape = np.array(np.shape(image)[0:2])# ---------------------------------------------------------## 在這里將圖像轉換成RGB圖像，防止灰度圖在預測時報錯。# 代碼僅僅支持RGB圖像的預測，所有其它類型的圖像都會轉化成RGB# ---------------------------------------------------------#image = cvtColor(image)# ---------------------------------------------------------## 給圖像增加灰條，實現不失真的resize# 也可以直接resize進行識別# ---------------------------------------------------------#image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)# ---------------------------------------------------------## 添加上batch_size維度# ---------------------------------------------------------#image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)with torch.no_grad():images = torch.from_numpy(image_data)if self.cuda:images = images.cuda()# ---------------------------------------------------------## 將圖像輸入網絡當中進行預測！# ---------------------------------------------------------#outputs = self.net(images)outputs = self.bbox_util.decode_box(outputs)# ---------------------------------------------------------## 將預測框進行堆疊，然后進行非極大抑制# ---------------------------------------------------------#results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,image_shape, self.letterbox_image, conf_thres=self.confidence,nms_thres=self.nms_iou)if results[0] is None:return imagetop_label = np.array(results[0][:, 6], dtype='int32')top_conf = results[0][:, 4] * results[0][:, 5]top_boxes = results[0][:, :4]# ---------------------------------------------------------## 設置字體與邊框厚度# ---------------------------------------------------------#font = ImageFont.truetype(font='model_data/simhei.ttf',size=np.floor(3e-2 * image.size[1] + 0.5).astype('int32'))thickness = int(max((image.size[0] + image.size[1]) // np.mean(self.input_shape), 1))# ---------------------------------------------------------## 圖像繪制# ---------------------------------------------------------#for i, c in list(enumerate(top_label)):predicted_class = self.class_names[int(c)]box = top_boxes[i]score = top_conf[i]top, left, bottom, right = boxtop = max(0, np.floor(top).astype('int32'))left = max(0, np.floor(left).astype('int32'))bottom = min(image.size[1], np.floor(bottom).astype('int32'))right = min(image.size[0], np.floor(right).astype('int32'))label = '{} {:.2f}'.format(predicted_class, score)draw = ImageDraw.Draw(image)label_size = draw.textsize(label, font)label = label.encode('utf-8')print(label, top, left, bottom, right)if top - label_size[1] >= 0:text_origin = np.array([left, top - label_size[1]])else:text_origin = np.array([left, top + 1])for i in range(thickness):draw.rectangle([left + i, top + i, right - i, bottom - i], outline=self.colors[c])draw.rectangle([tuple(text_origin), tuple(text_origin + label_size)], fill=self.colors[c])draw.text(text_origin, str(label, 'UTF-8'), fill=(0, 0, 0), font=font)del drawreturn imagedef get_FPS(self, image, test_interval):image_shape = np.array(np.shape(image)[0:2])# ---------------------------------------------------------## 在這里將圖像轉換成RGB圖像，防止灰度圖在預測時報錯。# 代碼僅僅支持RGB圖像的預測，所有其它類型的圖像都會轉化成RGB# ---------------------------------------------------------#image = cvtColor(image)# ---------------------------------------------------------## 給圖像增加灰條，實現不失真的resize# 也可以直接resize進行識別# ---------------------------------------------------------#image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)# ---------------------------------------------------------## 添加上batch_size維度# ---------------------------------------------------------#image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)with torch.no_grad():images = torch.from_numpy(image_data)if self.cuda:images = images.cuda()# ---------------------------------------------------------## 將圖像輸入網絡當中進行預測！# ---------------------------------------------------------#outputs = self.net(images)outputs = self.bbox_util.decode_box(outputs)# ---------------------------------------------------------## 將預測框進行堆疊，然后進行非極大抑制# ---------------------------------------------------------#results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,image_shape, self.letterbox_image, conf_thres=self.confidence,nms_thres=self.nms_iou)t1 = time.time()for _ in range(test_interval):with torch.no_grad():# ---------------------------------------------------------## 將圖像輸入網絡當中進行預測！# ---------------------------------------------------------#outputs = self.net(images)outputs = self.bbox_util.decode_box(outputs)# ---------------------------------------------------------## 將預測框進行堆疊，然后進行非極大抑制# ---------------------------------------------------------#results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,image_shape, self.letterbox_image,conf_thres=self.confidence, nms_thres=self.nms_iou)t2 = time.time()tact_time = (t2 - t1) / test_intervalreturn tact_timedef get_map_txt(self, image_id, image, class_names, map_out_path):f = open(os.path.join(map_out_path, "detection-results/" + image_id + ".txt"), "w")image_shape = np.array(np.shape(image)[0:2])# ---------------------------------------------------------## 在這里將圖像轉換成RGB圖像，防止灰度圖在預測時報錯。# 代碼僅僅支持RGB圖像的預測，所有其它類型的圖像都會轉化成RGB# ---------------------------------------------------------#image = cvtColor(image)# ---------------------------------------------------------## 給圖像增加灰條，實現不失真的resize# 也可以直接resize進行識別# ---------------------------------------------------------#image_data = resize_image(image, (self.input_shape[1], self.input_shape[0]), self.letterbox_image)# ---------------------------------------------------------## 添加上batch_size維度# ---------------------------------------------------------#image_data = np.expand_dims(np.transpose(preprocess_input(np.array(image_data, dtype='float32')), (2, 0, 1)), 0)with torch.no_grad():images = torch.from_numpy(image_data)if self.cuda:images = images.cuda()# ---------------------------------------------------------## 將圖像輸入網絡當中進行預測！# ---------------------------------------------------------#outputs = self.net(images)outputs = self.bbox_util.decode_box(outputs)# ---------------------------------------------------------## 將預測框進行堆疊，然后進行非極大抑制# ---------------------------------------------------------#results = self.bbox_util.non_max_suppression(torch.cat(outputs, 1), self.num_classes, self.input_shape,image_shape, self.letterbox_image, conf_thres=self.confidence,nms_thres=self.nms_iou)if results[0] is None:returntop_label = np.array(results[0][:, 6], dtype='int32')top_conf = results[0][:, 4] * results[0][:, 5]top_boxes = results[0][:, :4]for i, c in list(enumerate(top_label)):predicted_class = self.class_names[int(c)]box = top_boxes[i]score = str(top_conf[i])top, left, bottom, right = boxif predicted_class not in class_names:continuef.write("%s %s %s %s %s %s\n" % (predicted_class, score[:6], str(int(left)), str(int(top)), str(int(right)), str(int(bottom))))f.close()return

（3）運行后得到自動生成的掩碼圖像

（當模型沒有檢測到圖片中飛機的存在時，會將掩碼設置為全白）

2、使用pgd對數據集生成對抗樣本

運行train_PGD.py

#-------------------------------------# # 對數據集進行訓練 #-------------------------------------# import numpy as np from rsa import sign from sqlalchemy import false, true import torch import torch.backends.cudnn as cudnn import torch.optim as optim from torch.utils.data import DataLoaderfrom nets.yolo import YoloBody from nets.yolo_training1 import YOLOLoss, weights_init from utils.callbacks import LossHistory from utils.dataloader import YoloDataset, yolo_dataset_collate from utils.utils import get_anchors, get_classes from utils.utils_fit import fit_one_epochfrom PIL import Image from torchvision import transforms from torch import autograd import torch.nn.functional as F import math import os def getimg(img_path,height,width):#讀取圖片和mask，img = Image.open(img_path)img = img.resize((height,width),Image.ANTIALIAS)if isinstance(img, Image.Image):width_ = img.widthheight_ = img.heightimg = torch.ByteTensor(torch.ByteStorage.from_buffer(img.tobytes()))img = img.view(height_, width_, 3).transpose(0, 1).transpose(0, 2).contiguous()img = img.view(1, 3, height_, width_)img = img.float().div(255.0)elif type(img) == np.ndarray and len(img.shape) == 3: # cv2 imageimg = torch.from_numpy(img.transpose(2, 0, 1)).float().div(255.0).unsqueeze(0)elif type(img) == np.ndarray and len(img.shape) == 4:img = torch.from_numpy(img.transpose(0, 3, 1, 2)).float().div(255.0)elif type(img) == torch.Tensor and len(img.shape) == 4:img = imgelse:print("unknow image type")exit(-1)if cuda :img = img.cuda()return imgdef yolo_single_loss(output, conf_thresh, num_classes, num_anchors, only_objectness=1):# output 是不同大小的feature map（每次輸入一層feature map）if len(output.shape) == 3:output = np.expand_dims(output, axis=0)batch = output.shape[0] # patch數量，默認為1print(output.shape[1])print((5 + num_classes) * num_anchors)assert (output.shape[1] == (5 + num_classes) * num_anchors)h = output.shape[2] # feature map 的寬w = output.shape[3] # feature map 的高 (1, 0, 2)output = output.reshape(batch * num_anchors, 5 + num_classes, h * w).transpose(1, 0).reshape(5 + num_classes,batch * num_anchors * h * w) # 將 feature map 轉換為（80+5，*）det_confs = torch.sigmoid(output[4]) # 當前feature map該點處存在目標的概率 sigmoid(output[4])loss = 0.0idx = np.where((det_confs[:]).cpu().data > conf_thresh)loss += torch.sum(det_confs[idx])return lossdef getmask2():img = np.ones((1024,1024,3),dtype=float)img = torch.from_numpy(img.transpose(2, 0, 1)).float().div(255.0).unsqueeze(0)pmask = np.ones((1024,1024,3),dtype=float)pmask = torch.from_numpy(pmask.transpose(2, 0, 1)).float().div(255.0).unsqueeze(0)for i in range(input_shape[0]):for j in range(input_shape[1]):for k in range(3):if (i>512) & (j>512):pmask[0][k][i][j]=1else:pmask[0][k][i][j]=0print(img.shape)return img,pmask if __name__ == "__main__":epsilon=6eps=0.3alpha=2/255cuda = Trueconf_thresh=0.5norm_ord = 'L2'classes_path = '/home/yolo程序/yolov4/model_data/dota.txt'anchors_path = '/home/yolo程序/yolov4/model_data/yolo_anchors.txt'anchors_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]model_path = '/home/yolo程序/yolov4/model_data/ep098-loss0.443-val_loss1.670.pth'input_shape = [1024, 1024]Cosine_lr = Falsen_epochs = 10batch_size = 1lr = 1e-3num_workers = 4#----------------------------------------------------## 獲取classes和anchor#----------------------------------------------------#class_names, num_classes = get_classes(classes_path)anchors, num_anchors = get_anchors(anchors_path)val_annotation_path = '/home/yolo程序/yolov4/2007_test.txt'val_txt_name = '/home/yolo程序/yolov4/VOCdevkit/VOC2007/ImageSets/Main/test.txt'path = '/home/yolo程序/yolov4/patch/mask/'with open(val_annotation_path) as f:val_lines = f.readlines()line=f.readlineswith open(val_txt_name) as f:data = f.read().splitlines()num_val = len(val_lines)imgList=os.listdir(path)listLen = len(imgList) # 該文件夾圖片個數listNum = 0with open(val_annotation_path) as f:val_lines = f.readlines()num_val = len(val_lines)with open(val_txt_name) as f:data = f.read().splitlines()val_dataset = YoloDataset(val_lines, input_shape, num_classes, mosaic=False, train = False)gen_val = DataLoader(val_dataset , shuffle =False, batch_size = batch_size, num_workers = num_workers, pin_memory=True, drop_last=True, collate_fn=yolo_dataset_collate)p_img_batch,pmask= getmask2()#------------------------------------------------------## 創建yolo模型#------------------------------------------------------#model = YoloBody(anchors_mask, num_classes)weights_init(model)if model_path != '':#------------------------------------------------------## 權值文件請看README，百度網盤下載#------------------------------------------------------#print('Load weights {}.'.format(model_path))device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')model_dict = model.state_dict()pretrained_dict = torch.load(model_path, map_location = device)pretrained_dict = {k: v for k, v in pretrained_dict.items() if np.shape(model_dict[k]) == np.shape(v)}model_dict.update(pretrained_dict)model.load_state_dict(model_dict)yolo_loss = YOLOLoss(anchors, num_classes, input_shape, cuda, anchors_mask)if cuda:model_train = torch.nn.DataParallel(model)cudnn.benchmark = Truemodel_train = model_train.cuda()p_img_batch = p_img_batch.cuda()pmask= pmask.cuda()p_img_batch.requires_grad_(True)optimizer = optim.Adam([p_img_batch], lr, weight_decay = 5e-4)if Cosine_lr:lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=5, eta_min=1e-5)else:lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.94)model_train = model.eval()#tensor設置可求導迭代多少輪for epoch in range(n_epochs):loss = 0val_loss = 0print("epochs:"+str(epoch) )for iteration, batch in enumerate(gen_val):pm = getimg(path+data[iteration]+'.jpg',input_shape[0], input_shape[1])images, targets = batch[0], batch[1]#pm = getimg(path+data[iteration]+'.jpg',input_shape[0], input_shape[1])if cuda:images = torch.from_numpy(images).type(torch.FloatTensor).cuda()targets = [torch.from_numpy(ann).type(torch.FloatTensor).cuda() for ann in targets]else:images = torch.from_numpy(images).type(torch.FloatTensor)targets = [torch.from_numpy(ann).type(torch.FloatTensor) for ann in targets]images.requires_grad_(True)temp=p_img_batch*pmaskadv_images=p_img_batch*pm + images#adv_images = p_img_batch*pmask+images#adv_images = p_img_batch+imagesoutputs= model_train(adv_images)img1 = F.interpolate(adv_images, (input_shape[0], input_shape[1]))img1 = img1[0, :, :,]imm1 = transforms.ToPILImage('RGB')(img1)imm1.save("/home/yolo程序/yolov4/patch/PGD/img/"+ data[iteration] +".jpg")img1 = F.interpolate(p_img_batch, (input_shape[0], input_shape[1]))img1 = img1[0, :, :,]imm1 = transforms.ToPILImage('RGB')(img1)imm1.save("/home/yolo程序/yolov4/patch/PGD/adv/p_img_batch.jpg")# img3 = F.interpolate(temp, (input_shape[0], input_shape[1]))# img3 = img3[0, :, :,]# imm3 = transforms.ToPILImage('RGB')(img3)# imm3.save("D:/Anaconda3.8/python/yolov4/patch/PGDare/adv/pmask3"+ data[iteration] +".jpg")loss_value_all = 0num_pos_all = 0#----------------------## 計算損失#----------------------#for l in range(len(outputs)):loss, loss_loc ,loss_conf ,loss_cls,num_pos = yolo_loss(l,outputs[l],targets)loss_item=loss_confloss_value_all += loss_itemnum_pos_all += num_posloss_value = loss_value_all / num_pos_allgrad= torch.autograd.grad(loss_item,adv_images,retain_graph=False,create_graph=False)[0]adv_images = adv_images.detach()+alpha*grad.sign()delta = torch.clamp(adv_images-images, min=-eps, max=eps)adv_images = torch.clamp(images + delta, min=0, max=1).detach()p_img_batch=adv_images-images#p_img_batch.clip_(0,1)optimizer.zero_grad()loss += loss_value.item()print(loss)

得到的對抗樣本只有少部分被系統識別飛機并生成對抗
左圖沒有生成明顯對抗，右圖生成了明顯對抗

3、使用fgsm方法生成對抗樣本

將train_PGD.py修改為使用fgsm方法，形成train_fgsm.py文件

import numpy as np from rsa import sign from sqlalchemy import false, true import torch import torch.backends.cudnn as cudnn import torch.nn as nn import torch.optim as optim from torch.utils.data import DataLoaderfrom nets.yolo import YoloBody from nets.yolo_training1 import YOLOLoss, weights_init from utils.callbacks import LossHistory from utils.dataloader import YoloDataset, yolo_dataset_collate from utils.utils import get_anchors, get_classes from utils.utils_fit import fit_one_epochfrom PIL import Image from torchvision import transforms from torch import autograd import torch.nn.functional as F import math import osdef getimg(img_path,height,width):#讀取圖片和mask，img = Image.open(img_path)img = img.resize((height,width),Image.ANTIALIAS)if isinstance(img, Image.Image):width_ = img.widthheight_ = img.heightimg = torch.ByteTensor(torch.ByteStorage.from_buffer(img.tobytes()))img = img.view(height_, width_, 3).transpose(0, 1).transpose(0, 2).contiguous()img = img.view(1, 3, height_, width_)img = img.float().div(255.0)elif type(img) == np.ndarray and len(img.shape) == 3: # cv2 imageimg = torch.from_numpy(img.transpose(2, 0, 1)).float().div(255.0).unsqueeze(0)elif type(img) == np.ndarray and len(img.shape) == 4:img = torch.from_numpy(img.transpose(0, 3, 1, 2)).float().div(255.0)elif type(img) == torch.Tensor and len(img.shape) == 4:img = imgelse:print("unknow image type")exit(-1)if cuda :img = img.cuda()return imgdef fgsm_attack(image, epsilon, data_grad):# 使用sign（符號）函數，將對x求了偏導的梯度進行符號化(正數為1，零為0，負數為-1)sign_data_grad = data_grad.sign()# 通過epsilon生成對抗樣本perturbed_image = image + epsilon * sign_data_grad# 噪聲越來越大，機器越來越難以識別，但人眼可以看出差別# 做一個剪裁的工作，將torch.clamp內部大于1的數值變為1，小于0的數值等于0，防止image越界perturbed_image = torch.clamp(perturbed_image, 0, 1)# 返回對抗樣本return perturbed_imagedef getmask2():img = np.ones((1024,1024,3),dtype=float)img = torch.from_numpy(img.transpose(2, 0, 1)).float().div(255.0).unsqueeze(0)pmask = np.ones((1024,1024,3),dtype=float)pmask = torch.from_numpy(pmask.transpose(2, 0, 1)).float().div(255.0).unsqueeze(0)for i in range(input_shape[0]):for j in range(input_shape[1]):for k in range(3):if (i>512) & (j>512):pmask[0][k][i][j]=1else:pmask[0][k][i][j]=0print(img.shape)return img,pmaskif __name__ == "__main__":epsilon=6eps=0.3alpha=2/255cuda = Trueconf_thresh=0.5norm_ord = 'L2'classes_path = '/home/yolo程序/yolov4/model_data/dota.txt'anchors_path = '/home/yolo程序/yolov4/model_data/yolo_anchors.txt'anchors_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]model_path = '/home/yolo程序/yolov4/model_data/ep098-loss0.443-val_loss1.670.pth'input_shape = [1024, 1024]Cosine_lr = Falsen_epochs = 10batch_size = 1lr = 1e-3num_workers = 4# ----------------------------------------------------## 獲取classes和anchor# ----------------------------------------------------#class_names, num_classes = get_classes(classes_path)anchors, num_anchors = get_anchors(anchors_path)val_annotation_path = '/home/yolo程序/yolov4/2007_test.txt'val_txt_name = '/home/yolo程序/yolov4/VOCdevkit/VOC2007/ImageSets/Main/test.txt'path = '/home/yolo程序/yolov4/patch/mask/'with open(val_annotation_path) as f:val_lines = f.readlines()line = f.readlineswith open(val_txt_name) as f:data = f.read().splitlines()num_val = len(val_lines)imgList = os.listdir(path)listLen = len(imgList) # 該文件夾圖片個數listNum = 0with open(val_annotation_path) as f:val_lines = f.readlines()num_val = len(val_lines)with open(val_txt_name) as f:data = f.read().splitlines()val_dataset = YoloDataset(val_lines, input_shape, num_classes, mosaic=False, train=False)gen_val = DataLoader(val_dataset, shuffle=False, batch_size=batch_size, num_workers=num_workers, pin_memory=True,drop_last=True, collate_fn=yolo_dataset_collate)p_img_batch, pmask = getmask2()# ------------------------------------------------------## 創建yolo模型# ------------------------------------------------------#model = YoloBody(anchors_mask, num_classes)weights_init(model)if model_path != '':# ------------------------------------------------------## 權值文件請看README，百度網盤下載# ------------------------------------------------------#print('Load weights {}.'.format(model_path))device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')model_dict = model.state_dict()pretrained_dict = torch.load(model_path, map_location=device)pretrained_dict = {k: v for k, v in pretrained_dict.items() if np.shape(model_dict[k]) == np.shape(v)}model_dict.update(pretrained_dict)model.load_state_dict(model_dict)yolo_loss = YOLOLoss(anchors, num_classes, input_shape, cuda, anchors_mask)if cuda:model_train = torch.nn.DataParallel(model)cudnn.benchmark = Truemodel_train = model_train.cuda()p_img_batch = p_img_batch.cuda()pmask = pmask.cuda()p_img_batch.requires_grad_(True)# optimizer = optim.Adam([p_img_batch], lr, weight_decay=5e-4)# if Cosine_lr:# lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=5, eta_min=1e-5)# else:# lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.94)## model_train = model.eval()# ============================ step 3/5 損失函數 ============================criterion = nn.CrossEntropyLoss() # 選擇損失函數# ============================ step 4/5 優化器 ============================optimizer = optim.SGD(model.parameters(), lr=lr, momentum=0.9) # 選擇優化器scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1) # 設置學習率下降策略# ============================ step 5/5 訓練 ============================train_curve = list()valid_curve = list()for epoch in range(n_epochs):loss_mean = 0.correct = 0.total = 0.model.train()print("epochs:" + str(epoch))for iteration, batch in enumerate(gen_val):pm = getimg(path+data[iteration]+'.jpg',input_shape[0], input_shape[1])images, targets = batch[0], batch[1]#pm = getimg(path+data[iteration]+'.jpg',input_shape[0], input_shape[1])if cuda:images = torch.from_numpy(images).type(torch.FloatTensor).cuda()targets = [torch.from_numpy(ann).type(torch.FloatTensor).cuda() for ann in targets]else:images = torch.from_numpy(images).type(torch.FloatTensor)targets = [torch.from_numpy(ann).type(torch.FloatTensor) for ann in targets]images.requires_grad_(True)temp=p_img_batch*pmaskadv_images=p_img_batch*pm + imagesoutputs = model_train(adv_images)img1 = F.interpolate(adv_images, (input_shape[0], input_shape[1]))img1 = img1[0, :, :, ]imm1 = transforms.ToPILImage('RGB')(img1)imm1.save("/home/yolo程序/yolov4/patch/fgsm/img/" + data[iteration] + ".jpg")img1 = F.interpolate(p_img_batch, (input_shape[0], input_shape[1]))img1 = img1[0, :, :, ]imm1 = transforms.ToPILImage('RGB')(img1)imm1.save("/home/yolo程序/yolov4/patch/fgsm/adv/p_img_batch.jpg")loss_value_all = 0num_pos_all = 0# ----------------------## 計算損失# ----------------------#for l in range(len(outputs)):loss, loss_loc, loss_conf, loss_cls, num_pos = yolo_loss(l, outputs[l], targets)loss_item = loss_confloss_value_all += loss_itemnum_pos_all += num_posloss_value = loss_value_all / num_pos_all# loss = criterion(outputs, targets)# loss.backward()# 收集datagrad# data_grad = adv_images.grad.datagrad = torch.autograd.grad(loss, adv_images, retain_graph=False, create_graph=False)[0]# 調用FGSM攻擊perturbed_data = fgsm_attack(adv_images, eps, grad)adv_images = adv_images.detach() + alpha * grad.sign()delta = torch.clamp(adv_images - images, min=-eps, max=eps)adv_images = torch.clamp(images + delta, min=0, max=1).detach()p_img_batch = adv_images - images# p_img_batch.clip_(0,1)optimizer.zero_grad()loss += loss.item()# loss.backward()print(loss)

生成的對抗樣本對大部分圖片有明顯攻擊

4、使用map方法進行評估

（1）使用get_mappgd對生成的對抗樣本進行評估

import os import xml.etree.ElementTree as ETfrom PIL import Image from tqdm import tqdmfrom utils.utils import get_classes from utils.utils_map import get_coco_map, get_map from yolo import YOLOos.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'if __name__ == "__main__":'''Recall和Precision不像AP是一個面積的概念，在門限值不同時，網絡的Recall和Precision值是不同的。map計算結果中的Recall和Precision代表的是當預測時，門限置信度為0.5時，所對應的Recall和Precision值。此處獲得的./map_out/detection-results/里面的txt的框的數量會比直接predict多一些，這是因為這里的門限低，目的是為了計算不同門限條件下的Recall和Precision值，從而實現map的計算。'''#------------------------------------------------------------------------------------------------------------------## map_mode用于指定該文件運行時計算的內容# map_mode為0代表整個map計算流程，包括獲得預測結果、獲得真實框、計算VOC_map。# map_mode為1代表僅僅獲得預測結果。# map_mode為2代表僅僅獲得真實框。# map_mode為3代表僅僅計算VOC_map。# map_mode為4代表利用COCO工具箱計算當前數據集的0.50:0.95map。需要獲得預測結果、獲得真實框后并安裝pycocotools才行#-------------------------------------------------------------------------------------------------------------------#map_mode = 0#-------------------------------------------------------## 此處的classes_path用于指定需要測量VOC_map的類別# 一般情況下與訓練和預測所用的classes_path一致即可#-------------------------------------------------------#classes_path = '/home/yolo程序/yolov4/model_data/dota.txt'#-------------------------------------------------------## MINOVERLAP用于指定想要獲得的mAP0.x# 比如計算mAP0.75，可以設定MINOVERLAP = 0.75。#-------------------------------------------------------#MINOVERLAP = 0.5#-------------------------------------------------------## map_vis用于指定是否開啟VOC_map計算的可視化#-------------------------------------------------------#map_vis = False#-------------------------------------------------------## 指向VOC數據集所在的文件夾# 默認指向根目錄下的VOC數據集#-------------------------------------------------------#VOCdevkit_path = '/home/yolo程序/yolov4/VOCdevkit'#-------------------------------------------------------## 結果輸出的文件夾，默認為map_out#-------------------------------------------------------#map_out_path = '/home/yolo程序/yolov4/patch/PGD/map_oup-mom'image_ids = open(os.path.join(VOCdevkit_path, "VOC2007/ImageSets/Main/test.txt")).read().strip().split()if not os.path.exists(map_out_path):os.makedirs(map_out_path)if not os.path.exists(os.path.join(map_out_path, 'ground-truth')):os.makedirs(os.path.join(map_out_path, 'ground-truth'))if not os.path.exists(os.path.join(map_out_path, 'detection-results')):os.makedirs(os.path.join(map_out_path, 'detection-results'))if not os.path.exists(os.path.join(map_out_path, 'images-optional')):os.makedirs(os.path.join(map_out_path, 'images-optional'))class_names, _ = get_classes(classes_path)if map_mode == 0 or map_mode == 1:print("Load model.")yolo = YOLO(confidence = 0.001, nms_iou = 0.5)print("Load model done.")print("Get predict result.")for image_id in tqdm(image_ids):image_path = os.path.join("/home/yolo程序/yolov4/patch/PGD/img/"+image_id+".jpg")image = Image.open(image_path)if map_vis:image.save(os.path.join(map_out_path, "images-optional/" + image_id + ".jpg"))yolo.get_map_txt(image_id, image, class_names, map_out_path)print("Get predict result done.")if map_mode == 0 or map_mode == 2:print("Get ground truth result.")for image_id in tqdm(image_ids):with open(os.path.join(map_out_path, "ground-truth/"+image_id+".txt"), "w") as new_f:root = ET.parse(os.path.join(VOCdevkit_path, "VOC2007/Annotations/"+image_id+".xml")).getroot()for obj in root.findall('object'):difficult_flag = Falseif obj.find('difficult')!=None:difficult = obj.find('difficult').textif int(difficult)==1:difficult_flag = Trueobj_name = obj.find('name').textif obj_name not in class_names:continuebndbox = obj.find('bndbox')left = bndbox.find('xmin').texttop = bndbox.find('ymin').textright = bndbox.find('xmax').textbottom = bndbox.find('ymax').textif difficult_flag:new_f.write("%s %s %s %s %s difficult\n" % (obj_name, left, top, right, bottom))else:new_f.write("%s %s %s %s %s\n" % (obj_name, left, top, right, bottom))print("Get ground truth result done.")if map_mode == 0 or map_mode == 3:print("Get map.")get_map(MINOVERLAP, True, path = map_out_path)print("Get map done.")if map_mode == 4:print("Get map.")get_coco_map(class_names = class_names, path = map_out_path)print("Get map done.")#***********************************************************************************map_out_path = 'D:/yolov4/patch/PGD/map_oup-imgforloc1'image_ids = open(os.path.join(VOCdevkit_path, "VOC2007/ImageSets/Main/test.txt")).read().strip().split()if not os.path.exists(map_out_path):os.makedirs(map_out_path)if not os.path.exists(os.path.join(map_out_path, 'ground-truth')):os.makedirs(os.path.join(map_out_path, 'ground-truth'))if not os.path.exists(os.path.join(map_out_path, 'detection-results')):os.makedirs(os.path.join(map_out_path, 'detection-results'))if not os.path.exists(os.path.join(map_out_path, 'images-optional')):os.makedirs(os.path.join(map_out_path, 'images-optional'))class_names, _ = get_classes(classes_path)if map_mode == 0 or map_mode == 1:print("Load model.")yolo = YOLO(confidence = 0.001, nms_iou = 0.5)print("Load model done.")print("Get predict result.")for image_id in tqdm(image_ids):image_path = os.path.join(VOCdevkit_path, "../yolov4/patch/PGD/imgforloc1/"+image_id+".jpg")image = Image.open(image_path)if map_vis:image.save(os.path.join(map_out_path, "images-optional/" + image_id + ".jpg"))yolo.get_map_txt(image_id, image, class_names, map_out_path)print("Get predict result done.")if map_mode == 0 or map_mode == 2:print("Get ground truth result.")for image_id in tqdm(image_ids):with open(os.path.join(map_out_path, "ground-truth/"+image_id+".txt"), "w") as new_f:root = ET.parse(os.path.join(VOCdevkit_path, "VOC2007/Annotations/"+image_id+".xml")).getroot()for obj in root.findall('object'):difficult_flag = Falseif obj.find('difficult')!=None:difficult = obj.find('difficult').textif int(difficult)==1:difficult_flag = Trueobj_name = obj.find('name').textif obj_name not in class_names:continuebndbox = obj.find('bndbox')left = bndbox.find('xmin').texttop = bndbox.find('ymin').textright = bndbox.find('xmax').textbottom = bndbox.find('ymax').textif difficult_flag:new_f.write("%s %s %s %s %s difficult\n" % (obj_name, left, top, right, bottom))else:new_f.write("%s %s %s %s %s\n" % (obj_name, left, top, right, bottom))print("Get ground truth result done.")if map_mode == 0 or map_mode == 3:print("Get map.")get_map(MINOVERLAP, True, path = map_out_path)print("Get map done.")if map_mode == 4:print("Get map.")get_coco_map(class_names = class_names, path = map_out_path)print("Get map done.")

運行結果

（2）使用get_mapfgsm對生成的對抗樣本進行評估

import os import xml.etree.ElementTree as ETfrom PIL import Image from tqdm import tqdmfrom utils.utils import get_classes from utils.utils_map import get_coco_map, get_map from yolo import YOLOos.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'if __name__ == "__main__":'''Recall和Precision不像AP是一個面積的概念，在門限值不同時，網絡的Recall和Precision值是不同的。map計算結果中的Recall和Precision代表的是當預測時，門限置信度為0.5時，所對應的Recall和Precision值。此處獲得的./map_out/detection-results/里面的txt的框的數量會比直接predict多一些，這是因為這里的門限低，目的是為了計算不同門限條件下的Recall和Precision值，從而實現map的計算。'''#------------------------------------------------------------------------------------------------------------------## map_mode用于指定該文件運行時計算的內容# map_mode為0代表整個map計算流程，包括獲得預測結果、獲得真實框、計算VOC_map。# map_mode為1代表僅僅獲得預測結果。# map_mode為2代表僅僅獲得真實框。# map_mode為3代表僅僅計算VOC_map。# map_mode為4代表利用COCO工具箱計算當前數據集的0.50:0.95map。需要獲得預測結果、獲得真實框后并安裝pycocotools才行#-------------------------------------------------------------------------------------------------------------------#map_mode = 0#-------------------------------------------------------## 此處的classes_path用于指定需要測量VOC_map的類別# 一般情況下與訓練和預測所用的classes_path一致即可#-------------------------------------------------------#classes_path = '/home/yolo程序/yolov4/model_data/dota.txt'#-------------------------------------------------------## MINOVERLAP用于指定想要獲得的mAP0.x# 比如計算mAP0.75，可以設定MINOVERLAP = 0.75。#-------------------------------------------------------#MINOVERLAP = 0.5#-------------------------------------------------------## map_vis用于指定是否開啟VOC_map計算的可視化#-------------------------------------------------------#map_vis = False#-------------------------------------------------------## 指向VOC數據集所在的文件夾# 默認指向根目錄下的VOC數據集#-------------------------------------------------------#VOCdevkit_path = '/home/yolo程序/yolov4/VOCdevkit'#-------------------------------------------------------## 結果輸出的文件夾，默認為map_out#-------------------------------------------------------#map_out_path = '/home/yolo程序/yolov4/patch/fgsm/map_oup-fgsm'image_ids = open(os.path.join(VOCdevkit_path, "VOC2007/ImageSets/Main/test.txt")).read().strip().split()if not os.path.exists(map_out_path):os.makedirs(map_out_path)if not os.path.exists(os.path.join(map_out_path, 'ground-truth')):os.makedirs(os.path.join(map_out_path, 'ground-truth'))if not os.path.exists(os.path.join(map_out_path, 'detection-results')):os.makedirs(os.path.join(map_out_path, 'detection-results'))if not os.path.exists(os.path.join(map_out_path, 'images-optional')):os.makedirs(os.path.join(map_out_path, 'images-optional'))class_names, _ = get_classes(classes_path)if map_mode == 0 or map_mode == 1:print("Load model.")yolo = YOLO(confidence=0.001, nms_iou=0.5)print("Load model done.")print("Get predict result.")for image_id in tqdm(image_ids):image_path = os.path.join("/home/yolo程序/yolov4/patch/fgsm/img/" + image_id + ".jpg")image = Image.open(image_path)if map_vis:image.save(os.path.join(map_out_path, "images-optional/" + image_id + ".jpg"))yolo.get_map_txt(image_id, image, class_names, map_out_path)print("Get predict result done.")if map_mode == 0 or map_mode == 2:print("Get ground truth result.")for image_id in tqdm(image_ids):with open(os.path.join(map_out_path, "ground-truth/" + image_id + ".txt"), "w") as new_f:root = ET.parse(os.path.join(VOCdevkit_path, "VOC2007/Annotations/" + image_id + ".xml")).getroot()for obj in root.findall('object'):difficult_flag = Falseif obj.find('difficult') != None:difficult = obj.find('difficult').textif int(difficult) == 1:difficult_flag = Trueobj_name = obj.find('name').textif obj_name not in class_names:continuebndbox = obj.find('bndbox')left = bndbox.find('xmin').texttop = bndbox.find('ymin').textright = bndbox.find('xmax').textbottom = bndbox.find('ymax').textif difficult_flag:new_f.write("%s %s %s %s %s difficult\n" % (obj_name, left, top, right, bottom))else:new_f.write("%s %s %s %s %s\n" % (obj_name, left, top, right, bottom))print("Get ground truth result done.")if map_mode == 0 or map_mode == 3:print("Get map.")get_map(MINOVERLAP, True, path=map_out_path)print("Get map done.")if map_mode == 4:print("Get map.")get_coco_map(class_names=class_names, path=map_out_path)print("Get map done.")

結果

總結

以上是生活随笔為你收集整理的使用pgd和fgsm方法进行攻击并使用map方法评估的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。