import numpy as np import pandas as pd import tensorflow as tf from tensorflow import keras from keras.models import Sequential from keras.layers import Dense from keras.layers import Dropout from keras.layers import Flatten from keras.constraints import maxnorm from tensorflow.keras.optimizers import SGD from keras.layers.convolutional import Conv2D from keras.layers import Dense, Conv2D ,Flatten,Dropout,MaxPool2D, BatchNormalization from keras.utils import np_utils import tensorflow as tf from keras.preprocessing.image import ImageDataGenerator from tensorflow.keras.preprocessing import image_dataset_from_directory from tensorflow.keras.preprocessing import image from tensorflow.keras.applications.vgg19 import VGG19 import keras from PIL import Image import matplotlib.pyplot as plt import seaborn from sklearn.metrics import confusion_matrix , classification_report import os import cv2 from skimage.transform import resize import streamlit as st def get_output_layers(net): layer_names = net.getLayerNames() output_layers = [layer_names[i[0] - 1] for i in net.getUnconnectedOutLayers()] return output_layers # function to draw bounding box on the detected object with class name def draw_bounding_box(img, class_id, confidence, x, y, x_plus_w, y_plus_h, COLORS): label = f'damage:{confidence}' color = COLORS[class_id] cv2.rectangle(img, (x,y), (x_plus_w,y_plus_h), color, 2) cv2.putText(img, label, (x-10,y-10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2) # plt.imshow(img) # plt.show() def detection_inference(image, scale = 1/255, image_size = 416, conf_threshold = 0.1, nms_threshold = 0.4): Width = image.shape[1] Height = image.shape[0] net=cv2.dnn.readNet('yolov4-custom_best.weights','yolov4-custom.cfg') COLORS = np.random.uniform(0, 255, size=(1, 3)) blob = cv2.dnn.blobFromImage(image, scale, (image_size, image_size), (0,0,0), True, crop=False) net.setInput(blob) outs = net.forward(get_output_layers(net)) class_ids = [] confidences = [] boxes = [] for out in outs: for detection in out: scores=detection[5:] class_id=np.argmax(scores) confidence=scores[class_id] if confidence > 0.1: center_x = int(detection[0] * Width) center_y = int(detection[1] * Height) w = int(detection[2] * Width) h = int(detection[3] * Height) x = center_x - w / 2 y = center_y - h / 2 class_ids.append(class_id) confidences.append(float(confidence)) boxes.append([x, y, w, h]) indices = cv2.dnn.NMSBoxes(boxes, confidences, conf_threshold, nms_threshold) for i in indices: i = i[0] box = boxes[i] x = box[0] y = box[1] w = box[2] h = box[3] draw_bounding_box(image, class_ids[i], confidences[i], round(x), round(y), round(x+w), round(y+h), COLORS) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) plt.imshow(image) plt.show() return image # st.image(image, caption='Object detection output', use_column_width=True) def _predict(img, model): m = keras.models.load_model(model) img2 = img.resize((224, 224)) image_array = np.asarray(img2) new_one = image_array.reshape((1, 224, 224, 3)) y_pred = m(new_one) print(y_pred) val = np.argmax(y_pred, axis = 1) return y_pred, val @tf.custom_gradient def guidedRelu(x): def grad(dy): return tf.cast(dy>0,"float32") * tf.cast(x>0, "float32") * dy return tf.nn.relu(x), grad def gradcam(img, model): m = keras.models.load_model(model) LAYER_NAME = 'block5_conv4' gb_model = tf.keras.models.Model( inputs = [m.inputs], outputs = [m.get_layer(LAYER_NAME).output] ) layer_dict = [layer for layer in gb_model.layers[1:] if hasattr(layer,'activation')] for layer in layer_dict: if layer.activation == tf.keras.activations.relu: layer.activation = guidedRelu img2 = img.resize((224, 224)) image_array = np.asarray(img2) print(image_array.shape) new_one = image_array.reshape((1, 224, 224, 3)) with tf.GradientTape() as tape: inputs = tf.cast(new_one, tf.float32) tape.watch(inputs) outputs = gb_model(inputs)[0] grads = tape.gradient(outputs,inputs)[0] weights = tf.reduce_mean(grads, axis=(0, 1)) grad_cam = np.ones(outputs.shape[0: 2], dtype = np.float32) for i, w in enumerate(weights): grad_cam += w * outputs[:, :, i] grad_cam_img = cv2.resize(grad_cam.numpy(), (img.size[0], img.size[1])) grad_cam_img = np.maximum(grad_cam_img, 0) heatmap = (grad_cam_img - grad_cam_img.min()) / (grad_cam_img.max() - grad_cam_img.min()) grad_cam_img = cv2.applyColorMap(np.uint8(255*heatmap), cv2.COLORMAP_JET) output_image = cv2.addWeighted(np.asarray(img).astype('uint8'), 1, grad_cam_img, 0.4, 0) output_img = Image.fromarray(output_image) st.image(output_img, caption='Class Activation Visualization', use_column_width=True) plt.imshow(output_image) plt.axis("off") plt.show() # guided_back_prop = grads # guided_cam = np.maximum(grad_cam, 0) # guided_cam = guided_cam / np.max(guided_cam) # scale 0 to 1.0 # guided_cam = resize(guided_cam, (224,224), preserve_range=True) # #pointwise multiplcation of guided backprop and grad CAM # gd_gb = np.dstack(( # guided_back_prop[:, :, 0] * guided_cam, # guided_back_prop[:, :, 1] * guided_cam, # guided_back_prop[:, :, 2] * guided_cam, # )) # plt.imshow(gd_gb) # plt.axis("off") # plt.show() uploaded_file = st.file_uploader( "Choose an image of your infrastructure", type=['jpg', 'jpeg', 'png']) if uploaded_file is not None: img = Image.open(uploaded_file).convert('RGB') cv_img = np.array(img) cv_img = cv2.cvtColor(cv_img, cv2.COLOR_RGB2BGR) # img2 = Image.open('test.jpg') st.image(img, caption='Uploaded file of your infrastructure', use_column_width=True) # similarity = ssim(img, img2) # st.write("") # st.write(f'This is {similarity * 100}% histopathological image') # if similarity >= 0.85: st.write("") st.write("Classifying...") y_pred, val = _predict(img, 'damage-detections.h5') if val == 0: st.write(f'The infrastructure has damage.') final_img = detection_inference(cv_img) final_pil_image = Image.fromarray(final_img) gradcam(final_pil_image, 'damage-detections.h5') else: st.write(f'The infrastructure does not have damage.') gradcam(img, 'damage-detections.h5')