# %%
import shap
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import gradio as gr
import os
# %%
from joblib import load
# Assuming 'model' is your trained model
model = load('model_trained.joblib')
# %%
import pandas as pd
# First set of data
data1 = {'ratio':[0.053910],
'sunset_minutes': [1080.968557],
'Kitchen_night_temperature':[20.784371],
'uvindex':[4.365996],'visibility':[15.912960],
'illuminance':[10646.052788] ,
'Kitchen_night_illuminance':[76.282996],
'Lounge_night_illuminance':[83.069952] ,
'Kitchen_morning_temperature' :[83.069952] ,
'Bathroom_evening_temperature':[21.449295],
'Bathroom_night_activity':[19.367503],
'number_transitions':[222.017461],
'temperature_ratio':[2.004192],
'awake_ratio':[0.207502],
'rr_min':[9.178120],
'To_bed' :[161.162813],
'Lounge_afternoon_activity':[46.898529],
'rr_average':[14.402949],
'rr_min_std':[0.696502],
'Kitchen_afternoon_illuminance':[371.943435]}
data2 = {'ratio' :[0.030281],
'sunset_minutes' : [84.442677] ,
'Kitchen_night_temperature' :[2.082688],
'uvindex': [2.701415],
'visibility': [5.435793],
'illuminance': [7337.315382],
'Kitchen_night_illuminance': [54.300962],
'Lounge_night_illuminance' :[116.355455],
'Bathroom_evening_temperature':[2.087516],
'Kitchen_morning_temperature': [54.300962],
'Bathroom_night_activity':[35.206093] ,
'number_transitions':[94.177603],
'temperature_ratio':[4.045142],
'awake_ratio':[0.061002],
'rr_min' :[0.428977],
'To_bed':[10.510216],
'Lounge_afternoon_activity':[174.547164],
'rr_average':[0.599838],
'rr_min_std':[0.299745],
'Kitchen_afternoon_illuminance':[255.993848]}
mean_df = pd.DataFrame(data1)
std_df = pd.DataFrame(data2)
plot_feature_names = {
"uvindex": "UV index",
"sunset_minutes": "Sunset time",
"temperature_ratio": "Temperature ratio",
"ratio": "Illuminance ratio",
"illuminance": "Outdoor illuminance",
"cloudcover": "Cloudcover",
"visibility": "Visibility",
"number_transitions": "Number of transitions",
'awake_ratio' : 'Awake ratio',
'rr_min' : 'Minimum respiratory rate',
'rr_average' :'Average respiratory rate',
'rr_min_std' :'Variability of respiratory rate',"To_bed":"Bedtime","ratio":"Illuminance ratio"}
# %%
def normalize(example_data_df, mean_df, std_df, top_features):
scale_global = example_data_df
scale_global = (scale_global- mean_df) / std_df
scale_global = scale_global.dropna(subset=scale_global.columns, how='all')
return scale_global
# %%
# %%
top_features= ['ratio',
'sunset_minutes',
'Kitchen_night_temperature',
'uvindex',
'visibility',
'illuminance',
'Kitchen_night_illuminance',
'Lounge_night_illuminance',
'Bathroom_morning_temperature',
'Bathroom_evening_temperature',
'Bathroom_night_activity',
'number_transitions',
'temperature_ratio',
'awake_ratio',
'rr_min',
'To_bed',
'Lounge_afternoon_activity',
'rr_average',
'rr_min_std',
'Kitchen_afternoon_illuminance']
# %%
def update_sliders(example_df):
example_data= example_df
Kitchen_afternoon_illuminance = example_data['Kitchen_afternoon_illuminance'].astype(float).iloc[-1]
Kitchen_night_illuminance = example_data['Kitchen_night_illuminance'].astype(float).iloc[-1]
Lounge_night_illuminance = example_data['Lounge_night_illuminance'].astype(float).iloc[-1]
ratio = example_data['ratio'].astype(float).iloc[-1]
Kitchen_night_temperature= example_data['Kitchen_night_temperature'].astype(float).iloc[-1]
Kitchen_morning_temperature= example_data['Kitchen_morning_temperature'].astype(float).iloc[-1]
Bathroom_evening_temperature= example_data['Bathroom_evening_temperature'].astype(float).iloc[-1]
temperature_ratio = example_data['temperature_ratio'].astype(float).iloc[-1]
return Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio
# %%
sns.set_palette("Set1")
blue = sns.color_palette()[0]
red = sns.color_palette()[1]
# %%
def predict(example_df,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio):
example_data = example_df
example_df_norm = normalize(example_data, mean_df, std_df, top_features)
feature_values = example_df_norm[top_features].iloc[0]
df = example_df_norm[top_features]
pos_pred = model.predict(df)
proba = model.predict_proba(df)
overall_prediction = "positive" if pos_pred[0] == 1 else "negative"
class_index = 1 if overall_prediction == "positive" else 0
shap_values = explainer.shap_values(df)
class_shap_values = shap_values[class_index][0]
all_class_features = zip(class_shap_values, top_features, feature_values)
sorted_class_features = sorted(all_class_features, key=lambda x: (x[0]), reverse=True)
top_three_class_features = sorted_class_features[:3]
statements = []
if proba[0][1] > 0.7940:
statements.append("There is a high risk that the patient was agitated the past week. During the previous week:")
if 0.3437 < proba[0][1] <= 0.7940:
statements.append("There is a medium risk that the patient was agitated the past week. During the previous week:")
if proba[0][1] <= 0.3437:
statements.append("There is a low risk that the patient was agitated the past week. During the previous week:")
for i, (shap_value, feature, feature_value) in enumerate(top_three_class_features):
if feature_value > 0:
feature_direction = "positive"
else:
feature_direction = "negative"
# Additional information based on the feature and its direction
if feature == "ratio":
if feature_direction == "positive":
statement= "The environment had a higher illuminance ratio (better light quality)."
statements.append(statement)
else:
statement= "The environment had a lower illuminance ratio (worse light quality, perhaps accompanied by glare)."
statements.append(statement)
elif feature == "awake_ratio":
if feature_direction == "positive":
statement= "The patient was alert during their sleep."
statements.append(statement)
else:
statement= "The patient had lower levels of alertness during their sleep."
statements.append(statement)
elif feature == "rr_average":
if feature_direction == "positive":
statement= "The average respiratory rate during sleep was higher than usual."
statements.append(statement)
else:
statement= "The average respiratory rate during sleep was lower than usual. Perhaps the patient was experiencing breathing disruptions."
statements.append(statement)
elif feature == "Lounge_night_illuminance":
if feature_direction == "positive":
statement= "There was higher illuminance than usual observed in the lounge at night."
statements.append(statement)
else:
statement= "There was lower illuminance than usual observed in the lounge at night."
statements.append(statement)
elif feature == "Bathroom_evening_temperature":
if feature_direction == "positive":
statement= "The temperature in the bathroom in the evening was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the bathroom in the evening was lower than usual."
statements.append(statement)
elif feature == "Kitchen_morning_temperature":
if feature_direction == "positive":
statement= "The temperature in the bathroom in the night was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the bathroom in the night was lower than usual."
statements.append(statement)
elif feature == "Kitchen_afternoon_illuminance":
if feature_direction == "positive":
statement = "The indoor light exposure in the kitchen in the afternoon was higher than usual."
statements.append(statement)
else:
statement= "The indoor light exposure in the kitchen in the afternoon was lower than usual."
statements.append(statement)
elif feature == "number_transitions":
if feature_direction == "positive":
statement= "The patient had increased movements from room to room in the house."
statements.append(statement)
else:
statement="The patient tended to stay within specific rooms and did not move a lot from room to room."
statements.append(statement)
elif feature == "Bathroom_night_activity":
if feature_direction == "positive":
statement="More time than usual was spent in the bathroom in the night."
statements.append(statement)
else:
statement="Less time than usual was spent in the bathroom in the night."
statements.append(statement)
elif feature == "Lounge_afternoon_activity":
if feature_direction == "positive":
statement="More time than usual was spent in the lounge in the afternoon."
statements.append(statement)
else:
statement="Less time than usual was spent in the lounge in the afternoon."
statements.append(statement)
elif feature == "Kitchen_night_illuminance":
if feature_direction == "positive":
statement="The indoor light exposure in the kitchen at night was higher than usual."
statements.append(statement)
else:
statement="The indoor light exposure in the kitchen at night was lower than usual."
statements.append(statement)
elif feature == "Bathroom_night_temperature":
if feature_direction == "positive":
statement="The temperature in the bathroom at night was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the bathroom at night was lower than usual."
statements.append(statement)
elif feature == "rr_min":
if feature_direction == "positive":
statement="The minimum respiratory rate during sleep was higher than usual."
statements.append(statement)
else:
statement= "The minimum respiratory rate was lower than usual. Perhaps the patient was experiencing breathing disruptions."
statements.append(statement)
elif feature == "visibility":
if feature_direction == "positive":
statement= "The sky visibility was higher than usual."
statements.append(statement)
else:
statement= "The sky visibility was lower than usual."
statements.append(statement)
elif feature == "sunset_minutes":
if feature_direction == "positive":
statement= "The time of sunset was later than usual."
statements.append(statement)
else:
statement= "The time of sunset was earlier than usual."
statements.append(statement)
elif feature == "uvindex":
if feature_direction == "positive":
statement= "The UV index was higher than usual."
statements.append(statement)
else:
statement="The UV index was lower than usual."
statements.append(statement)
elif feature == "illuminance":
if feature_direction == "positive":
statement="Outdoor illuminance was higher than usual."
statements.append(statement)
else:
statements="Outdoor illuminance was lower than usual."
statements.append(statement)
elif feature == "Kitchen_night_illuminance":
if feature_direction == "positive":
statement = "There was higher illuminance than usual observed in the kitchen at night."
statements.append(statement)
else:
statement="There was lower illuminance than usual observed in the kitchen at night."
statements.append(statement)
elif feature == "temperature_ratio":
if feature_direction == "positive":
statement = "The environment a smaller difference between indoor and outdoor temperature than usual."
statements.append(statement)
else:
statement= "The environment a bigger difference between indoor and outdoor temperature than usual."
statements.append(statement)
elif feature == "rr_min_std":
if feature_direction == "positive":
statement = "The variability in the minimum respiratory rate through the week was higher than usual."
statements.append(statement)
else:
statement= "The variability in the minimum respiratory rate through the week was lower than usual"
statements.append(statement)
elif feature == "To_bed":
if feature_direction == "positive":
statement = "The bedtimes of the participant during the week were more inconsistent than usual"
statements.append(statement)
else:
statement= "The bedtimes of the participant during the week were more consistent than usual"
statements.append(statement)
html_output = "
".join(statements)
formatted_statements = gr.HTML(html_output)
return {"Agitated": float(proba[0][1]), "Non-agitated": float(proba[0][0])}, formatted_statements
# %%
import shap
import matplotlib.pyplot as plt
def interpret(example_df,Lounge_night_illuminance, Kitchen_night_illuminance, Kitchen_afternoon_illuminance, ratio, temperature_ratio, Kitchen_morning_temperature, Bathroom_evening_temperature, Kitchen_night_temperature):
# Read the example data
example_data = example_df
# Normalize the example data
example_df_norm = normalize(example_data, mean_df, std_df, top_features)
# Select top features
df = example_df_norm[top_features]
# Make predictions
predictions = model.predict(df)
# Calculate SHAP values for all data points
shap_values = explainer.shap_values(df)
base = explainer.expected_value
feature_values = example_df_norm[top_features].iloc[0]
# Plotting for positive class predictions (class label 1)
if predictions[0] == 1:
pos_shap_values = shap_values[1][0]
scores_desc = list(zip(pos_shap_values, top_features, feature_values))
scores_desc = sorted(scores_desc)
colors = [red if s[0] < 0 else blue for s in scores_desc]
plot = plt.figure(tight_layout=True, figsize=(18,18))
plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
plt.rcParams['font.family'] = 'Times New Roman'
bars = plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
for bar, score in zip(bars, scores_desc):
text_color = 'black'
text_value = f'{score[2]:.2f}'
text_position = bar.get_width() - 0.08 if score[0] < 0 else bar.get_width() + 0.08 # Adjust the padding as needed
plt.text(text_position, bar.get_y() + bar.get_height() / 2, text_value, ha='left' if score[0] < 0 else 'right', va='center', color=text_color, fontsize=20)
plt.title("Feature Shap Values for Positive Prediction",fontsize=30)
plt.ylabel("Feature", fontsize=28)
plt.xlabel("Shap Value",fontsize=28)
plt.xlim(min([bar.get_width() for bar in bars]) - 0.1, max([bar.get_width() for bar in bars]) + 0.1)
plt.xticks(fontsize=28)
plt.yticks(fontsize=28)
plt.tight_layout()
return plot
# Plotting for negative class predictions (class label 0)
else:
neg_shap_values = shap_values[0][0]
scores_desc = list(zip(neg_shap_values, top_features, feature_values))
scores_desc = sorted(scores_desc)
colors = [red if s[0] > 0 else blue for s in scores_desc]
plot = plt.figure(tight_layout=True, figsize=(18,18))
plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
plt.rcParams['font.family'] = 'Times New Roman'
bars = plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
for bar, score in zip(bars, scores_desc):
text_color = 'black'
text_value = f'{score[2]:.2f}'
text_position = bar.get_width() - 0.08 if score[0] < 0 else bar.get_width() + 0.08# Adjust the padding as needed
plt.text(text_position, bar.get_y() + bar.get_height() / 2, text_value, ha='left' if score[0] < 0 else 'right', va='center', color=text_color, fontsize=20)
plt.title("Feature Shap Values for Negative Prediction", fontsize=30)
plt.ylabel("Feature",fontsize=28 )
plt.xlabel("Shap Value",fontsize=28)
plt.xlim(min([bar.get_width() for bar in bars]) - 0.1, max([bar.get_width() for bar in bars]) + 0.1)
plt.xticks(fontsize=28)
plt.yticks(fontsize=28)
plt.tight_layout()
return plot
# %%
def update_data(example_df, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio):
example_data = example_df
example_data['Lounge_night_illuminance'].iloc[0] = Lounge_night_illuminance
example_data['Kitchen_night_illuminance'].iloc[0] = Kitchen_night_illuminance
example_data['Kitchen_afternoon_illuminance'].iloc[0] = Kitchen_afternoon_illuminance
example_data['ratio'].iloc[0] = ratio
example_data['Kitchen_morning_temperature'].iloc[0] = Kitchen_morning_temperature
example_data['Bathroom_evening_temperature'].iloc[0] = Bathroom_evening_temperature
example_data['Kitchen_night_temperature'].iloc[0] = Kitchen_night_temperature
example_data['temperature_ratio'].iloc[0] = temperature_ratio
example_data = example_data.apply(pd.to_numeric, errors='coerce')
example_df_norm = normalize(example_data, mean_df, std_df, top_features)
df = example_df_norm[top_features]
pos_pred = model.predict(df)
proba = model.predict_proba(df)
label = {"Agitated": float(proba[0][1]), "Non-agitated": float(proba[0][0])}
feature_values = example_df_norm[top_features].iloc[0]
df = example_df_norm[top_features]
pos_pred = model.predict(df)
proba = model.predict_proba(df)
overall_prediction = "positive" if pos_pred[0] == 1 else "negative"
class_index = 1 if overall_prediction == "positive" else 0
shap_values = explainer.shap_values(df)
class_shap_values = shap_values[class_index][0]
all_class_features = zip(class_shap_values, top_features, feature_values)
sorted_class_features = sorted(all_class_features, key=lambda x: (x[0]), reverse=True)
top_three_class_features = sorted_class_features[:3]
statements = []
if proba[0][1] > 0.7940:
statements.append("There is a high risk that the patient was agitated the past week. During the previous week:")
if 0.3437 < proba[0][1] <= 0.7940:
statements.append("There is a medium risk that the patient was agitated the past week. During the previous week:")
if proba[0][1] <= 0.3437:
statements.append("There is a low risk that the patient was agitated the past week. During the previous week:")
for i, (shap_value, feature, feature_value) in enumerate(top_three_class_features):
if feature_value > 0:
feature_direction = "positive"
else:
feature_direction = "negative"
# Additional information based on the feature and its direction
if feature == "ratio":
if feature_direction == "positive":
statement= "The environment had a higher illuminance ratio (better light quality)."
statements.append(statement)
else:
statement= "The environment had a lower illuminance ratio (worse light quality, perhaps accompanied by glare)."
statements.append(statement)
elif feature == "awake_ratio":
if feature_direction == "positive":
statement= "The patient was alert during their sleep."
statements.append(statement)
else:
statement= "The patient had lower levels of alertness during their sleep."
statements.append(statement)
elif feature == "rr_average":
if feature_direction == "positive":
statement= "The average respiratory rate during sleep was higher than usual."
statements.append(statement)
else:
statement= "The average respiratory rate during sleep was lower than usual. Perhaps the patient was experiencing breathing disruptions."
statements.append(statement)
elif feature == "Lounge_night_illuminance":
if feature_direction == "positive":
statement= "There was higher illuminance than usual observed in the lounge at night."
statements.append(statement)
else:
statement= "There was lower illuminance than usual observed in the lounge at night."
statements.append(statement)
elif feature == "Bathroom_evening_temperature":
if feature_direction == "positive":
statement= "The temperature in the bathroom in the evening was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the bathroom in the evening was lower than usual."
statements.append(statement)
elif feature == "Kitchen_morning_temperature":
if feature_direction == "positive":
statement= "The temperature in the bathroom in the night was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the bathroom in the night was lower than usual."
statements.append(statement)
elif feature == "Kitchen_afternoon_illuminance":
if feature_direction == "positive":
statement = "The indoor light exposure in the kitchen in the afternoon was higher than usual."
statements.append(statement)
else:
statement= "The indoor light exposure in the kitchen in the afternoon was lower than usual."
statements.append(statement)
elif feature == "number_transitions":
if feature_direction == "positive":
statement= "The patient had increased movements from room to room in the house."
statements.append(statement)
else:
statement="The patient tended to stay within specific rooms and did not move a lot from room to room."
statements.append(statement)
elif feature == "Bathroom_night_activity":
if feature_direction == "positive":
statement="More time than usual was spent in the bathroom in the night."
statements.append(statement)
else:
statement="Less time than usual was spent in the bathroom in the night."
statements.append(statement)
elif feature == "Lounge_afternoon_activity":
if feature_direction == "positive":
statement="More time than usual was spent in the lounge in the afternoon."
statements.append(statement)
else:
statement="Less time than usual was spent in the lounge in the afternoon."
statements.append(statement)
elif feature == "Kitchen_night_illuminance":
if feature_direction == "positive":
statement="The indoor light exposure in the kitchen at night was higher than usual."
statements.append(statement)
else:
statement="The indoor light exposure in the kitchen at night was lower than usual."
statements.append(statement)
elif feature == "Kitchen_morning_temperature":
if feature_direction == "positive":
statement="The temperature in the bathroom at night was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the bathroom at night was lower than usual."
statements.append(statement)
elif feature == "rr_min":
if feature_direction == "positive":
statement="The minimum respiratory rate during sleep was higher than usual."
statements.append(statement)
else:
statement= "The minimum respiratory rate was lower than usual. Perhaps the patient was experiencing breathing disruptions."
statements.append(statement)
elif feature == "visibility":
if feature_direction == "positive":
statement= "The sky visibility was higher than usual."
statements.append(statement)
else:
statement= "The sky visibility was lower than usual."
statements.append(statement)
elif feature == "sunset_minutes":
if feature_direction == "positive":
statement= "The time of sunset was later than usual."
statements.append(statement)
else:
statement= "The time of sunset was earlier than usual."
statements.append(statement)
elif feature == "uvindex":
if feature_direction == "positive":
statement= "The UV index was higher than usual."
statements.append(statement)
else:
statement="The UV index was lower than usual."
statements.append(statement)
elif feature == "illuminance":
if feature_direction == "positive":
statement="Outdoor illuminance was higher than usual."
statements.append(statement)
else:
statements="Outdoor illuminance was lower than usual."
statements.append(statement)
elif feature == "Kitchen_night_illuminance":
if feature_direction == "positive":
statement = "There was higher illuminance than usual observed in the kitchen at night."
statements.append(statement)
else:
statement="There was lower illuminance than usual observed in the kitchen at night."
statements.append(statement)
elif feature == "temperature_ratio":
if feature_direction == "positive":
statement = "The environment has a smaller difference between indoor and outdoor temperature than usual."
statements.append(statement)
else:
statement= "The environment has a bigger difference between indoor and outdoor temperature than usual."
statements.append(statement)
elif feature == "rr_min_std":
if feature_direction == "positive":
statement = "The variability in the minimum respiratory rate through the week was higher than usual."
statements.append(statement)
else:
statement= "The variability in the minimum respiratory rate through the week was lower than usual"
statements.append(statement)
elif feature == "To_bed":
if feature_direction == "positive":
statement = "The bedtimes of the participant during the week were more inconsistent than usual"
statements.append(statement)
else:
statement= "The bedtimes of the participant during the week were more consistent than usual"
statements.append(statement)
html_output = "
".join(statements)
formatted_statements = gr.HTML(html_output)
# Select top features
df = example_df_norm[top_features]
# Make predictions
predictions = model.predict(df)
feature_values = example_df_norm[top_features].iloc[0]
# Calculate SHAP values for all data points
shap_values = explainer.shap_values(df)
base = explainer.expected_value
# Plotting for positive class predictions (class label 1)
if predictions[0] == 1:
pos_shap_values = shap_values[1][0]
scores_desc = list(zip(pos_shap_values, top_features, feature_values))
scores_desc = sorted(scores_desc)
colors = [red if s[0] < 0 else blue for s in scores_desc]
plot = plt.figure(figsize=(18, 18))
plt.rcParams['font.family'] = 'Times New Roman'
plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
bars = plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
for bar, score in zip(bars, scores_desc):
text_color = 'black'
text_value = f'{score[2]:.2f}'
text_position = bar.get_width() - 0.015 if score[0] < 0 else bar.get_width() + 0.015 # Adjust the padding as needed
plt.text(text_position, bar.get_y() + bar.get_height() / 2, text_value, ha='left' if score[0] < 0 else 'right', va='center', color=text_color, fontsize=20)
plt.title("Feature Shap Values for Positive Prediction",fontsize=30)
plt.ylabel("Feature", fontsize=28)
plt.xlabel("SHAP Value", fontsize=28)
plt.xlim(min([bar.get_width() for bar in bars]) - 0.03, max([bar.get_width() for bar in bars]) + 0.03)
plt.xticks(fontsize=28)
plt.yticks(fontsize=28)
plt.tight_layout()
return {"Agitated": float(proba[0][1]), "Non-agitated": float(proba[0][0])},formatted_statements, plot
# Plotting for negative class predictions (class label 0)
else:
neg_shap_values = shap_values[0][0]
scores_desc = list(zip(neg_shap_values, top_features, feature_values))
scores_desc = sorted(scores_desc)
colors = [red if s[0] > 0 else blue for s in scores_desc]
plot = plt.figure(tight_layout=True,figsize=(18, 18))
plt.rcParams['font.family'] = 'Times New Roman'
plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
bars = plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
for bar, score in zip(bars, scores_desc):
text_color = 'black'
text_value = f'{score[2]:.2f}'
text_position = bar.get_width() - 0.015 if score[0] < 0 else bar.get_width() + 0.015 # Adjust the padding as needed
plt.text(text_position, bar.get_y() + bar.get_height() / 2, text_value, ha='left' if score[0] < 0 else 'right', va='center', color=text_color, fontsize=20)
plt.title("Feature Shap Values for Negative Prediction",fontsize=30)
plt.ylabel("Feature", fontsize=28)
plt.xlabel("SHAP Value", fontsize=28)
plt.xlim(min([bar.get_width() for bar in bars]) - 0.03, max([bar.get_width() for bar in bars]) + 0.03)
plt.xticks(fontsize=28)
plt.yticks(fontsize=28)
plt.tight_layout()
return {"Agitated": float(proba[0][1]), "Non-agitated": float(proba[0][0])},formatted_statements, plot
# %%
import pandas as pd
import numpy as np
def generate_data():
data = np.random.rand(1, len(top_features))
df = pd.DataFrame(data, columns=top_features)
# Assign specific values for each column
df['sunset_minutes'] = 900.015348
df['cloudcover'] = 41
df['visibility'] = 16
df['illuminance'] = 8000
df['UV_index'] = 2
df['rr_minimum'] = np.random.uniform(8, 18)
df['rr_average'] = np.random.uniform(8, 18)
df['awake_ratio'] = np.random.uniform(0, 1)# Random value between 0 and 40
df['Lounge_night_illuminance'] = np.random.uniform(0, 500)
df['Kitchen_night_illuminance'] = np.random.uniform(0, 500)
df['Kitchen_afternoon_illuminance'] = np.random.uniform(0, 5000)
df['ratio'] = np.random.uniform(0, 15)
df['temperature_ratio'] = np.random.uniform(-1.5, 1.5)
df['Kitchen_morning_temperature'] = np.random.uniform(0, 40) # Random value between 0 and 40
df['Bathroom_evening_temperature'] = np.random.uniform(0, 40)
df['Kitchen_night_temperature'] = np.random.uniform(0, 40)
df['number_transitions'] = np.random.uniform(0,500)
df['rr_min_std'] = np.random.uniform(0,1)
df['To_bed'] = np.random.uniform(50,360)
# Retrieve the values
Kitchen_afternoon_illuminance = df['Kitchen_afternoon_illuminance'].astype(float).iloc[-1]
Kitchen_night_illuminance = df['Kitchen_night_illuminance'].astype(float).iloc[-1]
Lounge_night_illuminance = df['Lounge_night_illuminance'].astype(float).iloc[-1]
ratio = df['ratio'].astype(float).iloc[-1]
Kitchen_night_temperature= df['Kitchen_night_temperature'].astype(float).iloc[-1]
Kitchen_morning_temperature= df['Kitchen_morning_temperature'].astype(float).iloc[-1]
Bathroom_evening_temperature= df['Bathroom_evening_temperature'].astype(float).iloc[-1]
temperature_ratio = df['temperature_ratio'].astype(float).iloc[-1]
return df,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio
# %%
import pandas as pd
import numpy as np
def return_original(example_df):
example_data = example_df
Lounge_night_illuminance = example_data['Lounge_night_illuminance'].astype(float).iloc[-1]
Kitchen_night_illuminance = example_data['Kitchen_night_illuminance'].astype(float).iloc[-1]
Kitchen_afternoon_illuminance = example_data['Kitchen_afternoon_illuminance'].astype(float).iloc[-1]
ratio = example_data['ratio'].astype(float).iloc[-1]
temperature_ratio = example_data['temperature_ratio'].astype(float).iloc[-1]
Kitchen_morning_temperature= example_data['Kitchen_morning_temperature'].astype(float).iloc[-1]
Bathroom_evening_temperature= example_data['Bathroom_evening_temperature'].astype(float).iloc[-1]
Kitchen_night_temperature= example_data['Kitchen_night_temperature'].astype(float).iloc[-1]
example_df_norm = normalize(example_data, mean_df, std_df, top_features)
df = example_df_norm[top_features]
pos_pred = model.predict(df)
proba = model.predict_proba(df)
# Normalize the example data
label = {"Agitated": float(proba[0][1]), "Non-agitated": float(proba[0][0])}
feature_values = example_df_norm[top_features].iloc[0]
df = example_df_norm[top_features]
pos_pred = model.predict(df)
proba = model.predict_proba(df)
overall_prediction = "positive" if pos_pred[0] == 1 else "negative"
class_index = 1 if overall_prediction == "positive" else 0
shap_values = explainer.shap_values(df)
class_shap_values = shap_values[class_index][0]
all_class_features = zip(class_shap_values, top_features, feature_values)
sorted_class_features = sorted(all_class_features, key=lambda x: (x[0]), reverse=True)
top_three_class_features = sorted_class_features[:3]
statements = []
if proba[0][1] > 0.7940:
statements.append("There is a high risk that the patient was agitated the past week. During the previous week:")
if 0.3437 < proba[0][1] <= 0.7940:
statements.append("There is a medium risk that the patient was agitated the past week. During the previous week:")
if proba[0][1] <= 0.3437:
statements.append("There is a low risk that the patient was agitated the past week. During the previous week:")
for i, (shap_value, feature, feature_value) in enumerate(top_three_class_features):
if feature_value > 0:
feature_direction = "positive"
else:
feature_direction = "negative"
if feature == "ratio":
if feature_direction == "positive":
statement= "The environment had a higher illuminance ratio (better light quality)."
statements.append(statement)
else:
statement= "The environment had a lower illuminance ratio (worse light quality, perhaps accompanied by glare)."
statements.append(statement)
elif feature == "awake_ratio":
if feature_direction == "positive":
statement= "The patient was alert during their sleep."
statements.append(statement)
else:
statement= "The patient had lower levels of alertness during their sleep."
statements.append(statement)
elif feature == "rr_average":
if feature_direction == "positive":
statement= "The average respiratory rate during sleep was higher than usual."
statements.append(statement)
else:
statement= "The average respiratory rate during sleep was lower than usual. Perhaps the patient was experiencing breathing disruptions."
statements.append(statement)
elif feature == "Lounge_night_illuminance":
if feature_direction == "positive":
statement= "There was higher illuminance than usual observed in the lounge at night."
statements.append(statement)
else:
statement= "There was lower illuminance than usual observed in the lounge at night."
statements.append(statement)
elif feature == "Bathroom_evening_temperature":
if feature_direction == "positive":
statement= "The temperature in the bathroom in the evening was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the bathroom in the evening was lower than usual."
statements.append(statement)
elif feature == "Kitchen_morning_temperature":
if feature_direction == "positive":
statement= "The temperature in the kitchen in the morning was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the kitchen in the morning was lower than usual."
statements.append(statement)
elif feature == "Kitchen_afternoon_illuminance":
if feature_direction == "positive":
statement = "The indoor light exposure in the kitchen in the afternoon was higher than usual."
statements.append(statement)
else:
statement= "The indoor light exposure in the kitchen in the afternoon was lower than usual."
statements.append(statement)
elif feature == "number_transitions":
if feature_direction == "positive":
statement= "The patient had increased movements from room to room in the house."
statements.append(statement)
else:
statement="The patient tended to stay within specific rooms and did not move a lot from room to room."
statements.append(statement)
elif feature == "Bathroom_night_activity":
if feature_direction == "positive":
statement="More time than usual was spent in the bathroom in the night."
statements.append(statement)
else:
statement="Less time than usual was spent in the bathroom in the night."
statements.append(statement)
elif feature == "Lounge_afternoon_activity":
if feature_direction == "positive":
statement="More time than usual was spent in the lounge in the afternoon."
statements.append(statement)
else:
statement="Less time than usual was spent in the lounge in the afternoon."
statements.append(statement)
elif feature == "Kitchen_night_illuminance":
if feature_direction == "positive":
statement="The indoor light exposure in the kitchen at night was higher than usual."
statements.append(statement)
else:
statement="The indoor light exposure in the kitchen at night was lower than usual."
statements.append(statement)
elif feature == "Bathroom_night_temperature":
if feature_direction == "positive":
statement="The temperature in the bathroom at night was higher than usual."
statements.append(statement)
else:
statement= "The temperature in the bathroom at night was lower than usual."
statements.append(statement)
elif feature == "rr_min":
if feature_direction == "positive":
statement="The minimum respiratory rate during sleep was higher than usual."
statements.append(statement)
else:
statement= "The minimum respiratory rate was lower than usual. Perhaps the patient was experiencing breathing disruptions."
statements.append(statement)
elif feature == "visibility":
if feature_direction == "positive":
statement= "The sky visibility was higher than usual."
statements.append(statement)
else:
statement= "The sky visibility was lower than usual."
statements.append(statement)
elif feature == "sunset_minutes":
if feature_direction == "positive":
statement= "The time of sunset was later than usual."
statements.append(statement)
else:
statement= "The time of sunset was earlier than usual."
statements.append(statement)
elif feature == "uvindex":
if feature_direction == "positive":
statement= "The UV index was higher than usual."
statements.append(statement)
else:
statement="The UV index was lower than usual."
statements.append(statement)
elif feature == "illuminance":
if feature_direction == "positive":
statement="Outdoor illuminance was higher than usual."
statements.append(statement)
else:
statements="Outdoor illuminance was lower than usual."
statements.append(statement)
elif feature == "Kitchen_night_illuminance":
if feature_direction == "positive":
statement = "There was higher illuminance than usual observed in the kitchen at night."
statements.append(statement)
else:
statement="There was lower illuminance than usual observed in the kitchen at night."
statements.append(statement)
elif feature == "temperature_ratio":
if feature_direction == "positive":
statement = "The environment a smaller difference between indoor and outdoor temperature than usual."
statements.append(statement)
else:
statement= "The environment a bigger difference between indoor and outdoor temperature than usual."
statements.append(statement)
elif feature == "rr_min_std":
if feature_direction == "positive":
statement = "The variability in the minimum respiratory rate through the week was higher than usual."
statements.append(statement)
else:
statement= "The variability in the minimum respiratory rate through the week was lower than usual"
statements.append(statement)
elif feature == "To_bed":
if feature_direction == "positive":
statement = "The bedtimes of the participant during the week were more inconsistent than usual"
statements.append(statement)
else:
statement= "The bedtimes of the participant during the week were more consistent than usual"
statements.append(statement)
html_output = "
".join(statements)
formatted_statements = gr.HTML(html_output)
# Select top features
df = example_df_norm[top_features]
# Make predictions
predictions = model.predict(df)
feature_values = example_df_norm[top_features].iloc[0]
# Calculate SHAP values for all data points
shap_values = explainer.shap_values(df)
base = explainer.expected_value
# Plotting for positive class predictions (class label 1)
if predictions[0] == 1:
pos_shap_values = shap_values[1][0]
scores_desc = list(zip(pos_shap_values, top_features, feature_values))
scores_desc = sorted(scores_desc)
colors = [red if s[0] < 0 else blue for s in scores_desc]
plot = plt.figure(tight_layout=True, figsize=(18,18))
plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
plt.rcParams['font.family'] = 'Times New Roman'
bars = plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
for bar, score in zip(bars, scores_desc):
text_color = 'black'
text_value = f'{score[2]:.2f}'
text_position = bar.get_width() - 0.08 if score[0] < 0 else bar.get_width() + 0.08 # Adjust the padding as needed
plt.text(text_position, bar.get_y() + bar.get_height() / 2, text_value, ha='left' if score[0] < 0 else 'right', va='center', color=text_color, fontsize=20)
plt.title("Feature Shap Values for Positive Prediction",fontsize=30)
plt.ylabel("Feature", fontsize=28)
plt.xlabel("Shap Value",fontsize=28)
plt.xlim(min([bar.get_width() for bar in bars]) - 0.1, max([bar.get_width() for bar in bars]) + 0.1)
plt.xticks(fontsize=28)
plt.yticks(fontsize=28)
plt.tight_layout()
return {"Agitated": float(proba[0][1]), "Non-agitated": float(proba[0][0])}, formatted_statements, plot,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio
# Plotting for negative class predictions (class label 0)
else:
neg_shap_values = shap_values[0][0]
scores_desc = list(zip(neg_shap_values, top_features, feature_values))
scores_desc = sorted(scores_desc)
colors = [red if s[0] > 0 else blue for s in scores_desc]
plot = plt.figure(tight_layout=True, figsize=(18,18))
plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
plt.rcParams['font.family'] = 'Times New Roman'
bars = plt.barh([plot_feature_names.get(s[1], s[1]) for s in scores_desc], [s[0] for s in scores_desc], color=colors)
for bar, score in zip(bars, scores_desc):
text_color = 'black'
text_value = f'{score[2]:.2f}'
text_position = bar.get_width() - 0.08 if score[0] < 0 else bar.get_width() + 0.08# Adjust the padding as needed
plt.text(text_position, bar.get_y() + bar.get_height() / 2, text_value, ha='left' if score[0] < 0 else 'right', va='center', color=text_color, fontsize=20)
plt.title("Feature Shap Values for Negative Prediction", fontsize=30)
plt.ylabel("Feature",fontsize=28 )
plt.xlabel("Shap Value",fontsize=28)
plt.xlim(min([bar.get_width() for bar in bars]) - 0.1, max([bar.get_width() for bar in bars]) + 0.1)
plt.xticks(fontsize=28)
plt.yticks(fontsize=28)
plt.tight_layout()
return {"Agitated": float(proba[0][1]), "Non-agitated": float(proba[0][0])}, formatted_statements, plot, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio
def save_file(example_df,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio):
example_data = example_df
example_data['Lounge_night_illuminance'] = Lounge_night_illuminance
example_data['Kitchen_night_illuminance'] = Kitchen_night_illuminance
example_data['Kitchen_afternoon_illuminance'] = Kitchen_afternoon_illuminance
example_data['ratio'] = ratio
example_data['temperature_ratio'] = temperature_ratio
example_data['Kitchen_morning_temperature'] = Kitchen_morning_temperature
example_data['Bathroom_evening_temperature'] = Bathroom_evening_temperature
example_data['Kitchen_night_temperature'] = Kitchen_night_temperature
current_path = os.getcwd()
output_filename = current_path + "Downloads/Data_modified.csv"
example_data.to_csv(output_filename, index=False)
return output_filename
explainer =shap.TreeExplainer(model)
# %%
with gr.Blocks(theme=gr.themes.Default(spacing_size="md", primary_hue="red", secondary_hue="blue")) as demo:
gr.HTML("""
An Interpretable Machine Learning Tool for In-Home Screening of Agitation Episodes in People Living with Dementia
""")
with gr.Row():
with gr.Column():
generate = gr.Button(value="Generate synthetic patient data")
csv_file = gr.DataFrame(headers=['ratio', 'sunset_minutes', 'Kitchen_night_temperature', 'uvindex', 'visibility','illuminance','Kitchen_night_illuminance','Lounge_night_illuminance','Kitchen_morning_temperature','Bathroom_evening_temperature', 'Bathroom_night_activity', 'number_transitions', 'temperature_ratio', 'awake_ratio','rr_min','To_bed', 'Lounge_afternoon_activity','rr_average',"rr_min_std",'Kitchen_afternoon_illuminance'],row_count=1 , col_count=20, datatype = "number", interactive=False)
modif = gr.Button(value="Change the modifiable parameters to see how agitation risk changes")
gr.Markdown(""" Light exposure
""")
Kitchen_afternoon_illuminance = gr.Slider(label="Mean afternoon illuminance in kitchen", minimum=0, maximum=3000, step=100)
Kitchen_night_illuminance = gr.Slider(label="Mean night illuminance in kitchen", minimum=0, maximum=3000, step=100)
Lounge_night_illuminance = gr.Slider(label="Mean night illuminance in lounge", minimum=0, maximum=3000, step=100, every=True)
ratio = gr.Slider(label="Ratio (indoor:outdoor illuminance)", minimum=0, maximum=0.35, step=0.1)
gr.Markdown(""" Ambient temperature
""")
Kitchen_morning_temperature = gr.Slider(label="Mean morning temperature in kitchen", minimum=-10, maximum=45, step=0.5)
Bathroom_evening_temperature = gr.Slider(label="Mean evening temperature in bathroom", minimum=-10, maximum=45, step=0.5)
Kitchen_night_temperature = gr.Slider(label="Mean night temperature in kitchen", minimum=-10, maximum=45, step=0.5)
temperature_ratio = gr.Slider(label="Ratio (indoor:outdoor temperature)", minimum=-500, maximum=60, step=0.1)
with gr.Row():
initialise = gr.Button(value="Return to original patient data")
flag = gr.Button(value="Save these combinations of parameters" ,variant='primary')
with gr.Column():
predict_btn = gr.Button(value="Screen patient for agitation")
label = gr.Label(scale= 0, label="Patient screening")
html_text = gr.HTML(label="Explanation")
interpret_btn = gr.Button(value="Show each feature's contribution for this prediction")
plot = gr.Plot(scale=2, label="Interpretation")
with gr.Row():
generate.click(generate_data, outputs=[csv_file,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio])
Lounge_night_illuminance.input(update_data, inputs=[csv_file, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text, plot])
Kitchen_night_illuminance.input(update_data, inputs=[csv_file, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text, plot])
Kitchen_afternoon_illuminance.input(update_data, inputs=[csv_file,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text,plot])
Kitchen_morning_temperature.input(update_data, inputs=[csv_file,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text,plot])
ratio.input(update_data, inputs=[csv_file,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text,plot])
temperature_ratio.input(update_data, inputs=[csv_file, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text,plot])
Kitchen_morning_temperature.input(update_data, inputs=[csv_file, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text,plot])
Bathroom_evening_temperature.input(update_data, inputs=[csv_file, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text,plot])
Kitchen_night_temperature.input(update_data, inputs=[csv_file, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio], outputs= [label,html_text,plot])
predict_btn.click(
predict,
inputs=[
csv_file, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio
],
outputs=[label,html_text]
)
interpret_btn.click(
interpret,
inputs=[
csv_file, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio
],
outputs=[plot],
)
initialise.click(
return_original,
inputs=[
csv_file
],
outputs=[label,html_text, plot, Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio],
)
flag.click(
save_file,
inputs= [csv_file,Kitchen_afternoon_illuminance, Lounge_night_illuminance, Kitchen_night_illuminance, ratio, Kitchen_night_temperature, Kitchen_morning_temperature, Bathroom_evening_temperature, temperature_ratio],
)
with gr.Row():
gr.Markdown("
")
gr.Markdown("
")
gr.Markdown("
")
gr.Markdown("
")
demo.launch(auth=(os.environ['user'], os.environ['password']))
# %%