third page

backend/disentangle_concepts.py

@@ -7,7 +7,7 @@ from PIL import Image
 
 
 
-def generate_composite_images(model, z, decision_boundaries, lambdas, latent_space='W'):
+def generate_composite_images(model, z, decision_boundaries, lambdas, latent_space='W', negative_colors=None):
     """
     The regenerate_images function takes a model, z, and decision_boundary as input.  It then
     constructs an inverse rotation/translation matrix and passes it to the generator.  The generator
@@ -33,9 +33,19 @@ def generate_composite_images(model, z, decision_boundaries, lambdas, latent_spa
     repetitions = 16 
     z_0 = z
     
-    for decision_boundary, lmbd in zip(decision_boundaries, lambdas):
-        decision_boundary = torch.from_numpy(decision_boundary.copy()).to(device)
-        z_0 = z_0 + int(lmbd) * decision_boundary
+    if negative_colors:
+        for decision_boundary, lmbd, neg_boundary in zip(decision_boundaries, lambdas, negative_colors):
+            decision_boundary = torch.from_numpy(decision_boundary.copy()).to(device)
+            if neg_boundary != 'None':
+                neg_boundary = torch.from_numpy(neg_boundary.copy()).to(device)
+            
+                z_0 = z_0 + int(lmbd) * (decision_boundary - (neg_boundary.T * decision_boundary) * neg_boundary)
+            else:
+                z_0 = z_0 + int(lmbd) * decision_boundary
+    else:            
+        for decision_boundary, lmbd in zip(decision_boundaries, lambdas):
+            decision_boundary = torch.from_numpy(decision_boundary.copy()).to(device)
+            z_0 = z_0 + int(lmbd) * decision_boundary
         
         
     if latent_space == 'Z':

pages/1_Textiles_Disentanglement.py

@@ -139,6 +139,11 @@ with input_col_4:
     with st.form('text_form_2'):
         st.write('Use best options')
         best = st.selectbox('Option:', tuple([True, False]), index=0)
+        sign = True
+        num_factors=10
+        cl_method='LR'
+        regularization=0.1
+        extremes=True
         if st.session_state.best is False:
             st.write('Options for StyleSpace (not available for Saturation and Value)')
             sign = st.selectbox('Sign option:', tuple([True, False]), index=1)

pages/3_Vectors_algebra.py

@@ -0,0 +1,186 @@
+import streamlit as st
+import pickle
+import pandas as pd
+import numpy as np
+import random
+import torch
+
+from matplotlib.backends.backend_agg import RendererAgg
+
+from backend.disentangle_concepts import *
+import torch_utils
+import dnnlib
+import legacy
+
+_lock = RendererAgg.lock
+
+
+st.set_page_config(layout='wide')
+BACKGROUND_COLOR = '#bcd0e7'
+SECONDARY_COLOR = '#bce7db'
+
+
+st.title('Disentanglement studies on the Textile Dataset')
+st.markdown(
+    """
+    This is a demo of the Disentanglement studies on the [iMET Textiles Dataset](https://www.metmuseum.org/art/collection/search/85531).
+    """,
+    unsafe_allow_html=False,)
+    
+annotations_file = './data/textile_annotated_files/seeds0000-100000_S.pkl'
+with open(annotations_file, 'rb') as f:
+    annotations = pickle.load(f)
+
+concept_vectors = pd.read_csv('./data/stored_vectors/scores_colors_hsv.csv')
+concept_vectors['vector'] = [np.array([float(xx) for xx in x]) for x in concept_vectors['vector'].str.split(', ')]
+concept_vectors['score'] = concept_vectors['score'].astype(float)
+
+concept_vectors['sign'] = [True if 'sign:True' in val else False for val in concept_vectors['kwargs']]
+concept_vectors['extremes'] = [True if 'extremes method:True' in val else False for val in concept_vectors['kwargs']]
+concept_vectors['regularization'] = [float(val.split(',')[1].strip('regularization: ')) if 'regularization:' in val else False for val in concept_vectors['kwargs']]
+concept_vectors['cl_method'] = [val.split(',')[0].strip('classification method:') if 'classification method:' in val else False for val in concept_vectors['kwargs']]
+concept_vectors['num_factors'] = [int(val.split(',')[1].strip('number of factors:')) if 'number of factors:' in val else False for val in concept_vectors['kwargs']]
+
+concept_vectors = concept_vectors.sort_values('score', ascending=False).reset_index()
+
+with dnnlib.util.open_url('./data/textile_model_files/network-snapshot-005000.pkl') as f:
+    model = legacy.load_network_pkl(f)['G_ema'].to('cpu') # type: ignore
+
+COLORS_LIST = ['Gray', 'Red Orange', 'Yellow', 'Green', 'Light Blue', 'Blue', 'Purple', 'Pink', 'Saturation', 'Value']
+COLORS_NEGATIVE = COLORS_LIST + ['None']
+
+if 'image_id' not in st.session_state:
+    st.session_state.image_id = 52921
+if 'colors' not in st.session_state:
+    st.session_state.colors = [COLORS_LIST[0]]
+if 'non_colors' not in st.session_state:
+    st.session_state.non_colors = ['None']
+if 'space_id' not in st.session_state:
+    st.session_state.space_id = 'W'
+if 'color_lambda' not in st.session_state:
+    st.session_state.color_lambda = 7
+if 'saturation_lambda' not in st.session_state:
+    st.session_state.saturation_lambda = 0
+if 'value_lambda' not in st.session_state:
+    st.session_state.value_lambda = 0
+if 'sign' not in st.session_state:
+    st.session_state.sign = False
+if 'extremes' not in st.session_state:
+    st.session_state.extremes = False
+if 'regularization' not in st.session_state:
+    st.session_state.regularization = False
+if 'cl_method' not in st.session_state:
+    st.session_state.cl_method = False
+if 'num_factors' not in st.session_state:
+    st.session_state.num_factors = False
+if 'best' not in st.session_state:
+    st.session_state.best = True
+    
+# def on_change_random_input():
+#     st.session_state.image_id = st.session_state.image_id
+
+# ----------------------------- INPUT ----------------------------------
+epsilon_container = st.empty()
+st.header('Image Manipulation with Vector Algebra')
+
+header_col_1, header_col_2, header_col_3, header_col_4 = st.columns([1,2,2,1])
+input_col_1, output_col_2, output_col_3, input_col_4 = st.columns([1,2,2,1])
+
+# --------------------------- INPUT column 1 ---------------------------
+with input_col_1:
+   with st.form('image_form'):
+        
+        # image_id = st.number_input('Image ID: ', format='%d', step=1)
+        st.write('**Choose or generate a random image to test the disentanglement**')
+        chosen_image_id_input = st.empty()
+        image_id = chosen_image_id_input.number_input('Image ID:', format='%d', step=1, value=st.session_state.image_id)
+        
+        choose_image_button = st.form_submit_button('Choose the defined image')
+        random_id = st.form_submit_button('Generate a random image')
+
+        if random_id:
+            image_id = random.randint(0, 100000)
+            st.session_state.image_id = image_id
+            chosen_image_id_input.number_input('Image ID:', format='%d', step=1, value=st.session_state.image_id)
+            
+        if choose_image_button:
+            image_id = int(image_id)
+            st.session_state.image_id = image_id
+
+with header_col_1:
+    st.write('Input image selection')
+
+if st.session_state.space_id == 'Z':
+    original_image_vec = annotations['z_vectors'][st.session_state.image_id]
+else:
+    original_image_vec = annotations['w_vectors'][st.session_state.image_id]
+    
+img = generate_original_image(original_image_vec, model, latent_space=st.session_state.space_id)
+
+with output_col_2:
+    st.image(img)
+
+with header_col_2:
+    st.write('Original image')
+    
+with input_col_4:
+    with st.form('text_form_1'):
+        
+        st.write('**Positive colors to vary (including Saturation and Value)**')
+        colors = st.multiselect('Color:', tuple(COLORS_LIST), default=[COLORS_LIST[0], COLORS_LIST[1]])
+        colors_button = st.form_submit_button('Choose the defined colors')
+        
+        st.session_state.image_id = image_id
+        st.session_state.colors = colors
+        st.session_state.color_lambda = [5]*len(colors)
+        st.session_state.non_colors = ['None']*len(colors)
+            
+        lambdas = []
+        negative_cols = []
+        for color in colors:
+            st.write(color)
+            st.write('**Set range of change**')
+            chosen_color_lambda_input = st.empty()
+            color_lambda = chosen_color_lambda_input.number_input('Lambda:', min_value=-100, step=1, value=5, key=color+'_number')
+            lambdas.append(color_lambda)
+                
+            st.write('**Set dimensions of change to not consider**')
+            chosen_color_negative_input = st.empty()
+            color_negative = chosen_color_negative_input.selectbox('Color:', tuple(COLORS_NEGATIVE), index=len(COLORS_NEGATIVE)-1, key=color+'_noncolor')
+            negative_cols.append(color_negative)
+                
+        lambdas_button = st.form_submit_button('Submit options')
+        if lambdas_button:
+            st.session_state.color_lambda = lambdas
+            st.session_state.non_colors = negative_cols
+                
+        
+# print(st.session_state.colors)
+# print(st.session_state.color_lambda)
+# print(st.session_state.non_colors)
+
+# ---------------------------- DISPLAY COL 1 ROW 1 ------------------------------
+
+with header_col_3:
+    separation_vectors = []
+    for col in st.session_state.colors:
+        separation_vector, score_1 = concept_vectors[concept_vectors['color'] == col].reset_index().loc[0, ['vector', 'score']]
+        separation_vectors.append(separation_vector)
+    
+    negative_separation_vectors = []
+    for non_col in st.session_state.non_colors:
+        if non_col != 'None':
+            negative_separation_vector, score_2 = concept_vectors[concept_vectors['color'] == non_col].reset_index().loc[0, ['vector', 'score']]
+            negative_separation_vectors.append(negative_separation_vector)
+        else:
+            negative_separation_vectors.append('None')
+    ## n1 − (n1T n2)n2
+    # print(negative_separation_vectors, separation_vectors)
+    st.write(f'Output Image, with positive {str(st.session_state.colors)}, and negative {str(st.session_state.non_colors)}')
+
+# ---------------------------- DISPLAY COL 2 ROW 1 ------------------------------
+
+
+with output_col_3:
+    image_updated = generate_composite_images(model, original_image_vec, separation_vectors, lambdas=st.session_state.color_lambda, negative_colors=negative_separation_vectors)
+    st.image(image_updated)