Initial commit.

.idea/workspace.xml

@@ -2,8 +2,10 @@
 <project version="4">
   <component name="ChangeListManager">
     <list default="true" id="5dd22f22-8223-4d55-99f9-57d1e00622d7" name="Default Changelist" comment="Initial commit.">
-      <change afterPath="$PROJECT_DIR$/examples/img5.jpeg" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/weights/segmenter_nusc.pth_variant_old.yml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
       <change beforePath="$PROJECT_DIR$/app.py" beforeDir="false" afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/weights/segmenter_nusc.pth_variant.yml" beforeDir="false" afterPath="$PROJECT_DIR$/weights/segmenter_nusc.pth_variant.yml" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -44,6 +46,30 @@
       <recent name="$PROJECT_DIR$/examples" />
     </key>
   </component>
+  <component name="RunManager">
+    <configuration name="app" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
+      <module name="DaS" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="$USER_HOME$/miniconda3/envs/pytorch/bin/python3" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="false" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/app.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+  </component>
   <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
   <component name="TaskManager">
     <task active="true" id="Default" summary="Default task">
@@ -52,14 +78,7 @@
       <option name="number" value="Default" />
       <option name="presentableId" value="Default" />
       <updated>1647350746642</updated>
-      <workItem from="1647350750956" duration="13983000" />
-    </task>
-    <task id="LOCAL-00006" summary="Initial commit.">
-      <created>1647354226094</created>
-      <option name="number" value="00006" />
-      <option name="presentableId" value="LOCAL-00006" />
-      <option name="project" value="LOCAL" />
-      <updated>1647354226094</updated>
+      <workItem from="1647350750956" duration="15168000" />
     </task>
     <task id="LOCAL-00007" summary="Initial commit.">
       <created>1647355534027</created>
@@ -397,7 +416,14 @@
       <option name="project" value="LOCAL" />
       <updated>1647366052086</updated>
     </task>
-    <option name="localTasksCounter" value="55" />
+    <task id="LOCAL-00055" summary="Initial commit.">
+      <created>1647366376267</created>
+      <option name="number" value="00055" />
+      <option name="presentableId" value="LOCAL-00055" />
+      <option name="project" value="LOCAL" />
+      <updated>1647366376267</updated>
+    </task>
+    <option name="localTasksCounter" value="56" />
     <servers />
   </component>
   <component name="TypeScriptGeneratedFilesManager">
@@ -419,4 +445,23 @@
     <MESSAGE value="Initial commit." />
     <option name="LAST_COMMIT_MESSAGE" value="Initial commit." />
   </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/app.py</url>
+          <line>66</line>
+          <option name="timeStamp" value="2" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/segmenter_model/utils.py</url>
+          <line>393</line>
+          <option name="timeStamp" value="3" />
+        </line-breakpoint>
+      </breakpoints>
+    </breakpoint-manager>
+  </component>
+  <component name="com.intellij.coverage.CoverageDataManagerImpl">
+    <SUITE FILE_PATH="coverage/DaS$app.coverage" NAME="app Coverage Results" MODIFIED="1647368417514" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+  </component>
 </project>

app.py

@@ -108,6 +108,7 @@ def create_model(resnet=False):
     # TODO: parse hyperparameters
     window_size = variant['inference_kwargs']["window_size"]
     window_stride = variant['inference_kwargs']["window_stride"]
+    im_size = variant['inference_kwargs']["im_size"]
 
     net_kwargs = variant["net_kwargs"]
     if not resnet:
@@ -126,19 +127,20 @@ def create_model(resnet=False):
 
     model.eval()
 
-    return model, window_size, window_stride
+    return model, window_size, window_stride, im_size
+
+
+download_weights()
+model, window_size, window_stride, im_size = create_model()
 
 
 def get_transformations():
     return transforms.Compose([
         transforms.ToTensor(),
+        transforms.Resize(im_size),
         transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
 
 
-download_weights()
-model, window_size, window_stride = create_model()
-
-
 def predict(input_img):
     input_img_pil = Image.open(input_img)
     transform = get_transformations()
@@ -159,7 +161,7 @@ def predict(input_img):
     print('CS colors done.')
 
     # drawing_pseudo = transforms.ToPILImage()(drawing_pseudo)
-    drawing_cs = transforms.ToPILImage()(drawing_cs)
+    drawing_cs = transforms.ToPILImage()(drawing_cs).resize(input_img_pil.size)
     drawing_cs_blend = blend_images(input_img_pil, drawing_cs)
     # return drawing_pseudo, drawing_cs
     return drawing_cs_blend
@@ -168,9 +170,9 @@ def predict(input_img):
 title = "Drive&Segment"
 description = 'Gradio Demo accompanying paper "Drive&Segment: Unsupervised Semantic Segmentation of Urban Scenes via Cross-modal Distillation"\nBecause of the CPU-only inference, it might take up to 20s for large images.'
 # article = "<p style='text-align: center'><a href='TODO' target='_blank'>Project Page</a> | <a href='codelink' target='_blank'>Github</a></p>"
-examples = ['examples/img5.jpeg','examples/100.jpeg', 'examples/39076.jpeg', 'examples/img1.jpg']  # , 'examples/img2.jpeg']
+examples = ['examples/img5.jpeg', 'examples/100.jpeg', 'examples/39076.jpeg', 'examples/img1.jpg']
 
-# predict(examples[0])
+predict(examples[0])
 
 iface = gr.Interface(predict, gr.inputs.Image(type='filepath'), "image", title=title, description=description,
                      examples=examples)

weights/segmenter_nusc.pth_variant.yml

@@ -3,9 +3,9 @@ dataset_kwargs:
   window_stride: 384
 debug: false
 inference_kwargs:
-  im_size: 1024
-  window_size: 512
-  window_stride: 256
+  im_size: 512
+  window_size: 256
+  window_stride: 128
 net_kwargs:
   backbone: dino_vits16
   d_model: 384

weights/segmenter_nusc.pth_variant_old.yml

@@ -0,0 +1,36 @@
+dataset_kwargs:
+  window_size: 512
+  window_stride: 384
+debug: false
+inference_kwargs:
+  im_size: 1024
+  window_size: 512
+  window_stride: 256
+net_kwargs:
+  backbone: dino_vits16
+  d_model: 384
+  decoder:
+    drop_path_rate: 0.0
+    dropout: 0.1
+    n_cls: 31
+    n_layers: 1
+    name: mask_transformer
+  distilled: false
+  drop_path_rate: 0.1
+  dropout: 0.0
+  image_size: !!python/tuple
+    - 256
+    - 256
+  n_cls: 31
+  n_heads: 3
+  n_layers: 12
+  normalization: deit
+  patch_size: 16
+val_dataset_kwargs:
+  batch_size: 1
+  crop_size: 512
+  dataset: cityscapes
+  image_size: 1024
+  normalization: deit
+  num_workers: 10
+  split: val