GuanHuaYu student commited on
Commit
61d758d
·
1 Parent(s): c4598a9
README.md ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ title: Fault Classification PMU
3
+ emoji: ⚡️
4
+ colorFrom: blue
5
+ colorTo: purple
6
+ sdk: gradio
7
+ sdk_version: "4.44.1"
8
+ app_file: app.py
9
+ pinned: false
10
+ ---
11
+
12
+ # Fault Classification for PMU and PV Systems
13
+
14
+ This Space hosts the Gradio front end used to retrain and serve fault classification models for transmission lines and grid-connected photovoltaic systems. It loads pre-trained Keras models and feature scalers, accepts bulk CSV uploads for continued training, and exposes interactive inference utilities for high-frequency PMU measurements.
15
+
16
+ Refer to `app.py` for the interface definition and to `fault_classification_pmu.py` for the training pipeline that supports CNN-LSTM, TCN, and SVM architectures.
app.py CHANGED
@@ -10,10 +10,15 @@ from __future__ import annotations
10
 
11
  import json
12
  import os
 
 
 
 
 
 
13
  import re
14
- import socket
15
  from pathlib import Path
16
- from typing import Dict, List, Optional, Sequence, Tuple
17
 
18
  import gradio as gr
19
  import joblib
@@ -22,29 +27,16 @@ import pandas as pd
22
  from huggingface_hub import hf_hub_download
23
  from tensorflow.keras.models import load_model
24
 
25
- # Reduce TensorFlow log verbosity
26
- os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "2")
 
 
 
27
 
28
  # --------------------------------------------------------------------------------------
29
  # Configuration
30
  # --------------------------------------------------------------------------------------
31
- DEFAULT_FEATURE_COLUMNS: List[str] = [
32
- "[325] UPMU_SUB22:FREQ",
33
- "[326] UPMU_SUB22:DFDT",
34
- "[327] UPMU_SUB22:FLAG",
35
- "[328] UPMU_SUB22-L1:MAG",
36
- "[329] UPMU_SUB22-L1:ANG",
37
- "[330] UPMU_SUB22-L2:MAG",
38
- "[331] UPMU_SUB22-L2:ANG",
39
- "[332] UPMU_SUB22-L3:MAG",
40
- "[333] UPMU_SUB22-L3:ANG",
41
- "[334] UPMU_SUB22-C1:MAG",
42
- "[335] UPMU_SUB22-C1:ANG",
43
- "[336] UPMU_SUB22-C2:MAG",
44
- "[337] UPMU_SUB22-C2:ANG",
45
- "[338] UPMU_SUB22-C3:MAG",
46
- "[339] UPMU_SUB22-C3:ANG",
47
- ]
48
  DEFAULT_SEQUENCE_LENGTH = 32
49
  DEFAULT_STRIDE = 4
50
 
@@ -70,23 +62,35 @@ def download_from_hub(filename: str) -> Optional[Path]:
70
  return None
71
  try:
72
  print(f"Downloading {filename} from {HUB_REPO} ...")
 
73
  path = hf_hub_download(repo_id=HUB_REPO, filename=filename)
74
  print("Downloaded", path)
75
  return Path(path)
76
  except Exception as exc: # pragma: no cover - logging convenience
77
  print("Failed to download", filename, "from", HUB_REPO, ":", exc)
 
78
  return None
79
 
80
 
81
  def resolve_artifact(local_name: str, env_var: str, hub_filename: str) -> Optional[Path]:
 
82
  candidates = [Path(local_name)] if local_name else []
83
  env_value = os.environ.get(env_var)
84
  if env_value:
85
  candidates.append(Path(env_value))
 
86
  for candidate in candidates:
87
  if candidate and candidate.exists():
 
88
  return candidate
89
- return download_from_hub(hub_filename)
 
 
 
 
 
 
 
90
 
91
 
92
  def load_metadata(path: Optional[Path]) -> Dict:
@@ -98,11 +102,62 @@ def load_metadata(path: Optional[Path]) -> Dict:
98
  return {}
99
 
100
 
101
- def try_load_model(path: Optional[Path]):
102
  if not path:
103
  return None
104
  try:
105
- model = load_model(path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
106
  print("Loaded model from", path)
107
  return model
108
  except Exception as exc: # pragma: no cover - runtime diagnostics
@@ -110,34 +165,360 @@ def try_load_model(path: Optional[Path]):
110
  return None
111
 
112
 
113
- def try_load_scaler(path: Optional[Path]):
114
- if not path:
115
- return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  try:
117
- scaler = joblib.load(path)
118
- print("Loaded scaler from", path)
119
- return scaler
120
- except Exception as exc:
121
- print("Failed to load scaler", path, exc)
 
 
 
 
 
122
  return None
123
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
 
125
- MODEL_PATH = resolve_artifact(LOCAL_MODEL_FILE, ENV_MODEL_PATH, HUB_MODEL_FILENAME)
126
- SCALER_PATH = resolve_artifact(LOCAL_SCALER_FILE, ENV_SCALER_PATH, HUB_SCALER_FILENAME)
127
- METADATA_PATH = resolve_artifact(LOCAL_METADATA_FILE, ENV_METADATA_PATH, HUB_METADATA_FILENAME)
128
 
129
- MODEL = try_load_model(MODEL_PATH)
130
- SCALER = try_load_scaler(SCALER_PATH)
131
- METADATA = load_metadata(METADATA_PATH)
132
 
133
- FEATURE_COLUMNS: List[str] = METADATA.get("feature_columns", DEFAULT_FEATURE_COLUMNS)
134
- LABEL_CLASSES: List[str] = [str(label) for label in METADATA.get("label_classes", [])]
135
- LABEL_COLUMN: str = METADATA.get("label_column", "Fault")
136
- SEQUENCE_LENGTH: int = int(METADATA.get("sequence_length", DEFAULT_SEQUENCE_LENGTH))
137
- DEFAULT_WINDOW_STRIDE: int = int(METADATA.get("stride", DEFAULT_STRIDE))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
138
 
139
- if MODEL is not None and not LABEL_CLASSES:
140
- LABEL_CLASSES = [str(i) for i in range(MODEL.output_shape[-1])]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
 
142
  # --------------------------------------------------------------------------------------
143
  # Pre-processing helpers
@@ -146,8 +527,11 @@ if MODEL is not None and not LABEL_CLASSES:
146
  def ensure_ready():
147
  if MODEL is None or SCALER is None:
148
  raise RuntimeError(
149
- "模型或特征缩放器未加载。请将 pmu_cnn_lstm_model.keras pmu_feature_scaler.pkl "
150
- "上传到 Space,或设置相关的 Hugging Face Hub 配置。"
 
 
 
151
  )
152
 
153
 
@@ -155,7 +539,7 @@ def parse_text_features(text: str) -> np.ndarray:
155
  cleaned = re.sub(r"[;\n\t]+", ",", text.strip())
156
  arr = np.fromstring(cleaned, sep=",")
157
  if arr.size == 0:
158
- raise ValueError("未解析到任何特征值,请输入以逗号分隔的数字。")
159
  return arr.astype(np.float32)
160
 
161
 
@@ -171,7 +555,8 @@ def apply_scaler(sequences: np.ndarray) -> np.ndarray:
171
  def make_sliding_windows(data: np.ndarray, sequence_length: int, stride: int) -> np.ndarray:
172
  if data.shape[0] < sequence_length:
173
  raise ValueError(
174
- f"数据行数 ({data.shape[0]}) 小于序列长度 ({sequence_length}),无法创建窗口。"
 
175
  )
176
  windows = [data[start : start + sequence_length] for start in range(0, data.shape[0] - sequence_length + 1, stride)]
177
  return np.stack(windows)
@@ -204,7 +589,8 @@ def dataframe_to_sequences(
204
  if sequence_length == 1 and array.shape[1] == n_features:
205
  return array.reshape(array.shape[0], 1, n_features)
206
  raise ValueError(
207
- "CSV 列与预期特征不匹配。请包含完整的 PMU 特征列,或提供整形后的窗口数据。"
 
208
  )
209
 
210
 
@@ -248,10 +634,18 @@ def probabilities_to_json(probabilities: np.ndarray) -> List[Dict[str, object]]:
248
  def predict_sequences(sequences: np.ndarray) -> Tuple[str, pd.DataFrame, List[Dict[str, object]]]:
249
  ensure_ready()
250
  sequences = apply_scaler(sequences.astype(np.float32))
251
- probs = MODEL.predict(sequences, verbose=0)
 
 
 
 
 
 
 
252
  table = format_predictions(probs)
253
  json_probs = probabilities_to_json(probs)
254
- status = f"共生成 {len(sequences)} 个窗口,模型输出维度 {probs.shape[1]}."
 
255
  return status, table, json_probs
256
 
257
 
@@ -260,21 +654,23 @@ def predict_from_text(text: str, sequence_length: int) -> Tuple[str, pd.DataFram
260
  n_features = len(FEATURE_COLUMNS)
261
  if arr.size % n_features != 0:
262
  raise ValueError(
263
- f"输入特征数量 {arr.size} 不是特征维度 {n_features} 的整数倍。请按照 {n_features} 个特征为一组输入。"
 
264
  )
265
  timesteps = arr.size // n_features
266
  if timesteps != sequence_length:
267
  raise ValueError(
268
- f"检测到 {timesteps} 个时间步,与当前设置的序列长度 {sequence_length} 不一致。"
 
269
  )
270
  sequences = arr.reshape(1, sequence_length, n_features)
271
  status, table, probs = predict_sequences(sequences)
272
- status = f"单窗口预测完成。{status}"
273
  return status, table, probs
274
 
275
 
276
  def predict_from_csv(file_obj, sequence_length: int, stride: int) -> Tuple[str, pd.DataFrame, List[Dict[str, object]]]:
277
- df = pd.read_csv(file_obj.name)
278
  sequences = dataframe_to_sequences(
279
  df,
280
  sequence_length=sequence_length,
@@ -282,82 +678,497 @@ def predict_from_csv(file_obj, sequence_length: int, stride: int) -> Tuple[str,
282
  feature_columns=FEATURE_COLUMNS,
283
  )
284
  status, table, probs = predict_sequences(sequences)
285
- status = f"CSV 处理完成,生成 {len(sequences)} 个窗口。{status}"
286
  return status, table, probs
287
 
288
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
289
  # --------------------------------------------------------------------------------------
290
  # Gradio interface
291
  # --------------------------------------------------------------------------------------
292
 
293
  def build_interface() -> gr.Blocks:
294
- with gr.Blocks(title="Fault Classification - PMU Data") as demo:
295
- gr.Markdown("# Fault Classification (PMU 数据)")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
296
  if MODEL is None or SCALER is None:
297
  gr.Markdown(
298
- "⚠️ **模型或缩放器未准备好。** 上传 `pmu_cnn_lstm_model.keras`、"
299
- "`pmu_feature_scaler.pkl` `pmu_metadata.json` Space 根目录,或配置环境变量以从 Hugging Face Hub 自动下载。"
 
300
  )
301
  else:
 
302
  gr.Markdown(
303
- "模型、特征缩放器与元数据均已加载。可以上传原始 PMU CSV 数据,或粘贴单个时间窗口的特征向量进行推理。"
 
 
 
304
  )
305
 
306
- with gr.Accordion("特征说明", open=False):
307
  gr.Markdown(
308
- f"输入窗口按以下特征顺序排列 (每个时间步共 {len(FEATURE_COLUMNS)} 个特征):\n"
309
  + "\n".join(f"- {name}" for name in FEATURE_COLUMNS)
310
  )
311
  gr.Markdown(
312
- f"训练时使用的窗口长度默认为 **{SEQUENCE_LENGTH}**,滑动步长默认为 **{DEFAULT_WINDOW_STRIDE}**。"
313
- )
314
-
315
- with gr.Row():
316
- file_in = gr.File(label="上传 PMU CSV", file_types=[".csv"])
317
- text_in = gr.Textbox(
318
- lines=4,
319
- label="或粘贴单个窗口的逗号分隔特征",
320
- placeholder="49.97772,1.215825E-38,...",
321
- )
322
-
323
- with gr.Row():
324
- sequence_length_input = gr.Slider(
325
- minimum=1,
326
- maximum=max(1, SEQUENCE_LENGTH * 2),
327
- step=1,
328
- value=SEQUENCE_LENGTH,
329
- label="序列长度 (timesteps)",
330
  )
331
- stride_input = gr.Slider(
332
- minimum=1,
333
- maximum=max(1, SEQUENCE_LENGTH),
334
- step=1,
335
- value=max(1, DEFAULT_WINDOW_STRIDE),
336
- label="CSV 滑动窗口步长",
337
- )
338
-
339
- predict_btn = gr.Button("执行预测", variant="primary")
340
- status_out = gr.Textbox(label="状态", interactive=False)
341
- table_out = gr.Dataframe(headers=["window", "predicted_label", "confidence", "top3"], label="预测结果", interactive=False)
342
- probs_out = gr.JSON(label="各窗口概率分布")
343
 
344
- def _run_prediction(file_obj, text, sequence_length, stride):
345
- sequence_length = int(sequence_length)
346
- stride = int(stride)
347
- try:
348
- if file_obj is not None:
349
- return predict_from_csv(file_obj, sequence_length, stride)
350
- if text and text.strip():
351
- return predict_from_text(text, sequence_length)
352
- return "请上传 CSV 或输入文本特征。", pd.DataFrame(), []
353
- except Exception as exc:
354
- return f"预测失败: {exc}", pd.DataFrame(), []
355
-
356
- predict_btn.click(
357
- _run_prediction,
358
- inputs=[file_in, text_in, sequence_length_input, stride_input],
359
- outputs=[status_out, table_out, probs_out],
360
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
361
 
362
  return demo
363
 
@@ -366,33 +1177,58 @@ def build_interface() -> gr.Blocks:
366
  # Launch helpers
367
  # --------------------------------------------------------------------------------------
368
 
369
- def find_free_port() -> int:
370
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
371
- s.bind(("", 0))
372
- return s.getsockname()[1]
373
-
374
-
375
- def choose_port() -> Optional[int]:
376
- for env_var in ("GRADIO_SERVER_PORT", "PORT"):
377
  value = os.environ.get(env_var)
378
  if value:
379
  try:
380
  return int(value)
381
  except ValueError:
382
- pass
383
- return find_free_port()
384
 
385
 
386
  def main():
387
- demo = build_interface()
 
 
 
 
 
 
 
 
 
 
388
  try:
389
- port = choose_port()
 
 
 
 
 
 
390
  print(f"Launching Gradio app on port {port}")
391
- demo.launch(server_name="0.0.0.0", server_port=port)
392
  except OSError as exc:
393
  print("Failed to launch on requested port:", exc)
394
- demo.launch()
 
 
 
 
 
 
 
395
 
396
 
397
  if __name__ == "__main__":
 
 
 
 
 
 
 
 
398
  main()
 
10
 
11
  import json
12
  import os
13
+ import shutil
14
+
15
+ os.environ.setdefault("CUDA_VISIBLE_DEVICES", "-1")
16
+ os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "2")
17
+ os.environ.setdefault("TF_ENABLE_ONEDNN_OPTS", "0")
18
+
19
  import re
 
20
  from pathlib import Path
21
+ from typing import Any, Dict, List, Optional, Sequence, Tuple
22
 
23
  import gradio as gr
24
  import joblib
 
27
  from huggingface_hub import hf_hub_download
28
  from tensorflow.keras.models import load_model
29
 
30
+ from fault_classification_pmu import (
31
+ DEFAULT_FEATURE_COLUMNS as TRAINING_DEFAULT_FEATURE_COLUMNS,
32
+ LABEL_GUESS_CANDIDATES as TRAINING_LABEL_GUESSES,
33
+ train_from_dataframe,
34
+ )
35
 
36
  # --------------------------------------------------------------------------------------
37
  # Configuration
38
  # --------------------------------------------------------------------------------------
39
+ DEFAULT_FEATURE_COLUMNS: List[str] = list(TRAINING_DEFAULT_FEATURE_COLUMNS)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
40
  DEFAULT_SEQUENCE_LENGTH = 32
41
  DEFAULT_STRIDE = 4
42
 
 
62
  return None
63
  try:
64
  print(f"Downloading {filename} from {HUB_REPO} ...")
65
+ # Add timeout to prevent hanging
66
  path = hf_hub_download(repo_id=HUB_REPO, filename=filename)
67
  print("Downloaded", path)
68
  return Path(path)
69
  except Exception as exc: # pragma: no cover - logging convenience
70
  print("Failed to download", filename, "from", HUB_REPO, ":", exc)
71
+ print("Continuing without pre-trained model...")
72
  return None
73
 
74
 
75
  def resolve_artifact(local_name: str, env_var: str, hub_filename: str) -> Optional[Path]:
76
+ print(f"Resolving artifact: {local_name}, env: {env_var}, hub: {hub_filename}")
77
  candidates = [Path(local_name)] if local_name else []
78
  env_value = os.environ.get(env_var)
79
  if env_value:
80
  candidates.append(Path(env_value))
81
+
82
  for candidate in candidates:
83
  if candidate and candidate.exists():
84
+ print(f"Found local artifact: {candidate}")
85
  return candidate
86
+
87
+ print(f"No local artifacts found, checking hub...")
88
+ # Only try to download if we have a hub repo configured
89
+ if HUB_REPO:
90
+ return download_from_hub(hub_filename)
91
+ else:
92
+ print("No HUB_REPO configured, skipping download")
93
+ return None
94
 
95
 
96
  def load_metadata(path: Optional[Path]) -> Dict:
 
102
  return {}
103
 
104
 
105
+ def try_load_scaler(path: Optional[Path]):
106
  if not path:
107
  return None
108
  try:
109
+ scaler = joblib.load(path)
110
+ print("Loaded scaler from", path)
111
+ return scaler
112
+ except Exception as exc:
113
+ print("Failed to load scaler", path, exc)
114
+ return None
115
+
116
+
117
+ # Initialize paths with error handling
118
+ print("Starting application initialization...")
119
+ try:
120
+ MODEL_PATH = resolve_artifact(LOCAL_MODEL_FILE, ENV_MODEL_PATH, HUB_MODEL_FILENAME)
121
+ print(f"Model path resolved: {MODEL_PATH}")
122
+ except Exception as e:
123
+ print(f"Model path resolution failed: {e}")
124
+ MODEL_PATH = None
125
+
126
+ try:
127
+ SCALER_PATH = resolve_artifact(LOCAL_SCALER_FILE, ENV_SCALER_PATH, HUB_SCALER_FILENAME)
128
+ print(f"Scaler path resolved: {SCALER_PATH}")
129
+ except Exception as e:
130
+ print(f"Scaler path resolution failed: {e}")
131
+ SCALER_PATH = None
132
+
133
+ try:
134
+ METADATA_PATH = resolve_artifact(LOCAL_METADATA_FILE, ENV_METADATA_PATH, HUB_METADATA_FILENAME)
135
+ print(f"Metadata path resolved: {METADATA_PATH}")
136
+ except Exception as e:
137
+ print(f"Metadata path resolution failed: {e}")
138
+ METADATA_PATH = None
139
+
140
+ try:
141
+ METADATA = load_metadata(METADATA_PATH)
142
+ print(f"Metadata loaded: {len(METADATA)} entries")
143
+ except Exception as e:
144
+ print(f"Metadata loading failed: {e}")
145
+ METADATA = {}
146
+
147
+ # Queuing configuration
148
+ QUEUE_MAX_SIZE = 32
149
+ # Apply a small per-event concurrency limit to avoid relying on the deprecated
150
+ # ``concurrency_count`` parameter when enabling Gradio's request queue.
151
+ EVENT_CONCURRENCY_LIMIT = 2
152
+
153
+ def try_load_model(path: Optional[Path], model_type: str, model_format: str):
154
+ if not path:
155
+ return None
156
+ try:
157
+ if model_type == "svm" or model_format == "joblib":
158
+ model = joblib.load(path)
159
+ else:
160
+ model = load_model(path)
161
  print("Loaded model from", path)
162
  return model
163
  except Exception as exc: # pragma: no cover - runtime diagnostics
 
165
  return None
166
 
167
 
168
+ FEATURE_COLUMNS: List[str] = list(DEFAULT_FEATURE_COLUMNS)
169
+ LABEL_CLASSES: List[str] = []
170
+ LABEL_COLUMN: str = "Fault"
171
+ SEQUENCE_LENGTH: int = DEFAULT_SEQUENCE_LENGTH
172
+ DEFAULT_WINDOW_STRIDE: int = DEFAULT_STRIDE
173
+ MODEL_TYPE: str = "cnn_lstm"
174
+ MODEL_FORMAT: str = "keras"
175
+
176
+ MODEL_FILENAME_BY_TYPE: Dict[str, str] = {
177
+ "cnn_lstm": LOCAL_MODEL_FILE,
178
+ "tcn": "pmu_tcn_model.keras",
179
+ "svm": "pmu_svm_model.joblib",
180
+ }
181
+
182
+ REQUIRED_PMU_COLUMNS: Tuple[str, ...] = tuple(DEFAULT_FEATURE_COLUMNS)
183
+ TRAINING_UPLOAD_DIR = Path(os.environ.get("PMU_TRAINING_UPLOAD_DIR", "training_uploads"))
184
+ TRAINING_UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
185
+
186
+
187
+ def _normalise_header(name: str) -> str:
188
+ return str(name).strip().lower()
189
+
190
+
191
+ def guess_label_from_columns(columns: Sequence[str], preferred: Optional[str] = None) -> Optional[str]:
192
+ if not columns:
193
+ return preferred
194
+
195
+ lookup = {_normalise_header(col): str(col) for col in columns}
196
+
197
+ if preferred:
198
+ preferred_stripped = preferred.strip()
199
+ for col in columns:
200
+ if str(col).strip() == preferred_stripped:
201
+ return str(col)
202
+ preferred_norm = _normalise_header(preferred)
203
+ if preferred_norm in lookup:
204
+ return lookup[preferred_norm]
205
+
206
+ for guess in TRAINING_LABEL_GUESSES:
207
+ guess_norm = _normalise_header(guess)
208
+ if guess_norm in lookup:
209
+ return lookup[guess_norm]
210
+
211
+ for col in columns:
212
+ if _normalise_header(col).startswith("fault"):
213
+ return str(col)
214
+
215
+ return str(columns[0])
216
+
217
+
218
+ def summarise_training_files(paths: Sequence[str], notes: Sequence[str]) -> str:
219
+ lines = [Path(path).name for path in paths]
220
+ lines.extend(notes)
221
+ return "\n".join(lines) if lines else "No training files selected."
222
+
223
+
224
+ def read_training_status(status_file_path: str) -> str:
225
+ """Read the current training status from file."""
226
  try:
227
+ if Path(status_file_path).exists():
228
+ with open(status_file_path, 'r') as f:
229
+ return f.read().strip()
230
+ except Exception:
231
+ pass
232
+ return "Training status unavailable"
233
+
234
+
235
+ def _persist_uploaded_file(file_obj) -> Optional[Path]:
236
+ if file_obj is None:
237
  return None
238
 
239
+ if isinstance(file_obj, (str, Path)):
240
+ source = Path(file_obj)
241
+ original_name = source.name
242
+ else:
243
+ source = Path(getattr(file_obj, "name", "") or getattr(file_obj, "path", ""))
244
+ original_name = getattr(file_obj, "orig_name", source.name) or source.name
245
+ if not source or not source.exists():
246
+ return None
247
+
248
+ original_name = Path(original_name).name or source.name
249
+
250
+ base_path = Path(original_name)
251
+ destination = TRAINING_UPLOAD_DIR / base_path.name
252
+ counter = 1
253
+ while destination.exists():
254
+ suffix = base_path.suffix or ".csv"
255
+ destination = TRAINING_UPLOAD_DIR / f"{base_path.stem}_{counter}{suffix}"
256
+ counter += 1
257
+
258
+ shutil.copy2(source, destination)
259
+ return destination
260
+
261
+
262
+ def append_training_files(new_files, existing_paths: Sequence[str], current_label: str):
263
+ if isinstance(existing_paths, (str, Path)):
264
+ paths: List[str] = [str(existing_paths)]
265
+ elif existing_paths is None:
266
+ paths = []
267
+ else:
268
+ paths = list(existing_paths)
269
+ if new_files:
270
+ for file in new_files:
271
+ persisted = _persist_uploaded_file(file)
272
+ if persisted is None:
273
+ continue
274
+ path_str = str(persisted)
275
+ if path_str not in paths:
276
+ paths.append(path_str)
277
+
278
+ valid_paths: List[str] = []
279
+ notes: List[str] = []
280
+ columns_map: Dict[str, str] = {}
281
+ for path in paths:
282
+ try:
283
+ df = load_measurement_csv(path)
284
+ except Exception as exc: # pragma: no cover - user file diagnostics
285
+ notes.append(f"⚠️ Skipped {Path(path).name}: {exc}")
286
+ try:
287
+ Path(path).unlink(missing_ok=True)
288
+ except Exception:
289
+ pass
290
+ continue
291
+ valid_paths.append(path)
292
+ for col in df.columns:
293
+ columns_map[_normalise_header(col)] = str(col)
294
+
295
+ paths = valid_paths
296
+ summary = summarise_training_files(paths, notes)
297
+ column_choices = sorted(columns_map.values())
298
+ preferred = current_label or LABEL_COLUMN
299
+ guessed = guess_label_from_columns(column_choices, preferred)
300
+ dropdown_choices = column_choices if column_choices else [preferred or LABEL_COLUMN]
301
+ dropdown_value = guessed or preferred or LABEL_COLUMN
302
 
303
+ return paths, summary, gr.update(choices=dropdown_choices, value=dropdown_value)
 
 
304
 
 
 
 
305
 
306
+ def clear_training_files():
307
+ default_label = LABEL_COLUMN or "Fault"
308
+ for cached_file in TRAINING_UPLOAD_DIR.glob("*"):
309
+ try:
310
+ if cached_file.is_file():
311
+ cached_file.unlink(missing_ok=True)
312
+ except Exception:
313
+ pass
314
+ return (
315
+ [],
316
+ "No training files selected.",
317
+ gr.update(choices=[default_label], value=default_label),
318
+ gr.update(value=None),
319
+ )
320
+
321
+ PROJECT_OVERVIEW_MD = """
322
+ ## Project Overview
323
+
324
+ This project focuses on classifying faults in electrical transmission lines and
325
+ grid-connected photovoltaic (PV) systems by combining ensemble learning
326
+ techniques with deep neural architectures.
327
+
328
+ ## Datasets
329
+
330
+ ### Transmission Line Fault Dataset
331
+ - 134,406 samples collected from Phasor Measurement Units (PMUs)
332
+ - 14 monitored channels covering currents, voltages, magnitudes, frequency, and phase angles
333
+ - Labels span symmetrical and asymmetrical faults: NF, L-G, LL, LL-G, LLL, and LLL-G
334
+ - Time span: 0 to 5.7 seconds with high-frequency sampling
335
+
336
+ ### Grid-Connected PV System Fault Dataset
337
+ - 2,163,480 samples from 16 experimental scenarios
338
+ - 14 features including PV array measurements (Ipv, Vpv, Vdc), three-phase currents/voltages, aggregate magnitudes (Iabc, Vabc), and frequency indicators (If, Vf)
339
+ - Captures array, inverter, grid anomaly, feedback sensor, and MPPT controller faults at 9.9989 μs sampling intervals
340
+
341
+ ## Data Format Quick Reference
342
+
343
+ Each measurement file may be comma or tab separated and typically exposes the
344
+ following ordered columns:
345
+
346
+ 1. `Timestamp`
347
+ 2. `[325] UPMU_SUB22:FREQ` – system frequency (Hz)
348
+ 3. `[326] UPMU_SUB22:DFDT` – frequency rate-of-change
349
+ 4. `[327] UPMU_SUB22:FLAG` – PMU status flag
350
+ 5. `[328] UPMU_SUB22-L1:MAG` – phase A voltage magnitude
351
+ 6. `[329] UPMU_SUB22-L1:ANG` – phase A voltage angle
352
+ 7. `[330] UPMU_SUB22-L2:MAG` – phase B voltage magnitude
353
+ 8. `[331] UPMU_SUB22-L2:ANG` – phase B voltage angle
354
+ 9. `[332] UPMU_SUB22-L3:MAG` – phase C voltage magnitude
355
+ 10. `[333] UPMU_SUB22-L3:ANG` – phase C voltage angle
356
+ 11. `[334] UPMU_SUB22-C1:MAG` – phase A current magnitude
357
+ 12. `[335] UPMU_SUB22-C1:ANG` – phase A current angle
358
+ 13. `[336] UPMU_SUB22-C2:MAG` – phase B current magnitude
359
+ 14. `[337] UPMU_SUB22-C2:ANG` – phase B current angle
360
+ 15. `[338] UPMU_SUB22-C3:MAG` – phase C current magnitude
361
+ 16. `[339] UPMU_SUB22-C3:ANG` – phase C current angle
362
+
363
+ Upload as many hourly CSV exports as needed—the training tab concatenates them
364
+ before building sliding windows.
365
+
366
+ ## Models Developed
367
+
368
+ 1. **Support Vector Machine (SVM)** – provides the classical machine learning baseline with balanced accuracy across both datasets (85% PMU / 83% PV).
369
+ 2. **CNN-LSTM** – couples convolutional feature extraction with temporal memory, achieving 92% PMU / 89% PV accuracy.
370
+ 3. **Temporal Convolutional Network (TCN)** – leverages dilated convolutions for long-range context and delivers the best trade-off between accuracy and training time (94% PMU / 91% PV).
371
+
372
+ ## Results Summary
373
+
374
+ - **Transmission Line Fault Classification**: SVM 85%, CNN-LSTM 92%, TCN 94%
375
+ - **PV System Fault Classification**: SVM 83%, CNN-LSTM 89%, TCN 91%
376
+
377
+ Use the **Inference** tab to score new PMU/PV windows and the **Training** tab to
378
+ fine-tune or retrain any of the supported models directly within Hugging Face
379
+ Spaces. The logs panel will surface TensorBoard archives whenever deep-learning
380
+ models are trained.
381
+ """
382
+
383
 
384
+ def load_measurement_csv(path: str) -> pd.DataFrame:
385
+ """Read a PMU/PV measurement file with flexible separators and column mapping."""
386
+
387
+ try:
388
+ df = pd.read_csv(path, sep=None, engine="python", encoding="utf-8-sig")
389
+ except Exception:
390
+ df = None
391
+ for separator in ("\t", ",", ";"):
392
+ try:
393
+ df = pd.read_csv(path, sep=separator, engine="python", encoding="utf-8-sig")
394
+ break
395
+ except Exception:
396
+ df = None
397
+ if df is None:
398
+ raise
399
+
400
+ # Clean column names
401
+ df.columns = [str(col).strip() for col in df.columns]
402
+
403
+ print(f"Loaded CSV with {len(df)} rows and {len(df.columns)} columns")
404
+ print(f"Columns: {list(df.columns)}")
405
+ print(f"Data shape: {df.shape}")
406
+
407
+ # Check if we have enough data for training
408
+ if len(df) < 100:
409
+ print(f"Warning: Only {len(df)} rows of data. Recommend at least 1000 rows for effective training.")
410
+
411
+ # Check for label column
412
+ has_label = any(col.lower() in ['fault', 'label', 'class', 'target'] for col in df.columns)
413
+ if not has_label:
414
+ print("Warning: No label column found. Adding dummy 'Fault' column with value 'Normal' for all samples.")
415
+ df['Fault'] = 'Normal' # Add dummy label for training
416
+
417
+ # Create column mapping - map similar column names to expected format
418
+ column_mapping = {}
419
+ expected_cols = list(REQUIRED_PMU_COLUMNS)
420
+
421
+ # If we have at least the right number of numeric columns after Timestamp, use positional mapping
422
+ if "Timestamp" in df.columns:
423
+ numeric_cols = [col for col in df.columns if col != "Timestamp"]
424
+ if len(numeric_cols) >= len(expected_cols):
425
+ # Map by position (after Timestamp)
426
+ for i, expected_col in enumerate(expected_cols):
427
+ if i < len(numeric_cols):
428
+ column_mapping[numeric_cols[i]] = expected_col
429
+
430
+ # Rename columns to match expected format
431
+ df = df.rename(columns=column_mapping)
432
+
433
+ # Check if we have the required columns after mapping
434
+ missing = [col for col in REQUIRED_PMU_COLUMNS if col not in df.columns]
435
+ if missing:
436
+ # If still missing, try a more flexible approach
437
+ available_numeric = df.select_dtypes(include=[np.number]).columns.tolist()
438
+ if len(available_numeric) >= len(expected_cols):
439
+ # Use the first N numeric columns
440
+ for i, expected_col in enumerate(expected_cols):
441
+ if i < len(available_numeric):
442
+ if available_numeric[i] not in df.columns:
443
+ continue
444
+ df = df.rename(columns={available_numeric[i]: expected_col})
445
+
446
+ # Recheck missing columns
447
+ missing = [col for col in REQUIRED_PMU_COLUMNS if col not in df.columns]
448
+
449
+ if missing:
450
+ missing_str = ", ".join(missing)
451
+ available_str = ", ".join(df.columns.tolist())
452
+ raise ValueError(
453
+ f"Missing required PMU feature columns: {missing_str}. "
454
+ f"Available columns: {available_str}. "
455
+ "Please ensure your CSV has the correct format with Timestamp followed by PMU measurements."
456
+ )
457
+
458
+ return df
459
+
460
+
461
+ def apply_metadata(metadata: Dict[str, Any]) -> None:
462
+ global FEATURE_COLUMNS, LABEL_CLASSES, LABEL_COLUMN, SEQUENCE_LENGTH, DEFAULT_WINDOW_STRIDE, MODEL_TYPE, MODEL_FORMAT
463
+ FEATURE_COLUMNS = [str(col) for col in metadata.get("feature_columns", DEFAULT_FEATURE_COLUMNS)]
464
+ LABEL_CLASSES = [str(label) for label in metadata.get("label_classes", [])]
465
+ LABEL_COLUMN = str(metadata.get("label_column", "Fault"))
466
+ SEQUENCE_LENGTH = int(metadata.get("sequence_length", DEFAULT_SEQUENCE_LENGTH))
467
+ DEFAULT_WINDOW_STRIDE = int(metadata.get("stride", DEFAULT_STRIDE))
468
+ MODEL_TYPE = str(metadata.get("model_type", "cnn_lstm")).lower()
469
+ MODEL_FORMAT = str(
470
+ metadata.get("model_format", "joblib" if MODEL_TYPE == "svm" else "keras")
471
+ ).lower()
472
+
473
+
474
+ apply_metadata(METADATA)
475
+
476
+ def sync_label_classes_from_model(model: Optional[object]) -> None:
477
+ global LABEL_CLASSES
478
+ if model is None:
479
+ return
480
+ if hasattr(model, "classes_"):
481
+ LABEL_CLASSES = [str(label) for label in getattr(model, "classes_")]
482
+ elif not LABEL_CLASSES and hasattr(model, "output_shape"):
483
+ LABEL_CLASSES = [str(i) for i in range(int(model.output_shape[-1]))]
484
+
485
+
486
+ # Load model and scaler with error handling
487
+ print("Loading model and scaler...")
488
+ try:
489
+ MODEL = try_load_model(MODEL_PATH, MODEL_TYPE, MODEL_FORMAT)
490
+ print(f"Model loaded: {MODEL is not None}")
491
+ except Exception as e:
492
+ print(f"Model loading failed: {e}")
493
+ MODEL = None
494
+
495
+ try:
496
+ SCALER = try_load_scaler(SCALER_PATH)
497
+ print(f"Scaler loaded: {SCALER is not None}")
498
+ except Exception as e:
499
+ print(f"Scaler loading failed: {e}")
500
+ SCALER = None
501
+
502
+ try:
503
+ sync_label_classes_from_model(MODEL)
504
+ print("Label classes synchronized")
505
+ except Exception as e:
506
+ print(f"Label sync failed: {e}")
507
+
508
+ print("Application initialization completed.")
509
+ print(f"Ready to start Gradio interface. Model available: {MODEL is not None}, Scaler available: {SCALER is not None}")
510
+
511
+
512
+ def refresh_artifacts(model_path: Path, scaler_path: Path, metadata_path: Path) -> None:
513
+ global MODEL_PATH, SCALER_PATH, METADATA_PATH, MODEL, SCALER, METADATA
514
+ MODEL_PATH = model_path
515
+ SCALER_PATH = scaler_path
516
+ METADATA_PATH = metadata_path
517
+ METADATA = load_metadata(metadata_path)
518
+ apply_metadata(METADATA)
519
+ MODEL = try_load_model(model_path, MODEL_TYPE, MODEL_FORMAT)
520
+ SCALER = try_load_scaler(scaler_path)
521
+ sync_label_classes_from_model(MODEL)
522
 
523
  # --------------------------------------------------------------------------------------
524
  # Pre-processing helpers
 
527
  def ensure_ready():
528
  if MODEL is None or SCALER is None:
529
  raise RuntimeError(
530
+ "The model and feature scaler are not available. Upload the trained model "
531
+ "(for example `pmu_cnn_lstm_model.keras`, `pmu_tcn_model.keras`, or `pmu_svm_model.joblib`), "
532
+ "the feature scaler (`pmu_feature_scaler.pkl`), and the metadata JSON (`pmu_metadata.json`) to the Space root "
533
+ "or configure the Hugging Face Hub environment variables so the artifacts can be downloaded "
534
+ "automatically."
535
  )
536
 
537
 
 
539
  cleaned = re.sub(r"[;\n\t]+", ",", text.strip())
540
  arr = np.fromstring(cleaned, sep=",")
541
  if arr.size == 0:
542
+ raise ValueError("No feature values were parsed. Please enter comma-separated numbers.")
543
  return arr.astype(np.float32)
544
 
545
 
 
555
  def make_sliding_windows(data: np.ndarray, sequence_length: int, stride: int) -> np.ndarray:
556
  if data.shape[0] < sequence_length:
557
  raise ValueError(
558
+ f"The dataset contains {data.shape[0]} rows which is less than the requested sequence "
559
+ f"length {sequence_length}. Provide more samples or reduce the sequence length."
560
  )
561
  windows = [data[start : start + sequence_length] for start in range(0, data.shape[0] - sequence_length + 1, stride)]
562
  return np.stack(windows)
 
589
  if sequence_length == 1 and array.shape[1] == n_features:
590
  return array.reshape(array.shape[0], 1, n_features)
591
  raise ValueError(
592
+ "CSV columns do not match the expected feature layout. Include the full PMU feature set "
593
+ "or provide pre-shaped sliding window data."
594
  )
595
 
596
 
 
634
  def predict_sequences(sequences: np.ndarray) -> Tuple[str, pd.DataFrame, List[Dict[str, object]]]:
635
  ensure_ready()
636
  sequences = apply_scaler(sequences.astype(np.float32))
637
+ if MODEL_TYPE == "svm":
638
+ flattened = sequences.reshape(sequences.shape[0], -1)
639
+ if hasattr(MODEL, "predict_proba"):
640
+ probs = MODEL.predict_proba(flattened)
641
+ else:
642
+ raise RuntimeError("Loaded SVM model does not expose predict_proba. Retrain with probability=True.")
643
+ else:
644
+ probs = MODEL.predict(sequences, verbose=0)
645
  table = format_predictions(probs)
646
  json_probs = probabilities_to_json(probs)
647
+ architecture = MODEL_TYPE.replace("_", "-").upper()
648
+ status = f"Generated {len(sequences)} windows. {architecture} model output dimension: {probs.shape[1]}."
649
  return status, table, json_probs
650
 
651
 
 
654
  n_features = len(FEATURE_COLUMNS)
655
  if arr.size % n_features != 0:
656
  raise ValueError(
657
+ f"The number of values ({arr.size}) is not a multiple of the feature dimension "
658
+ f"({n_features}). Provide values in groups of {n_features}."
659
  )
660
  timesteps = arr.size // n_features
661
  if timesteps != sequence_length:
662
  raise ValueError(
663
+ f"Detected {timesteps} timesteps which does not match the configured sequence length "
664
+ f"({sequence_length})."
665
  )
666
  sequences = arr.reshape(1, sequence_length, n_features)
667
  status, table, probs = predict_sequences(sequences)
668
+ status = f"Single window prediction complete. {status}"
669
  return status, table, probs
670
 
671
 
672
  def predict_from_csv(file_obj, sequence_length: int, stride: int) -> Tuple[str, pd.DataFrame, List[Dict[str, object]]]:
673
+ df = load_measurement_csv(file_obj.name)
674
  sequences = dataframe_to_sequences(
675
  df,
676
  sequence_length=sequence_length,
 
678
  feature_columns=FEATURE_COLUMNS,
679
  )
680
  status, table, probs = predict_sequences(sequences)
681
+ status = f"CSV processed successfully. Generated {len(sequences)} windows. {status}"
682
  return status, table, probs
683
 
684
 
685
+ # --------------------------------------------------------------------------------------
686
+ # Training helpers
687
+ # --------------------------------------------------------------------------------------
688
+
689
+
690
+ def classification_report_to_dataframe(report: Dict[str, Any]) -> pd.DataFrame:
691
+ rows: List[Dict[str, Any]] = []
692
+ for label, metrics in report.items():
693
+ if isinstance(metrics, dict):
694
+ row = {"label": label}
695
+ for key, value in metrics.items():
696
+ if key == "support":
697
+ row[key] = int(value)
698
+ else:
699
+ row[key] = round(float(value), 4)
700
+ rows.append(row)
701
+ else:
702
+ rows.append({"label": label, "accuracy": round(float(metrics), 4)})
703
+ return pd.DataFrame(rows)
704
+
705
+
706
+ def confusion_matrix_to_dataframe(confusion: Sequence[Sequence[float]], labels: Sequence[str]) -> pd.DataFrame:
707
+ if not confusion:
708
+ return pd.DataFrame()
709
+ df = pd.DataFrame(confusion, index=list(labels), columns=list(labels))
710
+ df.index.name = "True Label"
711
+ df.columns.name = "Predicted Label"
712
+ return df
713
+
714
+
715
  # --------------------------------------------------------------------------------------
716
  # Gradio interface
717
  # --------------------------------------------------------------------------------------
718
 
719
  def build_interface() -> gr.Blocks:
720
+ theme = gr.themes.Soft(primary_hue="sky", secondary_hue="blue", neutral_hue="gray").set(
721
+ body_background_fill="#1f1f1f",
722
+ body_text_color="#f5f5f5",
723
+ block_background_fill="#262626",
724
+ block_border_color="#333333",
725
+ button_primary_background_fill="#5ac8fa",
726
+ button_primary_background_fill_hover="#48b5eb",
727
+ button_primary_border_color="#38bdf8",
728
+ button_primary_text_color="#0f172a",
729
+ button_secondary_background_fill="#3f3f46",
730
+ button_secondary_text_color="#f5f5f5",
731
+ )
732
+ with gr.Blocks(title="Fault Classification - PMU Data", theme=theme) as demo:
733
+ gr.Markdown("# Fault Classification for PMU & PV Data")
734
+ gr.Markdown(
735
+ "🖥️ TensorFlow is locked to CPU execution so the Space can run without CUDA drivers."
736
+ )
737
  if MODEL is None or SCALER is None:
738
  gr.Markdown(
739
+ "⚠️ **Artifacts Missing** — Upload `pmu_cnn_lstm_model.keras`, "
740
+ "`pmu_feature_scaler.pkl`, and `pmu_metadata.json` to enable inference, "
741
+ "or configure the Hugging Face Hub environment variables so they can be downloaded."
742
  )
743
  else:
744
+ class_count = len(LABEL_CLASSES) if LABEL_CLASSES else "unknown"
745
  gr.Markdown(
746
+ f"Loaded a **{MODEL_TYPE.upper()}** model ({MODEL_FORMAT.upper()}) with "
747
+ f"{len(FEATURE_COLUMNS)} features, sequence length **{SEQUENCE_LENGTH}**, and "
748
+ f"{class_count} target classes. Use the tabs below to run inference or fine-tune "
749
+ "the model with your own CSV files."
750
  )
751
 
752
+ with gr.Accordion("Feature Reference", open=False):
753
  gr.Markdown(
754
+ f"Each time window expects **{len(FEATURE_COLUMNS)} features** ordered as follows:\n"
755
  + "\n".join(f"- {name}" for name in FEATURE_COLUMNS)
756
  )
757
  gr.Markdown(
758
+ f"Default training parameters: **sequence length = {SEQUENCE_LENGTH}**, "
759
+ f"**stride = {DEFAULT_WINDOW_STRIDE}**. Adjust them in the tabs as needed."
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
760
  )
 
 
 
 
 
 
 
 
 
 
 
 
761
 
762
+ with gr.Tabs():
763
+ with gr.Tab("Overview"):
764
+ gr.Markdown(PROJECT_OVERVIEW_MD)
765
+ with gr.Tab("Inference"):
766
+ gr.Markdown("## Run Inference")
767
+ with gr.Row():
768
+ file_in = gr.File(label="Upload PMU CSV", file_types=[".csv"])
769
+ text_in = gr.Textbox(
770
+ lines=4,
771
+ label="Or paste a single window (comma separated)",
772
+ placeholder="49.97772,1.215825E-38,...",
773
+ )
774
+
775
+ with gr.Row():
776
+ sequence_length_input = gr.Slider(
777
+ minimum=1,
778
+ maximum=max(1, SEQUENCE_LENGTH * 2),
779
+ step=1,
780
+ value=SEQUENCE_LENGTH,
781
+ label="Sequence length (timesteps)",
782
+ )
783
+ stride_input = gr.Slider(
784
+ minimum=1,
785
+ maximum=max(1, SEQUENCE_LENGTH),
786
+ step=1,
787
+ value=max(1, DEFAULT_WINDOW_STRIDE),
788
+ label="CSV window stride",
789
+ )
790
+
791
+ predict_btn = gr.Button("🚀 Run Inference", variant="primary")
792
+ status_out = gr.Textbox(label="Status", interactive=False)
793
+ table_out = gr.Dataframe(
794
+ headers=["window", "predicted_label", "confidence", "top3"],
795
+ label="Predictions",
796
+ interactive=False,
797
+ )
798
+ probs_out = gr.JSON(label="Per-window probabilities")
799
+
800
+ def _run_prediction(file_obj, text, sequence_length, stride):
801
+ sequence_length = int(sequence_length)
802
+ stride = int(stride)
803
+ try:
804
+ if file_obj is not None:
805
+ return predict_from_csv(file_obj, sequence_length, stride)
806
+ if text and text.strip():
807
+ return predict_from_text(text, sequence_length)
808
+ return "Please upload a CSV file or provide feature values.", pd.DataFrame(), []
809
+ except Exception as exc:
810
+ return f"Prediction failed: {exc}", pd.DataFrame(), []
811
+
812
+ predict_btn.click(
813
+ _run_prediction,
814
+ inputs=[file_in, text_in, sequence_length_input, stride_input],
815
+ outputs=[status_out, table_out, probs_out],
816
+ concurrency_limit=EVENT_CONCURRENCY_LIMIT,
817
+ )
818
+
819
+ with gr.Tab("Training"):
820
+ gr.Markdown("## Train or Fine-tune the Model")
821
+ gr.Markdown(
822
+ "Upload one or more PMU CSV files to create a combined training dataset. "
823
+ "The files will be concatenated in upload order before generating sliding windows."
824
+ )
825
+
826
+ training_files_state = gr.State([])
827
+ with gr.Row():
828
+ training_file_drop = gr.Files(
829
+ label="Drag and drop PMU training CSVs",
830
+ file_types=[".csv"],
831
+ file_count="multiple",
832
+ type="filepath",
833
+ )
834
+ with gr.Column(scale=1, min_width=180):
835
+ training_upload = gr.UploadButton(
836
+ "📂 Add training CSVs",
837
+ file_types=[".csv"],
838
+ file_count="multiple",
839
+ type="filepath",
840
+ variant="primary",
841
+ )
842
+ clear_training = gr.Button("Clear list", variant="secondary")
843
+
844
+ training_files_summary = gr.Textbox(
845
+ label="Selected training CSVs",
846
+ value="No training files selected.",
847
+ lines=4,
848
+ interactive=False,
849
+ )
850
+
851
+ with gr.Row():
852
+ label_input = gr.Dropdown(
853
+ value=LABEL_COLUMN,
854
+ choices=[LABEL_COLUMN],
855
+ allow_custom_value=True,
856
+ label="Label column name",
857
+ )
858
+ model_selector = gr.Radio(
859
+ choices=["CNN-LSTM", "TCN", "SVM"],
860
+ value=(
861
+ "TCN"
862
+ if MODEL_TYPE == "tcn"
863
+ else ("SVM" if MODEL_TYPE == "svm" else "CNN-LSTM")
864
+ ),
865
+ label="Model architecture",
866
+ )
867
+ sequence_length_train = gr.Slider(
868
+ minimum=4,
869
+ maximum=max(32, SEQUENCE_LENGTH * 2),
870
+ step=1,
871
+ value=SEQUENCE_LENGTH,
872
+ label="Sequence length",
873
+ )
874
+ stride_train = gr.Slider(
875
+ minimum=1,
876
+ maximum=max(32, SEQUENCE_LENGTH * 2),
877
+ step=1,
878
+ value=max(1, DEFAULT_WINDOW_STRIDE),
879
+ label="Stride",
880
+ )
881
+
882
+ model_default = (
883
+ str(MODEL_PATH)
884
+ if MODEL_PATH
885
+ else MODEL_FILENAME_BY_TYPE.get(MODEL_TYPE, LOCAL_MODEL_FILE)
886
+ )
887
+
888
+ with gr.Row():
889
+ validation_train = gr.Slider(
890
+ minimum=0.05,
891
+ maximum=0.4,
892
+ step=0.05,
893
+ value=0.2,
894
+ label="Validation split",
895
+ )
896
+ batch_train = gr.Slider(
897
+ minimum=32,
898
+ maximum=512,
899
+ step=32,
900
+ value=128,
901
+ label="Batch size",
902
+ )
903
+ epochs_train = gr.Slider(
904
+ minimum=5,
905
+ maximum=100,
906
+ step=5,
907
+ value=50,
908
+ label="Epochs",
909
+ )
910
+
911
+ with gr.Row():
912
+ model_name = gr.Textbox(value=model_default, label="Model output filename")
913
+ scaler_name = gr.Textbox(
914
+ value=str(SCALER_PATH or LOCAL_SCALER_FILE),
915
+ label="Scaler output filename",
916
+ )
917
+ metadata_name = gr.Textbox(
918
+ value=str(METADATA_PATH or LOCAL_METADATA_FILE),
919
+ label="Metadata output filename",
920
+ )
921
+
922
+ tensorboard_toggle = gr.Checkbox(
923
+ value=True,
924
+ label="Enable TensorBoard logging (creates downloadable archive)",
925
+ )
926
+
927
+ def _suggest_model_filename(choice: str, current_value: str):
928
+ choice_key = (choice or "cnn_lstm").lower().replace("-", "_")
929
+ suggested = MODEL_FILENAME_BY_TYPE.get(choice_key, LOCAL_MODEL_FILE)
930
+ known_defaults = {Path(name).name for name in MODEL_FILENAME_BY_TYPE.values()}
931
+ current_name = Path(current_value).name if current_value else ""
932
+ if current_name and current_name not in known_defaults:
933
+ return gr.update()
934
+ return gr.update(value=suggested)
935
+
936
+ model_selector.change(
937
+ _suggest_model_filename,
938
+ inputs=[model_selector, model_name],
939
+ outputs=model_name,
940
+ )
941
+
942
+ with gr.Row():
943
+ train_button = gr.Button("🛠️ Start Training", variant="primary")
944
+ progress_button = gr.Button("📊 Check Progress", variant="secondary")
945
+
946
+ # Training status display
947
+ training_status = gr.Textbox(label="Training Status", interactive=False)
948
+ report_output = gr.Dataframe(label="Classification report", interactive=False)
949
+ history_output = gr.JSON(label="Training history")
950
+ confusion_output = gr.Dataframe(label="Confusion matrix", interactive=False)
951
+ tensorboard_file = gr.File(
952
+ label="TensorBoard logs (.zip)",
953
+ interactive=False,
954
+ )
955
+
956
+ # Message area at the bottom for progress updates
957
+ with gr.Accordion("📋 Progress Messages", open=True):
958
+ progress_messages = gr.Textbox(
959
+ label="Training Messages",
960
+ lines=8,
961
+ max_lines=20,
962
+ interactive=False,
963
+ autoscroll=True,
964
+ placeholder="Click 'Check Progress' to see training updates..."
965
+ )
966
+ with gr.Row():
967
+ gr.Button("🗑️ Clear Messages", variant="secondary").click(
968
+ lambda: "",
969
+ outputs=[progress_messages]
970
+ )
971
+
972
+ def _run_training(
973
+ file_paths,
974
+ label_column,
975
+ model_choice,
976
+ sequence_length,
977
+ stride,
978
+ validation_split,
979
+ batch_size,
980
+ epochs,
981
+ model_filename,
982
+ scaler_filename,
983
+ metadata_filename,
984
+ enable_tensorboard,
985
+ ):
986
+ try:
987
+ # Create status file path for progress tracking
988
+ status_file = Path(model_filename).parent / "training_status.txt"
989
+
990
+ # Initialize status
991
+ with open(status_file, 'w') as f:
992
+ f.write("Starting training setup...")
993
+
994
+ if not file_paths:
995
+ raise ValueError("Add at least one training CSV via the uploader before starting.")
996
+
997
+ with open(status_file, 'w') as f:
998
+ f.write("Loading and validating CSV files...")
999
+
1000
+ available_paths = [path for path in file_paths if Path(path).exists()]
1001
+ missing_paths = [Path(path).name for path in file_paths if not Path(path).exists()]
1002
+ if not available_paths:
1003
+ raise ValueError("None of the referenced CSV files are available. Please upload them again.")
1004
+
1005
+ dfs = [load_measurement_csv(path) for path in available_paths]
1006
+ combined = pd.concat(dfs, ignore_index=True)
1007
+
1008
+ # Validate data size and provide recommendations
1009
+ total_samples = len(combined)
1010
+ if total_samples < 100:
1011
+ print(f"Warning: Only {total_samples} samples. Recommend at least 1000 for good results.")
1012
+ print("Automatically switching to SVM for small dataset compatibility.")
1013
+ if model_choice in ["cnn_lstm", "tcn"]:
1014
+ model_choice = "svm"
1015
+ print(f"Model type changed to SVM for better small dataset performance.")
1016
+ if total_samples < 10:
1017
+ raise ValueError(f"Insufficient data: {total_samples} samples. Need at least 10 samples for training.")
1018
+
1019
+ label_column = (label_column or LABEL_COLUMN).strip()
1020
+ if not label_column:
1021
+ raise ValueError("Label column name cannot be empty.")
1022
+
1023
+ model_choice = (model_choice or "CNN-LSTM").lower().replace("-", "_")
1024
+ if model_choice not in {"cnn_lstm", "tcn", "svm"}:
1025
+ raise ValueError("Select CNN-LSTM, TCN, or SVM for the model architecture.")
1026
+
1027
+ with open(status_file, 'w') as f:
1028
+ f.write(f"Starting {model_choice.upper()} training with {len(combined)} samples...")
1029
+
1030
+ # Start training
1031
+ result = train_from_dataframe(
1032
+ combined,
1033
+ label_column=label_column,
1034
+ feature_columns=None,
1035
+ sequence_length=int(sequence_length),
1036
+ stride=int(stride),
1037
+ validation_split=float(validation_split),
1038
+ batch_size=int(batch_size),
1039
+ epochs=int(epochs),
1040
+ model_type=model_choice,
1041
+ model_path=Path(model_filename),
1042
+ scaler_path=Path(scaler_filename),
1043
+ metadata_path=Path(metadata_filename),
1044
+ enable_tensorboard=bool(enable_tensorboard),
1045
+ )
1046
+
1047
+ refresh_artifacts(
1048
+ Path(result["model_path"]),
1049
+ Path(result["scaler_path"]),
1050
+ Path(result["metadata_path"]),
1051
+ )
1052
+
1053
+ report_df = classification_report_to_dataframe(result["classification_report"])
1054
+ confusion_df = confusion_matrix_to_dataframe(result["confusion_matrix"], result["class_names"])
1055
+ tensorboard_dir = result.get("tensorboard_log_dir")
1056
+ tensorboard_zip = result.get("tensorboard_zip_path")
1057
+
1058
+ architecture = result["model_type"].replace("_", "-").upper()
1059
+ status = (
1060
+ f"Training complete using a {architecture} architecture. "
1061
+ f"{result['num_sequences']} windows derived from "
1062
+ f"{result['num_samples']} rows across {len(available_paths)} file(s)."
1063
+ f" Artifacts saved to:"
1064
+ f"\n• Model: {result['model_path']}\n"
1065
+ f"• Scaler: {result['scaler_path']}\n"
1066
+ f"• Metadata: {result['metadata_path']}"
1067
+ )
1068
+
1069
+ status += f"\nLabel column used: {result.get('label_column', label_column)}"
1070
+
1071
+ if tensorboard_dir:
1072
+ status += (
1073
+ f"\nTensorBoard logs directory: {tensorboard_dir}"
1074
+ f"\nRun `tensorboard --logdir \"{tensorboard_dir}\"` to inspect the training curves."
1075
+ "\nDownload the archive below to explore the run offline."
1076
+ )
1077
+
1078
+ if missing_paths:
1079
+ skipped = ", ".join(missing_paths)
1080
+ status = f"⚠️ Skipped missing files: {skipped}\n" + status
1081
+
1082
+ return (
1083
+ status,
1084
+ report_df,
1085
+ result["history"],
1086
+ confusion_df,
1087
+ tensorboard_zip,
1088
+ gr.update(value=result.get("label_column", label_column)),
1089
+ )
1090
+ except Exception as exc:
1091
+ return (
1092
+ f"Training failed: {exc}",
1093
+ pd.DataFrame(),
1094
+ {},
1095
+ pd.DataFrame(),
1096
+ None,
1097
+ gr.update(),
1098
+ )
1099
+
1100
+ def _check_progress(model_filename, current_messages):
1101
+ """Check training progress by reading status file and accumulate messages."""
1102
+ status_file = Path(model_filename).parent / "training_status.txt"
1103
+ status_message = read_training_status(str(status_file))
1104
+
1105
+ # Add timestamp to the message
1106
+ from datetime import datetime
1107
+ timestamp = datetime.now().strftime("%H:%M:%S")
1108
+ new_message = f"[{timestamp}] {status_message}"
1109
+
1110
+ # Accumulate messages, keeping last 50 lines to prevent overflow
1111
+ if current_messages:
1112
+ lines = current_messages.split('\n')
1113
+ lines.append(new_message)
1114
+ # Keep only last 50 lines
1115
+ if len(lines) > 50:
1116
+ lines = lines[-50:]
1117
+ accumulated_messages = '\n'.join(lines)
1118
+ else:
1119
+ accumulated_messages = new_message
1120
+
1121
+ return accumulated_messages
1122
+
1123
+ train_button.click(
1124
+ _run_training,
1125
+ inputs=[
1126
+ training_files_state,
1127
+ label_input,
1128
+ model_selector,
1129
+ sequence_length_train,
1130
+ stride_train,
1131
+ validation_train,
1132
+ batch_train,
1133
+ epochs_train,
1134
+ model_name,
1135
+ scaler_name,
1136
+ metadata_name,
1137
+ tensorboard_toggle,
1138
+ ],
1139
+ outputs=[
1140
+ training_status,
1141
+ report_output,
1142
+ history_output,
1143
+ confusion_output,
1144
+ tensorboard_file,
1145
+ label_input,
1146
+ ],
1147
+ concurrency_limit=EVENT_CONCURRENCY_LIMIT,
1148
+ )
1149
+
1150
+ progress_button.click(
1151
+ _check_progress,
1152
+ inputs=[model_name, progress_messages],
1153
+ outputs=[progress_messages],
1154
+ )
1155
+
1156
+ training_upload.upload(
1157
+ append_training_files,
1158
+ inputs=[training_upload, training_files_state, label_input],
1159
+ outputs=[training_files_state, training_files_summary, label_input],
1160
+ concurrency_limit=EVENT_CONCURRENCY_LIMIT,
1161
+ )
1162
+ training_file_drop.upload(
1163
+ append_training_files,
1164
+ inputs=[training_file_drop, training_files_state, label_input],
1165
+ outputs=[training_files_state, training_files_summary, label_input],
1166
+ concurrency_limit=EVENT_CONCURRENCY_LIMIT,
1167
+ )
1168
+ clear_training.click(
1169
+ clear_training_files,
1170
+ outputs=[training_files_state, training_files_summary, label_input, training_file_drop],
1171
+ )
1172
 
1173
  return demo
1174
 
 
1177
  # Launch helpers
1178
  # --------------------------------------------------------------------------------------
1179
 
1180
+ def resolve_server_port() -> int:
1181
+ for env_var in ("PORT", "GRADIO_SERVER_PORT"):
 
 
 
 
 
 
1182
  value = os.environ.get(env_var)
1183
  if value:
1184
  try:
1185
  return int(value)
1186
  except ValueError:
1187
+ print(f"Ignoring invalid port value from {env_var}: {value}")
1188
+ return 7860
1189
 
1190
 
1191
  def main():
1192
+ print("Building Gradio interface...")
1193
+ try:
1194
+ demo = build_interface()
1195
+ print("Interface built successfully")
1196
+ except Exception as e:
1197
+ print(f"Failed to build interface: {e}")
1198
+ import traceback
1199
+ traceback.print_exc()
1200
+ return
1201
+
1202
+ print("Setting up queue...")
1203
  try:
1204
+ demo.queue(max_size=QUEUE_MAX_SIZE)
1205
+ print("Queue configured")
1206
+ except Exception as e:
1207
+ print(f"Failed to configure queue: {e}")
1208
+
1209
+ try:
1210
+ port = resolve_server_port()
1211
  print(f"Launching Gradio app on port {port}")
1212
+ demo.launch(server_name="0.0.0.0", server_port=port, show_error=True)
1213
  except OSError as exc:
1214
  print("Failed to launch on requested port:", exc)
1215
+ try:
1216
+ demo.launch(server_name="0.0.0.0", show_error=True)
1217
+ except Exception as e:
1218
+ print(f"Failed to launch completely: {e}")
1219
+ except Exception as e:
1220
+ print(f"Unexpected launch error: {e}")
1221
+ import traceback
1222
+ traceback.print_exc()
1223
 
1224
 
1225
  if __name__ == "__main__":
1226
+ print("="*50)
1227
+ print("PMU Fault Classification App Starting")
1228
+ print(f"Python version: {os.sys.version}")
1229
+ print(f"Working directory: {os.getcwd()}")
1230
+ print(f"HUB_REPO: {HUB_REPO}")
1231
+ print(f"Model available: {MODEL is not None}")
1232
+ print(f"Scaler available: {SCALER is not None}")
1233
+ print("="*50)
1234
  main()
fault_classification_pmu.py CHANGED
@@ -1,17 +1,19 @@
1
- """Fault classification training utilities for PMU data.
2
 
3
- This module trains a CNN-LSTM model on high-frequency PMU measurements to
4
- classify transmission line faults. It implements a full training pipeline
5
- including preprocessing, sequence generation, model definition, evaluation,
6
- and artifact export so the resulting model can be served via the Gradio app
7
- in this repository or on Hugging Face Spaces.
 
8
 
9
  Example
10
  -------
11
  python fault_classification_pmu.py \
12
  --data-path data/Fault_Classification_PMU_Data.csv \
13
  --label-column FaultType \
14
- --model-out pmu_cnn_lstm_model.keras \
 
15
  --scaler-out pmu_feature_scaler.pkl \
16
  --metadata-out pmu_metadata.json
17
 
@@ -22,24 +24,200 @@ via the ``--feature-columns`` argument. Data is automatically standardised
22
  and windowed to create temporal sequences that feed into the neural network.
23
 
24
  The exported metadata JSON file contains the feature ordering, label names,
25
- sequence length, and stride. The Gradio front-end consumes this file to
26
- replicate the same preprocessing steps during inference.
27
  """
28
  from __future__ import annotations
29
 
30
  import argparse
31
  import json
 
 
 
32
  from pathlib import Path
33
- from typing import List, Sequence, Tuple
 
 
 
 
 
 
34
 
35
  import joblib
36
  import numpy as np
37
  import pandas as pd
38
- from sklearn.metrics import classification_report, confusion_matrix
 
39
  from sklearn.model_selection import train_test_split
40
  from sklearn.preprocessing import LabelEncoder, StandardScaler
 
41
  from tensorflow.keras import callbacks, layers, models, optimizers
42
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  # Default PMU feature set as described in the user provided table. Timestamp is
44
  # intentionally omitted because it is not a model input feature.
45
  DEFAULT_FEATURE_COLUMNS: List[str] = [
@@ -60,6 +238,47 @@ DEFAULT_FEATURE_COLUMNS: List[str] = [
60
  "[339] UPMU_SUB22-C3:ANG",
61
  ]
62
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
 
64
  def _resolve_features(df: pd.DataFrame, feature_columns: Sequence[str] | None, label_column: str) -> List[str]:
65
  if feature_columns:
@@ -85,7 +304,7 @@ def load_dataset(
85
  *,
86
  feature_columns: Sequence[str] | None,
87
  label_column: str,
88
- ) -> Tuple[np.ndarray, np.ndarray, List[str]]:
89
  """Load the dataset from CSV.
90
 
91
  Parameters
@@ -105,15 +324,32 @@ def load_dataset(
105
  1-D array of label strings.
106
  columns: list[str]
107
  Actual feature ordering used.
 
 
108
  """
109
- df = pd.read_csv(csv_path)
110
- if label_column not in df.columns:
111
- raise ValueError(f"Label column '{label_column}' not found in {csv_path}")
 
 
 
 
 
 
 
 
 
 
 
 
 
112
 
113
- columns = _resolve_features(df, feature_columns, label_column)
 
 
114
  features = df[columns].astype(np.float32).values
115
- labels = df[label_column].astype(str).values
116
- return features, labels, columns
117
 
118
 
119
  def create_sequences(
@@ -175,6 +411,56 @@ def build_cnn_lstm(
175
  return model
176
 
177
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
178
  def train_model(
179
  sequences: np.ndarray,
180
  labels: np.ndarray,
@@ -182,40 +468,216 @@ def train_model(
182
  validation_split: float,
183
  batch_size: int,
184
  epochs: int,
185
- ) -> Tuple[models.Model, LabelEncoder, dict]:
186
- """Train the CNN-LSTM model and return training history and validation outputs."""
187
- label_encoder = LabelEncoder()
188
- y = label_encoder.fit_transform(labels)
189
-
190
- X_train, X_val, y_train, y_val = train_test_split(
191
- sequences, y, test_size=validation_split, stratify=y, random_state=42
192
- )
193
 
194
- model = build_cnn_lstm(input_shape=sequences.shape[1:], num_classes=len(label_encoder.classes_))
 
 
195
 
196
- callbacks_list = [
197
- callbacks.ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=5, min_lr=1e-5),
198
- callbacks.EarlyStopping(monitor="val_loss", patience=10, restore_best_weights=True),
199
- ]
200
 
201
- history = model.fit(
202
- X_train,
203
- y_train,
204
- validation_data=(X_val, y_val),
205
- epochs=epochs,
206
- batch_size=batch_size,
207
- callbacks=callbacks_list,
208
- verbose=2,
209
- )
210
 
211
- y_pred = model.predict(X_val, verbose=0).argmax(axis=1)
212
- metrics = {
213
- "history": history.history,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
214
  "validation": {
215
  "y_true": y_val,
216
  "y_pred": y_pred,
217
  "class_names": label_encoder.classes_.tolist(),
 
218
  },
 
 
 
219
  }
220
  return model, label_encoder, metrics
221
 
@@ -230,7 +692,7 @@ def standardise_sequences(sequences: np.ndarray) -> Tuple[np.ndarray, StandardSc
230
 
231
  def export_artifacts(
232
  *,
233
- model: models.Model,
234
  scaler: StandardScaler,
235
  label_encoder: LabelEncoder,
236
  feature_columns: Sequence[str],
@@ -246,9 +708,21 @@ def export_artifacts(
246
  model_path.parent.mkdir(parents=True, exist_ok=True)
247
  scaler_path.parent.mkdir(parents=True, exist_ok=True)
248
  metadata_path.parent.mkdir(parents=True, exist_ok=True)
249
- model.save(model_path)
 
 
 
 
250
  joblib.dump(scaler, scaler_path)
251
 
 
 
 
 
 
 
 
 
252
  metadata = {
253
  "feature_columns": list(feature_columns),
254
  "label_classes": label_encoder.classes_.tolist(),
@@ -258,28 +732,184 @@ def export_artifacts(
258
  "model_path": str(model_path),
259
  "scaler_path": str(scaler_path),
260
  "training_history": metrics["history"],
261
- "classification_report": classification_report(
262
- metrics["validation"]["y_true"], metrics["validation"]["y_pred"], target_names=label_encoder.classes_
263
- ),
264
- "confusion_matrix": metrics["validation"].get("confusion_matrix")
265
- if metrics["validation"].get("confusion_matrix") is not None
266
- else None,
267
  }
268
- # Add confusion matrix lazily to avoid recomputation.
269
- if metadata["confusion_matrix"] is None:
270
- cm = confusion_matrix(metrics["validation"]["y_true"], metrics["validation"]["y_pred"])
271
- metadata["confusion_matrix"] = cm.tolist()
272
 
273
  metadata_path.write_text(json.dumps(metadata, indent=2))
274
 
275
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
276
  def run_training(args: argparse.Namespace) -> None:
277
  csv_path = Path(args.data_path)
278
  model_out = Path(args.model_out)
279
  scaler_out = Path(args.scaler_out)
280
  metadata_out = Path(args.metadata_out)
281
 
282
- features, labels, feature_columns = load_dataset(
283
  csv_path, feature_columns=args.feature_columns, label_column=args.label_column
284
  )
285
 
@@ -291,12 +921,21 @@ def run_training(args: argparse.Namespace) -> None:
291
  )
292
 
293
  sequences, scaler = standardise_sequences(sequences)
 
 
 
 
 
 
294
  model, label_encoder, metrics = train_model(
295
  sequences,
296
  seq_labels,
297
  validation_split=args.validation_split,
298
  batch_size=args.batch_size,
299
  epochs=args.epochs,
 
 
 
300
  )
301
 
302
  export_artifacts(
@@ -304,7 +943,7 @@ def run_training(args: argparse.Namespace) -> None:
304
  scaler=scaler,
305
  label_encoder=label_encoder,
306
  feature_columns=feature_columns,
307
- label_column=args.label_column,
308
  sequence_length=args.sequence_length,
309
  stride=args.stride,
310
  model_path=model_out,
@@ -314,6 +953,7 @@ def run_training(args: argparse.Namespace) -> None:
314
  )
315
 
316
  print("Training complete")
 
317
  print(f"Model saved to : {model_out}")
318
  print(f"Scaler saved to : {scaler_out}")
319
  print(f"Metadata saved to : {metadata_out}")
@@ -322,10 +962,14 @@ def run_training(args: argparse.Namespace) -> None:
322
  metrics["validation"]["y_true"], metrics["validation"]["y_pred"], target_names=metrics["validation"]["class_names"]
323
  )
324
  print(report)
 
 
 
 
325
 
326
 
327
  def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
328
- parser = argparse.ArgumentParser(description="Train a CNN-LSTM model for PMU fault classification")
329
  parser.add_argument("--data-path", required=True, help="Path to Fault_Classification_PMU_Data CSV")
330
  parser.add_argument(
331
  "--label-column",
@@ -343,9 +987,27 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
343
  parser.add_argument("--validation-split", type=float, default=0.2, help="Validation set fraction")
344
  parser.add_argument("--batch-size", type=int, default=128, help="Training batch size")
345
  parser.add_argument("--epochs", type=int, default=50, help="Maximum number of training epochs")
 
 
 
 
 
 
346
  parser.add_argument("--model-out", default="pmu_cnn_lstm_model.keras", help="Path to save trained Keras model")
347
  parser.add_argument("--scaler-out", default="pmu_feature_scaler.pkl", help="Path to save fitted StandardScaler")
348
  parser.add_argument("--metadata-out", default="pmu_metadata.json", help="Path to save metadata JSON")
 
 
 
 
 
 
 
 
 
 
 
 
349
  return parser.parse_args(argv)
350
 
351
 
 
1
+ """Fault classification training utilities for PMU and PV datasets.
2
 
3
+ This module trains deep learning models on high-frequency PMU measurements and
4
+ supports classical machine learning baselines so the resulting artefacts can be
5
+ served via the Gradio app in this repository or on Hugging Face Spaces. It
6
+ implements a full training pipeline including preprocessing, sequence
7
+ generation, model definition (CNN-LSTM, Temporal Convolutional Network, or
8
+ Support Vector Machine), evaluation, and export of deployment metadata.
9
 
10
  Example
11
  -------
12
  python fault_classification_pmu.py \
13
  --data-path data/Fault_Classification_PMU_Data.csv \
14
  --label-column FaultType \
15
+ --model-type tcn \
16
+ --model-out pmu_tcn_model.keras \
17
  --scaler-out pmu_feature_scaler.pkl \
18
  --metadata-out pmu_metadata.json
19
 
 
24
  and windowed to create temporal sequences that feed into the neural network.
25
 
26
  The exported metadata JSON file contains the feature ordering, label names,
27
+ sequence length, stride, and chosen architecture. The Gradio front-end
28
+ consumes this file to replicate the same preprocessing steps during inference.
29
  """
30
  from __future__ import annotations
31
 
32
  import argparse
33
  import json
34
+ import os
35
+ import shutil
36
+ from datetime import datetime
37
  from pathlib import Path
38
+ from typing import Dict, List, Optional, Sequence, Tuple
39
+
40
+ import math
41
+
42
+ os.environ.setdefault("CUDA_VISIBLE_DEVICES", "-1")
43
+ os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "2")
44
+ os.environ.setdefault("TF_ENABLE_ONEDNN_OPTS", "0")
45
 
46
  import joblib
47
  import numpy as np
48
  import pandas as pd
49
+ from pandas.api.types import is_numeric_dtype
50
+ from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
51
  from sklearn.model_selection import train_test_split
52
  from sklearn.preprocessing import LabelEncoder, StandardScaler
53
+ from sklearn.svm import SVC
54
  from tensorflow.keras import callbacks, layers, models, optimizers
55
 
56
+
57
+
58
+ class ProgressCallback(callbacks.Callback):
59
+ """Custom callback to provide training progress updates."""
60
+
61
+ def __init__(
62
+ self,
63
+ total_epochs,
64
+ status_file_path=None,
65
+ *,
66
+ status_update_interval: float = 10.0,
67
+ batch_log_frequency: int = 10,
68
+ ):
69
+ super().__init__()
70
+ self.total_epochs = total_epochs
71
+ self.status_file_path = status_file_path
72
+ self.status_update_interval = max(1.0, float(status_update_interval))
73
+ self.batch_log_frequency = max(1, int(batch_log_frequency))
74
+ self.current_epoch = 0
75
+ self.train_start_time: Optional[float] = None
76
+ self.last_status_report: Optional[float] = None
77
+ self.total_batches_per_epoch = 0
78
+ self.batches_seen = 0
79
+
80
+ # ------------------------------------------------------------------
81
+ # Internal helpers
82
+ # ------------------------------------------------------------------
83
+ def _now(self) -> float:
84
+ import time
85
+
86
+ return time.perf_counter()
87
+
88
+ def _training_elapsed(self, now: Optional[float] = None) -> float:
89
+ if self.train_start_time is None:
90
+ return 0.0
91
+ if now is None:
92
+ now = self._now()
93
+ return max(0.0, now - self.train_start_time)
94
+
95
+ def _report_status(self, message: str, *, force: bool = False) -> None:
96
+ now = self._now()
97
+ if not force and self.last_status_report is not None:
98
+ if now - self.last_status_report < self.status_update_interval:
99
+ return
100
+
101
+ print(message, flush=True)
102
+
103
+ if self.status_file_path:
104
+ try:
105
+ with open(self.status_file_path, "w") as f:
106
+ f.write(message)
107
+ except Exception:
108
+ # Silently ignore status file failures; progress should still stream to stdout
109
+ pass
110
+
111
+ self.last_status_report = now
112
+
113
+ # ------------------------------------------------------------------
114
+ # Keras callback overrides
115
+ # ------------------------------------------------------------------
116
+ def on_train_begin(self, logs=None):
117
+ params = self.params or {}
118
+ steps = params.get("steps") or params.get("steps_per_epoch")
119
+ if steps:
120
+ self.total_batches_per_epoch = int(steps)
121
+ else:
122
+ samples = params.get("samples")
123
+ batch_size = params.get("batch_size") or 0
124
+ if samples and batch_size:
125
+ self.total_batches_per_epoch = math.ceil(samples / batch_size)
126
+ else:
127
+ self.total_batches_per_epoch = 0
128
+
129
+ self.batches_seen = 0
130
+ self.last_status_report = None
131
+ self.train_start_time = self._now()
132
+
133
+ def on_epoch_begin(self, epoch, logs=None):
134
+ import time
135
+
136
+ now = self._now()
137
+ if self.train_start_time is None:
138
+ self.train_start_time = now
139
+
140
+ self.current_epoch = epoch + 1
141
+ self.batches_seen = 0
142
+
143
+ progress_pct = (self.current_epoch / self.total_epochs) * 100
144
+ elapsed_time = self._training_elapsed(now)
145
+ status_msg = (
146
+ f"Training epoch {self.current_epoch}/{self.total_epochs} "
147
+ f"({progress_pct:.1f}%) - {elapsed_time:.1f}s elapsed"
148
+ )
149
+ self._report_status(status_msg, force=True)
150
+
151
+ if self.current_epoch == 1:
152
+ wall_clock = time.strftime("%H:%M:%S")
153
+ print(f"Starting first epoch at {wall_clock}", flush=True)
154
+
155
+ def on_batch_begin(self, batch, logs=None):
156
+ if self.current_epoch == 1 and batch % self.batch_log_frequency == 0:
157
+ elapsed = self._training_elapsed()
158
+ print(f"Epoch {self.current_epoch}, Batch {batch} started - {elapsed:.1f}s elapsed", flush=True)
159
+
160
+ def on_batch_end(self, batch, logs=None):
161
+ self.batches_seen = batch + 1
162
+
163
+ if self.current_epoch == 1 and batch % self.batch_log_frequency == 0:
164
+ logs = logs or {}
165
+ loss = logs.get("loss", 0)
166
+ elapsed = self._training_elapsed()
167
+ print(
168
+ f"Epoch {self.current_epoch}, Batch {batch} completed - Loss: {loss:.4f}, {elapsed:.1f}s elapsed",
169
+ flush=True,
170
+ )
171
+
172
+ total_batches = self.total_batches_per_epoch or 0
173
+ if not total_batches:
174
+ params = self.params or {}
175
+ total_batches = (
176
+ params.get("steps")
177
+ or params.get("steps_per_epoch")
178
+ or 0
179
+ )
180
+
181
+ if total_batches:
182
+ epoch_fraction = min(1.0, (batch + 1) / total_batches)
183
+ else:
184
+ epoch_fraction = 0.0
185
+
186
+ overall_progress = (
187
+ (self.current_epoch - 1 + epoch_fraction) / self.total_epochs * 100
188
+ )
189
+ elapsed_time = self._training_elapsed()
190
+ status_msg = (
191
+ f"Epoch {self.current_epoch}/{self.total_epochs} - Batch {batch + 1}/{total_batches or '?'} "
192
+ f"({overall_progress:.1f}%) - {elapsed_time:.1f}s elapsed"
193
+ )
194
+ self._report_status(status_msg)
195
+
196
+ def on_epoch_end(self, epoch, logs=None):
197
+ logs = logs or {}
198
+ loss = logs.get("loss", 0)
199
+ val_loss = logs.get("val_loss", 0)
200
+ accuracy = logs.get("accuracy", logs.get("acc", 0))
201
+ val_accuracy = logs.get("val_accuracy", logs.get("val_acc", 0))
202
+ _ = epoch # Suppress unused variable warning
203
+
204
+ elapsed_time = self._training_elapsed()
205
+ status_msg = (
206
+ f"Epoch {self.current_epoch}/{self.total_epochs} completed - "
207
+ f"Loss: {loss:.4f}, Val Loss: {val_loss:.4f}, "
208
+ f"Acc: {accuracy:.4f}, Val Acc: {val_accuracy:.4f} - {elapsed_time:.1f}s total"
209
+ )
210
+ self._report_status(status_msg, force=True)
211
+
212
+ def on_train_end(self, logs=None):
213
+ total_elapsed = self._training_elapsed()
214
+ final_message = (
215
+ f"Training finished after {self.total_epochs} epoch(s) - "
216
+ f"{total_elapsed:.1f}s total elapsed"
217
+ )
218
+ self._report_status(final_message, force=True)
219
+
220
+
221
  # Default PMU feature set as described in the user provided table. Timestamp is
222
  # intentionally omitted because it is not a model input feature.
223
  DEFAULT_FEATURE_COLUMNS: List[str] = [
 
238
  "[339] UPMU_SUB22-C3:ANG",
239
  ]
240
 
241
+ LABEL_GUESS_CANDIDATES: Tuple[str, ...] = ("Fault", "FaultType", "Label", "Target", "Class")
242
+
243
+
244
+ def _normalise_column_name(name: str) -> str:
245
+ return str(name).strip().lower()
246
+
247
+
248
+ def _resolve_label_column(df: pd.DataFrame, requested: str) -> str:
249
+ columns = [str(col) for col in df.columns]
250
+ if not columns:
251
+ raise ValueError("Provided dataframe does not contain any columns.")
252
+
253
+ requested = str(requested or "").strip()
254
+ if requested and requested in df.columns:
255
+ return requested
256
+
257
+ if requested:
258
+ for col in df.columns:
259
+ if str(col).strip() == requested:
260
+ return str(col)
261
+ lowered = requested.lower()
262
+ lowered_map = {_normalise_column_name(col): str(col) for col in df.columns}
263
+ if lowered in lowered_map:
264
+ return lowered_map[lowered]
265
+
266
+ lowered_map = {_normalise_column_name(col): str(col) for col in df.columns}
267
+ for guess in LABEL_GUESS_CANDIDATES:
268
+ key = guess.lower()
269
+ if key in lowered_map:
270
+ return lowered_map[key]
271
+
272
+ for col in reversed(df.columns):
273
+ if not is_numeric_dtype(df[col]):
274
+ return str(col)
275
+
276
+ available = ", ".join(columns)
277
+ raise ValueError(
278
+ f"Label column '{requested or ' '}' not found in provided dataframe. "
279
+ f"Available columns: {available}"
280
+ )
281
+
282
 
283
  def _resolve_features(df: pd.DataFrame, feature_columns: Sequence[str] | None, label_column: str) -> List[str]:
284
  if feature_columns:
 
304
  *,
305
  feature_columns: Sequence[str] | None,
306
  label_column: str,
307
+ ) -> Tuple[np.ndarray, np.ndarray, List[str], str]:
308
  """Load the dataset from CSV.
309
 
310
  Parameters
 
324
  1-D array of label strings.
325
  columns: list[str]
326
  Actual feature ordering used.
327
+ resolved_label: str
328
+ The column name that supplied the labels.
329
  """
330
+ df = pd.read_csv(csv_path, sep=None, engine="python")
331
+ resolved_label = _resolve_label_column(df, label_column)
332
+
333
+ columns = _resolve_features(df, feature_columns, resolved_label)
334
+ features = df[columns].astype(np.float32).values
335
+ labels = df[resolved_label].astype(str).values
336
+ return features, labels, columns, resolved_label
337
+
338
+
339
+ def load_dataset_from_dataframe(
340
+ df: pd.DataFrame,
341
+ *,
342
+ feature_columns: Sequence[str] | None,
343
+ label_column: str,
344
+ ) -> Tuple[np.ndarray, np.ndarray, List[str], str]:
345
+ """Load dataset arrays directly from a DataFrame."""
346
 
347
+ resolved_label = _resolve_label_column(df, label_column)
348
+
349
+ columns = _resolve_features(df, feature_columns, resolved_label)
350
  features = df[columns].astype(np.float32).values
351
+ labels = df[resolved_label].astype(str).values
352
+ return features, labels, columns, resolved_label
353
 
354
 
355
  def create_sequences(
 
411
  return model
412
 
413
 
414
+ def build_tcn(
415
+ input_shape: Tuple[int, int],
416
+ num_classes: int,
417
+ *,
418
+ filters: int = 64,
419
+ kernel_size: int = 3,
420
+ dilations: Sequence[int] = (1, 2, 4, 8),
421
+ dropout: float = 0.2,
422
+ ) -> models.Model:
423
+ """Construct a lightweight Temporal Convolutional Network."""
424
+
425
+ inputs = layers.Input(shape=input_shape)
426
+ x = inputs
427
+ for dilation in dilations:
428
+ residual = x
429
+ x = layers.Conv1D(
430
+ filters,
431
+ kernel_size,
432
+ padding="causal",
433
+ activation="relu",
434
+ dilation_rate=dilation,
435
+ )(x)
436
+ x = layers.BatchNormalization()(x)
437
+ x = layers.Dropout(dropout)(x)
438
+ x = layers.Conv1D(
439
+ filters,
440
+ kernel_size,
441
+ padding="causal",
442
+ activation="relu",
443
+ dilation_rate=dilation,
444
+ )(x)
445
+ x = layers.BatchNormalization()(x)
446
+ if residual.shape[-1] != filters:
447
+ residual = layers.Conv1D(filters, 1, padding="same")(residual)
448
+ x = layers.Add()([x, residual])
449
+ x = layers.Activation("relu")(x)
450
+
451
+ x = layers.GlobalAveragePooling1D()(x)
452
+ x = layers.Dropout(dropout)(x)
453
+ outputs = layers.Dense(num_classes, activation="softmax")(x)
454
+
455
+ model = models.Model(inputs, outputs)
456
+ model.compile(
457
+ optimizer=optimizers.Adam(learning_rate=1e-3),
458
+ loss="sparse_categorical_crossentropy",
459
+ metrics=["accuracy"],
460
+ )
461
+ return model
462
+
463
+
464
  def train_model(
465
  sequences: np.ndarray,
466
  labels: np.ndarray,
 
468
  validation_split: float,
469
  batch_size: int,
470
  epochs: int,
471
+ model_type: str = "cnn_lstm",
472
+ tensorboard_log_dir: Optional[Path] = None,
473
+ status_file_path: Optional[Path] = None,
474
+ ) -> Tuple[object, LabelEncoder, Dict[str, object]]:
475
+ """Train a sequence model and return training history and validation outputs."""
 
 
 
476
 
477
+ model_type = model_type.lower().strip()
478
+ if model_type not in {"cnn_lstm", "tcn", "svm"}:
479
+ raise ValueError("model_type must be either 'cnn_lstm', 'tcn', or 'svm'")
480
 
481
+ # Handle status file for progress tracking
482
+ status_file = status_file_path if status_file_path else None
 
 
483
 
484
+ label_encoder = LabelEncoder()
485
+ y = label_encoder.fit_transform(labels)
 
 
 
 
 
 
 
486
 
487
+ if model_type == "svm":
488
+ features = sequences.reshape(sequences.shape[0], -1)
489
+ else:
490
+ features = sequences
491
+
492
+ tb_dir: Optional[str] = None
493
+ if model_type != "svm" and tensorboard_log_dir is not None:
494
+ tensorboard_log_dir.mkdir(parents=True, exist_ok=True)
495
+ tb_dir = str(tensorboard_log_dir.resolve())
496
+ else:
497
+ tensorboard_log_dir = None
498
+
499
+ # Check if we can use stratification (each class needs at least 2 samples)
500
+ unique_labels, label_counts = np.unique(y, return_counts=True)
501
+ min_samples_per_class = np.min(label_counts)
502
+
503
+ print(f"Label distribution: {dict(zip(unique_labels, label_counts))}")
504
+ print(f"Minimum samples per class: {min_samples_per_class}")
505
+ print(f"Total sequences: {len(sequences)}, Features per sequence: {sequences.shape[1:]}")
506
+
507
+ # Check for potential memory issues
508
+ import sys
509
+ data_size_mb = sequences.nbytes / (1024 * 1024)
510
+ print(f"Data size: {data_size_mb:.2f} MB")
511
+ if data_size_mb > 1000: # > 1GB
512
+ print("Warning: Large dataset detected. Consider reducing batch size or sequence length.")
513
+
514
+ # Validate data ranges
515
+ if np.any(np.isnan(sequences)) or np.any(np.isinf(sequences)):
516
+ print("Warning: NaN or Inf values detected in sequences")
517
+ sequences = np.nan_to_num(sequences, nan=0.0, posinf=1e6, neginf=-1e6)
518
+
519
+ # Use stratification only if each class has at least 2 samples
520
+ if min_samples_per_class >= 2:
521
+ X_train, X_val, y_train, y_val = train_test_split(
522
+ features, y, test_size=validation_split, stratify=y, random_state=42
523
+ )
524
+ else:
525
+ print(f"Warning: Some classes have only {min_samples_per_class} sample(s). Using simple random split instead of stratified split.")
526
+
527
+ # If validation split would result in empty validation set for some classes,
528
+ # reduce validation split or use a minimum number of samples
529
+ total_samples = len(y)
530
+ if validation_split * total_samples < len(unique_labels):
531
+ # Ensure at least one sample per class in validation if possible
532
+ adjusted_split = max(0.1, len(unique_labels) / total_samples)
533
+ adjusted_split = min(adjusted_split, 0.3) # Cap at 30%
534
+ print(f"Adjusting validation split from {validation_split} to {adjusted_split}")
535
+ validation_split = adjusted_split
536
+
537
+ X_train, X_val, y_train, y_val = train_test_split(
538
+ features, y, test_size=validation_split, random_state=42
539
+ )
540
+
541
+ if model_type == "cnn_lstm":
542
+ print("Building CNN-LSTM model...")
543
+
544
+ # Optimize model for large datasets
545
+ if len(sequences) > 100000:
546
+ print("Using lightweight CNN-LSTM for large dataset")
547
+ model = build_cnn_lstm(
548
+ input_shape=sequences.shape[1:],
549
+ num_classes=len(label_encoder.classes_),
550
+ conv_filters=64, # Reduce from 128
551
+ lstm_units=64, # Reduce from 128
552
+ dropout=0.2 # Reduce dropout
553
+ )
554
+ else:
555
+ model = build_cnn_lstm(
556
+ input_shape=sequences.shape[1:], num_classes=len(label_encoder.classes_)
557
+ )
558
+ print(f"CNN-LSTM model built. Input shape: {sequences.shape[1:]}, Classes: {len(label_encoder.classes_)}")
559
+ print(f"Model parameters: {model.count_params():,}")
560
+
561
+ # Adjust callbacks for dataset size
562
+ if len(sequences) > 100000:
563
+ callbacks_list = [
564
+ ProgressCallback(total_epochs=epochs, status_file_path=str(status_file) if status_file else None),
565
+ callbacks.ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=2, min_lr=1e-5),
566
+ callbacks.EarlyStopping(monitor="val_loss", patience=3, restore_best_weights=True), # More aggressive
567
+ ]
568
+ print("Using aggressive callbacks for large dataset")
569
+ else:
570
+ callbacks_list = [
571
+ ProgressCallback(total_epochs=epochs, status_file_path=str(status_file) if status_file else None),
572
+ callbacks.ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=3, min_lr=1e-5),
573
+ callbacks.EarlyStopping(monitor="val_loss", patience=6, restore_best_weights=True),
574
+ ]
575
+ if tensorboard_log_dir is not None:
576
+ callbacks_list.insert(-2, callbacks.TensorBoard(log_dir=tb_dir, histogram_freq=0, write_graph=False)) # Reduce TensorBoard overhead
577
+
578
+ print(f"Starting CNN-LSTM training with {len(X_train)} training samples, {len(X_val)} validation samples")
579
+ print(f"Batch size: {batch_size}, Epochs: {epochs}")
580
+
581
+ if status_file:
582
+ with open(status_file, 'w') as f:
583
+ f.write(f"CNN-LSTM training started - {len(X_train)} train, {len(X_val)} val samples, batch_size={batch_size}")
584
+
585
+ history = model.fit(
586
+ X_train,
587
+ y_train,
588
+ validation_data=(X_val, y_val),
589
+ epochs=epochs,
590
+ batch_size=batch_size,
591
+ callbacks=callbacks_list,
592
+ verbose=2,
593
+ )
594
+
595
+ print("CNN-LSTM training completed, starting prediction...")
596
+ if status_file:
597
+ with open(status_file, 'w') as f:
598
+ f.write("CNN-LSTM training completed, evaluating model...")
599
+
600
+ print(f"Making predictions on {len(X_val)} validation samples...")
601
+ if status_file:
602
+ with open(status_file, 'w') as f:
603
+ f.write(f"Making predictions on {len(X_val)} validation samples...")
604
+ y_pred = model.predict(X_val, verbose=0).argmax(axis=1)
605
+ print("Predictions completed")
606
+ training_history: Dict[str, object] = history.history
607
+ elif model_type == "tcn":
608
+ print("Building TCN model...")
609
+ model = build_tcn(input_shape=sequences.shape[1:], num_classes=len(label_encoder.classes_))
610
+ print(f"TCN model built. Input shape: {sequences.shape[1:]}, Classes: {len(label_encoder.classes_)}")
611
+
612
+ callbacks_list = [
613
+ ProgressCallback(total_epochs=epochs, status_file_path=str(status_file) if status_file else None),
614
+ callbacks.ReduceLROnPlateau(monitor="val_loss", factor=0.5, patience=3, min_lr=1e-5),
615
+ callbacks.EarlyStopping(monitor="val_loss", patience=6, restore_best_weights=True),
616
+ ]
617
+ if tensorboard_log_dir is not None:
618
+ callbacks_list.insert(-2, callbacks.TensorBoard(log_dir=tb_dir, histogram_freq=0, write_graph=False)) # Reduce TensorBoard overhead
619
+
620
+ print(f"Starting TCN training with {len(X_train)} training samples, {len(X_val)} validation samples")
621
+ print(f"Batch size: {batch_size}, Epochs: {epochs}")
622
+
623
+ if status_file:
624
+ with open(status_file, 'w') as f:
625
+ f.write(f"TCN training started - {len(X_train)} train, {len(X_val)} val samples, batch_size={batch_size}")
626
+
627
+ history = model.fit(
628
+ X_train,
629
+ y_train,
630
+ validation_data=(X_val, y_val),
631
+ epochs=epochs,
632
+ batch_size=batch_size,
633
+ callbacks=callbacks_list,
634
+ verbose=2,
635
+ )
636
+
637
+ print("TCN training completed, starting prediction...")
638
+ if status_file:
639
+ with open(status_file, 'w') as f:
640
+ f.write("TCN training completed, evaluating model...")
641
+
642
+ print(f"Making TCN predictions on {len(X_val)} validation samples...")
643
+ if status_file:
644
+ with open(status_file, 'w') as f:
645
+ f.write(f"Making TCN predictions on {len(X_val)} validation samples...")
646
+ y_pred = model.predict(X_val, verbose=0).argmax(axis=1)
647
+ print("TCN predictions completed")
648
+ training_history = history.history
649
+ else: # svm
650
+ print("Training SVM model...", flush=True)
651
+ if status_file:
652
+ with open(status_file, 'w') as f:
653
+ f.write("Training SVM model...")
654
+
655
+ model = SVC(kernel="rbf", probability=True, class_weight="balanced")
656
+ model.fit(X_train, y_train)
657
+
658
+ print("SVM training completed. Evaluating...", flush=True)
659
+ if status_file:
660
+ with open(status_file, 'w') as f:
661
+ f.write("SVM training completed. Evaluating...")
662
+
663
+ y_pred = model.predict(X_val)
664
+ training_history = {
665
+ "train_accuracy": float(model.score(X_train, y_train)),
666
+ "val_accuracy": float(accuracy_score(y_val, y_pred)),
667
+ }
668
+
669
+ cm = confusion_matrix(y_val, y_pred)
670
+ metrics: Dict[str, object] = {
671
+ "history": training_history,
672
  "validation": {
673
  "y_true": y_val,
674
  "y_pred": y_pred,
675
  "class_names": label_encoder.classes_.tolist(),
676
+ "confusion_matrix": cm,
677
  },
678
+ "model_type": model_type,
679
+ "input_shape": list(sequences.shape[1:]),
680
+ "tensorboard_log_dir": tb_dir,
681
  }
682
  return model, label_encoder, metrics
683
 
 
692
 
693
  def export_artifacts(
694
  *,
695
+ model: object,
696
  scaler: StandardScaler,
697
  label_encoder: LabelEncoder,
698
  feature_columns: Sequence[str],
 
708
  model_path.parent.mkdir(parents=True, exist_ok=True)
709
  scaler_path.parent.mkdir(parents=True, exist_ok=True)
710
  metadata_path.parent.mkdir(parents=True, exist_ok=True)
711
+ model_type = str(metrics.get("model_type", "cnn_lstm"))
712
+ if model_type == "svm":
713
+ joblib.dump(model, model_path)
714
+ else:
715
+ model.save(model_path)
716
  joblib.dump(scaler, scaler_path)
717
 
718
+ validation = metrics["validation"]
719
+ report_dict = classification_report(
720
+ validation["y_true"],
721
+ validation["y_pred"],
722
+ target_names=label_encoder.classes_,
723
+ output_dict=True,
724
+ )
725
+
726
  metadata = {
727
  "feature_columns": list(feature_columns),
728
  "label_classes": label_encoder.classes_.tolist(),
 
732
  "model_path": str(model_path),
733
  "scaler_path": str(scaler_path),
734
  "training_history": metrics["history"],
735
+ "classification_report": report_dict,
736
+ "model_type": model_type,
737
+ "model_format": "joblib" if model_type == "svm" else "keras",
738
+ "input_shape": metrics.get("input_shape"),
739
+ "tensorboard_log_dir": metrics.get("tensorboard_log_dir"),
 
740
  }
741
+ confusion = validation.get("confusion_matrix")
742
+ if confusion is None:
743
+ confusion = confusion_matrix(validation["y_true"], validation["y_pred"])
744
+ metadata["confusion_matrix"] = np.asarray(confusion).tolist()
745
 
746
  metadata_path.write_text(json.dumps(metadata, indent=2))
747
 
748
 
749
+ def train_from_dataframe(
750
+ df: pd.DataFrame,
751
+ *,
752
+ label_column: str,
753
+ feature_columns: Sequence[str] | None = None,
754
+ sequence_length: int = 32,
755
+ stride: int = 4,
756
+ validation_split: float = 0.2,
757
+ batch_size: int = 128,
758
+ epochs: int = 50,
759
+ model_type: str = "cnn_lstm",
760
+ model_path: Path | str = "pmu_cnn_lstm_model.keras",
761
+ scaler_path: Path | str = "pmu_feature_scaler.pkl",
762
+ metadata_path: Path | str = "pmu_metadata.json",
763
+ enable_tensorboard: bool = True,
764
+ tensorboard_root: Path | str | None = None,
765
+ ) -> dict:
766
+ """Train a PMU fault classification model using an in-memory dataframe."""
767
+
768
+ model_path = Path(model_path)
769
+ scaler_path = Path(scaler_path)
770
+ metadata_path = Path(metadata_path)
771
+
772
+ # Create status file for progress tracking
773
+ status_file = model_path.parent / "training_status.txt"
774
+ print(f"Training progress will be written to: {status_file}")
775
+
776
+ tensorboard_log_dir: Optional[Path] = None
777
+ if enable_tensorboard and model_type.lower() != "svm":
778
+ base_dir = Path(tensorboard_root) if tensorboard_root is not None else Path("tensorboard_runs")
779
+ timestamp = datetime.utcnow().strftime("%Y%m%d-%H%M%S")
780
+ tensorboard_log_dir = base_dir / f"run-{timestamp}"
781
+
782
+ features, labels, used_columns, resolved_label = load_dataset_from_dataframe(
783
+ df, feature_columns=feature_columns, label_column=label_column
784
+ )
785
+
786
+ print(f"Input data: {len(features)} samples")
787
+ print(f"Creating sequences with length={sequence_length}, stride={stride}")
788
+
789
+ sequences, seq_labels = create_sequences(
790
+ features,
791
+ labels,
792
+ sequence_length=sequence_length,
793
+ stride=stride,
794
+ )
795
+
796
+ print(f"Generated {len(sequences)} sequences")
797
+
798
+ # Validate sequence count and adjust parameters if necessary
799
+ if len(sequences) < 10:
800
+ raise ValueError(
801
+ f"Only {len(sequences)} sequences generated. Need at least 10 for training. "
802
+ f"Try reducing sequence_length (currently {sequence_length}) or stride (currently {stride}), "
803
+ "or provide more data."
804
+ )
805
+
806
+ # If very few sequences, recommend SVM instead of deep learning
807
+ if len(sequences) < 100 and model_type in ['cnn_lstm', 'tcn']:
808
+ print(f"Warning: Only {len(sequences)} sequences available. Consider using SVM for small datasets.")
809
+
810
+ sequences, scaler = standardise_sequences(sequences)
811
+
812
+ # Adjust training parameters based on data size
813
+ original_batch_size = batch_size
814
+ original_epochs = epochs
815
+ original_validation_split = validation_split
816
+
817
+ # Handle large datasets (>100K sequences) - optimize for memory and speed
818
+ if len(sequences) > 100000:
819
+ print(f"Large dataset detected ({len(sequences)} sequences). Optimizing parameters...")
820
+ batch_size = min(batch_size * 2, 512) # Increase batch size for efficiency
821
+ epochs = min(epochs, 30) # Reduce epochs for large datasets
822
+ print(f"Adjusted parameters for large dataset:")
823
+ print(f" Batch size: {original_batch_size} -> {batch_size}")
824
+ print(f" Epochs: {original_epochs} -> {epochs}")
825
+
826
+ # Force garbage collection
827
+ import gc
828
+ gc.collect()
829
+
830
+ elif len(sequences) < 100:
831
+ # For very small datasets
832
+ batch_size = max(min(batch_size, len(sequences) // 4), 4) # Ensure batch_size >= 4
833
+ epochs = min(epochs, 20) # Reduce epochs to prevent overfitting
834
+ validation_split = min(validation_split, 0.3) # Reduce validation split
835
+ print(f"Adjusted parameters for small dataset:")
836
+ print(f" Batch size: {original_batch_size} -> {batch_size}")
837
+ print(f" Epochs: {original_epochs} -> {epochs}")
838
+ print(f" Validation split: {original_validation_split} -> {validation_split}")
839
+
840
+ model, label_encoder, metrics = train_model(
841
+ sequences,
842
+ seq_labels,
843
+ validation_split=validation_split,
844
+ batch_size=batch_size,
845
+ epochs=epochs,
846
+ model_type=model_type,
847
+ tensorboard_log_dir=tensorboard_log_dir,
848
+ status_file_path=status_file,
849
+ )
850
+
851
+ export_artifacts(
852
+ model=model,
853
+ scaler=scaler,
854
+ label_encoder=label_encoder,
855
+ feature_columns=used_columns,
856
+ label_column=resolved_label,
857
+ sequence_length=sequence_length,
858
+ stride=stride,
859
+ model_path=model_path,
860
+ scaler_path=scaler_path,
861
+ metadata_path=metadata_path,
862
+ metrics=metrics,
863
+ )
864
+
865
+ tensorboard_zip_path: Optional[str] = None
866
+ if tensorboard_log_dir and tensorboard_log_dir.exists():
867
+ try:
868
+ tensorboard_zip_path = shutil.make_archive(
869
+ base_name=str(tensorboard_log_dir.parent / tensorboard_log_dir.name),
870
+ format="zip",
871
+ root_dir=str(tensorboard_log_dir.parent),
872
+ base_dir=tensorboard_log_dir.name,
873
+ )
874
+ tensorboard_zip_path = str(Path(tensorboard_zip_path).resolve())
875
+ except Exception:
876
+ tensorboard_zip_path = None
877
+
878
+ report_dict = classification_report(
879
+ metrics["validation"]["y_true"],
880
+ metrics["validation"]["y_pred"],
881
+ target_names=metrics["validation"]["class_names"],
882
+ output_dict=True,
883
+ )
884
+ confusion = metrics["validation"].get("confusion_matrix")
885
+ if confusion is None:
886
+ confusion = confusion_matrix(metrics["validation"]["y_true"], metrics["validation"]["y_pred"])
887
+
888
+ return {
889
+ "num_samples": int(df.shape[0]),
890
+ "num_sequences": int(sequences.shape[0]),
891
+ "feature_columns": used_columns,
892
+ "class_names": label_encoder.classes_.tolist(),
893
+ "model_path": str(model_path.resolve()),
894
+ "scaler_path": str(scaler_path.resolve()),
895
+ "metadata_path": str(metadata_path.resolve()),
896
+ "history": metrics["history"],
897
+ "model_type": metrics.get("model_type", model_type),
898
+ "classification_report": report_dict,
899
+ "confusion_matrix": np.asarray(confusion).tolist(),
900
+ "tensorboard_log_dir": metrics.get("tensorboard_log_dir"),
901
+ "tensorboard_zip_path": tensorboard_zip_path,
902
+ "label_column": resolved_label,
903
+ }
904
+
905
+
906
  def run_training(args: argparse.Namespace) -> None:
907
  csv_path = Path(args.data_path)
908
  model_out = Path(args.model_out)
909
  scaler_out = Path(args.scaler_out)
910
  metadata_out = Path(args.metadata_out)
911
 
912
+ features, labels, feature_columns, resolved_label = load_dataset(
913
  csv_path, feature_columns=args.feature_columns, label_column=args.label_column
914
  )
915
 
 
921
  )
922
 
923
  sequences, scaler = standardise_sequences(sequences)
924
+ tensorboard_log_dir: Optional[Path] = None
925
+ if args.tensorboard and args.model_type != "svm":
926
+ if args.tensorboard_log_dir:
927
+ tensorboard_log_dir = Path(args.tensorboard_log_dir)
928
+ else:
929
+ tensorboard_log_dir = Path("tensorboard_runs") / datetime.utcnow().strftime("%Y%m%d-%H%M%S")
930
  model, label_encoder, metrics = train_model(
931
  sequences,
932
  seq_labels,
933
  validation_split=args.validation_split,
934
  batch_size=args.batch_size,
935
  epochs=args.epochs,
936
+ model_type=args.model_type,
937
+ tensorboard_log_dir=tensorboard_log_dir,
938
+ status_file_path=None, # No status file for CLI usage
939
  )
940
 
941
  export_artifacts(
 
943
  scaler=scaler,
944
  label_encoder=label_encoder,
945
  feature_columns=feature_columns,
946
+ label_column=resolved_label,
947
  sequence_length=args.sequence_length,
948
  stride=args.stride,
949
  model_path=model_out,
 
953
  )
954
 
955
  print("Training complete")
956
+ print(f"Model architecture : {args.model_type}")
957
  print(f"Model saved to : {model_out}")
958
  print(f"Scaler saved to : {scaler_out}")
959
  print(f"Metadata saved to : {metadata_out}")
 
962
  metrics["validation"]["y_true"], metrics["validation"]["y_pred"], target_names=metrics["validation"]["class_names"]
963
  )
964
  print(report)
965
+ if metrics.get("tensorboard_log_dir"):
966
+ tb_dir = metrics["tensorboard_log_dir"]
967
+ print(f"TensorBoard logs written to: {tb_dir}")
968
+ print(f"Launch TensorBoard with: tensorboard --logdir \"{tb_dir}\"")
969
 
970
 
971
  def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
972
+ parser = argparse.ArgumentParser(description="Train a sequence model for PMU fault classification")
973
  parser.add_argument("--data-path", required=True, help="Path to Fault_Classification_PMU_Data CSV")
974
  parser.add_argument(
975
  "--label-column",
 
987
  parser.add_argument("--validation-split", type=float, default=0.2, help="Validation set fraction")
988
  parser.add_argument("--batch-size", type=int, default=128, help="Training batch size")
989
  parser.add_argument("--epochs", type=int, default=50, help="Maximum number of training epochs")
990
+ parser.add_argument(
991
+ "--model-type",
992
+ choices=["cnn_lstm", "tcn", "svm"],
993
+ default="cnn_lstm",
994
+ help="Model architecture to train (choices: cnn_lstm, tcn, svm)",
995
+ )
996
  parser.add_argument("--model-out", default="pmu_cnn_lstm_model.keras", help="Path to save trained Keras model")
997
  parser.add_argument("--scaler-out", default="pmu_feature_scaler.pkl", help="Path to save fitted StandardScaler")
998
  parser.add_argument("--metadata-out", default="pmu_metadata.json", help="Path to save metadata JSON")
999
+ parser.add_argument(
1000
+ "--tensorboard-log-dir",
1001
+ default=None,
1002
+ help="Optional directory to write TensorBoard logs (defaults to tensorboard_runs/<timestamp>)",
1003
+ )
1004
+ parser.add_argument(
1005
+ "--no-tensorboard",
1006
+ dest="tensorboard",
1007
+ action="store_false",
1008
+ help="Disable TensorBoard logging for neural network models",
1009
+ )
1010
+ parser.set_defaults(tensorboard=True)
1011
  return parser.parse_args(argv)
1012
 
1013
 
lstm_cnn_gradio_notebook.ipynb CHANGED
File without changes
requirements.txt CHANGED
@@ -1,4 +1,4 @@
1
- gradio>=3.0
2
  tensorflow>=2.6
3
  numpy
4
  pandas
 
1
+ gradio>=4.44,<5
2
  tensorflow>=2.6
3
  numpy
4
  pandas
tcn_app.py CHANGED
File without changes
tcn_gradio_notebook.ipynb CHANGED
File without changes