JoaquinVanschoren commited on
Commit
b218e8e
·
1 Parent(s): a5b79af

fix for repeated restarts on HF

Browse files
Files changed (1) hide show
  1. validation.py +7 -6
validation.py CHANGED
@@ -32,12 +32,12 @@ def validate_croissant(json_data):
32
  error_message = f"Unexpected error during validation: {str(e)}\n\n{error_details}"
33
  return False, error_message
34
 
35
- def try_generate_record(record_iterator):
36
  try:
37
- next(record_iterator)
 
38
  return "success"
39
  except Exception as e:
40
- # Return the exception object to the outer function
41
  return e
42
 
43
  def validate_records(json_data):
@@ -53,9 +53,10 @@ def validate_records(json_data):
53
 
54
  for record_set in record_sets:
55
  try:
56
- records = dataset.records(record_set=record_set.uuid)
57
-
58
- result = func_timeout.func_timeout(WAIT_TIME, try_generate_record, args=(iter(records),))
 
59
 
60
  if isinstance(result, Exception):
61
  raise result # re-raise actual error outside timeout
 
32
  error_message = f"Unexpected error during validation: {str(e)}\n\n{error_details}"
33
  return False, error_message
34
 
35
+ def try_generate_record(record_collection):
36
  try:
37
+ iterator = iter(record_collection)
38
+ next(iterator)
39
  return "success"
40
  except Exception as e:
 
41
  return e
42
 
43
  def validate_records(json_data):
 
53
 
54
  for record_set in record_sets:
55
  try:
56
+ result = func_timeout.func_timeout(
57
+ WAIT_TIME,
58
+ lambda: try_generate_record(dataset.records(record_set=record_set.uuid))
59
+ )
60
 
61
  if isinstance(result, Exception):
62
  raise result # re-raise actual error outside timeout