|
""" |
|
A function to parse an lm_eval text outputs into json format |
|
""" |
|
import os |
|
import json |
|
|
|
|
|
def txt2json(file): |
|
"""Convert lm_eval text file to json format""" |
|
with open(file) as fh: |
|
lang = file.split('_')[-1].split('.txt')[0] |
|
data = fh.read().split('hf (')[1:] |
|
print(len(data)) |
|
for evaluation in data: |
|
metadata = {} |
|
results = {} |
|
lines = evaluation.split('\n') |
|
header, batch_size = lines[0].split('batch_size: ') |
|
metadata['batch_size'] = batch_size |
|
header = header.split(',') |
|
for entry in header: |
|
entry = entry.strip() |
|
if not entry: continue |
|
if '=' in entry: |
|
key, value = entry.split('=') |
|
elif 'batch_size: ' in entry: |
|
key = 'batch_size' |
|
value = entry.split('batch_size: ')[1] |
|
print(key, value) |
|
else: |
|
key, value = entry.split(': ') |
|
if value.endswith(')'): |
|
value = value[:-1] |
|
if value.startswith('('): |
|
value = value[1:] |
|
if not key == 'dtype': |
|
try: |
|
value = eval(value) |
|
except Exception as e: |
|
pass |
|
if key == 'pretrained': |
|
value = value.split('/')[-1] |
|
pretrained = value |
|
metadata[key] = value |
|
print(metadata) |
|
task = '' |
|
alias = '' |
|
for line in lines[1:]: |
|
if line.startswith('|'): |
|
columns = line.split('|') |
|
_, tasks, version, filter, nshot, metric, _1, value, _2, stderr, _3 = columns |
|
tasks = columns[1].strip() |
|
if tasks == 'Tasks': continue |
|
if '--' in tasks: continue |
|
if tasks == 'Groups': continue |
|
aliases = tasks |
|
tasks = tasks.split('- ')[-1] |
|
if tasks: |
|
task = tasks |
|
alias = aliases |
|
results[task] = {} |
|
|
|
|
|
|
|
if version.strip(): |
|
try: |
|
results[task]['version'] = float(version.strip()) |
|
except Exception as e: |
|
print(e) |
|
if nshot.strip(): |
|
results[task]['nshot'] = int(nshot.strip()) |
|
|
|
metric = metric.strip() |
|
value = value.strip() |
|
filter = filter.strip() |
|
|
|
results[task]['alias'] = alias |
|
results[task][f'{metric},{filter}'] = float(value.strip()) |
|
results[task]['stderr'] = float(stderr.strip()) |
|
|
|
output = {'config': metadata, "results": results} |
|
print(output) |
|
pretrained = pretrained[0].upper()+pretrained[1:-1]+ pretrained[-1].upper() |
|
with open(f'{pretrained}_{lang}.json', 'w') as f: |
|
json.dump(output, f, ensure_ascii=False, indent=2) |
