Spaces:
Runtime error
Runtime error
File size: 1,008 Bytes
65e9efa b8d16b2 65e9efa b8d16b2 65e9efa b8d16b2 65e9efa b8d16b2 65e9efa b8d16b2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
import numpy as np
import pandas as pd
import yaml
def load_config(config_file: str) -> dict:
with open(config_file) as f:
config = yaml.safe_load(f)
return config
INPUT_FILE = "data/raw/Wellcome-grants-awarded-1-October-2005-to-04-05-2022.csv"
OUTPUT_FILE = "data/processed/wellcome_grant_descriptions.csv"
def subset_docs(input_file: str, output_file: str, sample: int):
print(f"Reading data from {input_file}")
data = pd.read_csv(input_file)
data = (
data[["Description"]]
.replace("Not available", np.nan)
.dropna()
.drop_duplicates()
.reset_index(drop=True)
.sample(sample)
)
print(f"Number of rows: {data.shape[0]}")
print(f"Number of unique rows: {data['Description'].nunique()}")
print(f"Saving file to {output_file}")
data.to_csv(output_file, index=False)
if __name__ == "__main__":
params = load_config("params.yaml")
subset_docs(INPUT_FILE, OUTPUT_FILE, sample=params["n_docs"])
|