|
|
|
|
|
|
|
|
preprocess_code = """ |
|
|
def preprocess(text): |
|
|
import re |
|
|
import string |
|
|
import spacy |
|
|
|
|
|
try: |
|
|
|
|
|
# Checking if it the string |
|
|
text = str(text) |
|
|
|
|
|
# remove html |
|
|
text = re.sub(r"<.*?>","", text) |
|
|
|
|
|
# Remove URL |
|
|
url_pattern = r"https?://\\S+|www\\.\\S+|\\S+\\.\\S{2," |
|
|
text = re.sub(url_pattern,"", text) |
|
|
|
|
|
# Remove Punctuation |
|
|
translator = str.maketrans("","", string.punctuation) |
|
|
text.translate(translator) |
|
|
|
|
|
# Lower case |
|
|
text.lower().strip() |
|
|
|
|
|
# Remove Unicodes - only applicable for english language. Because other language letters represented as unicodes. |
|
|
unicode_pattern = str.maketrans("","","\\xa0") |
|
|
text.translate(unicode_pattern) |
|
|
|
|
|
# Remove Escape sequences (\\n, \\t, \\r) |
|
|
text = re.sub(r"\\[nt\\r]"," ",text) |
|
|
|
|
|
# Remove Stop words using spacy |
|
|
|
|
|
spacy.prefer_gpu() # using GPU if available. may reduce the run time. |
|
|
nlp = spacy.load("en_core_web_sm") |
|
|
doc = nlp(text) |
|
|
text = " ".join([token.text for token in doc if not token.is_stop]) |
|
|
|
|
|
# Remove irrelevant white spaces |
|
|
text = re.sub(r"\\s+"," ",text) |
|
|
except: |
|
|
print(f"error occured") |
|
|
|
|
|
return text |
|
|
""" |
|
|
|
|
|
|
|
|
postprocess_code = """ |
|
|
def post_process(output): |
|
|
import torch |
|
|
classes = ['ACCOUNTANT', 'ADVOCATE', 'AGRICULTURE', 'APPAREL', 'ARTS', 'AUTOMOBILE', 'AVIATION', 'BANKING', 'BPO', 'BUSINESS-DEVELOPMENT', 'CHEF', 'CONSTRUCTION', 'CONSULTANT', 'DESIGNER', 'DIGITAL-MEDIA', 'ENGINEERING', 'FINANCE', 'FITNESS', 'HEALTHCARE', 'HR', 'INFORMATION-TECHNOLOGY', 'PUBLIC-RELATIONS', 'SALES', 'TEACHER'] |
|
|
try: |
|
|
logits = output.logits |
|
|
sigmoid = torch.nn.Sigmoid() |
|
|
probs = sigmoid(logits.squeeze().cpu()) |
|
|
temp = probs.sort() |
|
|
return classes[temp[-1][-1].item()] |
|
|
except: |
|
|
print("Some Error occured") |
|
|
""" |
|
|
|
|
|
|
|
|
|
|
|
from transformers import PretrainedConfig, AutoModel |
|
|
|
|
|
class CustomConfig(PretrainedConfig): |
|
|
|
|
|
def __init__(self, preprocess_function = None, postprocess_function = None, **kwargs): |
|
|
super().__init__(**kwargs) |
|
|
self.preprocess_function = preprocess_function |
|
|
self.postprocess_function = postprocess_function |
|
|
|
|
|
|
|
|
config = CustomConfig(preprocess_function= preprocess_code, postprocess_function=postprocess_code) |
|
|
|
|
|
|
|
|
|
|
|
config.save_pretrained("config with functions") |
