rachith commited on
Commit
76e5451
·
1 Parent(s): e7d3e05

adding two models to check speed

Browse files
Files changed (1) hide show
  1. app.py +21 -15
app.py CHANGED
@@ -3,28 +3,34 @@ from transformers import AutoModel, AutoTokenizer
3
  from sklearn.neighbors import NearestNeighbors
4
 
5
 
 
 
6
 
7
- MODEL = "cardiffnlp/twitter-roberta-base-jun2022"
8
- model = AutoModel.from_pretrained(MODEL)
9
- tokenizer = AutoTokenizer.from_pretrained(MODEL)
10
- embedding_matrix = model.embeddings.word_embeddings.weight
11
- embedding_matrix = embedding_matrix.detach().numpy()
 
12
 
13
- knn_model = NearestNeighbors(n_neighbors=500,
14
- metric='cosine',
15
- algorithm='auto',
16
- n_jobs=3)
17
-
18
- nbrs = knn_model.fit(embedding_matrix)
19
 
20
- distances, indices = nbrs.kneighbors(embedding_matrix)
21
 
 
22
 
23
- title = "How does a word's meaning change with time?"
24
 
 
25
 
26
- def topk(word):
27
  outs = []
 
 
28
  index = tokenizer.encode(f'{word}')
29
  for i in indices[index[1]]:
30
  outs.append(tokenizer.decode(i))
@@ -42,6 +48,6 @@ with gr.Blocks() as demo:
42
  with gr.Row():
43
  greet_btn = gr.Button("Compute")
44
  with gr.Row():
45
- greet_btn.click(fn=topk, inputs=[word], outputs=gr.outputs.Textbox())
46
 
47
  demo.launch()
 
3
  from sklearn.neighbors import NearestNeighbors
4
 
5
 
6
+ models = ['cardiffnlp/twitter-roberta-base-jun2022',
7
+ 'cardiffnlp/twitter-roberta-base-2019-90m']
8
 
9
+ def topk_model(MODEL):
10
+ # MODEL = "cardiffnlp/twitter-roberta-base-jun2022"
11
+ model = AutoModel.from_pretrained(MODEL)
12
+ tokenizer = AutoTokenizer.from_pretrained(MODEL)
13
+ embedding_matrix = model.embeddings.word_embeddings.weight
14
+ embedding_matrix = embedding_matrix.detach().numpy()
15
 
16
+ knn_model = NearestNeighbors(n_neighbors=500,
17
+ metric='cosine',
18
+ algorithm='auto',
19
+ n_jobs=3)
20
+
21
+ nbrs = knn_model.fit(embedding_matrix)
22
 
23
+ distances, indices = nbrs.kneighbors(embedding_matrix)
24
 
25
+ return distances,indices,tokenizer
26
 
 
27
 
28
+ title = "How does a word's meaning change with time?"
29
 
30
+ def topk(word,model):
31
  outs = []
32
+ distances, indices, tokenizer = topk_model(model)
33
+
34
  index = tokenizer.encode(f'{word}')
35
  for i in indices[index[1]]:
36
  outs.append(tokenizer.decode(i))
 
48
  with gr.Row():
49
  greet_btn = gr.Button("Compute")
50
  with gr.Row():
51
+ greet_btn.click(fn=topk, inputs=[word,gr.Dropdown(models)], outputs=gr.outputs.Textbox())
52
 
53
  demo.launch()