Upload 28 files

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/abstracts.csv filter=lfs diff=lfs merge=lfs -text
+data/authors.csv filter=lfs diff=lfs merge=lfs -text
+data/clustering.csv filter=lfs diff=lfs merge=lfs -text
+data/embeddings.csv filter=lfs diff=lfs merge=lfs -text
+data/geospatial_clustering_data.csv filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,273 @@
+import os
+import numpy as np
+import pandas as pd
+import streamlit as st
+import plotly.express as px
+from pyspark.sql import functions as F
+from pyspark.sql import SparkSession
+from sentence_transformers import SentenceTransformer
+from sklearn.metrics.pairwise import cosine_similarity
+
+from spark_setup import create_spark_session, load_data, file_paths
+
+############################
+# Caching and Setup
+############################
+
+@st.cache_data(show_spinner=False)
+def get_model(model_path="embedding_model"):
+    if not os.path.exists(model_path):
+        model = SentenceTransformer('all-mpnet-base-v2')
+        model.save(model_path)
+    return SentenceTransformer(model_path)
+
+@st.cache_data(show_spinner=False)
+def get_embeddings(embedding_path="embeddings.npy"):
+    return np.load(embedding_path)
+
+@st.cache_resource(show_spinner=False)
+def get_spark_session():
+    return create_spark_session()
+
+@st.cache_resource(show_spinner=False)
+def get_data(_spark):
+    return load_data(_spark, file_paths)
+
+@st.cache_data(show_spinner=False)
+def get_index_to_pub_id(_publications_spark_df):
+    pub_ids = _publications_spark_df.select("publication_id").rdd.map(lambda x: x.publication_id).collect()
+    return {idx: pub_id for idx, pub_id in enumerate(pub_ids)}
+
+############################
+# Main Code
+############################
+
+embeddings = get_embeddings()
+model = get_model()
+
+spark = get_spark_session()
+dataframes = get_data(spark)
+
+data = dataframes["geospatial_clustering_data"]
+publications_df = dataframes["clustering"]
+
+# Create the mapping from embedding index to publication_id
+index_to_pub_id = get_index_to_pub_id(publications_df)
+
+# Rename clusters as fields of study
+field_topics = {
+    0: "Phylogenetics and Species Diversity",
+    1: "Advanced Materials and Nanotechnology",
+    2: "Bioactive Compounds and Antioxidant Studies",
+    3: "Catalysis and Energy Conversion",
+    4: "Machine Learning and Image Processing",
+    5: "Clinical and Epidemiological Studies",
+    6: "Social and Behavioral Research",
+    7: "Environmental Risk and Water Management",
+    8: "Microbiology and Antibiotic Resistance",
+    9: "Systems Engineering and Optimization",
+    10: "Virology and Infectious Diseases",
+    11: "Oral and Dental Research",
+    12: "Surgery and Clinical Outcomes",
+    13: "Composite Materials and Structural Engineering",
+    14: "Cancer Research and Cellular Mechanisms",
+    15: "Particle Physics and Cosmology",
+    16: "Psychiatry and Cognitive Disorders"
+}
+
+# Page configuration
+st.set_page_config(
+    page_title="🌏 Chulalongkorn University Global Collaboration Explorer",
+    layout="wide",
+    page_icon="🌏"
+)
+
+# Initialize variables
+field_id, field_name = -1, "All Fields"
+keyword = None
+
+# Sidebar
+with st.sidebar:
+    st.title("🌟 Global Collaboration Explorer")
+    st.markdown("""
+        **Explore Chulalongkorn University's global academic collaborations**  
+        Use the options below to choose a field of study or explore by keyword.
+    """)
+
+    # Add a search mode radio button
+    search_mode = st.radio(
+        "Exploration Mode:",
+        options=["Explore by Field of Study", "Explore by Keyword"],
+        index=0
+    )
+
+    if search_mode == "Explore by Field of Study":
+        st.markdown("#### 🎓 Select a Field of Study")
+        if "selected_field" not in st.session_state:
+            st.session_state.selected_field = -1
+
+        search_query = st.selectbox(
+            "Field of Study:",
+            options=[(-1, "All Fields")] + list(field_topics.items()),
+            format_func=lambda x: "All Fields" if x[0] == -1 else f"Field {x[0]}: {x[1]}",
+            index=st.session_state.selected_field + 1
+        )
+        field_id, field_name = search_query
+        st.session_state.selected_field = field_id
+
+        # Filter data based on selected field
+        if field_id == -1:
+            filtered_map_data_spark = data
+        else:
+            filtered_map_data_spark = data.filter(F.col("cluster") == field_id)
+
+    elif search_mode == "Explore by Keyword":
+        st.markdown("#### 🔍 Enter a Keyword")
+        keyword = st.text_input("Keyword:")
+        if keyword:
+            input_embedding = model.encode(keyword)
+            cos_similarities = cosine_similarity([input_embedding], embeddings)[0]
+
+            # Create similarity DataFrame
+            similarity_df = pd.DataFrame({
+                "publication_id": [index_to_pub_id[i] for i in range(len(embeddings))],
+                "similarity": cos_similarities
+            })
+
+            # Threshold filtering
+            similarity_threshold = 0.38
+            similarity_df = similarity_df[similarity_df["similarity"] >= similarity_threshold]
+
+            if similarity_df.empty:
+                filtered_map_data_spark = data.limit(0)
+            else:
+                # Convert to Spark DF and join all matched publications
+                similarity_spark_df = spark.createDataFrame(similarity_df)
+                joined_df = data.join(similarity_spark_df, on="publication_id", how="inner")
+
+                # Sort by similarity descending
+                filtered_map_data_spark = joined_df.orderBy(F.col("similarity").desc())
+        else:
+            filtered_map_data_spark = data.limit(0)
+
+    # Function to get unique affiliation count as points
+    def get_country_points(_filtered_spark_df):
+        return (
+            _filtered_spark_df.groupBy("country")
+            .agg(F.countDistinct("affiliation_id").alias("points"))
+            .orderBy(F.col("points").desc())
+        )
+
+    country_points_spark = get_country_points(filtered_map_data_spark)
+    filtered_map_data_pd = country_points_spark.toPandas()
+
+    def get_dynamic_country_options(pdf):
+        return [("All Countries", 0)] + [(row["country"], row["points"]) for _, row in pdf.iterrows()]
+
+    if "selected_country" not in st.session_state:
+        st.session_state.selected_country = "All Countries"
+
+    country_options = get_dynamic_country_options(filtered_map_data_pd)
+
+    selected_country = st.selectbox(
+        "Select a Country:",
+        options=country_options,
+        format_func=lambda x: f"{x[0]} ({x[1]} unique affiliations)" if x[0] != "All Countries" else "All Countries",
+        index=0
+    )
+
+    selected_country_name = selected_country[0]
+    st.session_state.selected_country = selected_country_name
+
+    # Statistics Table Section
+    st.markdown("#### 📊 Show Country Statistics")
+    show_stats = st.checkbox("Show Table", value=True)
+
+# Main Title and Description
+st.title("🌏 Chulalongkorn University's Global Research Collaborations")
+
+if search_mode == "Explore by Field of Study":
+    st.markdown(
+        f"**Exploring collaborations in:** {'All Fields' if field_id == -1 else field_name} "
+        f"**|** {'All Countries' if selected_country_name == 'All Countries' else selected_country_name}"
+    )
+else:
+    st.markdown(
+        f"**Exploring collaborations by keyword:** {'None' if not keyword else keyword} "
+        f"**|** {'All Countries' if selected_country_name == 'All Countries' else selected_country_name}"
+    )
+
+# Filter by selected country if needed
+if selected_country_name != "All Countries":
+    filtered_map_data_spark = filtered_map_data_spark.filter(F.col("country") == selected_country_name)
+    filtered_map_data_pd = (
+        filtered_map_data_spark.groupBy("country")
+        .agg(F.countDistinct("affiliation_id").alias("points"))
+        .orderBy(F.col("points").desc())
+        .toPandas()
+    )
+
+if search_mode == "Explore by Field of Study":
+    title_text = f"Chulalongkorn University's Global Collaborations by {'All Fields' if field_id == -1 else field_name}"
+else:
+    title_text = "Chulalongkorn University's Global Collaborations by Keyword"
+
+fig = px.choropleth(
+    filtered_map_data_pd,
+    locations="country",
+    locationmode="country names",
+    color="points",
+    color_continuous_scale="Greens",
+    title=title_text,
+    labels={'points': 'Unique Affiliations'},
+)
+
+fig.update_geos(
+    showcountries=True,
+    countrycolor="Black",
+    showcoastlines=True,
+    coastlinecolor="Gray",
+    showland=True,
+    landcolor="white",
+    showocean=True,
+    oceancolor="lightblue",
+    projection_type="natural earth"
+)
+
+fig.update_layout(
+    title_font=dict(size=24, family="Arial"),
+    margin={"r": 10, "t": 50, "l": 10, "b": 10},
+    coloraxis_colorbar=dict(
+        title="Unique Affiliations",
+        title_font=dict(size=16, family="Arial"),
+        tickfont=dict(size=12, family="Arial"),
+    )
+)
+
+# Display the map
+st.plotly_chart(fig, use_container_width=True)
+
+# Show top 10 rows
+top_10_pd = filtered_map_data_spark.limit(10).toPandas()
+
+# Select only needed columns for preview and rename them
+# Original columns: header -> affiliation, city -> city, country -> country, title_x -> title
+display_df = top_10_pd[["header", "city", "country", "title_x"]].copy()
+display_df.rename(columns={
+    "header": "affiliation",
+    "city": "city",
+    "country": "country",
+    "title_x": "title"
+}, inplace=True)
+
+st.markdown("### 📜 Example Papers from Chulalongkorn University and Its Partners")
+st.dataframe(display_df)
+
+# Display Country Statistics if enabled
+if show_stats:
+    st.markdown("---")
+    st.subheader("🌐 Country Statistics (Unique Affiliations)")
+    if filtered_map_data_pd.empty:
+        st.write("No data available for the selected filters.")
+    else:
+        st.dataframe(filtered_map_data_pd.style.format(precision=0).set_properties(**{'text-align': 'left'}))

data/abstracts.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87f092b08f6a0997fb7e7dc8a69bb2a0bfb50f1f97a74e766650816d7fc67937
+size 27109000

data/authors.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcf90cce3f96116cf903568cc968c046f04fe0b2fdc94d8823035483820f3d9a
+size 87265279

data/clustering.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e2fe64e97eac26714a67cf25c339310e5c1d0d28c7d66af44e918a3d7d7adf4
+size 62658453

data/embeddings.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b92d220f5a9a6bf5a6bc19845024f0d66708fb64b69a79a7f2782170f36bad96
+size 87835864

data/geospatial_clustering_data.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:086e0645f51d81341aef6d5981ae862b048df69a4dc5b211d1a07ce003dc97c1
+size 90943859

embedding_model/1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false,
+  "include_prompt": true
+}

embedding_model/README.md

@@ -0,0 +1,177 @@
+---
+language: en
+license: apache-2.0
+library_name: sentence-transformers
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+datasets:
+- s2orc
+- flax-sentence-embeddings/stackexchange_xml
+- ms_marco
+- gooaq
+- yahoo_answers_topics
+- code_search_net
+- search_qa
+- eli5
+- snli
+- multi_nli
+- wikihow
+- natural_questions
+- trivia_qa
+- embedding-data/sentence-compression
+- embedding-data/flickr30k-captions
+- embedding-data/altlex
+- embedding-data/simple-wiki
+- embedding-data/QQP
+- embedding-data/SPECTER
+- embedding-data/PAQ_pairs
+- embedding-data/WikiAnswers
+pipeline_tag: sentence-similarity
+---
+
+
+# all-mpnet-base-v2
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+## Usage (Sentence-Transformers)
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+import torch.nn.functional as F
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-mpnet-base-v2')
+model = AutoModel.from_pretrained('sentence-transformers/all-mpnet-base-v2')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+# Normalize embeddings
+sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+## Evaluation Results
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name=sentence-transformers/all-mpnet-base-v2)
+
+------
+
+## Background
+
+The project aims to train sentence embedding models on very large sentence level datasets using a self-supervised 
+contrastive learning objective. We used the pretrained [`microsoft/mpnet-base`](https://huggingface.co/microsoft/mpnet-base) model and fine-tuned in on a 
+1B sentence pairs dataset. We use a contrastive learning objective: given a sentence from the pair, the model should predict which out of a set of randomly sampled other sentences, was actually paired with it in our dataset.
+
+We developped this model during the 
+[Community week using JAX/Flax for NLP & CV](https://discuss.huggingface.co/t/open-to-the-community-community-week-using-jax-flax-for-nlp-cv/7104), 
+organized by Hugging Face. We developped this model as part of the project:
+[Train the Best Sentence Embedding Model Ever with 1B Training Pairs](https://discuss.huggingface.co/t/train-the-best-sentence-embedding-model-ever-with-1b-training-pairs/7354). We benefited from efficient hardware infrastructure to run the project: 7 TPUs v3-8, as well as intervention from Googles Flax, JAX, and Cloud team member about efficient deep learning frameworks.
+
+## Intended uses
+
+Our model is intented to be used as a sentence and short paragraph encoder. Given an input text, it ouptuts a vector which captures 
+the semantic information. The sentence vector may be used for information retrieval, clustering or sentence similarity tasks.
+
+By default, input text longer than 384 word pieces is truncated.
+
+
+## Training procedure
+
+### Pre-training 
+
+We use the pretrained [`microsoft/mpnet-base`](https://huggingface.co/microsoft/mpnet-base) model. Please refer to the model card for more detailed information about the pre-training procedure.
+
+### Fine-tuning 
+
+We fine-tune the model using a contrastive objective. Formally, we compute the cosine similarity from each possible sentence pairs from the batch.
+We then apply the cross entropy loss by comparing with true pairs.
+
+#### Hyper parameters
+
+We trained ou model on a TPU v3-8. We train the model during 100k steps using a batch size of 1024 (128 per TPU core).
+We use a learning rate warm up of 500. The sequence length was limited to 128 tokens. We used the AdamW optimizer with
+a 2e-5 learning rate. The full training script is accessible in this current repository: `train_script.py`.
+
+#### Training data
+
+We use the concatenation from multiple datasets to fine-tune our model. The total number of sentence pairs is above 1 billion sentences.
+We sampled each dataset given a weighted probability which configuration is detailed in the `data_config.json` file.
+
+
+| Dataset                                                  | Paper                                    | Number of training tuples  |
+|--------------------------------------------------------|:----------------------------------------:|:--------------------------:|
+| [Reddit comments (2015-2018)](https://github.com/PolyAI-LDN/conversational-datasets/tree/master/reddit) | [paper](https://arxiv.org/abs/1904.06472) | 726,484,430 |
+| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Abstracts) | [paper](https://aclanthology.org/2020.acl-main.447/) | 116,288,806 |
+| [WikiAnswers](https://github.com/afader/oqa#wikianswers-corpus) Duplicate question pairs | [paper](https://doi.org/10.1145/2623330.2623677) | 77,427,422 |
+| [PAQ](https://github.com/facebookresearch/PAQ) (Question, Answer) pairs | [paper](https://arxiv.org/abs/2102.07033) | 64,371,441 |
+| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Titles) | [paper](https://aclanthology.org/2020.acl-main.447/) | 52,603,982 |
+| [S2ORC](https://github.com/allenai/s2orc) (Title, Abstract) | [paper](https://aclanthology.org/2020.acl-main.447/) | 41,769,185 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Body) pairs  | - | 25,316,456 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title+Body, Answer) pairs  | - | 21,396,559 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Answer) pairs  | - | 21,396,559 |
+| [MS MARCO](https://microsoft.github.io/msmarco/) triplets | [paper](https://doi.org/10.1145/3404835.3462804) | 9,144,553 |
+| [GOOAQ: Open Question Answering with Diverse Answer Types](https://github.com/allenai/gooaq) | [paper](https://arxiv.org/pdf/2104.08727.pdf) | 3,012,496 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 1,198,260 |
+| [Code Search](https://huggingface.co/datasets/code_search_net) | - | 1,151,414 |
+| [COCO](https://cocodataset.org/#home) Image captions | [paper](https://link.springer.com/chapter/10.1007%2F978-3-319-10602-1_48) | 828,395|
+| [SPECTER](https://github.com/allenai/specter) citation triplets | [paper](https://doi.org/10.18653/v1/2020.acl-main.207) | 684,100 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Question, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 681,164 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Question) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 659,896 |
+| [SearchQA](https://huggingface.co/datasets/search_qa) | [paper](https://arxiv.org/abs/1704.05179) | 582,261 |
+| [Eli5](https://huggingface.co/datasets/eli5) | [paper](https://doi.org/10.18653/v1/p19-1346) | 325,475 |
+| [Flickr 30k](https://shannon.cs.illinois.edu/DenotationGraph/) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/229/33) | 317,695 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles) | | 304,525 |
+| AllNLI ([SNLI](https://nlp.stanford.edu/projects/snli/) and [MultiNLI](https://cims.nyu.edu/~sbowman/multinli/) | [paper SNLI](https://doi.org/10.18653/v1/d15-1075), [paper MultiNLI](https://doi.org/10.18653/v1/n18-1101) | 277,230 | 
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (bodies) | | 250,519 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles+bodies) | | 250,460 |
+| [Sentence Compression](https://github.com/google-research-datasets/sentence-compression) | [paper](https://www.aclweb.org/anthology/D13-1155/) | 180,000 |
+| [Wikihow](https://github.com/pvl/wikihow_pairs_dataset) | [paper](https://arxiv.org/abs/1810.09305) | 128,542 |
+| [Altlex](https://github.com/chridey/altlex/) | [paper](https://aclanthology.org/P16-1135.pdf) | 112,696 |
+| [Quora Question Triplets](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) | - | 103,663 |
+| [Simple Wikipedia](https://cs.pomona.edu/~dkauchak/simplification/) | [paper](https://www.aclweb.org/anthology/P11-2117/) | 102,225 |
+| [Natural Questions (NQ)](https://ai.google.com/research/NaturalQuestions) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/1455) | 100,231 |
+| [SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) | [paper](https://aclanthology.org/P18-2124.pdf) | 87,599 |
+| [TriviaQA](https://huggingface.co/datasets/trivia_qa) | - | 73,346 |
+| **Total** | | **1,170,060,424** |

embedding_model/config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "sentence-transformers/all-mpnet-base-v2",
+  "architectures": [
+    "MPNetModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "mpnet",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "relative_attention_num_buckets": 32,
+  "torch_dtype": "float32",
+  "transformers_version": "4.47.0",
+  "vocab_size": 30527
+}

embedding_model/config_sentence_transformers.json

@@ -0,0 +1,10 @@
+{
+  "__version__": {
+    "sentence_transformers": "3.3.1",
+    "transformers": "4.47.0",
+    "pytorch": "2.5.1+cpu"
+  },
+  "prompts": {},
+  "default_prompt_name": null,
+  "similarity_fn_name": "cosine"
+}

embedding_model/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b3c8c717335c801abb15983036a6f1df4b6943fd6b93717969efd96d22eeec6
+size 437967672

embedding_model/modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

embedding_model/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 384,
+  "do_lower_case": false
+}

embedding_model/special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

embedding_model/tokenizer_config.json

@@ -0,0 +1,73 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "104": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30526": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<s>",
+  "do_lower_case": true,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "mask_token": "<mask>",
+  "max_length": 128,
+  "model_max_length": 384,
+  "pad_to_multiple_of": null,
+  "pad_token": "<pad>",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
+  "sep_token": "</s>",
+  "stride": 0,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "MPNetTokenizer",
+  "truncation_side": "right",
+  "truncation_strategy": "longest_first",
+  "unk_token": "[UNK]"
+}

embeddings.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18b6bf845c4c24135a9f4637328e2e738717e7862034f8a48261c76f534a24ce
+size 61437056

spark_setup.py

@@ -0,0 +1,40 @@
+# spark_setup.py
+from pyspark.sql import SparkSession
+
+# Initialize Spark session
+def create_spark_session(app_name="University Research Analysis", master="local"):
+    spark = SparkSession.builder \
+        .appName(app_name) \
+        .master(master) \
+        .getOrCreate()
+    return spark
+
+# Load data into Spark DataFrames and return a dictionary of DataFrames
+def load_data(spark, file_paths):
+    dataframes = {}
+    for name, path in file_paths.items():
+        dataframes[name] = spark.read.csv(path, header=True, inferSchema=True)
+        # Register as a temporary view
+        dataframes[name].createOrReplaceTempView(name)
+    return dataframes
+
+# File paths for each dataset
+file_paths = {
+    "author_affiliations": "data/author_affiliations.csv",
+    "affiliations": "data/affiliations.csv",
+    "subject_areas": "data/subject_areas.csv",
+    "keywords": "data/keywords.csv",
+    "publications": "data/publications.csv",
+    "authors": "data/authors.csv",
+    "embeddings": "data/embeddings.csv",
+    "clustering": "data/clustering.csv",
+    "abstracts": "data/abstracts.csv",
+    "geospatial_clustering_data": "data/geospatial_clustering_data.csv",
+    "geospatial_data_by_publication": "data/geospatial_data_by_publication.csv",
+    "scopus_affiliation_data": "data/scopus_affiliation_data.csv"
+}
+
+if __name__ == "__main__":
+    spark = create_spark_session()
+    dataframes = load_data(spark, file_paths)
+    print("Spark session initialized and data loaded.")