refactor data processing in process_csvs_store.py; update dataset paths

data/process_csvs_store.py

@@ -1,33 +1,39 @@
-import pandas as pd
-import time
+import json
 import os
-from buster.documents_manager import DeepLakeDocumentsManager
+import time
 
-# from deeplake.core.vectorstore import VectorStore
-# from langchain.embeddings.openai import OpenAIEmbeddings
 import numpy as np
+import pandas as pd
+from buster.documents_manager import DeepLakeDocumentsManager
 
-# from openai import OpenAI
-
-DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "ai-tutor-dataset")
+DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "ai-tutor-dataset-2")
 DEEPLAKE_ORG = os.getenv("DEEPLAKE_ORG", "towards_ai")
 
-# df1 = pd.read_csv("./data/jobs.csv", encoding='ISO-8859-1')  # or 'latin1' or 'cp1252'
-# df2 = pd.read_csv("./data/hf_transformers.csv")
-# df3 = pd.read_csv("./data/langchain_course.csv")
-# df4 = pd.read_csv("./data/filtered_tai_v2.csv")
-# df5 = pd.read_csv("./data/wiki.csv")  # , encoding="ISO-8859-1")
-# df6 = pd.read_csv("./data/openai.csv")
-df1 = pd.read_csv("./advanced_rag_course.csv")
-
-# print(len(df1), len(df2), len(df3), len(df4), len(df5), len(df6))
-print(len(df1))
-
-dataset_path = f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
+df1 = pd.read_csv("data/advanced_rag_course.csv")
+df2 = pd.read_csv("data/hf_transformers.csv")
+df3 = pd.read_csv("data/langchain_course.csv")
+df4 = pd.read_csv("data/filtered_tai_v2.csv")
+df5 = pd.read_csv("data/wiki.csv")  # , encoding="ISO-8859-1")
+# df6 = pd.read_csv("data/openai.csv")  # Broken
+df7 = pd.read_csv("data/activeloop.csv")
+df8 = pd.read_csv("data/llm_course.csv")
+df9 = pd.read_csv("data/langchain_docs.csv")  # , encoding="ISO-8859-1")
+
+print(len(df1), df1.columns)
+print(len(df2), df2.columns)
+print(len(df3), df3.columns)
+print(len(df4), df4.columns)
+print(len(df5), df5.columns)
+# print(len(df6), df6.columns)
+print(len(df7), df7.columns)
+print(len(df8), df8.columns)
+print(len(df9), df9.columns)
+
+
+# dataset_path = f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
+dataset_path = f"local_dataset"
 # dataset_path = f"{DEEPLAKE_DATASET}"
-# because wrong name
-# df1['content'] = df1['cleaned_description']
-# print(np.sum(df1.content.isna()), len(df1) )
+
 
 dm = DeepLakeDocumentsManager(
     vector_store_path=dataset_path,
@@ -35,11 +41,71 @@ dm = DeepLakeDocumentsManager(
     required_columns=["url", "content", "source", "title"],
 )
 
+
 dm.batch_add(
     df=df1,
     batch_size=3000,
-    min_time_interval=60,
-    num_workers=32,
+    min_time_interval=5,
+    num_workers=15,
+    csv_overwrite=False,
+)
+dm.batch_add(
+    df=df2,
+    batch_size=3000,
+    min_time_interval=5,
+    num_workers=15,
+    csv_overwrite=False,
+)
+dm.batch_add(
+    df=df3,
+    batch_size=3000,
+    min_time_interval=5,
+    num_workers=15,
+    csv_overwrite=False,
+)
+dm.batch_add(
+    df=df4,
+    batch_size=3000,
+    min_time_interval=5,
+    num_workers=15,
+    csv_overwrite=False,
+)
+dm.batch_add(
+    df=df5,
+    batch_size=3000,
+    min_time_interval=5,
+    num_workers=15,
+    csv_overwrite=False,
+)
+
+# ERROR DO NOT ADD
+# dm.batch_add(
+#     df=df6,
+#     batch_size=3000,
+#     min_time_interval=5,
+#     num_workers=15,
+#     csv_overwrite=False,
+# )
+
+dm.batch_add(
+    df=df7,
+    batch_size=3000,
+    min_time_interval=5,
+    num_workers=15,
+    csv_overwrite=False,
+)
+dm.batch_add(
+    df=df8,
+    batch_size=3000,
+    min_time_interval=5,
+    num_workers=15,
+    csv_overwrite=False,
+)
+dm.batch_add(
+    df=df9,
+    batch_size=3000,
+    min_time_interval=5,
+    num_workers=15,
     csv_overwrite=False,
 )