move over to activeloop online hosting (#14)

cfg.py

@@ -10,32 +10,22 @@ from buster.tokenizers import GPTTokenizer
 from buster.validators import QuestionAnswerValidator, Validator
 from huggingface_hub import hf_hub_download
 
-from utils import extract_zip
-
 logger = logging.getLogger(__name__)
 logging.basicConfig(level=logging.INFO)
 
-# For authentication
-USERNAME = os.getenv("BUSTER_USERNAME")
-PASSWORD = os.getenv("BUSTER_PASSWORD")
-
-HUB_TOKEN = os.getenv("HUB_TOKEN")
-REPO_ID = os.getenv("HF_DATASET")
+# required
+ACTIVELOOP_TOKEN = os.getenv("ACTIVELOOP_TOKEN")
+if ACTIVELOOP_TOKEN is None:
+    logger.warning("No activeloop token found, you will not be able to fetch data.")
 
 DEEPLAKE_DATASET = os.getenv("DEEPLAKE_DATASET", "wiki_tai_langchain")
-ZIP_FILE = DEEPLAKE_DATASET + ".zip"
+DEEPLAKE_ORG = os.getenv("DEEPLAKE_ORG", "towards_ai")
 
-logger.info(f"Downloading {ZIP_FILE} from hub...")
-hf_hub_download(
-    repo_id=REPO_ID,
-    repo_type="dataset",
-    filename=ZIP_FILE,
-    token=HUB_TOKEN,
-    local_dir=".",
+# if you want to use a local dataset, set the env. variable, it overrides all others
+DEEPLAKE_DATASET_PATH = os.getenv(
+    "DEEPLAKE_DATASET_PATH", f"hub://{DEEPLAKE_ORG}/{DEEPLAKE_DATASET}"
 )
 
-extract_zip(zip_file_path=ZIP_FILE, output_path=DEEPLAKE_DATASET)
-
 example_questions = [
     "What is the LLama model?",
     "What is a LLM?",