src/constants.py

import os
import src.web_crawler as web_crawler_utils
import src.weather as weather_utils

# Wheater to load the existing index store or create from scratch?
LOAD_FROM_EXISTING_INDEX_STORE = True
INDEX_TYPE = 'FAISS'

# Path from where to load the data (from the local directory)
DATA_PATH = './data/'

OUTPUT_DIR = './output/'
if not os.path.exists(OUTPUT_DIR):
    os.makedirs(OUTPUT_DIR)
# Path to store the index/vector db
OUTPUT_PATH = os.path.join(OUTPUT_DIR, INDEX_TYPE)
# Create OUTPUT_PATH directory if not present
if not os.path.exists(OUTPUT_PATH):
    os.makedirs(OUTPUT_PATH)

# Output path to store the feedback for the answers received by KCCs-FTAs
OUTPUT_PATH_ANSWER_FEEDBACK = os.path.join(OUTPUT_DIR, 'answers_feedback')
if not os.path.exists(OUTPUT_PATH_ANSWER_FEEDBACK):
    os.makedirs(OUTPUT_PATH_ANSWER_FEEDBACK)
OUTPUT_PATH_ANSWER_FEEDBACK_FILE_PREFIX = 'answers_feedback'
OUTPUT_PATH_ANSWER_FEEDBACK_FILE_SAVE_SEPARATOR = '\t'

# Index categories (There would be an index for each category. On asking the query, App will search for the relevant docs/information only from the respective index category.)
INDEX_CATEGORY = [
    'crops',
    'fruits',
    'pest_management',
    'govt_policy',
    'insurance',
    'soil',
    'general',
    'vegetables'
]

# Doctype of the master index of each index category. Master index for each index category would be stored under this key.
INDEX_CATEGORY_MASTER_INDEX_DOC_TYPE = 'master'

# List of data sources/types & from where to load the data and create the index/vector store
# 2nd item is the type of source from where the data would be loaded. Currently it could come from either a file or URL.
DATA_SOURCES = {
    'PDF': 'pdf',
    'Text File': 'textfile',
    'Online PDF': 'online_pdf', # web_crawler_utils.get_ipm_packages_pdfs_urls()[:1]
    'URLs': 'urls',
}

# LangChain related constants
SIMILARITY_TOP_K = 1
MODE = 'embedding'
RESPONSE_MODE = 'default'
TEXT_SPLITTER_CHUNK_SIZE = 1000
TEXT_SPLITTER_CHUNK_OVERLAP = 0
TEXT_SPLITTER_SEPARATOR = '\n\n'


URLS = [
    # Govt. Schemes
    'https://agricoop.nic.in/en/Major#gsc.tab=0'
    'https://agricoop.nic.in/#gsc.tab=0',

    'https://dmi.gov.in/Documents/GrantCAGrapes.pdf',
    'https://dmi.gov.in/Documents/organicfaq.pdf',
    'https://dmi.gov.in/Documents/CAGMOrganic-III.pdf',
    'https://dmi.gov.in/GradesStandard.aspx',
    'https://www.india.gov.in/topics/agriculture',
    'https://www.india.gov.in/farmers-portal',
    
    # Pest Management related
    'https://niphm.gov.in/IPMPackages/Maize.pdf',

    # Banned Pesticides
    'https://ppqs.gov.in/divisions/cib-rc/registered-products',     # Online PDF links on the page
    
    # Mandi Price related
    'https://agmarknet.gov.in/',
    
    # General information related: Information of interests are present on the 2nd level url
    'https://www.manage.gov.in/nf/nf.asp',
    
    # Weather forecast related
    'https://nwp.imd.gov.in/blf/blf_temp/',   # need to select state -> district (on the new page) -> displays detailed table -> can get info at the block level as well from the same page on selection
    'https://nwp.imd.gov.in/blf/blf_temp/dis.php?value=12gujarat',   # to get weather forecast for the given state
    'https://nwp.imd.gov.in/blf/blf_temp/block.php?dis=12BHAVNAGAR',   # to get the weather forecast for the given district
]


# Supported Indian laguages for translating the English text to Indian language
INDIC_LANGUAGE = {
    'Hindi': 'hi',
    'Gujarati': 'gu',
    'Kannada': 'kn',
    'Marathi': 'mr',
    'Panjabi': 'pa',
    'Bengali': "bn",
    'Telugu': 'te',
    'Tamil': 'ta',
    'Malayalam': 'ml',
}

# State list used in the Mandi Price widget dropdown list
MANDI_PRICE_STATES = [
    'ANDAMAN AND NICOBAR ISLANDS',
    'ANDHRA PRADESH',
    'ASSAM',
    'BIHAR',
    'CHANDIGARH',
    'CHHATTISGARH',
    'GOA',
    'GUJARAT',
    'HARYANA',
    'HIMACHAL PRADESH',
    'JAMMU AND KASHMIR',
    'JHARKHAND',
    'KARNATAKA',
    'KERALA',
    'MADHYA PRADESH',
    'MAHARASHTRA',
    'NAGALAND',
    'ODISHA',
    'PUDUCHERRY',
    'PUNJAB',
    'RAJASTHAN',
    'TAMIL NADU',
    'TELANGANA',
    'TRIPURA',
    'UTTAR PRADESH',
    'UTTARAKHAND',
    'WEST BENGAL'
]

# State list used in the Weather forecast widget dropdown list
weather_utils_obj = weather_utils.WEATHER()
WEATHER_FORECAST_STATE_CODES = weather_utils_obj.get_state_names_codes()

# LIST OF  PESTICIDES WHICH ARE BANNED AND RESTRICTED USE (List created from: https://pib.gov.in/PressReleaseIframePage.aspx?PRID=1896140)
BANNED_PESTICIDES_FORMULATIONS = [
    'Alachlor',
    'Aldicarb',
    'Aldrin',
    'Benzene Hexachloride',
    'Benomyl',
    'Calcium Cyanide',
    'Carbaryl',
    'Chlorbenzilate',
    'Chlordane',
    'Chlorofenvinphos',
    'Copper Acetoarsenite',
]