Update app.py

app.py

@@ -1,64 +1,508 @@
 import os
-from flask import Flask, render_template, request, jsonify
+from flask import Flask, render_template, request, jsonify, send_file
 import requests
 import pandas as pd
 from datetime import datetime
+import plotly.express as px
+import plotly.io as pio
+import numpy as np
+import dotenv
+import json
+import gtts
+import uuid
+from pathlib import Path
+
+dotenv.load_dotenv()
+
 app = Flask(__name__)
 
-# Function to fetch data from the API
-def fetch_market_data(state=None, district=None, market=None, commodity=None):  
-    api_key = os.getenv("GOV_API_KEY")
-    api_key = api_key
-    base_url = "https://api.data.gov.in/resource/9ef84268-d588-465a-a308-a864a43d0070"
+# Create audio directory if it doesn't exist
+AUDIO_DIR = Path("static/audio")
+AUDIO_DIR.mkdir(parents=True, exist_ok=True)
+
+def fetch_market_data(state=None, district=None, market=None, commodity=None):
+    """Fetch data from the agricultural market API.
+    If the API fails or returns empty data, fallback to the CSV file.
+    Filters (state, district, market, commodity) are applied manually on CSV data.
+    """
+    api_key = "579b464db66ec23bdd000001189bbb99e979428764bdbe8fdd44ebb7"
+    base_url = "https://api.data.gov.in/resource/9ef84268-d588-465a-a308-a864a43d007"
+    
     params = {
         "api-key": api_key,
         "format": "json",
-        "limit": 15000,
+        "limit": 1000,
     }
+    
+    if state:
+        params["filters[state]"] = state
+    if district:
+        params["filters[district]"] = district
+    if market:
+        params["filters[market]"] = market
+    if commodity:
+        params["filters[commodity]"] = commodity
 
+    try:
+        response = requests.get(base_url, params=params)
+        if response.status_code == 200:
+            data = response.json()
+            records = data.get("records", [])
+            df = pd.DataFrame(records)
+        else:
+            print(f"API Error: {response.status_code}")
+            raise Exception(f"API Error: {response.status_code}")
+    except Exception as e:
+        print(f"Error fetching data from API: {str(e)}. Falling back to CSV file.")
+        df = pd.read_csv("final_price_data.csv")
+        if 'min_price' not in df.columns:
+            rename_mapping = {
+                'State': 'state',
+                'District': 'district',
+                'Market': 'market',
+                'Commodity': 'commodity',
+                'Variety': 'variety',
+                'Grade': 'grade',
+                'Arrival_Date': 'arrival_date',
+                'Min_x0020_Price': 'min_price',
+                'Max_x0020_Price': 'max_price',
+                'Modal_x0020_Price': 'modal_price'
+            }
+            df.rename(columns=rename_mapping, inplace=True)
+    
+    if df.empty:
+        print("API returned empty data. Falling back to CSV file.")
+        df = pd.read_csv("final_price_data.csv")
+        if 'min_price' not in df.columns:
+            rename_mapping = {
+                'State': 'state',
+                'District': 'district',
+                'Market': 'market',
+                'Commodity': 'commodity',
+                'Variety': 'variety',
+                'Grade': 'grade',
+                'Arrival_Date': 'arrival_date',
+                'Min_x0020_Price': 'min_price',
+                'Max_x0020_Price': 'max_price',
+                'Modal_x0020_Price': 'modal_price'
+            }
+            df.rename(columns=rename_mapping, inplace=True)
+    
     if state:
-        params["filters[state.keyword]"] = state
+        df = df[df['state'] == state]
     if district:
-        params["filters[district.keyword]"] = district
+        df = df[df['district'] == district]
     if market:
-        params["filters[market.keyword]"] = market
+        df = df[df['market'] == market]
     if commodity:
-        params["filters[commodity.keyword]"] = commodity
+        df = df[df['commodity'] == commodity]
+    
+    return df
+
+def get_ai_insights(market_data, state, district, market=None, commodity=None, language="English"):
+    """Get enhanced insights from Gemini API with focus on profitable suggestions for farmers.
+       Supports multiple languages through the prompt.
+       Returns dynamic insights only. If something goes wrong, returns an empty string.
+    """
+    if not state or not district or market_data.empty:
+        return ""
+    
+    try:
+        # Filter data based on provided parameters
+        district_data = market_data[market_data['district'] == district]
+        if district_data.empty:
+            return ""
+        
+        # Apply market filter if provided
+        if market and not market_data[market_data['market'] == market].empty:
+            market_specific = True
+            district_data = district_data[district_data['market'] == market]
+        else:
+            market_specific = False
+            
+        # Apply commodity filter if provided
+        if commodity and not market_data[market_data['commodity'] == commodity].empty:
+            commodity_specific = True
+            district_data = district_data[district_data['commodity'] == commodity]
+        else:
+            commodity_specific = False
+        
+        # Calculate price trends
+        price_trends = district_data.groupby('commodity').agg({
+            'modal_price': ['mean', 'min', 'max', 'std']
+        }).round(2)
+        
+        # Using environment variable for Gemini API key
+        api_key = "AIzaSyBtXV2xJbrWVV57B5RWy_meKXOA59HFMeY"
+        if not api_key:
+            print("Warning: Gemini API key not set")
+            return ""
+            
+        price_trends['price_stability'] = (price_trends['modal_price']['std'] / 
+                                         price_trends['modal_price']['mean']).round(2)
+        
+        district_data['arrival_date'] = pd.to_datetime(district_data['arrival_date'])
+        district_data['month'] = district_data['arrival_date'].dt.month
+        monthly_trends = district_data.groupby(['commodity', 'month'])['modal_price'].mean().round(2)
+        
+        market_competition = len(district_data['market'].unique())
+        top_commodities = district_data.groupby('commodity')['modal_price'].mean().nlargest(5).index.tolist()
+        
+        # Get min and max prices for key commodities
+        price_range_info = {}
+        for commodity in top_commodities[:3]:
+            comm_data = district_data[district_data['commodity'] == commodity]
+            if not comm_data.empty:
+                price_range_info[commodity] = {
+                    'min': comm_data['modal_price'].min(),
+                    'max': comm_data['modal_price'].max(),
+                    'avg': comm_data['modal_price'].mean()
+                }
+        
+        # Calculate market-specific metrics if market is selected
+        market_details = ""
+        if market_specific:
+            market_details = f"""
+            Market-specific information for {market}:
+            - Number of commodities: {len(district_data['commodity'].unique())}
+            - Most expensive commodity: {district_data.groupby('commodity')['modal_price'].mean().idxmax()}
+            - Cheapest commodity: {district_data.groupby('commodity')['modal_price'].mean().idxmin()}
+            """
+        
+        # Commodity-specific details if commodity is selected
+        commodity_details = ""
+        if commodity_specific:
+            commodity_data = district_data[district_data['commodity'] == commodity]
+            best_market = commodity_data.loc[commodity_data['modal_price'].idxmin()]['market']
+            worst_market = commodity_data.loc[commodity_data['modal_price'].idxmax()]['market']
+            
+            commodity_details = f"""
+            Commodity-specific information for {commodity}:
+            - Best market to buy (lowest price): {best_market}
+            - Highest priced market: {worst_market}
+            - Price variance across markets: {commodity_data['modal_price'].std().round(2)}
+            """
+        
+        # Improved prompt for better structured output with language support
+        prompt = f"""
+        Analyze the following agricultural market data for {district}, {state} and provide insights in {language} language.
+        
+        Market data:
+        - Active markets: {market_competition}
+        - Top crops: {', '.join(top_commodities[:5])}
+        - Data from {len(price_trends.index)} crops and {len(monthly_trends)} monthly entries.
+        
+        Price information:
+        {json.dumps(price_range_info, indent=2)}
+        
+        {market_details}
+        {commodity_details}
+        
+        Analyze this data and provide insights about crop market trends and profitability. 
+        Include specific numbers from the data about prices.
+        
+        Provide structured insights with clear sections. Use this exact format with bullet points:
+        
+        Crop Profitability Analysis:
+        * [First insight about profitable crops with specific prices mentioned]
+        * [Second insight]
+        
+        Market Price Analysis:
+        * [First insight about markets with specific price ranges]
+        * [Second insight]
+        
+        Recommendations for Farmers:
+        * [Action item 1]
+        * [Action item 2]
+        """
+        
+        api_url = "https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-pro:generateContent"
+        headers = {"Content-Type": "application/json"}
+        
+        payload = {
+            "contents": [
+                {
+                    "parts": [
+                        {"text": prompt}
+                    ]
+                }
+            ],
+            "generationConfig": {
+                "temperature": 0.4,
+                "maxOutputTokens": 1024
+            }
+        }
+        
+        response = requests.post(
+            f"{api_url}?key={api_key}", 
+            headers=headers, 
+            json=payload, 
+            timeout=20
+        )
+        
+        if response.status_code == 200:
+            response_data = response.json()
+            if 'candidates' in response_data and len(response_data['candidates']) > 0:
+                content = response_data['candidates'][0]['content']
+                if 'parts' in content and len(content['parts']) > 0:
+                    insights = content['parts'][0]['text']
+                    return format_ai_insights(insights, language)
+            print(f"API Response issue: {response.text[:100]}")
+        else:
+            print(f"Gemini API Error: {response.status_code} - {response.text[:100]}")
+        
+        return ""
+    
+    except Exception as e:
+        print(f"Error generating insights: {str(e)}")
+        return ""
+
+def extract_text_from_insights(insights_html):
+    """Extract pure text content from HTML insights for text-to-speech conversion."""
+    # Simple HTML tag removal - for production, consider using BeautifulSoup for better parsing
+    import re
+    text = re.sub(r'<.*?>', ' ', insights_html)
+    text = re.sub(r'\s+', ' ', text)  # Remove extra whitespace
+    return text.strip()
+
+def create_audio_from_text(text, language_code="en"):
+    """Generate audio file from text using gTTS."""
+    if not text:
+        return None
+    
+    # Map UI language selection to gTTS language codes
+    language_map = {
+        "English": "en",
+        "Hindi": "hi",
+        "Tamil": "ta",
+        "Telugu": "te",
+        "Marathi": "mr",
+        "Bengali": "bn",
+        "Gujarati": "gu",
+        "Kannada": "kn", 
+        "Malayalam": "ml",
+        "Punjabi": "pa"
+    }
+    
+    tts_lang = language_map.get(language_code, "en")
+    
+    # Generate unique filename
+    filename = f"{uuid.uuid4()}.mp3"
+    filepath = AUDIO_DIR / filename
+    
+    try:
+        tts = gtts.gTTS(text, lang=tts_lang, slow=False)
+        tts.save(str(filepath))
+        return f"/static/audio/{filename}"
+    except Exception as e:
+        print(f"Error creating audio: {str(e)}")
+        return None
+
+def create_audio_local_fallback(text, language_code="en"):
+    """Local fallback for TTS when network is unavailable."""   
+    try:
+        # This requires pyttsx3 to be installed
+        import pyttsx3
+        engine = pyttsx3.init()
+        
+        # Generate unique filename
+        filename = f"{uuid.uuid4()}.mp3"
+        filepath = AUDIO_DIR / filename
+        
+        engine.save_to_file(text, str(filepath))
+        engine.runAndWait()
+        
+        return f"/static/audio/{filename}"
+    except Exception as e:
+        print(f"Local TTS fallback failed: {str(e)}")
+        return None
 
-    response = requests.get(base_url, params=params)
+def format_ai_insights(insights_data, language="English"):
+    """Format AI insights into structured HTML.
+       Returns an empty string if no valid insights are provided.
+    """
+    if not insights_data or not insights_data.strip():
+        return ""
+    
+    # Process the insights text - each bullet point becomes a formatted item
+    formatted_content = ""
+    
+    # Split by bullet points
+    bullet_points = insights_data.split('*')
+    
+    # Filter out empty items and process each bullet point
+    bullet_points = [point.strip() for point in bullet_points if point.strip()]
+    
+    # Check if any section headers exist in the content
+    sections = {}
+    current_section = "Recommendations"
+    
+    for point in bullet_points:
+        if ":" in point and len(point.split(":")[0]) < 30:  # Likely a section header
+            current_section = point.split(":")[0].strip()
+            # Start a new section
+            if current_section not in sections:
+                sections[current_section] = []
+        else:
+            # Add to current section
+            if current_section not in sections:
+                sections[current_section] = []
+            sections[current_section].append(point)
+    
+    # Now build the HTML with proper sections
+    for section, points in sections.items():
+        formatted_content += f'<div class="insight-card"><h5>{section}</h5><ul class="insight-list">'
+        for point in points:
+            # Highlight prices with special styling
+            if "₹" in point:
+                # Replace price mentions with highlighted spans
+                parts = point.split("₹")
+                styled_point = parts[0]
+                for i in range(1, len(parts)):
+                    # Extract the price value
+                    price_text = parts[i].split()[0]
+                    # Add the highlighted price and the rest of the text
+                    styled_point += f'<span class="price-highlight">₹{price_text}</span>' + parts[i][len(price_text):]
+                formatted_content += f'<li>{styled_point}</li>'
+            else:
+                formatted_content += f'<li>{point}</li>'
+        formatted_content += '</ul></div>'
+    
+    # Create the plain text version for audio generation
+    plain_text = f"Market Insights for {language}.\n\n"
+    for section, points in sections.items():
+        plain_text += f"{section}:\n"
+        for point in points:
+            # Clean up for speech
+            clean_point = point.replace("₹", " rupees ")
+            plain_text += f"• {clean_point}\n"
+        plain_text += "\n"
+    
+    # Generate audio file
+    audio_path = create_audio_from_text(plain_text, language)
+    if audio_path is None:
+        audio_path = create_audio_local_fallback(plain_text)
+    
+    # Add a wrapper for the insights with audio player
+    audio_player = ""
+    if audio_path:
+        audio_player = f"""
+        <div class="audio-player-container">
+            <h4>Listen to Insights</h4>
+            <audio id="insightsAudio" controls>
+                <source src="{audio_path}" type="audio/mpeg">
+                Your browser does not support the audio element.
+            </audio>
+            <button class="btn btn-sm btn-custom mt-2" id="playAudioBtn">
+                <i class="fa fa-play"></i> Play Audio
+            </button>
+        </div>
+        """
+    
+    html = f"""
+    <div class="insights-header">
+        <h3>AI Market Insights</h3>
+        {audio_player}
+    </div>
+    <div class="insight-section">
+        {formatted_content}
+    </div>
+    """
+    
+    return html
 
-    if response.status_code == 200:
-        data = response.json()
-        records = data.get("records", [])
-        df = pd.DataFrame(records)
-        return df
-    else:
-        return pd.DataFrame()
+def generate_plots(df):
+    """Generate all plots in English"""
+    if df.empty:
+        return {}, "No data available"
+    
+    price_cols = ['min_price', 'max_price', 'modal_price']
+    for col in price_cols:
+        df[col] = pd.to_numeric(df[col], errors='coerce')
+    
+    colors = ["#4CAF50", "#8BC34A", "#CDDC39", "#FFC107", "#FF5722"]
+    
+    df_bar = df.groupby('commodity')['modal_price'].mean().reset_index()
+    fig_bar = px.bar(df_bar, 
+                     x='commodity', 
+                     y='modal_price',
+                     title="Average Price by Commodity",
+                     color_discrete_sequence=colors)
+    
+    fig_line = None
+    if 'commodity' in df.columns and len(df['commodity'].unique()) == 1:
+        df['arrival_date'] = pd.to_datetime(df['arrival_date'])
+        df_line = df.sort_values('arrival_date')
+        fig_line = px.line(df_line, 
+                          x='arrival_date', 
+                          y='modal_price',
+                          title="Price Trend",
+                          color_discrete_sequence=colors)
+    
+    fig_box = px.box(df, 
+                     x='commodity', 
+                     y='modal_price',
+                     title="Price Distribution",
+                     color='commodity',
+                     color_discrete_sequence=colors)
+    
+    plots = {
+        'bar': pio.to_html(fig_bar, full_html=False),
+        'box': pio.to_html(fig_box, full_html=False)
+    }
+    if fig_line:
+        plots['line'] = pio.to_html(fig_line, full_html=False)
+    
+    return plots
 
-# Fetch initial state data
-@app.route("/")
+@app.route('/')
 def index():
-    # Fetch distinct states for dropdown
-    market_data = fetch_market_data()
-    states = market_data['state'].dropna().unique().tolist()
-    today = datetime.today().strftime('%Y-%m-%d')
-    return render_template('index.html', states=states, today=today)
-
-# Handle AJAX requests to filter market data
-@app.route("/filter_data", methods=["POST"])
+    try:
+        initial_data = fetch_market_data()
+        states = sorted(initial_data['state'].dropna().unique()) if not initial_data.empty else []
+    except Exception as e:
+        print(f"Error fetching initial data: {str(e)}")
+        states = []
+    
+    return render_template('index.html', 
+                           states=states,
+                           today=datetime.today().strftime('%Y-%m-%d'))
+
+@app.route('/filter_data', methods=['POST'])
 def filter_data():
-    state = request.form.get("state")
-    district = request.form.get("district")
-    market = request.form.get("market")
-    commodity = request.form.get("commodity")
-
-    # Fetch filtered data
-    filtered_data = fetch_market_data(state, district, market, commodity)
-
-    # Generate HTML for the filtered table
-    table_html = ""
-    for _, row in filtered_data.iterrows():
-        table_html += f"""
+    state = request.form.get('state')
+    district = request.form.get('district')
+    market = request.form.get('market')
+    commodity = request.form.get('commodity')
+    language = request.form.get('language', 'English')  # Default to English
+    
+    df = fetch_market_data(state, district, market, commodity)
+    plots = generate_plots(df)
+    # Pass market and commodity to get_ai_insights
+    insights = get_ai_insights(df, state, district, market, commodity, language) if state and district and not df.empty else ""
+    
+    market_table_html = """
+    <div class="table-responsive">
+        <table class="table table-striped table-bordered">
+            <thead>
+                <tr>
+                    <th>State</th>
+                    <th>District</th>
+                    <th>Market</th>
+                    <th>Commodity</th>
+                    <th>Variety</th>
+                    <th>Grade</th>
+                    <th>Arrival Date</th>
+                    <th>Min Price</th>
+                    <th>Max Price</th>
+                    <th>Modal Price</th>
+                </tr>
+            </thead>
+            <tbody>
+    """
+    
+    for _, row in df.iterrows():
+        market_table_html += f"""
             <tr>
                 <td>{row['state']}</td>
                 <td>{row['district']}</td>
@@ -67,61 +511,110 @@ def filter_data():
                 <td>{row['variety']}</td>
                 <td>{row['grade']}</td>
                 <td>{row['arrival_date']}</td>
-                <td>{row['min_price']}</td>
-                <td>{row['max_price']}</td>
-                <td>{row['modal_price']}</td>
+                <td>₹{row['min_price']}</td>
+                <td>₹{row['max_price']}</td>
+                <td>₹{row['modal_price']}</td>
             </tr>
         """
+    market_table_html += "</tbody></table></div>"
 
-    # Get top 5 cheapest crops by modal price
-    cheapest_crops = filtered_data.sort_values("modal_price", ascending=True).head(5)
-    cheapest_html = ""
+    cheapest_crops = df.sort_values('modal_price', ascending=True).head(5)
+    cheapest_table_html = """
+    <div class="table-responsive">
+        <table class="table table-sm table-bordered">
+            <thead>
+                <tr>
+                    <th>Commodity</th>
+                    <th>Market</th>
+                    <th>Modal Price</th>
+                </tr>
+            </thead>
+            <tbody>
+    """
+    
     for _, row in cheapest_crops.iterrows():
-        cheapest_html += f"""
+        cheapest_table_html += f"""
             <tr>
                 <td>{row['commodity']}</td>
-                <td>{row['modal_price']}</td>
+                <td>{row['market']}</td>
+                <td>₹{row['modal_price']}</td>
             </tr>
         """
+    cheapest_table_html += "</tbody></table></div>"
 
-    # Get top 5 costliest crops by modal price
-    costliest_crops = filtered_data.sort_values("modal_price", ascending=False).head(5)
-    costliest_html = ""
+    costliest_crops = df.sort_values('modal_price', ascending=False).head(5)
+    costliest_table_html = """
+    <div class="table-responsive">
+        <table class="table table-sm table-bordered">
+            <thead>
+                <tr>
+                    <th>Commodity</th>
+                    <th>Market</th>
+                    <th>Modal Price</th>
+                </tr>
+            </thead>
+            <tbody>
+    """
+    
     for _, row in costliest_crops.iterrows():
-        costliest_html += f"""
+        costliest_table_html += f"""
             <tr>
                 <td>{row['commodity']}</td>
-                <td>{row['modal_price']}</td>
+                <td>{row['market']}</td>
+                <td>₹{row['modal_price']}</td>
             </tr>
         """
+    costliest_table_html += "</tbody></table></div>"
 
-    return jsonify({
-        "market_html": table_html,
-        "cheapest_html": cheapest_html,
-        "costliest_html": costliest_html
-    })
+    market_stats = {
+        'total_commodities': len(df['commodity'].unique()),
+        'avg_modal_price': f"₹{df['modal_price'].mean():.2f}",
+        'price_range': f"₹{df['modal_price'].min():.2f} - ₹{df['modal_price'].max():.2f}",
+        'total_markets': len(df['market'].unique())
+    }
 
-# Handle AJAX requests for dropdown filtering
-@app.route("/get_districts", methods=["POST"])
+    response = {
+        'plots': plots,
+        'insights': insights,
+        'success': not df.empty,
+        'hasStateDistrict': bool(state and district),
+        'market_html': market_table_html,
+        'cheapest_html': cheapest_table_html,
+        'costliest_html': costliest_table_html,
+        'market_stats': market_stats
+    }
+    
+    return jsonify(response)
+
+@app.route('/get_districts', methods=['POST'])
 def get_districts():
-    state = request.form.get("state")
-    market_data = fetch_market_data(state=state)
-    districts = market_data["district"].dropna().unique().tolist()
+    state = request.form.get('state')
+    df = fetch_market_data(state=state)
+    districts = sorted(df['district'].dropna().unique())
     return jsonify(districts)
 
-@app.route("/get_markets", methods=["POST"])
+@app.route('/get_markets', methods=['POST'])
 def get_markets():
-    district = request.form.get("district")
-    market_data = fetch_market_data(district=district)
-    markets = market_data["market"].dropna().unique().tolist()
+    district = request.form.get('district')
+    df = fetch_market_data(district=district)
+    markets = sorted(df['market'].dropna().unique())
     return jsonify(markets)
 
-@app.route("/get_commodities", methods=["POST"])
+@app.route('/get_commodities', methods=['POST'])
 def get_commodities():
-    market = request.form.get("market")
-    market_data = fetch_market_data(market=market)
-    commodities = market_data["commodity"].dropna().unique().tolist()
+    market = request.form.get('market')
+    df = fetch_market_data(market=market)
+    commodities = sorted(df['commodity'].dropna().unique())
     return jsonify(commodities)
 
-if __name__ == "__main__":
-     app.run(debug=True, host="0.0.0.0", port=7860)
+@app.route('/static/audio/<filename>')
+def serve_audio(filename):
+    try:
+        return send_file(f"static/audio/{filename}", mimetype="audio/mpeg")
+    except Exception as e:
+        print(f"Error serving audio file: {str(e)}")
+        return "Audio file not found", 404
+
+if __name__ == '__main__':
+    pio.templates.default = "plotly_white"
+    app.run(debug=True, host='0.0.0.0', port=7860)