Upload 6 files

app.py

@@ -57,10 +57,11 @@ st.markdown(html_title, unsafe_allow_html=True)
 
 menu_options = [
     {"label": "Internal users", "icon": "📄", "description": "Upload a document and schedule it for email"},
-    {"label": "External users", "icon": "📊", "description": "Schedule an email with BigQuery data"}
+    {"label": "External users", "icon": "📊", "description": "Schedule an email with BigQuery data"},
+    {"label": "Recon checking", "icon": "📊", "description": "Schedule an email with BigQuery data"}
 ]
 
-        # Create the custom option menu
+# Create the custom option menu
 selected_option = option_menu(
     menu_title="Select Integration",  # Title of the menu
     options=[option["label"] for option in menu_options],  # Displayed options
@@ -379,3 +380,7 @@ if selected_option == "External users":
 elif selected_option == "Internal users":
     with open('ap.py') as file:
         exec(file.read())
+
+elif selected_option == "Recon checking":
+    with open('recon.py') as file:
+        exec(file.read())

queries.py

@@ -0,0 +1,344 @@
+
+queries = {
+    # Query 1
+    '1. fynd-db.finance_recon_tool_asia.01_finance_avis_data_final': '''
+        select
+        bag_id,
+        concat(bag_id,Recon_Status,Settlement_type,Transaction_Type) as Merged,
+        count(concat(bag_id,Recon_Status,Settlement_type,Transaction_Type))
+        from `fynd-db.finance_recon_tool_asia.01_finance_avis_data_final`
+        group by
+        1,2
+        having count(concat(bag_id,Recon_Status,Settlement_type,Transaction_Type)) not in (1)
+    ''',
+
+    # Query 2
+    '2. Seller fees date validation': '''
+        select
+        *
+        from `fynd-db.finance_recon_tool_asia.11_seller_fees_daily`
+        where
+        invoice_generation_date is null
+    ''',
+
+    # Query 3
+    '3. Transaction components validation': '''
+        select
+        bag_id,
+        transaction_type,
+        inserted_date,
+        round(transaction_fee+packaging_fee+logistics_charges+refund_support_fees+sla_charges-net_charges,0) as diff
+        from `fynd-db.finance_recon_tool_asia.11_seller_fees_daily`
+        where
+        round(transaction_fee+packaging_fee+logistics_charges+refund_support_fees+sla_charges-net_charges,0) not in (0,-1,-1)
+        and inserted_date > '2023-09-30'
+        group by 1,2,3, transaction_fee,packaging_fee,refund_support_fees,sla_charges,net_charges,logistics_charges
+    ''',
+
+    # Query 4
+    '4. Transactions in net collection validation': '''
+        select
+        concat(bag_id,Settlement_type) as Merged,
+        count(concat(bag_id,Settlement_type))
+        from `fynd-db.finance_recon_tool_asia.09_seller_net_collection_daily`
+        group by 1
+        having  
+        count(concat(bag_id,Settlement_type)) not in (1)
+    ''',
+
+    # Query 5
+    '5. Transactions in seller fees validation': '''
+        select
+        bag_id,
+        concat(bag_id,transaction_type) as Merged,
+        count(concat(bag_id,transaction_type))
+        from `fynd-db.finance_recon_tool_asia.11_seller_fees_daily`
+        group by 1,2
+        having  
+        count(concat(bag_id,transaction_type)) not in (1)
+    ''',
+
+
+    # Query 6
+    '6. Transactions in seller claims validation': '''
+        select
+        current_shipment_id,
+        concat(current_shipment_id,transaction_type) as Merged,
+        count(concat(current_shipment_id,transaction_type))
+        from `fynd-db.finance_recon_tool_asia.12_seller_claims_daily`
+        group by 1,2
+        having  
+        count(concat(current_shipment_id,transaction_type)) not in (1)
+    ''',
+
+
+    # Query 7
+    '7. Aggregate liability validation': '''
+        select
+        *
+        from `fynd-db.finance_recon_tool_asia.12_seller_claims_daily`
+        where
+        aggregate_liability is null
+    ''',
+
+
+    # Query 8
+    '8. Lost claim validation': '''
+        select
+        *
+        from `fynd-db.finance_recon_tool_asia.01_finance_avis_data_final` as A
+        left join
+        `fynd-db.finance_recon_tool_asia.12_seller_claims_daily` as B
+        on
+        A.current_shipment_id = B.current_shipment_id
+        where
+        A.claim_settle_date is not null
+        and A.transaction_type = 'Claim'
+        and A.dp_partner = 'fynd'
+        and B.current_shipment_id is null
+    ''',
+    
+
+
+    # Query 9
+    '9. Gstin validation': '''
+        select
+        company_id,
+        company_name,
+        ordering_channel
+        from `fynd-db.finance_recon_tool_asia.11_seller_fees_daily`
+        where
+        company_gstn is null
+        group by
+        1,2,3
+    ''',
+
+
+
+    # Query 10
+    '10. Positive transaction components validation': '''
+        select
+        *
+        from `fynd-db.finance_recon_tool_asia.11_seller_fees_daily`
+        where
+        transaction_type in ('Sale','SLA','Claim')
+        and net_charges > 0
+    ''',
+
+
+     # Query 11
+    '11. Negative transaction components validation': '''
+        select
+        *
+        from `fynd-db.finance_recon_tool_asia.11_seller_fees_daily`
+        where
+        transaction_type = 'Return'
+        and net_charges < 0
+    ''',
+
+
+    # Query 12
+    '12. GST Tag validation': '''
+        select
+        company_id,
+        company_name,
+        ordering_channel,
+        company_gstn,
+        gst_tag,
+        case WHEN LENGTH(company_gstn) = 10 then "SGST"
+        when SUBSTRING(company_gstn,1,2) = '27' then "SGST" else "IGST" end as Tag,
+        case when gst_tag = (case WHEN LENGTH(company_gstn) = 10 then "SGST"
+        when SUBSTRING(company_gstn,1,2) = '27' then "SGST" else "IGST" end) then "Match" else "Not_Match" end as CC
+        from `fynd-db.finance_recon_tool_asia.11_seller_fees_daily`
+        where
+        (case when gst_tag = (case WHEN LENGTH(company_gstn) = 10 then "SGST"
+        when SUBSTRING(company_gstn,1,2) = '27' then "SGST" else "IGST" end) then "Match" else "Not_Match" end) = "Not_Match"
+        group by 1,2,3,4,5,6
+    ''',
+
+     # Query 13
+    '13. 09_Net collection all data validation': '''
+        select
+        A.company_id,
+        A.company_name,
+        A.ordering_channel,
+        A.bag_id,
+        A.Settlement_type,
+        A.recon_status,
+        A.recon_date,
+        A.inserted_date,
+        B.bag_id,
+        B.Settlement_type
+        from `fynd-db.finance_recon_tool_asia.01_finance_avis_data_final` as A
+        left join
+        `fynd-db.finance_recon_tool_asia.09_seller_net_collection_daily` as B
+        on
+        A.bag_id = B.bag_id
+        and A.Settlement_type = B.Settlement_type
+        where
+        A.Settlement_type not in ('NA','SLA')
+        and A.recon_status in ('delivery_done','return_bag_delivered','return_bag_picked')
+        and B.bag_id is null 
+        and (case when A.Settlement_type = 'collection' and A.collection_partner = 'fynd' then 'yes' 
+        when A.Settlement_type = 'refund' and A.refund_partner = 'fynd' then 'yes' else 'no' end) = 'yes'
+        group by
+        1,2,3,4,5,6,7,8,9,10
+    ''',
+
+
+     # Query 14
+    '14. Net collection collection & refund validation': '''
+        select
+        bag_id,
+        settlement_type,
+        collection_partner,
+        refund_partner,
+        case when settlement_type in ('collection','Claim',"dispute","rectify","rectify_R") and collection_partner = 'fynd' then 'Yes' when settlement_type = 'refund' and refund_partner = 'fynd' then 'Yes' else 'No' end as Comment
+        from `fynd-db.finance_recon_tool_asia.09_seller_net_collection_daily`
+        where
+        (case when settlement_type in ('collection','Claim',"dispute","rectify","rectify_R") and collection_partner = 'fynd' then 'Yes' when settlement_type = 'refund' and refund_partner = 'fynd' then 'Yes' else 'No' end) = 'No'
+    ''',
+
+
+    # Query 15
+    '15. 09_Net collection collection & refund validation': '''
+        select
+        A.bag_id,
+        B.bag_id,
+        A.settlement_type,
+        B.settlement_type,
+        A.transaction_type,
+        A.collection_partner,
+        A.refund_partner,
+        case when A.settlement_type = 'collection' and A.collection_partner = 'fynd' then 'Yes' when A.settlement_type = 'refund' and A.refund_partner = 'fynd' then 'Yes' else 'No' end as Comment
+        from `fynd-db.finance_recon_tool_asia.01_finance_avis_data_final` as A
+        left join
+        `fynd-db.finance_recon_tool_asia.09_seller_net_collection_daily` as B
+        on
+        A.bag_id = B.bag_id 
+        and A.settlement_type = B.settlement_type
+        where
+        A.settlement_type <> 'NA'
+        and A.recon_status not in ("bag_lost","return_bag_lost")
+        and (case when A.settlement_type = 'collection' and A.collection_partner = 'fynd' then 'yes' when A.settlement_type = 'refund' and A.refund_partner = 'fynd' then 'yes' else 'No' end) = 'yes'  
+        and (case when A.transaction_type = 'Sale' and A.collection_partner = 'fynd' then 'Yes' when A.transaction_type = 'Return' and A.refund_partner = 'fynd' then 'Yes' when A.transaction_type = 'Claim' and A.collection_partner = 'fynd' then 'Yes' else 'No' end) = 'Yes'
+        and B.bag_id is null
+    ''',
+
+
+    # Query 16
+    '16. NA settlement validation': '''
+        select
+        *
+        from `fynd-db.finance_recon_tool_asia.09_seller_net_collection_daily`
+        where
+        -- settlement_type = 'NA'
+        (case when settlement_type in ('collection','Claim','rectify','rectify_R') and collection_partner = 'fynd' then 'Yes' when settlement_type = 'refund' and refund_partner = 'fynd' then 'Yes' else 'No'end ) = 'No'
+    ''',
+
+
+    # Query 17
+    '17. Fynd collection placed bags validation': '''
+        select
+        A.bag_id,
+        B.bag_id,
+        A.order_type,
+        A.transaction_type,
+        case when B.settlement_type = 'collection' then 'Sale' else 'Return' end as transaction_type,
+        from `fynd-db.finance_recon_tool_asia.11_seller_fees_daily` as A
+        left join
+        `fynd-db.finance_recon_tool_asia.09_seller_net_collection_daily` as B
+        on
+        A.bag_id = B.bag_id
+        and A.transaction_type = (case when B.settlement_type = 'collection' then 'Sale' when B.settlement_type = 'refund' then 'Return' else 'NA' end )
+        where
+        transaction_type <> 'SLA'
+        and (case when A.transaction_type = 'Sale' and A.collection_partner = 'fynd' then 'yes' when A.transaction_type = 'Return' and A.refund_partner = 'fynd' then 'yes' else 'no'end) = 'yes'
+        and B.bag_id is null
+    ''',
+
+
+    #  # Query 18
+    # '18. Disbursement table validation': '''
+    #     select
+    #     A.sett_id,
+    #     B.sett_id,
+    #     NP,
+    #     net_amount,
+    #     case when A.sett_id = B.sett_id and round(NP - net_amount,0) in (0,1) then 'Match' else "Not_Match" end as Diff
+    #     from
+    #     (SELECT
+    #     sett_id,
+    #     SUM(seller_net_collection) AS NP
+    #     FROM
+    #     `fynd-db.finance_recon_tool_asia.09_seller_net_collection_daily`
+    #     GROUP BY
+    #     1
+    #     UNION ALL
+    #     SELECT
+    #     sett_id,
+    #     SUM(total_charges) AS NP
+    #     FROM
+    #     `fynd-db.finance_recon_tool_asia.11_seller_fees_daily`
+    #     GROUP BY
+    #     1
+    #     UNION ALL
+    #     SELECT
+    #     sett_id,
+    #     SUM(claimable_amt) AS NP
+    #     FROM
+    #     `fynd-db.finance_recon_tool_asia.12_seller_claims_daily`
+    #     GROUP BY
+    #     1
+    #     UNION ALL
+    #     SELECT
+    #     sett_id,
+    #     SUM(dispute_amount) AS NP
+    #     FROM
+    #     `fynd-db.finance_recon_tool_asia.17_seller_manual_Dispute`
+    #     GROUP BY
+    #     1)  as A
+    #     --left join
+    #     right join
+    #     (select
+    #     sett_id,
+    #     sum(net_amount) as net_amount
+    #     from  
+    #     `fynd-db.finance_recon_tool_asia.disbursement_summary`
+    #     group by
+    #     1) as B
+    #     on
+    #     A.sett_id = B.sett_id
+    #     where
+    #     (case when A.sett_id = B.sett_id and round(NP - net_amount,0) in (0,1) then 'Match' else "Not_Match" end) = 'Not_Match'
+    # ''',
+
+
+    # Query 19
+    '18. PPD validation': '''
+        select
+        *
+        from
+        (SELECT
+        *
+        FROM
+        `fynd-db.Outstanding.09_Payable_File`
+        where
+        expected_payout_date <= current_date()
+        and order_type = "PPD"
+        and lower(collection_partner) = "seller") as A
+        left join
+        (select
+        bag_id,
+        txn_id,
+        collected_amount
+        from
+        `fynd-db.finance_recon_tool_asia.05_partner_collection`) as B
+        on 
+        A.bag_id = B.bag_id
+        where
+        B.bag_id is not null
+    ''',
+
+    # Add more queries as needed
+}

recon.py

@@ -0,0 +1,285 @@
+from google.cloud import bigquery
+import functions_framework
+from queries import queries
+from google.oauth2 import service_account
+import json
+import requests
+import streamlit as st
+import pyperclip
+from ap import send_message_via_webhook, Webhook_urls
+
+ # Dropdown for channels/members
+webhook_url = list(Webhook_urls.keys())
+html_subject = """
+<html>
+<head>
+<style>
+    .button {
+        display: inline-block;
+        padding: 10px 20px;
+        border-radius: 12px;
+        background: linear-gradient(to bottom, #f8f9fa, #e0e0e0);
+        box-shadow: 
+            0 6px 12px rgba(0, 0, 0, 0.3), 
+            0 8px 16px rgba(0, 0, 0, 0.2), 
+            inset 0 -2px 4px rgba(255, 255, 255, 0.6);
+        text-align: center;
+        position: relative;
+        transform: translateY(4px);
+        transition: transform 0.2s ease-in-out, box-shadow 0.2s ease-in-out;
+        cursor: pointer;
+        user-select: none;
+    }
+    .button:hover {
+        box-shadow: 
+            0 8px 16px rgba(0, 0, 0, 0.3), 
+            0 12px 24px rgba(0, 0, 0, 0.2);
+        transform: translateY(2px);
+    }
+    .button:active {
+        box-shadow: 
+            0 4px 8px rgba(0, 0, 0, 0.3), 
+            0 6px 12px rgba(0, 0, 0, 0.2);
+        transform: translateY(0);
+    }
+</style>
+</head>
+<body>
+    <div class="button">
+        <h3 style="
+            font-size: 20px;
+            color: #ffffff;
+            background-image: linear-gradient(to right, #800000, #ff0000, #ffdab9);
+            background-clip: text;
+            -webkit-background-clip: text;
+            text-fill-color: transparent;
+            -webkit-text-fill-color: transparent;
+            margin: 0;
+            text-shadow: 0 2px 5px rgba(0, 0, 0, 0.4);
+        ">Select channels/members</h3>
+    </div>
+</body>
+</html>
+"""
+
+
+st.markdown(html_subject, unsafe_allow_html=True)
+selection = st.multiselect("", webhook_url)
+# SLACK_WEBHOOK_URL = "https://hooks.slack.com/services/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"
+
+
+
+def check_duplicates(credentials_file):
+    """Check for duplicates using BigQuery with the provided credentials file."""
+    results = {}
+    credentials = service_account.Credentials.from_service_account_info(json.loads(credentials_file))
+    client = bigquery.Client(credentials=credentials, project=credentials.project_id)
+
+    for i, (query_name, query) in enumerate(queries.items()):
+        
+        query_job = client.query(query)
+        df = query_job.result().to_dataframe()
+        # For debugging, write the DataFrame to the Streamlit app
+        st.write(f"{query_name}:", df)
+
+        button_styles = """
+                    <style>
+                    div.stButton > button {
+                        color: #ffffff; /* Text color */
+                        font-size: 30px;
+                        background-image: linear-gradient(to right, #800000, #ff0000); /* Maroon to light red gradient */
+                        border: none;
+                        padding: 10px 20px;
+                        cursor: pointer;
+                        border-radius: 15px;
+                        display: inline-block;
+                        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1), 0 8px 15px rgba(0, 0, 0, 0.1); /* Box shadow */
+                        transition: all 0.3s ease; /* Smooth transition on hover */
+                    }
+                    div.stButton > button:hover {
+                        background-color: #00ff00; /* Hover background color */
+                        color: #ff0000; /* Hover text color */
+                        box-shadow: 0 6px 10px rgba(0, 0, 0, 0.2), 0 12px 20px rgba(0, 0, 0, 0.2); /* Box shadow on hover */
+                    }
+                    </style>
+                """
+        st.markdown(button_styles, unsafe_allow_html=True)
+        if st.button(f"Copy Query", key=f"copy_query_{i}"):
+            pyperclip.copy(query)
+            st.success('Query copied to clipboard!')
+
+        if not df.empty:
+            duplicate_count = len(df)
+            results[query_name] = duplicate_count 
+
+    return results
+
+# Streamlit UI
+html_subject = """
+    <html>
+    <head>
+    <style>
+        .button {
+            display: inline-block;
+            padding: 10px 20px;
+            border-radius: 12px;
+            background: linear-gradient(to bottom, #f8f9fa, #e0e0e0);
+            box-shadow: 
+                0 6px 12px rgba(0, 0, 0, 0.3), 
+                0 8px 16px rgba(0, 0, 0, 0.2), 
+                inset 0 -2px 4px rgba(255, 255, 255, 0.6);
+            text-align: center;
+            position: relative;
+            transform: translateY(4px);
+            transition: transform 0.2s ease-in-out, box-shadow 0.2s ease-in-out;
+            cursor: pointer;
+            user-select: none;
+        }
+        .button:hover {
+            box-shadow: 
+                0 8px 16px rgba(0, 0, 0, 0.3), 
+                0 12px 24px rgba(0, 0, 0, 0.2);
+            transform: translateY(2px);
+        }
+        .button:active {
+            box-shadow: 
+                0 4px 8px rgba(0, 0, 0, 0.3), 
+                0 6px 12px rgba(0, 0, 0, 0.2);
+            transform: translateY(0);
+        }
+    </style>
+    </head>
+    <body>
+        <div class="button">
+            <h3 style="
+                font-size: 20px;
+                color: #ffffff;
+                background-image: linear-gradient(to right, #800000, #ff0000, #ffdab9);
+                background-clip: text;
+                -webkit-background-clip: text;
+                text-fill-color: transparent;
+                -webkit-text-fill-color: transparent;
+                margin: 0;
+                text-shadow: 0 2px 5px rgba(0, 0, 0, 0.4);
+            ">Upload service a/c credentials</h3>
+        </div>
+    </body>
+    </html>
+    """
+
+st.markdown(html_subject, unsafe_allow_html=True)
+
+# Upload credentials file
+credentials_file = st.file_uploader("", type="json")
+
+if credentials_file is not None:
+    # Read the credentials file
+    credentials_data = credentials_file.read().decode("utf-8")
+    
+    # Check for duplicates
+    results = check_duplicates(credentials_data)
+    st.write("")
+    st.write("")
+    
+    if results:
+    # Define the HTML message with gradient text and extra spacing
+        html_subject = """
+        <html>
+        <head>
+        <style>
+            .button {
+                display: inline-block;
+                padding: 10px 20px;
+                border-radius: 12px;
+                background: linear-gradient(to bottom, #f8f9fa, #e0e0e0);
+                box-shadow: 
+                    0 6px 12px rgba(0, 0, 0, 0.3), 
+                    0 8px 16px rgba(0, 0, 0, 0.2), 
+                    inset 0 -2px 4px rgba(255, 255, 255, 0.6);
+                text-align: center;
+                position: relative;
+                transform: translateY(4px);
+                transition: transform 0.2s ease-in-out, box-shadow 0.2s ease-in-out;
+                cursor: pointer;
+                user-select: none;
+            }
+            .button:hover {
+                box-shadow: 
+                    0 8px 16px rgba(0, 0, 0, 0.3), 
+                    0 12px 24px rgba(0, 0, 0, 0.2);
+                transform: translateY(2px);
+            }
+            .button:active {
+                box-shadow: 
+                    0 4px 8px rgba(0, 0, 0, 0.3), 
+                    0 6px 12px rgba(0, 0, 0, 0.2);
+                transform: translateY(0);
+            }
+            .spacing {
+                margin: 20px 0; /* Adjust the value as needed */
+            }
+            .result-box {
+                border: 1px solid #ddd;
+                border-radius: 8px;
+                background: #f9f9f9;
+                box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+                padding: 10px;
+                margin-bottom: 10px;
+                font-size: 16px;
+            }
+            .result-title {
+                font-weight: bold;
+                color: #333;
+            }
+        </style>
+        </head>
+        <body>
+            <div class="button">
+                <h3 style="
+                    font-size: 20px;
+                    color: #ffffff;
+                    background-image: linear-gradient(to right, #800000, #ff0000, #ffdab9);
+                    background-clip: text;
+                    -webkit-background-clip: text;
+                    text-fill-color: transparent;
+                    -webkit-text-fill-color: transparent;
+                    margin: 0;
+                    text-shadow: 0 2px 5px rgba(0, 0, 0, 0.4);
+                ">Duplicate Counts for Queries:</h3>
+            </div>
+            <div class="spacing"></div> <!-- Adds spacing between the heading and the content -->
+        </body>
+        </html>
+        """
+
+        # Display HTML message with spacing
+        st.markdown(html_subject, unsafe_allow_html=True)
+
+        # Prepare the plain text message with styled boxes
+        message_html = ""
+        for query_name, count in results.items():
+            message_html += f"""
+            <div class="result-box">
+                <div class="result-title">{query_name}</div>
+                <div>{count} duplicate rows</div>
+            </div>
+            """
+
+        # Display the styled message in Streamlit
+        st.markdown(message_html, unsafe_allow_html=True)
+
+        # Prepare plain text for Slack messages
+        message_text = ""
+        for query_name, count in results.items():
+            message_text += f"*{query_name}*\n{count} duplicate rows\n\n"
+
+        if not results:
+            message_text = "No duplicates found in the queries."
+
+        # Send the message to selected channels
+        for channel in selection:
+            webhook_url = Webhook_urls.get(channel)
+        if webhook_url:
+            send_message_via_webhook(message_text, webhook_url)
+        else:
+            st.error(f"Webhook URL not found for channel: {channel}")

requirements.txt

@@ -124,6 +124,7 @@ pyngrok==7.2.0
 pyOpenSSL==24.2.1
 pyparsing==3.1.2
 PyPDF2==3.0.1
+pyperclip==1.9.0
 PySocks==1.7.1
 pytesseract==0.3.10
 python-dateutil==2.9.0.post0
@@ -184,6 +185,7 @@ websocket-client==1.8.0
 Werkzeug==3.0.3
 wrapt==1.16.0
 wsproto==1.2.0
+xlrd==2.0.1
 XlsxWriter==3.2.0
 xxhash==3.4.1
 yarl==1.9.4