init

.gitignore

@@ -0,0 +1,131 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+secret*
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.streamlit
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

app.py

@@ -0,0 +1,545 @@
+from youtube_transcript_api import YouTubeTranscriptApi as yta
+from youtube_transcript_api import NoTranscriptFound, TranscriptsDisabled
+import streamlit as st
+from yt_stats import YTstats
+from datetime import datetime
+import isodate
+import pandas as pd
+import deeppunkt
+import time
+import lexrank 
+import mysheet 
+
+def time_it(func):
+    def wrapper(*args, **kwargs):
+        start = time.time()
+        result = func(*args, **kwargs)
+        end = time.time()
+        elapsed = end - start
+        #st.write(f"Elapsed time: {end - start}")
+        st.write('Load time: '+str(round(elapsed,1))+' sec')
+        return result
+    return wrapper
+
+def reset_session():
+    if 'punkt' in st.session_state:
+        del st.session_state.punkt
+    if 'extract' in st.session_state:
+        del st.session_state.extract
+    if 'channel_id' in st.session_state:
+        del st.session_state.channel_id
+
+def update_param_example():
+    #st.session_state.url_vid = st.session_state.ex_vid
+    video_id = get_id_from_link(st.session_state.ex_vid) 
+    st.experimental_set_query_params(vid=video_id)
+    reset_session()
+
+def update_param_textinput():
+    #st.session_state.url_vid = st.session_state.ti_vid
+    video_id = get_id_from_link(st.session_state.ti_vid) 
+    st.experimental_set_query_params(vid=video_id)
+    reset_session()
+
+def get_link_from_id(video_id):
+    if "v=" not in video_id:
+        return 'https://www.youtube.com/watch?v='+video_id
+    else:
+        return video_id
+
+
+def get_id_from_link(link):
+    if "v=" in link:
+        return link.split("v=")[1].split("&")[0]
+    elif len(link)==11:
+        return link
+    else:
+        return "Error: Invalid Link."
+
+# @st.cache(allow_output_mutation=True, suppress_st_warning=True)
+# def retry_access_yt_object(url, max_retries=5, interval_secs=5, on_progress_callback=None):
+#     """
+#     Retries creating a YouTube object with the given URL and accessing its title several times
+#     with a given interval in seconds, until it succeeds or the maximum number of attempts is reached.
+#     If the object still cannot be created or the title cannot be accessed after the maximum number
+#     of attempts, the last exception is raised.
+#     """
+#     last_exception = None
+#     for i in range(max_retries):
+#         try:
+#             yt = YouTube(url, on_progress_callback=on_progress_callback)
+#             #title = yt.title  # Access the title of the YouTube object.
+#             #views = yt.views
+#             return yt  # Return the YouTube object if successful.
+#         except Exception as err:
+#             last_exception = err  # Keep track of the last exception raised.
+#             st.write(f"Failed to create YouTube object or access title. Retrying... ({i+1}/{max_retries})")
+#             time.sleep(interval_secs)  # Wait for the specified interval before retrying.
+
+#     # If the YouTube object still cannot be created or the title cannot be accessed after the maximum number of attempts, raise the last exception.
+#     raise last_exception
+
+@st.cache_data()
+def get_video_data(_yt, video_id):
+
+    yt_img = f'http://img.youtube.com/vi/{video_id}/mqdefault.jpg'
+    yt_img_html = '<img src='+yt_img+' width="250" height="150" />'
+    yt_img_html_link = '<a href='+url+'>'+yt_img_html+'</a>'
+
+    snippet = yt._get_single_video_data(video_id,'snippet')
+    yt_publish_date = snippet['publishedAt']
+    yt_title = snippet['title']
+    yt_author = snippet['channelTitle']
+    yt_channel_id = snippet['channelId'] 
+
+    try:
+        yt_keywords = snippet['tags']
+    except:
+        yt_keywords = []
+    
+
+    statistics = yt._get_single_video_data(video_id,'statistics')
+    yt_views = statistics['viewCount']
+    contentDetails = yt._get_single_video_data(video_id,'contentDetails')
+    yt_length = contentDetails['duration']
+    yt_length_isodate = isodate.parse_duration(yt_length)
+    yt_length_isoformat = isodate.duration_isoformat(yt_length_isodate, "%H:%M:%S")[1:]
+
+    data = {'Video':[yt_img_html_link],
+            'Author': [yt_author],
+            'Title': [yt_title],
+            'Published': [datetime.strptime(yt_publish_date, '%Y-%m-%dT%H:%M:%SZ').strftime('%B %d, %Y')],
+            'Views':[format(int(yt_views), ",").replace(",", "'")],
+            'Length':[yt_length_isoformat]}
+    
+    return data, yt_keywords, yt_channel_id
+
+
+@st.cache_data()
+def get_video_data_from_gsheed(df, video_id):
+    
+    yt_img_html_link = df.loc[df["ID"] == video_id]['Video'].to_list()[0]
+    yt_author = df.loc[df["ID"] == video_id]['Author'].to_list()[0]
+    yt_title = df.loc[df["ID"] == video_id]['Title'].to_list()[0]
+    yt_publish_date = df.loc[df["ID"] == video_id]['Published'].to_list()[0]
+    yt_views = df.loc[df["ID"] == video_id]['Views'].to_list()[0]
+    yt_length_isoformat = df.loc[df["ID"] == video_id]['Length'].to_list()[0]
+    yt_keywords = df.loc[df["ID"] == video_id]['Keywords'].to_list()[0].split(';')
+    yt_channel_id = df.loc[df["ID"] == video_id]['Channel'].to_list()[0]
+
+    data = {'Video':[yt_img_html_link],
+            'Author': [yt_author],
+            'Title': [yt_title],
+            'Published': [yt_publish_date],
+            'Views':[yt_views],
+            'Length':[yt_length_isoformat]}
+    
+    return data, yt_keywords, yt_channel_id
+
+@time_it
+def get_punctuated_text(raw_text):    
+    response = deeppunkt.predict('sentences',raw_text)
+    st.session_state['punkt'] = response
+
+
+def get_punctuated_text_to_dict(raw_text):
+    #st.session_state['punkt'] = {'data':[raw_text,0,0,0,0], 'duration':0}
+    st.session_state['punkt'] = [raw_text,0,0,0,0]
+
+
+@time_it
+def get_extracted_text(raw_text):
+
+    response = lexrank.summarize(raw_text)
+    st.session_state['extract'] = response
+
+def get_extracted_text_to_dict(raw_text):
+    st.session_state['extract'] = [raw_text,0,0,0,0]
+
+
+#######################################################################################
+# Application Start 
+#######################################################################################
+
+
+st.title("Transcriptifier")
+st.subheader("Youtube Transcript Downloader")
+
+example_urls = [
+    'https://www.youtube.com/watch?v=8uQDDUfGNPA', # blog
+    'https://www.youtube.com/watch?v=ofZEo0Rzo5s', # h-educate
+    'https://www.youtube.com/watch?v=ReHGSGwV4-A', #wholesale ted
+    'https://www.youtube.com/watch?v=n8JHnLgodRI', #kevindavid
+    'https://www.youtube.com/watch?v=6MI0f6YjJIk', # Nicholas
+    'https://www.youtube.com/watch?v=nr4kmlTr9xw', # Linus
+    'https://www.youtube.com/watch?v=64Izfm24FKA', # Yannic
+    'https://www.youtube.com/watch?v=Mt1P7p9HmkU', # Fogarty
+    'https://www.youtube.com/watch?v=bj9snrsSook', #Geldschnurrbart
+    'https://www.youtube.com/watch?v=0kJz0q0pvgQ', # fcc
+    'https://www.youtube.com/watch?v=gNRGkMeITVU', # iman
+    'https://www.youtube.com/watch?v=vAuQuL8dlXo', #ghiorghiu
+    'https://www.youtube.com/watch?v=5scEDopRAi0', #infohaus
+    'https://www.youtube.com/watch?v=lCnHfTHkhbE', #fcc tutorial
+    'https://www.youtube.com/watch?v=QI2okshNv_4'
+]
+
+
+par_vid = st.experimental_get_query_params().get("vid")
+if par_vid:
+    par_url = par_vid[0]
+else:
+    par_url = None
+
+select_examples = st.selectbox(label="Choose an example",options=example_urls, key='ex_vid', on_change=update_param_example)
+url = st.text_input("Or Enter the YouTube video URL or ID:", value=par_url if par_url else select_examples, key='ti_vid', on_change=update_param_textinput)
+
+
+########################
+# Load the data for a given video
+########################
+
+
+API_KEY = st.secrets["api_key"]
+yt = YTstats(API_KEY)
+#yt = retry_access_yt_object(get_link_from_id(url))
+
+if url:
+    video_id = get_id_from_link(url)
+    
+    if 'gsheed' not in st.session_state:
+        df = mysheet.read_gspread()
+        st.session_state.gsheed = df 
+        #st.write("reading spradsheet")
+    else:
+        df = st.session_state.gsheed
+        #st.write("getting spreadsheed from session_state") 
+    
+    gslist=[]
+    try:
+        gslist = df.ID.to_list()
+    except:
+        st.write('no items available.')
+        
+    if video_id in gslist:
+        #st.write(df.loc[df["ID"] == video_id])
+        st.write("reading from sheet")
+        #transcript_item_is_generated = False
+        #transcript_text = df.loc[df["ID"] == video_id]['Punkttext'].to_list()[0]
+        #get_punctuated_text_to_dict(transcript_text)
+        extracted_text = df.loc[df["ID"] == video_id]['Lextext'].to_list()[0]
+        get_extracted_text_to_dict(extracted_text)
+
+        video_data, yt_keywords, yt_channel_id = get_video_data_from_gsheed(df, video_id)
+    else:
+        st.write("reading from api")
+        video_data, yt_keywords, yt_channel_id = get_video_data(yt, video_id)
+
+    st.session_state["video_data"] = video_data
+    st.session_state["keywords"] = yt_keywords
+    st.session_state["channel_id"] = yt_channel_id 
+
+
+df = pd.DataFrame(st.session_state["video_data"])
+st.markdown(df.style.hide(axis="index").to_html(), unsafe_allow_html=True)
+st.write("")
+
+###########################
+# Load Transcript
+###########################
+
+transcript_list = yta.list_transcripts(video_id)
+
+transcript_raw = None
+transcript_item = transcript_list.find_transcript(['en'])
+transcript_item_is_generated = transcript_item.is_generated
+transcript_raw = transcript_item.fetch()
+
+if transcript_raw is None:
+    st.error("No transcript available.")
+    st.stop()
+
+transcript_text = '\n'.join([i['text'].replace('\n',' ') for i in transcript_raw])
+
+########################
+# Load Author Keywords, that are not viewable by users
+########################
+
+keywords_data = {'Authors Keywords':yt_keywords}
+st.table(keywords_data)
+st.write("")
+
+# TODO
+# or this video (bj9snrsSook) transcripts are available in the following languages:
+
+# (MANUALLY CREATED)
+# None
+
+# (GENERATED)
+#  - de ("Deutsch (automatisch erzeugt)")[TRANSLATABLE]
+
+# (TRANSLATION LANGUAGES)
+#  - af ("Afrikaans")
+
+
+########################
+# Display the transcript along with the download button
+########################
+
+with st.expander('Preview Transcript'):
+    st.code(transcript_text, language=None)
+st.download_button('Download Transcript', transcript_text)
+
+########################
+# API Call to deeppunkt-gr
+########################
+
+
+st.subheader("Restore Punctuations of Transcript")
+if not transcript_item_is_generated:
+    st.write("Transcript is punctuated by author.")
+    # TODO
+    #check if the transcript contains more than 5 sentences
+
+if st.button('Load Punctuated Transcript'):
+    with st.spinner('Loading Punctuation...'):
+        if 'punkt' not in st.session_state:
+            # first figure out if transcript is already punctuated
+            if transcript_item_is_generated:
+                get_punctuated_text(transcript_text)
+            else:
+                get_punctuated_text_to_dict(transcript_text)
+    #st.write('Load time: '+str(round(st.session_state.punkt['duration'],1))+' sec')
+    metrics_data = {'Words':[int(st.session_state.punkt[1])],
+            'Sentences': [int(st.session_state.punkt[2])],
+            'Characters': [int(st.session_state.punkt[3])],
+            'Tokens':[int(st.session_state.punkt[4])]}
+    df = pd.DataFrame(metrics_data)
+    st.markdown(df.style.hide(axis="index").to_html(), unsafe_allow_html=True)
+    st.write("")
+    with st.expander('Preview Transcript'):
+        st.code(st.session_state.punkt[0], language=None)
+
+########################
+#  Call to lexrank-gr
+########################
+
+st.subheader("Extract Core Sentences from Transcript")
+
+if st.button('Extract Sentences'):
+    # decide if the extract is already available, if not, text has to be punctuated first
+    with st.spinner('Loading Extractions ...'):
+        if 'extract' not in st.session_state:
+            with st.spinner('Loading Punctuation for Extraction ...'):
+                if 'punkt' not in st.session_state:
+                    # first figure out if transcript is already punctuated
+                    if transcript_item_is_generated:
+                        get_punctuated_text(transcript_text)
+                    else:
+                        get_punctuated_text_to_dict(transcript_text)
+
+            get_extracted_text(st.session_state.punkt[0])
+
+    metrics_data = {'Words':[int(st.session_state.extract[1])],
+            'Sentences': [int(st.session_state.extract[2])],
+            'Characters': [int(st.session_state.extract[3])],
+            'Tokens':[int(st.session_state.extract[4])]}
+    
+    df = pd.DataFrame(metrics_data)
+    st.markdown(df.style.hide(axis="index").to_html(), unsafe_allow_html=True)
+    st.write("")
+    
+    with st.expander('Preview Transcript'):
+        st.code(st.session_state.extract[0], language=None)
+
+    ################
+    if 'extract' not in st.session_state:
+        st.error('Please run extraction first.', icon="🚨")
+    else:
+        transcript_info = {'Words':[int(st.session_state.extract[1])],
+                'Sentences': [int(st.session_state.extract[2])],
+                'Characters': [int(st.session_state.extract[3])],
+                'Tokens':[int(st.session_state.extract[4])],
+                'Lextext':[st.session_state.extract[0]]}
+        
+        yt_img = f'http://img.youtube.com/vi/{video_id}/mqdefault.jpg'
+        yt_img_html = '<img src='+yt_img+' width="250" height="150" />'
+        yt_img_html_link = '<a href='+url+'>'+yt_img_html+'</a>'
+        video_info = {'ID': [video_id],
+                'Video':[yt_img_html_link],
+                'Author': [st.session_state["video_data"]["Author"][0]],
+                'Channel':[st.session_state["channel_id"]],
+                'Title': [st.session_state["video_data"]["Title"][0]],
+                'Published': [st.session_state["video_data"]["Published"][0]],
+                'Views':[st.session_state["video_data"]["Views"][0]],
+                'Length':[st.session_state["video_data"]["Length"][0]],
+                'Keywords':['; '.join(st.session_state["keywords"])]}
+        df_current_ts = pd.DataFrame({**video_info,**transcript_info})
+
+        # initial write.
+        #df_new_sheet = pd.concat([df_current_ts])
+        #mysheet.write_gspread(df_new_sheet)
+        #st.write(video_info)
+
+        if 'gsheed' not in st.session_state:
+            df = mysheet.read_gspread()
+            st.session_state.gsheed = df
+
+        df_sheet = st.session_state.gsheed
+        df_current_ts_id = list(df_current_ts.ID)[0]
+        if df_current_ts_id not in list(df_sheet.ID):
+            df_new_sheet = pd.concat([df_sheet,df_current_ts])
+            mysheet.write_gspread(df_new_sheet)
+            st.session_state.gsheed = df_new_sheet
+            st.write('video added to sheet')
+        #else:
+        #    st.write('video already in sheet')
+        #    st.write(df_sheet)
+
+
+#######################
+# write to gspread file
+########################
+
+if st.button('Read Spreadsheet'):
+
+    if 'gsheed' not in st.session_state:
+        df = mysheet.read_gspread()
+        st.session_state.gsheed = df
+    
+    st.write(st.session_state.gsheed)
+
+
+#if st.button('Add to Spreadsheet'):
+    
+    
+
+
+#######################
+# API Call to summarymachine
+########################
+
+# def get_summarized_text(raw_text):
+#     response = requests.post("https://wldmr-summarymachine.hf.space/run/predict", json={
+#         "data": [
+#             raw_text,
+#         ]})
+#     #response_id = response
+#     if response.status_code == 504:
+#         raise "Error: Request took too long (>60sec), please try a shorter text."
+#     return response.json()
+
+# st.subheader("Summarize Extracted Sentences with Flan-T5-large")
+
+# if st.button('Summarize Sentences'):
+#     command = 'Summarize the transcript in one sentence:\n\n'
+#     with st.spinner('Loading Punctuation (Step 1/3)...'):
+#         if 'punkt' not in st.session_state:
+#             # first figure out if transcript is already punctuated
+#             if transcript_item.is_generated:
+#                 get_punctuated_text(transcript_text)
+#             else:
+#                 get_punctuated_text_to_dict(transcript_text)
+#     with st.spinner('Loading Extraction (Step 2/3)...'):
+#         if 'extract' not in st.session_state:
+#             get_extracted_text(st.session_state.punkt['data'][0])
+#     with st.spinner('Loading Summary (Step 3/3)...'):
+#         summary_text = get_summarized_text(command+st.session_state.extract['data'][0])
+#     st.write('Load time: '+str(round(summary_text['duration'],1))+' sec')
+#     with st.expander('Preview Transcript'):
+#         st.write(summary_text['data'][0], language=None)
+
+########################
+# Channel
+########################
+
+
+st.subheader("Other Videos of the Channel")
+#st.write(st.session_state["channel_id"])
+if 'channel_id' not in st.session_state:
+    st.error('Channel ID not available.', icon="🚨")
+else:
+    yt.get_channel_statistics(st.session_state["channel_id"])
+    stats_data = {'Channel ID': [st.session_state["channel_id"]],
+                'Total Views':[format(int(yt.channel_statistics["viewCount"]), ",").replace(",", "'")],
+                'Total Subscribers':[format(int(yt.channel_statistics["subscriberCount"]), ",").replace(",", "'")],
+                'Total Videos':[format(int(yt.channel_statistics["videoCount"]), ",").replace(",", "'")],
+                }
+    df = pd.DataFrame(stats_data)
+    st.markdown(df.style.hide(axis="index").to_html(), unsafe_allow_html=True)
+st.write("")
+
+
+if st.button('Load Videos'):
+
+    progress_text = 'Loading...'
+    loading_bar = st.progress(0, text=progress_text)
+    item_limit=3
+    yt.get_channel_video_data(st.session_state["channel_id"],loading_bar, progress_text, item_limit)
+
+    #with st.spinner('Loading...'):
+        #yt.get_channel_video_data(st.session_state["channel_id"])
+        #videos = scrapetube.get_channel(yt.channel_id, limit=3, sleep=2)
+
+
+    vids_thumbnails = []
+    vids_videoIds = []
+    vids_titles = []
+    vids_lengths = []
+    vids_published= []
+    vids_views= []
+    item=0
+    for video in yt.video_data:
+        if item == item_limit:
+            break
+        item = item+1
+
+        vids_video_id = video
+        vids_url = 'https://www.youtube.com/watch?v='+vids_video_id
+
+        yt_img = f'http://img.youtube.com/vi/{vids_video_id}/mqdefault.jpg'
+        yt_img_html = '<img src='+yt_img+' width="250" height="150" />'
+        yt_img_html_link = '<a href='+vids_url+'>'+yt_img_html+'</a>'
+        vids_thumbnails.append(yt_img_html_link)
+        
+        vids_video_id_link = '<a target="_self" href="/?vid='+vids_video_id+'">'+vids_video_id+'</a>'
+        vids_videoIds.append(vids_video_id_link)
+
+        vids_titles.append(yt.video_data[video]['title'])
+
+        yt_length = yt.video_data[video]['duration']
+        yt_length_isodate = isodate.parse_duration(yt_length)
+        yt_length_isoformat = isodate.duration_isoformat(yt_length_isodate, "%H:%M:%S")[1:]
+        vids_lengths.append(yt_length_isoformat)
+        
+        yt_publish_date = yt.video_data[video]['publishedAt']
+        yt_publish_date_formatted = datetime.strptime(yt_publish_date, '%Y-%m-%dT%H:%M:%SZ').strftime('%B %d, %Y')
+        vids_published.append(yt_publish_date_formatted)
+        
+        yt_views = yt.video_data[video]['viewCount']
+        yt_viws_formatted = format(int(yt_views), ",").replace(",", "'")
+        vids_views.append(yt_viws_formatted)
+
+    df_videos = {'Video': vids_thumbnails,
+                'Video ID':vids_videoIds,
+                'Title':vids_titles,
+                'Published':vids_published,
+                'Views':vids_views,
+                'Length':vids_lengths}
+
+
+    dataset = pd.DataFrame(df_videos)
+    st.markdown(dataset.style.hide(axis="index").to_html(), unsafe_allow_html=True)
+
+
+
+###############
+# End of File #
+###############
+# hide_streamlit_style = """
+# <style>
+# #MainMenu {visibility: hidden;}
+# footer {visibility: hidden;}
+# </style>
+# """
+# st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+

deeppunkt.py

@@ -0,0 +1,73 @@
+from deepmultilingualpunctuation import PunctuationModel
+import re
+import metrics
+        
+def remove_filler_words(transcript):
+    
+    # preserve line brakes
+    transcript_hash = " # ".join(transcript.strip().splitlines())
+    # preprocess the text by removing filler words
+    # Define a list of filler words to remove
+    filler_words = ["um", "uh", "hmm", "ha", "er", "ah", "yeah"]
+    words = transcript_hash.split()
+    clean_words = [word for word in words if word.lower() not in filler_words]
+    input_text_clean = ' '.join(clean_words)
+    # restore the line brakes
+    input_text= input_text_clean.replace(' # ','\n')
+    return input_text
+    # Define a regular expression pattern that matches any filler word surrounded by whitespace or punctuation
+    #pattern = r"(?<=\s|\b)(" + "|".join(fillers) + r")(?=\s|\b)"
+    # Use re.sub to replace the filler words with empty strings
+    #clean_input_text = re.sub(pattern, "", input_text)    
+
+def predict(brakes, transcript):
+
+    input_text = remove_filler_words(transcript)
+    # Do the punctuation restauration
+    model = PunctuationModel()
+    output_text = model.restore_punctuation(input_text)
+
+    # if any of the line brake methods are implemented,
+    # return the text as a single line
+    pcnt_file_cr = output_text
+
+    if 'textlines' in brakes:
+
+        # preserve line brakes
+        srt_file_hash = '# '.join(input_text.strip().splitlines())
+        #srt_file_sub=re.sub('\s*\n\s*','# ',srt_file_strip)
+        srt_file_array=srt_file_hash.split()
+        pcnt_file_array=output_text.split()
+
+        # goal: restore the break points i.e. the same number of lines as the srt file
+        # this is necessary, because each line in the srt file corresponds to a frame from the video
+        if len(srt_file_array)!=len(pcnt_file_array):
+            return "AssertError: The length of the transcript and the punctuated file should be the same: ",len(srt_file_array),len(pcnt_file_array)
+        
+        pcnt_file_array_hash = []
+        for idx, item in enumerate(srt_file_array):
+            if item.endswith('#'):
+                pcnt_file_array_hash.append(pcnt_file_array[idx]+'#')
+            else:
+                pcnt_file_array_hash.append(pcnt_file_array[idx])
+    
+        # assemble the array back to a string
+        pcnt_file_cr=' '.join(pcnt_file_array_hash).replace('#','\n')
+
+    elif 'sentences' in brakes:
+        split_text = output_text.split('. ')
+        pcnt_file_cr = '.\n'.join(split_text)
+    
+    regex1 = r"\bi\b"
+    regex2 = r"(?<=[.?!;])\s*\w"
+    regex3 = r"^\w"
+    pcnt_file_cr_cap = re.sub(regex3, lambda x: x.group().upper(), re.sub(regex2, lambda x: x.group().upper(), re.sub(regex1, "I", pcnt_file_cr)))
+
+    metrics.load_nltk()
+    n_tokens= metrics.num_tokens(pcnt_file_cr_cap)
+    n_sents = metrics.num_sentences(pcnt_file_cr_cap)
+    n_words = metrics.num_words(pcnt_file_cr_cap)
+    n_chars = metrics.num_chars(pcnt_file_cr_cap)
+
+    return pcnt_file_cr_cap, n_words, n_sents, n_chars, n_tokens
+ 

lexrank.py

@@ -0,0 +1,93 @@
+
+
+from sumy.parsers.html import HtmlParser
+from sumy.parsers.plaintext import PlaintextParser
+from sumy.nlp.tokenizers import Tokenizer
+from sumy.summarizers.lex_rank import LexRankSummarizer
+from sumy.nlp.stemmers import Stemmer
+from sumy.utils import get_stop_words
+import metrics
+import os
+import nltk
+
+def summarize(in_text):
+
+    if len(in_text)==0:
+        return 'Error: No text provided', None
+
+    nltk_file = '/home/user/nltk_data/tokenizers/punkt.zip'
+    if os.path.exists(nltk_file):
+        print('nltk punkt file exists in ', nltk_file)
+    else:
+        print("downloading punkt file")
+        nltk.download('punkt')
+
+    in_longtext = []
+    # Discard all senteces that have less than 10 words in them
+    in_text_sentenses = in_text.split('.')
+    
+    for sen in in_text_sentenses:
+        sen_split = sen.split()
+        len_sen_split = len(sen_split)
+        if len_sen_split > 10:
+            in_longtext.append(sen)
+    in_text = '.'.join(in_longtext)+'.'
+    
+    # The size of the summary is limited to 1024
+    # The Lexrank algorith accepts only sentences as a limit
+    # We start with one sentece and check the token size
+    # Then increase the number of sentences until the tokensize 
+    # of the next sentence exceed the limit
+    target_tokens = 1024
+
+    in_sents = metrics.num_sentences(in_text)
+
+    out_text = get_Summary(in_text,1)
+    n_tokens= metrics.num_tokens(out_text)
+    prev_n_tokens=0
+    for sen in range(2, in_sents):
+        if n_tokens >= target_tokens:
+            n_tokens = prev_n_tokens
+            break
+        else:
+            out_text = get_Summary(in_text,sen)
+            prev_n_tokens = n_tokens
+            n_tokens= metrics.num_tokens(out_text)
+    
+    n_sents = metrics.num_sentences(out_text)
+    n_words = metrics.num_words(out_text)
+    n_chars = metrics.num_chars(out_text)
+
+    return out_text, n_words, n_sents, n_chars, n_tokens
+
+def get_Summary(in_text, nr_sentences):
+    
+    #sentences = in_text.split('. ')
+    # summarize small part of the text
+    #nr_sentences = 1 #len(sentences)
+    #print('nr_sentences: '+str(nr_sentences))
+    
+    if nr_sentences == 0:
+        return 'Error: No sentences available', None 
+    list_summary = get_Lexrank(in_text,nr_sentences)
+    # it can happen that for lexrank a sentence consists of multiple actual sentences, 
+    # that are separated with full stops. Then the correspoinding timestamp cannot be found
+    # all items from the lexrank summary must be concatinated and split up by full stops.
+    concat_list_summary = '. '.join([str(item).replace('.','') for item in list_summary])#.split('. ')
+    concat_list_summary = concat_list_summary.replace('\\n','')
+    concat_list_summary = concat_list_summary.replace('. ','.\n')+'.'
+    
+    return concat_list_summary
+
+def get_Lexrank(text, nr_sentences):
+    summary=[]
+    LANGUAGE = "english"
+    SENTENCES_COUNT = nr_sentences
+    parser = PlaintextParser.from_string(text, Tokenizer(LANGUAGE))
+    stemmer = Stemmer(LANGUAGE)
+    summarizer = LexRankSummarizer(stemmer)
+    summarizer.stop_words = get_stop_words(LANGUAGE)
+    for sentence in summarizer(parser.document, SENTENCES_COUNT):
+        summary.append(sentence)
+
+    return summary

metrics.py

@@ -0,0 +1,69 @@
+# Import nltk library for natural language processing
+import nltk
+import os
+from transformers import AutoTokenizer
+
+def load_nltk():
+    nltk_file = '/home/user/nltk_data/tokenizers/punkt.zip'
+    if os.path.exists(nltk_file):
+        print('nltk punkt file exists in ', nltk_file)
+    else:
+        print("downloading punkt file")
+        nltk.download('punkt')
+
+
+# Define a function that takes some text as input and returns the number of tokens
+def token_count(text):
+    # Import the Encoder class from bpe
+    from bpe import Encoder
+    # Create an encoder object with a vocabulary size of 10
+    encoder = Encoder(vocab_size=14735746)
+
+    # Train the encoder on the text
+    encoder.fit(text.split())
+
+    # Encode the text into tokens
+    tokens = encoder.tokenize(text)
+
+    # Return the number of tokens
+    return tokens
+
+def num_tokens(text):
+
+    tokenizer = AutoTokenizer.from_pretrained("gpt2")
+
+    token_ids = tokenizer.encode(text)
+
+    token_size = len(token_ids)
+
+    return token_size
+
+def num_words(text):
+    sentences = nltk.sent_tokenize(text)
+    # Tokenize each sentence into words using nltk.word_tokenize()
+    words = []
+    for sentence in sentences:
+        words.extend(nltk.word_tokenize(sentence))
+    
+    num_words = len(words)
+
+    return num_words
+
+def num_sentences(text):
+    # Tokenize the text into sentences using nltk.sent_tokenize()
+    sentences = nltk.sent_tokenize(text)
+    num_sentences = len(sentences)
+    return num_sentences
+
+
+def num_chars(text):
+    num_characters = len(text)
+    return num_characters
+
+
+# Print out the results
+# print(f"Number of sentences: {num_sentences}")
+# print(f"Number of words: {num_words}")
+# print(f"Number of tokens: {num_tokens}")
+# print(f"Number of trans_tokens: {trans_tokens}")
+# print(f"Number of characters: {num_characters}")

mysheet.py

@@ -0,0 +1,41 @@
+import streamlit as st
+#from google.oauth2 import service_account
+import pandas as pd
+import gspread
+import json
+
+def get_gspread_connection():
+    # Create a connection object.
+    # credentials = service_account.Credentials.from_service_account_info(
+    #     st.secrets["gcp_service_account"],
+    #     scopes=[
+    #         "https://www.googleapis.com/auth/spreadsheets",
+    #     ],
+    # )
+    #client = gspread.authorize(credentials)
+
+    st_credentials = st.secrets["gcp_service_account"]
+    if type(st_credentials) is str:
+        print("INFO: transforming str to dict")
+        credentials_dict = json.loads(st_credentials, strict=False)
+        client = gspread.service_account_from_dict(credentials_dict)
+    else:
+        print("INFO: using credentials in dict")
+        client = gspread.service_account_from_dict(st_credentials)
+
+
+    st_sheet_url = st.secrets["private_gsheets_url"]
+    spreadsheet = client.open_by_url(st_sheet_url)
+    worksheet = spreadsheet.get_worksheet(0)
+    return worksheet
+
+#@st.cache_data
+def read_gspread():
+    worksheet = get_gspread_connection()
+    df = pd.DataFrame(worksheet.get_all_records())
+    return df
+
+def write_gspread(df):
+    #df.loc[len(df)] = ['Mia','worst']
+    worksheet = get_gspread_connection()
+    worksheet.update([df.columns.values.tolist()] + df.values.tolist())

requirements.txt

@@ -0,0 +1,15 @@
+streamlit
+youtube-transcript-api
+pandas
+requests
+transformers
+torch
+sentencepiece
+deepmultilingualpunctuation
+nltk
+sumy
+google
+google-auth
+google-auth-oauthlib
+gspread
+isodate

yt_stats.py

@@ -0,0 +1,160 @@
+import json
+import requests
+from tqdm import tqdm
+import isodate
+
+class YTstats:
+
+    def __init__(self, api_key):
+        self.api_key = api_key
+        self.channel_statistics = None
+        self.video_data = None
+
+    def extract_all(self, channel_id):
+        self.get_channel_statistics(channel_id)
+        self.get_channel_video_data(channel_id)
+
+    def get_channel_statistics(self, channel_id):
+        """Extract the channel statistics"""
+        print('get channel statistics...')
+        url = f'https://www.googleapis.com/youtube/v3/channels?part=statistics&id={channel_id}&key={self.api_key}'
+        #pbar = tqdm(total=1)
+        
+        json_url = requests.get(url)
+        data = json.loads(json_url.text)
+        try:
+            data = data['items'][0]['statistics']
+        except KeyError:
+            print('Could not get channel statistics')
+            data = {}
+
+        self.channel_statistics = data
+        #pbar.update()
+        #pbar.close()
+        return data
+
+    def get_channel_video_data(self, channel_id, loading_bar, progress_text, item_limit=3):
+        "Extract all video information of the channel"
+        print('get video data...')
+        channel_videos, channel_playlists = self._get_channel_content(channel_id, limit=50)
+
+        channel_videos_out = dict()
+
+        total_items = len(channel_videos)
+        item = 0
+        step_size=0
+        step=0
+        if total_items!=0:
+            step_size=round(1/total_items,4)
+        #step = step_size
+        parts=["snippet", "statistics","contentDetails", "topicDetails"]
+        for video_id in tqdm(channel_videos):
+            if item == item_limit:
+                break
+            
+            loading_bar.progress(step, text=progress_text)
+
+            for part in parts:
+                data = self._get_single_video_data(video_id, part)
+                channel_videos[video_id].update(data)
+            
+            duration = isodate.parse_duration(channel_videos[video_id]['duration'])
+            short_duration = isodate.parse_duration('PT4M')
+
+            if duration > short_duration:
+                item = item+1
+                step = step +step_size
+                channel_videos_out[video_id] = channel_videos[video_id]
+                
+
+        step=1.0
+        loading_bar.progress(step, text=progress_text)
+        self.video_data = channel_videos_out
+        
+
+    def _get_single_video_data(self, video_id, part):
+        """
+        Extract further information for a single video
+        parts can be: 'snippet', 'statistics', 'contentDetails', 'topicDetails'
+        """
+
+        url = f"https://www.googleapis.com/youtube/v3/videos?part={part}&id={video_id}&key={self.api_key}"
+        json_url = requests.get(url)
+        data = json.loads(json_url.text)
+        try:
+            data = data['items'][0][part]
+        except KeyError as e:
+            print(f'Error! Could not get {part} part of data: \n{data}')
+            data = dict()
+        return data
+
+    def _get_channel_content(self, channel_id, limit=None, check_all_pages=True):
+        """
+        Extract all videos and playlists, can check all available search pages
+        channel_videos = videoId: title, publishedAt
+        channel_playlists = playlistId: title, publishedAt
+        return channel_videos, channel_playlists
+        """
+        url = f"https://www.googleapis.com/youtube/v3/search?key={self.api_key}&channelId={channel_id}&part=snippet,id&order=date"
+        if limit is not None and isinstance(limit, int):
+            url += "&maxResults=" + str(limit)
+
+        vid, pl, npt = self._get_channel_content_per_page(url)
+        idx = 0
+        while(check_all_pages and npt is not None and idx < 10):
+            nexturl = url + "&pageToken=" + npt
+            next_vid, next_pl, npt = self._get_channel_content_per_page(nexturl)
+            vid.update(next_vid)
+            pl.update(next_pl)
+            idx += 1
+
+        return vid, pl
+
+    def _get_channel_content_per_page(self, url):
+        """
+        Extract all videos and playlists per page
+        return channel_videos, channel_playlists, nextPageToken
+        """
+        json_url = requests.get(url)
+        data = json.loads(json_url.text)
+        channel_videos = dict()
+        channel_playlists = dict()
+        if 'items' not in data:
+            print('Error! Could not get correct channel data!\n', data)
+            return channel_videos, channel_videos, None
+
+        nextPageToken = data.get("nextPageToken", None)
+
+        item_data = data['items']
+        for item in item_data:
+            try:
+                kind = item['id']['kind']
+                published_at = item['snippet']['publishedAt']
+                title = item['snippet']['title']
+                if kind == 'youtube#video':
+                    video_id = item['id']['videoId']
+                    channel_videos[video_id] = {'publishedAt': published_at, 'title': title}
+                elif kind == 'youtube#playlist':
+                    playlist_id = item['id']['playlistId']
+                    channel_playlists[playlist_id] = {'publishedAt': published_at, 'title': title}
+            except KeyError as e:
+                print('Error! Could not extract data from item:\n', item)
+
+        return channel_videos, channel_playlists, nextPageToken
+
+    def dump(self, channel_id):
+        """Dumps channel statistics and video data in a single json file"""
+        if self.channel_statistics is None or self.video_data is None:
+            print('data is missing!\nCall get_channel_statistics() and get_channel_video_data() first!')
+            return
+
+        fused_data = {channel_id: {"channel_statistics": self.channel_statistics,
+                              "video_data": self.video_data}}
+
+        channel_title = self.video_data.popitem()[1].get('channelTitle', channel_id)
+        channel_title = channel_title.replace(" ", "_").lower()
+        filename = channel_title + '.json'
+        with open(filename, 'w') as f:
+            json.dump(fused_data, f, indent=4)
+        
+        print('file dumped to', filename)