Duplicate from peb-peb/shravan

Co-authored-by: Harsh <[email protected]>

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+gradio*

CODE_OF_CONDUCT.md

@@ -0,0 +1,66 @@
+## Code of Conduct
+
+### Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+### Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+### Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+### Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+### Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [INSERT EMAIL ADDRESS]. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.

CONTRIBUTING.md

@@ -0,0 +1,15 @@
+# Contributing
+
+When contributing to this repository, please first discuss the change you wish to make via issue,
+email, or any other method with the owners of this repository before making a change. 
+
+Please note we have a code of conduct, please follow it in all your interactions with the project.
+
+## Pull Request Process
+
+1. Ensure any install or build dependencies are removed before the end of the layer when doing a 
+   build.
+2. Update the README.md with details of changes to the interface, this includes new environment 
+   variables, exposed ports, useful file locations and container parameters.
+3. You may merge the Pull Request in once you have the sign-off of two other developers, or if you 
+   do not have permission to do that, you may request the second reviewer to merge it for you.

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Harsh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,14 @@
+---
+title: Shravan
+emoji: 👀
+colorFrom: indigo
+colorTo: yellow
+sdk: gradio
+sdk_version: 3.35.2
+app_file: app.py
+pinned: false
+license: mit
+duplicated_from: peb-peb/shravan
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,57 @@
+import gradio as gr
+import requests
+
+from transcribe import transcribe
+from sentiment_analysis import sentiment_analyser
+from summary import summarizer
+from topic import topic_gen
+
+
+def main(audio_file, number_of_speakers):
+  # Audio to Text Converter
+  text_data = transcribe(audio_file, number_of_speakers)
+  topic = topic_gen(text_data)[0]["generated_text"]
+  summary = summarizer(text_data)[0]["summary_text"]
+  sent_analy = sentiment_analyser(text_data)
+  sent_analysis = sent_analy[0]["label"] + " (" + str(float(sent_analy[0]["score"]) * 100) + "%)"
+  return topic, summary, sent_analysis
+
+# UI Interface on the Hugging Face Page
+with gr.Blocks() as demo:
+  gr.Markdown("""
+  # Shravan - Unlocking Value from Call Data
+
+  > **NOTE:** You need to give a `.wav` audio file and the audio file should be `monochannel`.
+  """)
+  with gr.Box():
+    with gr.Row():
+      with gr.Column():
+        audio_file = gr.Audio(label="Upload an Audio file (.wav)", source="upload", type="filepath")
+        number_of_speakers = gr.Number(label="Number of Speakers", value=2)
+        with gr.Row():
+          btn_clear = gr.ClearButton(value="Clear", components=[audio_file, number_of_speakers])
+          btn_submit = gr.Button(value="Submit")
+      with gr.Column():
+        topic = gr.Textbox(label="Title", placeholder="Title for Conversation")
+        summary = gr.Textbox(label="Short Summary", placeholder="Short Summary for Conversation")
+        sentiment_analysis = gr.Textbox(label="Sentiment Analysis", placeholder="Sentiment Analysis for Conversation")
+      btn_submit.click(fn=main, inputs=[audio_file, number_of_speakers], outputs=[topic, summary, sentiment_analysis])
+    gr.Markdown("## Examples")
+    gr.Examples( 
+      examples=[
+        ["./examples/sample4.wav", 2],
+        ["./examples/monopos.wav", 2],
+        ["./examples/mononeg.wav", 2],
+      ],
+      inputs=[audio_file, number_of_speakers],
+      outputs=[topic, summary, sentiment_analysis],
+      fn=main,
+    )
+  gr.Markdown(
+    """
+    NOTE: The Tool takes around 5mins to run. So be patient! ;)
+    See [https://github.com/peb-peb/shravan](https://github.com/peb-peb/shravan) for more details.
+    """
+  )
+
+demo.launch()

data.py

@@ -0,0 +1,24 @@
+data = """
+SPEAKER 1 0:00:00
+Thank you for calling Martha's Flowers. How may I assist you? 
+SPEAKER 2 0:00:04
+Hello, I'd like to order flowers and I think you have what I'm looking for. 
+SPEAKER 1 0:00:08
+I'd be happy to take care of your order. May I have your name please? 
+SPEAKER 2 0:00:10
+Randall Thomas. 
+SPEAKER 1 0:00:12
+Randall Thomas. Can you spell that for me? 
+SPEAKER 2 0:00:16
+Randall, R-A-N-D-A-L-L, Thomas, T-H-O-M-A-N. 
+SPEAKER 1 0:00:22
+Thank you for that information, Randall. May I have your home or office number and area code first? 
+SPEAKER 2 0:00:28
+Area code 409, then 866-5088. 
+SPEAKER 1 0:00:34
+That's 409-866-5088. Do you have a fax number or email address? 
+SPEAKER 2 0:00:42
+My email is randall.thomas at gmail.com. 
+SPEAKER 1 0:00:46
+randall.thomas at gmail.com. 
+"""

examples/mononeg.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48ff84445d6358625065758ff2911259481b66151c054b712e752a6aa02f2f21
+size 2939982

examples/monopos.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebd054310c0b00ea193bc2dba688a95abccb236fd477df51fd0fd277c15748cf
+size 3787854

examples/sample4.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:839bd656f8baf785086c86ef835c7b9ad7118c6adf14bb6bdf7607a327c6129b
+size 8820078

requirements.txt

@@ -0,0 +1,4 @@
+git+https://github.com/pyannote/pyannote-audio
+git+https://github.com/openai/whisper.git
+git+https://github.com/huggingface/transformers
+gradio

sentiment_analysis.py

@@ -0,0 +1,5 @@
+from transformers import pipeline
+
+def sentiment_analyser(text):
+  sent = pipeline("sentiment-analysis",model="siebert/sentiment-roberta-large-english")
+  return sent(text)

summary.py

@@ -0,0 +1,5 @@
+from transformers import pipeline
+
+def summarizer(text):
+  summ = pipeline("summarization", model="knkarthick/MEETING-SUMMARY-BART-LARGE-XSUM-SAMSUM-DIALOGSUM")
+  return summ(text)

topic.py

@@ -0,0 +1,5 @@
+from transformers import pipeline
+
+def topic_gen(text):
+  topic = pipeline("text2text-generation", model="knkarthick/TOPIC-DIALOGSUM")
+  return topic(text)

transcribe.py

@@ -0,0 +1,96 @@
+import whisper
+import datetime
+import subprocess
+import wave
+import contextlib
+
+
+import torch
+import pyannote.audio
+from pyannote.audio.pipelines.speaker_verification import PretrainedSpeakerEmbedding
+from pyannote.audio import Audio
+from pyannote.core import Segment
+from sklearn.cluster import AgglomerativeClustering
+import numpy as np
+
+model = whisper.load_model("large-v2")
+embedding_model = PretrainedSpeakerEmbedding( 
+    "speechbrain/spkrec-ecapa-voxceleb",
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+)
+
+def transcribe(audio, num_speakers):
+  print(type(audio))
+  path, error = convert_to_wav(audio)
+  if error is not None:
+    return error
+
+  duration = get_duration(path)
+  if duration > 4 * 60 * 60:
+    return "Audio duration too long"
+
+  result = model.transcribe(path)
+  segments = result["segments"]
+
+  num_speakers = min(max(round(num_speakers), 1), len(segments))
+  if len(segments) == 1:
+    segments[0]['speaker'] = 'SPEAKER 1'
+  else:
+    embeddings = make_embeddings(path, segments, duration)
+    add_speaker_labels(segments, embeddings, num_speakers)
+  output = get_output(segments)
+  return output
+
+def convert_to_wav(path):
+  if path[-3:] != 'wav':
+    new_path = '.'.join(path.split('.')[:-1]) + '.wav'
+    try:
+      subprocess.call(['ffmpeg', '-i', path, new_path, '-y'])
+    except:
+      return path, 'Error: Could not convert file to .wav'
+    path = new_path
+  return path, None
+
+def get_duration(path):
+  with contextlib.closing(wave.open(path,'r')) as f:
+    frames = f.getnframes()
+    rate = f.getframerate()
+    return frames / float(rate)
+
+def make_embeddings(path, segments, duration):
+  embeddings = np.zeros(shape=(len(segments), 192))
+  for i, segment in enumerate(segments):
+    embeddings[i] = segment_embedding(path, segment, duration)
+  return np.nan_to_num(embeddings)
+
+audio = Audio()
+
+def segment_embedding(path, segment, duration):
+  start = segment["start"]
+  # Whisper overshoots the end timestamp in the last segment
+  end = min(duration, segment["end"])
+  clip = Segment(start, end)
+  waveform, sample_rate = audio.crop(path, clip)
+  return embedding_model(waveform[None])
+
+def add_speaker_labels(segments, embeddings, num_speakers):
+  """Add speaker labels"""
+  clustering = AgglomerativeClustering(num_speakers).fit(embeddings)
+  labels = clustering.labels_
+  for i in range(len(segments)):
+    segments[i]["speaker"] = 'SPEAKER ' + str(labels[i] + 1)
+
+def time(secs):
+  """Function to return time delta"""
+  return datetime.timedelta(seconds=round(secs))
+
+def get_output(segments):
+  """Format and generate the output string"""
+  output = ''
+  for (i, segment) in enumerate(segments):
+    if i == 0 or segments[i - 1]["speaker"] != segment["speaker"]:
+      if i != 0:
+        output += '\n\n'
+      output += segment["speaker"] + ' ' + str(time(segment["start"])) + '\n'
+    output += segment["text"][1:] + ' '
+  return output