hello

.github/workflows/build-push.yml

@@ -0,0 +1,25 @@
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches: [main]
+
+env:
+  IMAGE_NAME: mbonea/trannybrickleberry
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and Push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: ${{ env.IMAGE_NAME }}:latest

.gitignore

@@ -0,0 +1 @@
+*.pyc

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "cSpell.words": ["websockets"]
+}

App/Analytics/AnalyticsRoutes.py

@@ -0,0 +1,7 @@
+from fastapi import APIRouter, status
+from .Schemas import BaseRequest, editRequest
+from .Model import Comments
+from App.Users.Model import User
+from App.Post.Model import Post
+
+analytics_router = APIRouter(tags=["Analytics"])

App/Analytics/Model.py

@@ -0,0 +1,23 @@
+import asyncio
+import orm
+import psycopg2
+import datetime
+import pydantic
+from App.modelInit import database, models
+from App.Users.Model import User
+
+
+class Analytics(orm.Model):
+    tablename = "Analytics"
+    registry = models
+    fields = {
+        "id": orm.Integer(primary_key=True),
+        "user": orm.ForeignKey(
+            User, on_delete=orm.CASCADE, allow_null=True
+        ),  # Optional for unknown users.
+        "post": orm.ForeignKey(User, on_delete=orm.CASCADE),
+        "ip": orm.String(max_length=100),
+        "device": orm.String(max_length=100),
+        "country": orm.String(max_length=100),
+        "createdAt": orm.DateTime(index=True, default=datetime.datetime.now),
+    }

App/Analytics/Schemas.py

@@ -0,0 +1,13 @@
+from typing import List, Optional
+from pydantic import EmailStr, BaseModel
+from datetime import date, datetime, time, timedelta
+
+
+class BaseRequest(BaseModel):
+    user: Optional[int]
+    id: Optional[int]
+    content: Optional[str]
+
+
+class editRequest(BaseRequest):
+    updatedAt: datetime = datetime.now()

App/Authentication.py

@@ -0,0 +1,4 @@
+from fastapi import APIRouter, status
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+
+oauth_scheme = OAuth2PasswordBearer(tokenUrl="/user/login")

App/Settings.py

@@ -0,0 +1,3 @@
+class Settings:
+    ALGORITHM = "HS256"
+    HASH = "86c5ceb27e1bf441130299c0209e5f35b88089f62c06b2b09d65772274f12057"

App/Transcription/Model.py

@@ -0,0 +1,21 @@
+import asyncio
+import orm
+import psycopg2
+import datetime
+import ujson
+import pydantic
+from passlib.context import CryptContext
+from App.modelInit import database, models
+
+
+class Transcriptions(orm.Model):
+    tablename = "transcriptions"
+    registry = models
+    fields = {
+        "id": orm.Integer(primary_key=True),
+        "tl_file_id": orm.String(max_length=100, index=True, default=""),
+        "user": orm.ForeignKey(User, on_delete=orm.CASCADE),
+        "createdAt": orm.DateTime(index=True, default=datetime.datetime.now),
+        "content": orm.JSON(default={}),
+        "transcription_state": orm.String(max_length=100, index=True, default="qued"),
+    }

App/Transcription/Schemas.py

@@ -0,0 +1,8 @@
+from typing import List, Optional
+from pydantic import EmailStr, BaseModel
+from passlib.context import CryptContext
+from fastapi import UploadFile
+
+
+class UserDetails(BaseModel):
+    userId: str

App/Transcription/TranscriptionRoutes.py

@@ -0,0 +1,38 @@
+from fastapi import APIRouter, status, Form, UploadFile, File
+from typing_extensions import Annotated
+from .Schemas import UserDetails
+import aiofiles
+from celery.result import AsyncResult
+from App.Worker import transcription_task
+
+# from .Model import User
+# from sqlalchemy import and_
+
+
+transcription_router = APIRouter(tags=["User"])
+
+
+@transcription_router.post("/uploadfile/")
+async def create_file(file: UploadFile, userId: int = 1):
+    # Read the file contents
+    contents = await file.read()
+
+    # Write the file to disk asynchronously
+    async with aiofiles.open(file.filename, "wb") as f:
+        await f.write(contents)
+
+    # celery task
+    task = transcription_task.delay(file.filename)
+
+    return {"file_size": file.size, "file_name": file.filename, "task_id": task.id}
+
+
+@transcription_router.get("/tasks/{task_id}")
+async def get_status(task_id):
+    task_result = AsyncResult(task_id)
+    result = {
+        "task_id": task_id,
+        "task_status": task_result.status,
+        "task_result": task_result.result,
+    }
+    return result

App/Transcription/Utils/audio_transcription.py

@@ -0,0 +1,42 @@
+from faster_whisper import WhisperModel
+from tqdm import tqdm
+import os
+
+model_size = "tiny"
+
+
+def transcribe_file(state, file_path, model_size="tiny"):
+    result = {}
+    model = WhisperModel(model_size, device="cpu", compute_type="int8")
+    segments, info = model.transcribe(file_path, beam_size=5)
+
+    total_duration = round(info.duration, 2)
+    state.update_state(
+        state="PROGRESS",
+        meta={
+            "logs": "Detected language '%s' with probability %f"
+            % (info.language, info.language_probability),
+        },
+    )
+
+
+    with tqdm(total=total_duration, unit=" seconds") as pbar:
+        for segment in segments:
+            segment_duration = segment.end - segment.start
+            time_stamp = "[%.2fs -> %.2fs]" % (segment.start, segment.end)
+            result[time_stamp] = segment.text
+            state.update_state(
+                state="PROGRESS",
+                meta={
+                    "done": segment.end,
+                    "total": total_duration,
+                    "percentage": f"{((segment.end / total_duration)*100)}",
+                },
+            )
+            pbar.update(segment_duration)
+            
+    #delete file
+    os.remove(file_path)
+
+
+    return result

App/Users/Model.py

@@ -0,0 +1,28 @@
+import asyncio
+import orm
+import psycopg2
+import datetime
+import pydantic
+from passlib.context import CryptContext
+from App.modelInit import database, models
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+
+class User(orm.Model):
+    tablename = "users"
+    registry = models
+    fields = {
+        "id": orm.Integer(primary_key=True),
+        "name": orm.String(max_length=100, index=True),
+        "email": orm.String(max_length=100, index=True, unique=True),
+        "password": orm.String(max_length=100, index=True),
+        "phoneNumber": orm.String(max_length=100, index=True, allow_null=True),
+        "account_type": orm.Integer(index=True, default=1),
+        "createdAt": orm.DateTime(index=True, default=datetime.datetime.now),
+        "updatedAt": orm.DateTime(index=True, default=datetime.datetime.now),
+        "lastLogin": orm.DateTime(index=True, default=datetime.datetime.now),
+    }
+
+    def verify_password(self, plain_password):
+        return pwd_context.verify(plain_password, self.password)

App/Users/Schemas.py

@@ -0,0 +1,15 @@
+from typing import List, Optional
+from pydantic import EmailStr, BaseModel
+from passlib.context import CryptContext
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+
+class BaseRequest(BaseModel):
+    email: EmailStr
+    name: str
+    password: str
+    phoneNumber: Optional[str]
+
+    def hash_password(self):
+        self.password = pwd_context.hash(self.password)

App/Users/UserRoutes.py

@@ -0,0 +1,28 @@
+from fastapi import APIRouter, status
+from .Schemas import BaseRequest
+from .Model import User
+from sqlalchemy import and_
+from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+
+
+user_router = APIRouter(tags=["User"])
+
+
+@user_router.post("/user/register")
+async def register_user(user: BaseRequest):
+    data = await User.objects.filter(email=user.email).first()
+    if data != None:
+        return {"code": 400, "message": "user exists", "payload": None}
+    else:
+        user.hash_password()
+    sample = await User.objects.create(**user.dict())
+    return {"code": 200, "message": "success", "payload": None}
+
+
+@user_router.post("/user/login")
+async def register_user(user: BaseRequest):
+    db_user = await User.objects.filter(email=user.email).first()
+    if db_user:
+        if db_user.verify_password(user.password):
+            return {"code": 200, "message": "success", "payload": db_user.__dict__}
+    return {"code": 401, "message": "Invalid Credentials", "payload": None}

App/Worker.py

@@ -0,0 +1,13 @@
+from celery import Celery
+import os
+import time
+from App import celery_config
+from App.Transcription.Utils.audio_transcription import transcribe_file
+
+celery = Celery()
+celery.config_from_object(celery_config)
+
+
+@celery.task(name="create_task", bind=True)
+def transcription_task(self, file_path):
+    return transcribe_file(state=self, file_path=file_path, model_size="tiny")

App/app.py

@@ -0,0 +1,30 @@
+from fastapi import FastAPI
+from .Users.UserRoutes import user_router
+from .modelInit import models, database
+from .Transcription.TranscriptionRoutes import transcription_router
+
+app = FastAPI()
+
+
+@app.on_event("startup")
+async def startup_event():
+    await models.create_all()
+    if not database.is_connected:
+        await database.connect()
+        print("connected!")
+
+
+@app.on_event("shutdown")
+async def shutdown_event():
+    if not database.is_connected:
+        await database.disconnect()
+        print("shutting down!")
+
+
+@app.get("/")
+async def landing_page():
+    return {"code": 200, "message": "still running"}
+
+
+app.include_router(user_router)
+app.include_router(transcription_router)

App/celery_config.py

@@ -0,0 +1,21 @@
+from os import environ
+import ssl
+
+task_serializer = "json"
+result_serializer = "json"
+accept_content = ["json"]
+timezone = "Europe/Oslo"
+enable_utc = True
+
+broker_url = f"rediss://default:59c9ffda43f61cc18b44a3407a2a7793@master.transcription--legal-stuff--a96n-7tyr.addon.code.run:6379?ssl_cert_reqs=none"
+
+
+result_backend = f"rediss://default:59c9ffda43f61cc18b44a3407a2a7793@master.transcription--legal-stuff--a96n-7tyr.addon.code.run:6379?ssl_cert_reqs=none"
+
+# SSL/TLS and SNI configuration
+broker_use_ssl = {
+    "ssl_cert_reqs": ssl.CERT_NONE,
+    "ssl_match_hostname": False,
+    "ssl_check_hostname": False,
+    "ssl_sni": "master.transcription--legal-stuff--a96n-7tyr.addon.code.run",
+}

App/modelInit.py

@@ -0,0 +1,22 @@
+import databases
+import orm
+import psycopg2
+
+
+
+
+#deployment
+database = databases.Database(
+    "postgresql+asyncpg://postgres:[email protected]:5432/postgres"
+)
+
+
+#development
+
+# database = databases.Database(
+#     "postgresql+asyncpg://postgres:[email protected]:5432/postgres"
+# )
+
+
+
+models = orm.ModelRegistry(database=database)

App/utils.py

@@ -0,0 +1,17 @@
+from App.Users.Model import User
+from App.Post.Model import Post
+import asyncio
+from fastapi import HTTPException
+
+
+async def get_user_and_post(content):
+    try:
+        #    user = None
+        #    post = await Post.objects.get(id=content.postId)
+        #    print(post.id)
+        user, post = await asyncio.gather(
+            *[User.objects.get(id=content.userId), Post.objects.get(id=content.postId)]
+        )
+    except:
+        raise HTTPException(status_code=400, detail="Invalid data")
+    return user, post

Dockerfile

@@ -0,0 +1,20 @@
+# Builder stage
+FROM python:3.9.13-slim as builder
+
+WORKDIR /srv
+
+RUN apt-get update && apt-get install -y git ffmpeg aria2 \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+
+
+
+COPY . /srv
+
+
+
+CMD  uvicorn App.app:app --host 0.0.0.0 --port 7860 & celery -A App.Worker.celery worker -c 1 --loglevel=info
+EXPOSE 7860

README.md

@@ -1,11 +1 @@
----
-title: Tranny
-emoji: 🌖
-colorFrom: red
-colorTo: red
-sdk: docker
-pinned: false
-license: creativeml-openrail-m
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Trasnscription

data.txt

@@ -0,0 +1 @@
+

okteto-stack.yaml

@@ -0,0 +1,16 @@
+version: 1
+services:
+  app:
+    build: .
+    command: uvicorn App.app:app --host 0.0.0.0 --port 8000
+    environment:
+      - DATABASE_URL=postgresql://user:password@db:5432/mydatabase
+    ports:
+      - 8000:8000
+
+  worker:
+    build: .
+    command: celery -A App.Worker.celery worker -c 1 --loglevel=info
+    resources:
+      memory: 3Gi
+      cpu: 1000m

requirements.txt

@@ -0,0 +1,26 @@
+asyncpg==0.27.0
+click==8.1.3
+databases==0.7.0
+fastapi==0.92.0
+Flask==2.2.2
+greenlet==2.0.2
+itsdangerous==2.1.2
+orm==0.3.1
+faster-whisper
+aiofiles
+psycopg2-binary==2.9.5
+SQLAlchemy==1.4.46
+starlette==0.25.0
+typesystem==0.3.1
+Werkzeug==2.2.2
+passlib                # for password hashing
+pydantic[email]
+uvicorn==0.21.1
+gunicorn
+ujson
+orm[mysql]
+celery
+python-multipart
+redis==4.5.5
+python-multipart
+asyncmy

stress_test.py

@@ -0,0 +1,30 @@
+import asyncio
+import aiohttp
+
+analysis = {"failed": 0, "passed": 0}
+
+
+async def send_post_request(session, number=0):
+    async with session.post(
+        url="https://app-mbonea-mjema.cloud.okteto.net/post/create",
+        json={"content": {}, "recommendations": {}},
+    ) as response:
+        response_text = await response.text()
+        if response.status == 200:
+            analysis["passed"] += 1
+            print(f"Response from {number}: {response_text}")
+        else:
+            analysis["failed"] += 1
+
+
+async def send_post_requests_async():
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for number in range(1_000):
+            tasks.append(asyncio.ensure_future(send_post_request(session, number)))
+        await asyncio.gather(*tasks)
+
+
+if __name__ == "__main__":
+    asyncio.run(send_post_requests_async())
+    print(analysis)