ChandimaPrabath
commited on
Commit
•
151773c
1
Parent(s):
815f274
0.0.2.7 V Beta
Browse files- LoadBalancer.py +43 -68
- TODO.md +1 -1
- app.py +4 -3
- hf_scrapper.py +1 -177
- indexer.py +0 -1
- old.LoadBalancer.py +308 -20
- old.app.py +65 -267
- utils.py +64 -0
LoadBalancer.py
CHANGED
@@ -1,25 +1,21 @@
|
|
1 |
import os
|
2 |
import json
|
3 |
-
import urllib.request
|
4 |
-
import concurrent.futures
|
5 |
-
from tqdm import tqdm
|
6 |
from indexer import indexer
|
7 |
import re
|
8 |
from tvdb import fetch_and_cache_json
|
9 |
-
from threading import Event
|
10 |
import time
|
11 |
import logging
|
12 |
-
from
|
13 |
from api import InstancesAPI
|
14 |
|
15 |
-
|
16 |
CACHE_DIR = os.getenv("CACHE_DIR")
|
17 |
|
18 |
download_progress = {}
|
19 |
|
20 |
class LoadBalancer:
|
21 |
def __init__(self, cache_dir, index_file, token, repo, polling_interval=4, max_retries=3, initial_delay=1):
|
22 |
-
self.version = "0.0.2.
|
23 |
self.instances = []
|
24 |
self.instances_health = {}
|
25 |
self.polling_interval = polling_interval
|
@@ -34,24 +30,52 @@ class LoadBalancer:
|
|
34 |
self.FILM_STORE = {}
|
35 |
self.TV_STORE = {}
|
36 |
self.file_structure = None
|
|
|
37 |
|
38 |
# Ensure CACHE_DIR exists
|
39 |
if not os.path.exists(self.CACHE_DIR):
|
40 |
os.makedirs(self.CACHE_DIR)
|
41 |
|
42 |
-
# Index the file structure
|
43 |
indexer()
|
44 |
|
45 |
# Load the file structure JSON
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
46 |
if not os.path.exists(self.INDEX_FILE):
|
47 |
raise FileNotFoundError(f"{self.INDEX_FILE} not found. Please make sure the file exists.")
|
48 |
|
49 |
with open(self.INDEX_FILE, 'r') as f:
|
50 |
self.file_structure = json.load(f)
|
|
|
51 |
|
52 |
-
|
53 |
-
|
54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
55 |
|
56 |
def register_instance(self, instance_url):
|
57 |
if instance_url not in self.instances:
|
@@ -124,38 +148,9 @@ class LoadBalancer:
|
|
124 |
logging.info("Stopping polling.")
|
125 |
self.stop_event.set()
|
126 |
|
127 |
-
|
128 |
-
|
129 |
-
|
130 |
-
"""
|
131 |
-
Retrieves the system's HTTP and HTTPS proxies.
|
132 |
-
|
133 |
-
Returns:
|
134 |
-
dict: A dictionary containing the proxies.
|
135 |
-
"""
|
136 |
-
try:
|
137 |
-
proxies = urllib.request.getproxies()
|
138 |
-
print("System proxies:", proxies)
|
139 |
-
return {
|
140 |
-
"http": proxies.get("http"),
|
141 |
-
"https": proxies.get("http")
|
142 |
-
}
|
143 |
-
except Exception as e:
|
144 |
-
print(f"Error getting system proxies: {e}")
|
145 |
-
return {}
|
146 |
-
|
147 |
-
@staticmethod
|
148 |
-
def is_valid_url(url):
|
149 |
-
# Simple URL validation (could be more complex if needed)
|
150 |
-
regex = re.compile(
|
151 |
-
r'^(?:http|ftp)s?://' # http:// or https://
|
152 |
-
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
|
153 |
-
r'localhost|' # localhost...
|
154 |
-
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
|
155 |
-
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
|
156 |
-
r'(?::\d+)?' # optional port
|
157 |
-
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
158 |
-
return re.match(regex, url) is not None
|
159 |
|
160 |
#################################################################
|
161 |
|
@@ -183,8 +178,8 @@ class LoadBalancer:
|
|
183 |
|
184 |
# Calculate free space for each instance
|
185 |
for instance_url, space_info in self.instances_health.items():
|
186 |
-
total_space =
|
187 |
-
used_space =
|
188 |
free_space = total_space - used_space
|
189 |
|
190 |
if free_space > max_free_space:
|
@@ -227,8 +222,8 @@ class LoadBalancer:
|
|
227 |
|
228 |
# Calculate free space for each instance
|
229 |
for instance_url, space_info in self.instances_health.items():
|
230 |
-
total_space =
|
231 |
-
used_space =
|
232 |
free_space = total_space - used_space
|
233 |
|
234 |
if free_space > max_free_space:
|
@@ -251,12 +246,6 @@ class LoadBalancer:
|
|
251 |
logging.error("No suitable instance found for downloading the film.")
|
252 |
return {"error": "No suitable instance found for downloading the film."}
|
253 |
|
254 |
-
def _convert_to_gb(self, space_str):
|
255 |
-
"""
|
256 |
-
Converts a space string like '50 GB' or '3.33 GB' to a float representing the number of GB.
|
257 |
-
"""
|
258 |
-
return float(space_str.split()[0])
|
259 |
-
|
260 |
#################################################################
|
261 |
def find_movie_path(self, title):
|
262 |
"""Find the path of the movie in the JSON data based on the title."""
|
@@ -316,16 +305,6 @@ class LoadBalancer:
|
|
316 |
|
317 |
fetch_and_cache_json(original_title, title, media_type, year)
|
318 |
|
319 |
-
def bytes_to_human_readable(self, num, suffix="B"):
|
320 |
-
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
|
321 |
-
if abs(num) < 1024.0:
|
322 |
-
return f"{num:3.1f} {unit}{suffix}"
|
323 |
-
num /= 1024.0
|
324 |
-
return f"{num:.1f} Y{suffix}"
|
325 |
-
|
326 |
-
def encode_episodeid(self, title, season, episode):
|
327 |
-
return f"{title}_{season}_{episode}"
|
328 |
-
|
329 |
def get_all_tv_shows(self):
|
330 |
"""Get all TV shows from the indexed cache structure JSON file."""
|
331 |
tv_shows = {}
|
@@ -355,8 +334,4 @@ class LoadBalancer:
|
|
355 |
for sub_directory in directory['contents']:
|
356 |
if sub_directory['type'] == 'directory':
|
357 |
films.append(sub_directory['path'])
|
358 |
-
return films
|
359 |
-
|
360 |
-
def start_prefetching(self):
|
361 |
-
"""Start the metadata prefetching in a separate thread."""
|
362 |
-
self.prefetch_metadata()
|
|
|
1 |
import os
|
2 |
import json
|
|
|
|
|
|
|
3 |
from indexer import indexer
|
4 |
import re
|
5 |
from tvdb import fetch_and_cache_json
|
6 |
+
from threading import Event, Thread
|
7 |
import time
|
8 |
import logging
|
9 |
+
from utils import convert_to_gb
|
10 |
from api import InstancesAPI
|
11 |
|
|
|
12 |
CACHE_DIR = os.getenv("CACHE_DIR")
|
13 |
|
14 |
download_progress = {}
|
15 |
|
16 |
class LoadBalancer:
|
17 |
def __init__(self, cache_dir, index_file, token, repo, polling_interval=4, max_retries=3, initial_delay=1):
|
18 |
+
self.version = "0.0.2.7 V Beta"
|
19 |
self.instances = []
|
20 |
self.instances_health = {}
|
21 |
self.polling_interval = polling_interval
|
|
|
30 |
self.FILM_STORE = {}
|
31 |
self.TV_STORE = {}
|
32 |
self.file_structure = None
|
33 |
+
self.index_file_last_modified = None
|
34 |
|
35 |
# Ensure CACHE_DIR exists
|
36 |
if not os.path.exists(self.CACHE_DIR):
|
37 |
os.makedirs(self.CACHE_DIR)
|
38 |
|
39 |
+
# Index the file structure initially
|
40 |
indexer()
|
41 |
|
42 |
# Load the file structure JSON
|
43 |
+
self.load_file_structure()
|
44 |
+
|
45 |
+
# Start polling and file checking in separate threads
|
46 |
+
polling_thread = Thread(target=self.start_polling)
|
47 |
+
polling_thread.daemon = True
|
48 |
+
polling_thread.start()
|
49 |
+
|
50 |
+
file_checking_thread = Thread(target=self.check_file_updates)
|
51 |
+
file_checking_thread.daemon = True
|
52 |
+
file_checking_thread.start()
|
53 |
+
|
54 |
+
def load_file_structure(self):
|
55 |
if not os.path.exists(self.INDEX_FILE):
|
56 |
raise FileNotFoundError(f"{self.INDEX_FILE} not found. Please make sure the file exists.")
|
57 |
|
58 |
with open(self.INDEX_FILE, 'r') as f:
|
59 |
self.file_structure = json.load(f)
|
60 |
+
logging.info("File structure loaded successfully.")
|
61 |
|
62 |
+
def check_file_updates(self):
|
63 |
+
while not self.stop_event.is_set():
|
64 |
+
if self.index_file_last_modified != os.path.getmtime(self.INDEX_FILE):
|
65 |
+
logging.info(f"{self.INDEX_FILE} has been updated. Re-indexing...")
|
66 |
+
indexer() # Re-run the indexer
|
67 |
+
self.load_file_structure() # Reload the file structure
|
68 |
+
self.index_file_last_modified = os.path.getmtime(self.INDEX_FILE)
|
69 |
+
|
70 |
+
# Restart prefetching thread
|
71 |
+
if hasattr(self, 'prefetch_thread') and self.prefetch_thread.is_alive():
|
72 |
+
self.prefetch_thread.join()
|
73 |
+
|
74 |
+
self.prefetch_thread = Thread(target=self.start_prefetching)
|
75 |
+
self.prefetch_thread.daemon = True
|
76 |
+
self.prefetch_thread.start()
|
77 |
+
|
78 |
+
time.sleep(120) # Check every 2 minutes
|
79 |
|
80 |
def register_instance(self, instance_url):
|
81 |
if instance_url not in self.instances:
|
|
|
148 |
logging.info("Stopping polling.")
|
149 |
self.stop_event.set()
|
150 |
|
151 |
+
def start_prefetching(self):
|
152 |
+
"""Start the metadata prefetching in a separate thread."""
|
153 |
+
self.prefetch_metadata()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
154 |
|
155 |
#################################################################
|
156 |
|
|
|
178 |
|
179 |
# Calculate free space for each instance
|
180 |
for instance_url, space_info in self.instances_health.items():
|
181 |
+
total_space = convert_to_gb(space_info['total'])
|
182 |
+
used_space = convert_to_gb(space_info['used'])
|
183 |
free_space = total_space - used_space
|
184 |
|
185 |
if free_space > max_free_space:
|
|
|
222 |
|
223 |
# Calculate free space for each instance
|
224 |
for instance_url, space_info in self.instances_health.items():
|
225 |
+
total_space = convert_to_gb(space_info['total'])
|
226 |
+
used_space = convert_to_gb(space_info['used'])
|
227 |
free_space = total_space - used_space
|
228 |
|
229 |
if free_space > max_free_space:
|
|
|
246 |
logging.error("No suitable instance found for downloading the film.")
|
247 |
return {"error": "No suitable instance found for downloading the film."}
|
248 |
|
|
|
|
|
|
|
|
|
|
|
|
|
249 |
#################################################################
|
250 |
def find_movie_path(self, title):
|
251 |
"""Find the path of the movie in the JSON data based on the title."""
|
|
|
305 |
|
306 |
fetch_and_cache_json(original_title, title, media_type, year)
|
307 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
308 |
def get_all_tv_shows(self):
|
309 |
"""Get all TV shows from the indexed cache structure JSON file."""
|
310 |
tv_shows = {}
|
|
|
334 |
for sub_directory in directory['contents']:
|
335 |
if sub_directory['type'] == 'directory':
|
336 |
films.append(sub_directory['path'])
|
337 |
+
return films
|
|
|
|
|
|
|
|
TODO.md
CHANGED
@@ -1 +1 @@
|
|
1 |
-
# implement a better way
|
|
|
1 |
+
# implement a better way clear instance cache and refresh
|
app.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1 |
from flask import Flask, jsonify, request, send_from_directory
|
2 |
from flask_cors import CORS
|
|
|
3 |
import os
|
4 |
import json
|
5 |
from threading import Thread
|
@@ -97,7 +98,7 @@ def get_episode_id_api(title,season,episode):
|
|
97 |
"""Endpoint to get the episode ID by providing the TV show title, season, and episode."""
|
98 |
if not title or not season or not episode:
|
99 |
return jsonify({"error": "Title, season, and episode parameters are required"}), 400
|
100 |
-
episode_id =
|
101 |
return jsonify({"episode_id": episode_id})
|
102 |
|
103 |
@app.route('/api/cache/size', methods=['GET'])
|
@@ -107,7 +108,7 @@ def get_cache_size_api():
|
|
107 |
for f in filenames:
|
108 |
fp = os.path.join(dirpath, f)
|
109 |
total_size += os.path.getsize(fp)
|
110 |
-
readable_size =
|
111 |
return jsonify({"cache_size": readable_size})
|
112 |
|
113 |
@app.route('/api/cache/clear', methods=['POST'])
|
@@ -192,7 +193,7 @@ def register_instance():
|
|
192 |
return jsonify({"error": "No URL provided"}), 400
|
193 |
|
194 |
url = data["url"]
|
195 |
-
if not
|
196 |
return jsonify({"error": "Invalid URL"}), 400
|
197 |
|
198 |
# Register the instance
|
|
|
1 |
from flask import Flask, jsonify, request, send_from_directory
|
2 |
from flask_cors import CORS
|
3 |
+
from utils import is_valid_url, bytes_to_human_readable, encode_episodeid
|
4 |
import os
|
5 |
import json
|
6 |
from threading import Thread
|
|
|
98 |
"""Endpoint to get the episode ID by providing the TV show title, season, and episode."""
|
99 |
if not title or not season or not episode:
|
100 |
return jsonify({"error": "Title, season, and episode parameters are required"}), 400
|
101 |
+
episode_id = encode_episodeid(title,season,episode)
|
102 |
return jsonify({"episode_id": episode_id})
|
103 |
|
104 |
@app.route('/api/cache/size', methods=['GET'])
|
|
|
108 |
for f in filenames:
|
109 |
fp = os.path.join(dirpath, f)
|
110 |
total_size += os.path.getsize(fp)
|
111 |
+
readable_size = bytes_to_human_readable(total_size)
|
112 |
return jsonify({"cache_size": readable_size})
|
113 |
|
114 |
@app.route('/api/cache/clear', methods=['POST'])
|
|
|
193 |
return jsonify({"error": "No URL provided"}), 400
|
194 |
|
195 |
url = data["url"]
|
196 |
+
if not is_valid_url(url):
|
197 |
return jsonify({"error": "Invalid URL"}), 400
|
198 |
|
199 |
# Register the instance
|
hf_scrapper.py
CHANGED
@@ -29,178 +29,6 @@ def get_system_proxies():
|
|
29 |
print(f"Error getting system proxies: {e}")
|
30 |
return {}
|
31 |
|
32 |
-
def download_film(file_url, token, cache_path, proxies, film_id, title, chunk_size=100 * 1024 * 1024):
|
33 |
-
"""
|
34 |
-
Downloads a file from the specified URL and saves it to the cache path.
|
35 |
-
Tracks the download progress.
|
36 |
-
|
37 |
-
Args:
|
38 |
-
file_url (str): The URL of the file to download.
|
39 |
-
token (str): The authorization token for the request.
|
40 |
-
cache_path (str): The path to save the downloaded file.
|
41 |
-
proxies (dict): Proxies for the request.
|
42 |
-
film_id (str): Unique identifier for the film download.
|
43 |
-
title (str): The title of the film.
|
44 |
-
chunk_size (int): Size of each chunk to download.
|
45 |
-
"""
|
46 |
-
print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
|
47 |
-
headers = {'Authorization': f'Bearer {token}'}
|
48 |
-
try:
|
49 |
-
response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
|
50 |
-
response.raise_for_status()
|
51 |
-
|
52 |
-
total_size = int(response.headers.get('content-length', 0))
|
53 |
-
download_progress[film_id] = {"total": total_size, "downloaded": 0, "status": "Downloading", "start_time": time.time()}
|
54 |
-
|
55 |
-
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
56 |
-
with open(cache_path, 'wb') as file, tqdm(total=total_size, unit='B', unit_scale=True, desc=cache_path) as pbar:
|
57 |
-
for data in response.iter_content(chunk_size=chunk_size):
|
58 |
-
file.write(data)
|
59 |
-
pbar.update(len(data))
|
60 |
-
download_progress[film_id]["downloaded"] += len(data)
|
61 |
-
|
62 |
-
print(f'File cached to {cache_path} successfully.')
|
63 |
-
update_film_store_json(title, cache_path)
|
64 |
-
download_progress[film_id]["status"] = "Completed"
|
65 |
-
except RequestException as e:
|
66 |
-
print(f"Error downloading file: {e}")
|
67 |
-
download_progress[film_id]["status"] = "Failed"
|
68 |
-
except IOError as e:
|
69 |
-
print(f"Error writing file {cache_path}: {e}")
|
70 |
-
download_progress[film_id]["status"] = "Failed"
|
71 |
-
finally:
|
72 |
-
if download_progress[film_id]["status"] != "Downloading":
|
73 |
-
download_progress[film_id]["end_time"] = time.time()
|
74 |
-
|
75 |
-
def get_download_progress(id):
|
76 |
-
"""
|
77 |
-
Gets the download progress for a specific film.
|
78 |
-
|
79 |
-
Args:
|
80 |
-
film_id (str): The unique identifier for the film download.
|
81 |
-
|
82 |
-
Returns:
|
83 |
-
dict: A dictionary containing the total size, downloaded size, progress percentage, status, and ETA.
|
84 |
-
"""
|
85 |
-
if id in download_progress:
|
86 |
-
total = download_progress[id]["total"]
|
87 |
-
downloaded = download_progress[id]["downloaded"]
|
88 |
-
status = download_progress[id].get("status", "In Progress")
|
89 |
-
progress = (downloaded / total) * 100 if total > 0 else 0
|
90 |
-
|
91 |
-
eta = None
|
92 |
-
if status == "Downloading" and downloaded > 0:
|
93 |
-
elapsed_time = time.time() - download_progress[id]["start_time"]
|
94 |
-
estimated_total_time = elapsed_time * (total / downloaded)
|
95 |
-
eta = estimated_total_time - elapsed_time
|
96 |
-
elif status == "Completed":
|
97 |
-
eta = 0
|
98 |
-
|
99 |
-
return {"total": total, "downloaded": downloaded, "progress": progress, "status": status, "eta": eta}
|
100 |
-
return {"total": 0, "downloaded": 0, "progress": 0, "status": "Not Found", "eta": None}
|
101 |
-
|
102 |
-
def update_film_store_json(title, cache_path):
|
103 |
-
"""
|
104 |
-
Updates the film store JSON with the new file.
|
105 |
-
|
106 |
-
Args:
|
107 |
-
title (str): The title of the film.
|
108 |
-
cache_path (str): The local path where the file is saved.
|
109 |
-
"""
|
110 |
-
FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
|
111 |
-
|
112 |
-
film_store_data = {}
|
113 |
-
if os.path.exists(FILM_STORE_JSON_PATH):
|
114 |
-
with open(FILM_STORE_JSON_PATH, 'r') as json_file:
|
115 |
-
film_store_data = json.load(json_file)
|
116 |
-
|
117 |
-
film_store_data[title] = cache_path
|
118 |
-
|
119 |
-
with open(FILM_STORE_JSON_PATH, 'w') as json_file:
|
120 |
-
json.dump(film_store_data, json_file, indent=2)
|
121 |
-
print(f'Film store updated with {title}.')
|
122 |
-
|
123 |
-
|
124 |
-
###############################################################################
|
125 |
-
def download_episode(file_url, token, cache_path, proxies, episode_id, title, chunk_size=100 * 1024 * 1024):
|
126 |
-
"""
|
127 |
-
Downloads a file from the specified URL and saves it to the cache path.
|
128 |
-
Tracks the download progress.
|
129 |
-
|
130 |
-
Args:
|
131 |
-
file_url (str): The URL of the file to download.
|
132 |
-
token (str): The authorization token for the request.
|
133 |
-
cache_path (str): The path to save the downloaded file.
|
134 |
-
proxies (dict): Proxies for the request.
|
135 |
-
film_id (str): Unique identifier for the film download.
|
136 |
-
title (str): The title of the film.
|
137 |
-
chunk_size (int): Size of each chunk to download.
|
138 |
-
"""
|
139 |
-
print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
|
140 |
-
headers = {'Authorization': f'Bearer {token}'}
|
141 |
-
try:
|
142 |
-
response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
|
143 |
-
response.raise_for_status()
|
144 |
-
|
145 |
-
total_size = int(response.headers.get('content-length', 0))
|
146 |
-
download_progress[episode_id] = {"total": total_size, "downloaded": 0, "status": "Downloading", "start_time": time.time()}
|
147 |
-
|
148 |
-
os.makedirs(os.path.dirname(cache_path), exist_ok=True)
|
149 |
-
with open(cache_path, 'wb') as file, tqdm(total=total_size, unit='B', unit_scale=True, desc=cache_path) as pbar:
|
150 |
-
for data in response.iter_content(chunk_size=chunk_size):
|
151 |
-
file.write(data)
|
152 |
-
pbar.update(len(data))
|
153 |
-
download_progress[episode_id]["downloaded"] += len(data)
|
154 |
-
|
155 |
-
print(f'File cached to {cache_path} successfully.')
|
156 |
-
update_tv_store_json(title, cache_path)
|
157 |
-
download_progress[episode_id]["status"] = "Completed"
|
158 |
-
except RequestException as e:
|
159 |
-
print(f"Error downloading file: {e}")
|
160 |
-
download_progress[episode_id]["status"] = "Failed"
|
161 |
-
except IOError as e:
|
162 |
-
print(f"Error writing file {cache_path}: {e}")
|
163 |
-
download_progress[episode_id]["status"] = "Failed"
|
164 |
-
finally:
|
165 |
-
if download_progress[episode_id]["status"] != "Downloading":
|
166 |
-
download_progress[episode_id]["end_time"] = time.time()
|
167 |
-
|
168 |
-
|
169 |
-
def update_tv_store_json(title, cache_path):
|
170 |
-
"""
|
171 |
-
Updates the TV store JSON with the new file, organizing by title, season, and episode.
|
172 |
-
|
173 |
-
Args:
|
174 |
-
title (str): The title of the TV show.
|
175 |
-
cache_path (str): The local path where the file is saved.
|
176 |
-
"""
|
177 |
-
TV_STORE_JSON_PATH = os.path.join(CACHE_DIR, "tv_store.json")
|
178 |
-
|
179 |
-
tv_store_data = {}
|
180 |
-
if os.path.exists(TV_STORE_JSON_PATH):
|
181 |
-
with open(TV_STORE_JSON_PATH, 'r') as json_file:
|
182 |
-
tv_store_data = json.load(json_file)
|
183 |
-
|
184 |
-
# Extract season and episode information from the cache_path
|
185 |
-
season_part = os.path.basename(os.path.dirname(cache_path)) # Extracts 'Season 1'
|
186 |
-
episode_part = os.path.basename(cache_path) # Extracts 'Grand Blue Dreaming - S01E01 - Deep Blue HDTV-720p.mp4'
|
187 |
-
|
188 |
-
# Create the structure if not already present
|
189 |
-
if title not in tv_store_data:
|
190 |
-
tv_store_data[title] = {}
|
191 |
-
|
192 |
-
if season_part not in tv_store_data[title]:
|
193 |
-
tv_store_data[title][season_part] = {}
|
194 |
-
|
195 |
-
# Assuming episode_part is unique for each episode within a season
|
196 |
-
tv_store_data[title][season_part][episode_part] = cache_path
|
197 |
-
|
198 |
-
with open(TV_STORE_JSON_PATH, 'w') as json_file:
|
199 |
-
json.dump(tv_store_data, json_file, indent=2)
|
200 |
-
|
201 |
-
print(f'TV store updated with {title}, {season_part}, {episode_part}.')
|
202 |
-
|
203 |
-
###############################################################################
|
204 |
def get_file_structure(repo, token, path="", proxies=None):
|
205 |
"""
|
206 |
Fetches the file structure of a specified Hugging Face repository.
|
@@ -242,8 +70,4 @@ def write_file_structure_to_json(file_structure, file_path):
|
|
242 |
|
243 |
if __name__ == "__main__":
|
244 |
file_url = "https://huggingface.co/Unicone-Studio/jellyfin_media/resolve/main/films/Funky%20Monkey%202004/Funky%20Monkey%20(2004)%20Web-dl%201080p.mp4"
|
245 |
-
token = os.getenv("TOKEN")
|
246 |
-
cache_path = os.path.join(CACHE_DIR, "films/Funky Monkey 2004/Funky Monkey (2004) Web-dl 1080p.mp4")
|
247 |
-
proxies = get_system_proxies()
|
248 |
-
film_id = "funky_monkey_2004" # Unique identifier for the film download
|
249 |
-
download_film(file_url, token, cache_path, proxies=proxies, film_id=film_id)
|
|
|
29 |
print(f"Error getting system proxies: {e}")
|
30 |
return {}
|
31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
32 |
def get_file_structure(repo, token, path="", proxies=None):
|
33 |
"""
|
34 |
Fetches the file structure of a specified Hugging Face repository.
|
|
|
70 |
|
71 |
if __name__ == "__main__":
|
72 |
file_url = "https://huggingface.co/Unicone-Studio/jellyfin_media/resolve/main/films/Funky%20Monkey%202004/Funky%20Monkey%20(2004)%20Web-dl%201080p.mp4"
|
73 |
+
token = os.getenv("TOKEN")
|
|
|
|
|
|
|
|
indexer.py
CHANGED
@@ -24,7 +24,6 @@ def indexer():
|
|
24 |
token = os.getenv("TOKEN")
|
25 |
repo = os.getenv("REPO")
|
26 |
output_path = os.getenv("INDEX_FILE")
|
27 |
-
|
28 |
proxies = get_system_proxies()
|
29 |
full_structure = index_repository(token, repo, "", proxies)
|
30 |
write_file_structure_to_json(full_structure, output_path)
|
|
|
24 |
token = os.getenv("TOKEN")
|
25 |
repo = os.getenv("REPO")
|
26 |
output_path = os.getenv("INDEX_FILE")
|
|
|
27 |
proxies = get_system_proxies()
|
28 |
full_structure = index_repository(token, repo, "", proxies)
|
29 |
write_file_structure_to_json(full_structure, output_path)
|
old.LoadBalancer.py
CHANGED
@@ -1,17 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
import time
|
2 |
import logging
|
3 |
-
from threading import Thread, Event
|
4 |
from api import InstancesAPI
|
5 |
|
|
|
|
|
|
|
|
|
|
|
6 |
class LoadBalancer:
|
7 |
-
def __init__(self, polling_interval=
|
8 |
-
self.version = "0.0.
|
9 |
self.instances = []
|
|
|
10 |
self.polling_interval = polling_interval
|
11 |
self.max_retries = max_retries
|
12 |
self.initial_delay = initial_delay
|
13 |
self.stop_event = Event()
|
14 |
self.instances_api = InstancesAPI(self.instances)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
15 |
|
16 |
def register_instance(self, instance_url):
|
17 |
if instance_url not in self.instances:
|
@@ -23,28 +62,55 @@ class LoadBalancer:
|
|
23 |
def remove_instance(self, instance_url):
|
24 |
if instance_url in self.instances:
|
25 |
self.instances.remove(instance_url)
|
|
|
26 |
logging.info(f"Removed instance {instance_url}")
|
27 |
else:
|
28 |
logging.info(f"Instance {instance_url} not found for removal.")
|
29 |
|
30 |
def get_reports(self):
|
31 |
reports = self.instances_api.fetch_reports()
|
|
|
|
|
|
|
|
|
|
|
32 |
for instance_url in self.instances[:]: # Copy list to avoid modification during iteration
|
33 |
if instance_url in reports:
|
34 |
report = reports[instance_url]
|
35 |
logging.info(f"Report from {instance_url}: {report}")
|
36 |
-
self.process_report(instance_url, report)
|
37 |
else:
|
38 |
logging.error(f"Failed to get report from {instance_url}. Removing instance.")
|
39 |
self.remove_instance(instance_url)
|
40 |
|
41 |
-
|
42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
logging.info(f"Processing report from {instance_url}")
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
48 |
|
49 |
def start_polling(self):
|
50 |
logging.info("Starting polling.")
|
@@ -57,17 +123,239 @@ class LoadBalancer:
|
|
57 |
logging.info("Stopping polling.")
|
58 |
self.stop_event.set()
|
59 |
|
60 |
-
|
61 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
62 |
|
63 |
-
|
|
|
64 |
|
65 |
-
|
66 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
67 |
|
68 |
-
|
69 |
-
|
70 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
71 |
|
72 |
-
|
73 |
-
|
|
|
|
1 |
+
import os
|
2 |
+
import json
|
3 |
+
import urllib.request
|
4 |
+
from tqdm import tqdm
|
5 |
+
from indexer import indexer
|
6 |
+
import re
|
7 |
+
from tvdb import fetch_and_cache_json
|
8 |
+
from threading import Event
|
9 |
import time
|
10 |
import logging
|
11 |
+
from threading import Thread, Event
|
12 |
from api import InstancesAPI
|
13 |
|
14 |
+
|
15 |
+
CACHE_DIR = os.getenv("CACHE_DIR")
|
16 |
+
|
17 |
+
download_progress = {}
|
18 |
+
|
19 |
class LoadBalancer:
|
20 |
+
def __init__(self, cache_dir, index_file, token, repo, polling_interval=4, max_retries=3, initial_delay=1):
|
21 |
+
self.version = "0.0.2.6 V Beta"
|
22 |
self.instances = []
|
23 |
+
self.instances_health = {}
|
24 |
self.polling_interval = polling_interval
|
25 |
self.max_retries = max_retries
|
26 |
self.initial_delay = initial_delay
|
27 |
self.stop_event = Event()
|
28 |
self.instances_api = InstancesAPI(self.instances)
|
29 |
+
self.CACHE_DIR = cache_dir
|
30 |
+
self.INDEX_FILE = index_file
|
31 |
+
self.TOKEN = token
|
32 |
+
self.REPO = repo
|
33 |
+
self.FILM_STORE = {}
|
34 |
+
self.TV_STORE = {}
|
35 |
+
self.file_structure = None
|
36 |
+
|
37 |
+
# Ensure CACHE_DIR exists
|
38 |
+
if not os.path.exists(self.CACHE_DIR):
|
39 |
+
os.makedirs(self.CACHE_DIR)
|
40 |
+
|
41 |
+
# Index the file structure
|
42 |
+
indexer()
|
43 |
+
|
44 |
+
# Load the file structure JSON
|
45 |
+
if not os.path.exists(self.INDEX_FILE):
|
46 |
+
raise FileNotFoundError(f"{self.INDEX_FILE} not found. Please make sure the file exists.")
|
47 |
+
|
48 |
+
with open(self.INDEX_FILE, 'r') as f:
|
49 |
+
self.file_structure = json.load(f)
|
50 |
+
|
51 |
+
prefetch_thread = Thread(target=self.start_prefetching)
|
52 |
+
prefetch_thread.daemon = True
|
53 |
+
prefetch_thread.start()
|
54 |
|
55 |
def register_instance(self, instance_url):
|
56 |
if instance_url not in self.instances:
|
|
|
62 |
def remove_instance(self, instance_url):
|
63 |
if instance_url in self.instances:
|
64 |
self.instances.remove(instance_url)
|
65 |
+
self.instances_health.pop(instance_url, None)
|
66 |
logging.info(f"Removed instance {instance_url}")
|
67 |
else:
|
68 |
logging.info(f"Instance {instance_url} not found for removal.")
|
69 |
|
70 |
def get_reports(self):
|
71 |
reports = self.instances_api.fetch_reports()
|
72 |
+
|
73 |
+
# Initialize temporary JSON data holders
|
74 |
+
temp_film_store = {}
|
75 |
+
temp_tv_store = {}
|
76 |
+
|
77 |
for instance_url in self.instances[:]: # Copy list to avoid modification during iteration
|
78 |
if instance_url in reports:
|
79 |
report = reports[instance_url]
|
80 |
logging.info(f"Report from {instance_url}: {report}")
|
81 |
+
self.process_report(instance_url, report, temp_film_store, temp_tv_store)
|
82 |
else:
|
83 |
logging.error(f"Failed to get report from {instance_url}. Removing instance.")
|
84 |
self.remove_instance(instance_url)
|
85 |
|
86 |
+
self.FILM_STORE = temp_film_store
|
87 |
+
self.TV_STORE = temp_tv_store
|
88 |
+
|
89 |
+
def process_report(self, instance_url, report, temp_film_store, temp_tv_store):
|
90 |
+
film_store = report.get('film_store', {})
|
91 |
+
tv_store = report.get('tv_store', {})
|
92 |
+
cache_size = report.get('cache_size')
|
93 |
+
|
94 |
logging.info(f"Processing report from {instance_url}")
|
95 |
+
|
96 |
+
# Update temporary film store
|
97 |
+
for title, path in film_store.items():
|
98 |
+
url = f"{instance_url}/api/film/{title.replace(' ', '%20')}"
|
99 |
+
temp_film_store[title] = url
|
100 |
+
|
101 |
+
# Update temporary TV store
|
102 |
+
for title, seasons in tv_store.items():
|
103 |
+
if title not in temp_tv_store:
|
104 |
+
temp_tv_store[title] = {}
|
105 |
+
for season, episodes in seasons.items():
|
106 |
+
if season not in temp_tv_store[title]:
|
107 |
+
temp_tv_store[title][season] = {}
|
108 |
+
for episode, path in episodes.items():
|
109 |
+
url = f"{instance_url}/api/tv/{title.replace(' ', '%20')}/{season.replace(' ', '%20')}/{episode.replace(' ', '%20')}"
|
110 |
+
temp_tv_store[title][season][episode] = url
|
111 |
+
|
112 |
+
logging.info("Film and TV Stores processed successfully.")
|
113 |
+
self.update_instances_health(instance=instance_url, cache_size=cache_size)
|
114 |
|
115 |
def start_polling(self):
|
116 |
logging.info("Starting polling.")
|
|
|
123 |
logging.info("Stopping polling.")
|
124 |
self.stop_event.set()
|
125 |
|
126 |
+
######################################################################
|
127 |
+
@staticmethod
|
128 |
+
def get_system_proxies():
|
129 |
+
"""
|
130 |
+
Retrieves the system's HTTP and HTTPS proxies.
|
131 |
+
|
132 |
+
Returns:
|
133 |
+
dict: A dictionary containing the proxies.
|
134 |
+
"""
|
135 |
+
try:
|
136 |
+
proxies = urllib.request.getproxies()
|
137 |
+
print("System proxies:", proxies)
|
138 |
+
return {
|
139 |
+
"http": proxies.get("http"),
|
140 |
+
"https": proxies.get("http")
|
141 |
+
}
|
142 |
+
except Exception as e:
|
143 |
+
print(f"Error getting system proxies: {e}")
|
144 |
+
return {}
|
145 |
+
|
146 |
+
@staticmethod
|
147 |
+
def is_valid_url(url):
|
148 |
+
# Simple URL validation (could be more complex if needed)
|
149 |
+
regex = re.compile(
|
150 |
+
r'^(?:http|ftp)s?://' # http:// or https://
|
151 |
+
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
|
152 |
+
r'localhost|' # localhost...
|
153 |
+
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
|
154 |
+
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
|
155 |
+
r'(?::\d+)?' # optional port
|
156 |
+
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
157 |
+
return re.match(regex, url) is not None
|
158 |
+
|
159 |
+
#################################################################
|
160 |
+
|
161 |
+
def update_instances_health(self, instance, cache_size):
|
162 |
+
self.instances_health[instance] = {"used":cache_size["cache_size"],
|
163 |
+
"total": "50 GB"}
|
164 |
+
logging.info(f"Updated instance {instance} with cache size {cache_size}")
|
165 |
+
|
166 |
+
|
167 |
+
def download_film_to_best_instance(self, title):
|
168 |
+
"""
|
169 |
+
Downloads a film to the first instance that has more free space on the self.instance_health list variable.
|
170 |
+
The instance_health looks like this:
|
171 |
+
{
|
172 |
+
"https://unicone-studio-instance1.hf.space": {
|
173 |
+
"total": "50 GB",
|
174 |
+
"used": "3.33 GB"
|
175 |
+
}
|
176 |
+
}
|
177 |
+
Args:
|
178 |
+
title (str): The title of the film.
|
179 |
+
"""
|
180 |
+
best_instance = None
|
181 |
+
max_free_space = -1
|
182 |
+
|
183 |
+
# Calculate free space for each instance
|
184 |
+
for instance_url, space_info in self.instances_health.items():
|
185 |
+
total_space = self._convert_to_gb(space_info['total'])
|
186 |
+
used_space = self._convert_to_gb(space_info['used'])
|
187 |
+
free_space = total_space - used_space
|
188 |
+
|
189 |
+
if free_space > max_free_space:
|
190 |
+
max_free_space = free_space
|
191 |
+
best_instance = instance_url
|
192 |
+
|
193 |
+
if best_instance:
|
194 |
+
result = self.instances_api.download_film(best_instance, title)
|
195 |
+
film_id = result["film_id"]
|
196 |
+
status = result["status"]
|
197 |
+
progress_url = f'{best_instance}/api/progress/{film_id}'
|
198 |
+
response = {
|
199 |
+
"film_id":film_id,
|
200 |
+
"status":status,
|
201 |
+
"progress_url":progress_url
|
202 |
+
}
|
203 |
+
|
204 |
+
return response
|
205 |
+
else:
|
206 |
+
logging.error("No suitable instance found for downloading the film.")
|
207 |
+
return {"error": "No suitable instance found for downloading the film."}
|
208 |
+
|
209 |
+
def download_episode_to_best_instance(self, title, season, episode):
|
210 |
+
"""
|
211 |
+
Downloads a episode to the first instance that has more free space on the self.instance_health list variable.
|
212 |
+
The instance_health looks like this:
|
213 |
+
{
|
214 |
+
"https://unicone-studio-instance1.hf.space": {
|
215 |
+
"total": "50 GB",
|
216 |
+
"used": "3.33 GB"
|
217 |
+
}
|
218 |
+
}
|
219 |
+
Args:
|
220 |
+
title (str): The title of the Tv show.
|
221 |
+
season (str): The season of the Tv show.
|
222 |
+
episode (str): The title of the Tv show.
|
223 |
+
"""
|
224 |
+
best_instance = None
|
225 |
+
max_free_space = -1
|
226 |
+
|
227 |
+
# Calculate free space for each instance
|
228 |
+
for instance_url, space_info in self.instances_health.items():
|
229 |
+
total_space = self._convert_to_gb(space_info['total'])
|
230 |
+
used_space = self._convert_to_gb(space_info['used'])
|
231 |
+
free_space = total_space - used_space
|
232 |
+
|
233 |
+
if free_space > max_free_space:
|
234 |
+
max_free_space = free_space
|
235 |
+
best_instance = instance_url
|
236 |
+
|
237 |
+
if best_instance:
|
238 |
+
result = self.instances_api.download_episode(best_instance, title, season, episode)
|
239 |
+
episode_id = result["episode_id"]
|
240 |
+
status = result["status"]
|
241 |
+
progress_url = f'{best_instance}/api/progress/{episode_id}'
|
242 |
+
response = {
|
243 |
+
"episode_id":episode_id,
|
244 |
+
"status":status,
|
245 |
+
"progress_url":progress_url
|
246 |
+
}
|
247 |
+
|
248 |
+
return response
|
249 |
+
else:
|
250 |
+
logging.error("No suitable instance found for downloading the film.")
|
251 |
+
return {"error": "No suitable instance found for downloading the film."}
|
252 |
+
|
253 |
+
def _convert_to_gb(self, space_str):
|
254 |
+
"""
|
255 |
+
Converts a space string like '50 GB' or '3.33 GB' to a float representing the number of GB.
|
256 |
+
"""
|
257 |
+
return float(space_str.split()[0])
|
258 |
+
|
259 |
+
#################################################################
|
260 |
+
def find_movie_path(self, title):
|
261 |
+
"""Find the path of the movie in the JSON data based on the title."""
|
262 |
+
for directory in self.file_structure:
|
263 |
+
if directory['type'] == 'directory' and directory['path'] == 'films':
|
264 |
+
for sub_directory in directory['contents']:
|
265 |
+
if sub_directory['type'] == 'directory':
|
266 |
+
for item in sub_directory['contents']:
|
267 |
+
if item['type'] == 'file' and title.lower() in item['path'].lower():
|
268 |
+
return item['path']
|
269 |
+
return None
|
270 |
+
|
271 |
+
def find_tv_path(self, title):
|
272 |
+
"""Find the path of the TV show in the JSON data based on the title."""
|
273 |
+
for directory in self.file_structure:
|
274 |
+
if directory['type'] == 'directory' and directory['path'] == 'tv':
|
275 |
+
for sub_directory in directory['contents']:
|
276 |
+
if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
|
277 |
+
return sub_directory['path']
|
278 |
+
return None
|
279 |
+
|
280 |
+
def get_tv_structure(self, title):
|
281 |
+
"""Find the path of the TV show in the JSON data based on the title."""
|
282 |
+
for directory in self.file_structure:
|
283 |
+
if directory['type'] == 'directory' and directory['path'] == 'tv':
|
284 |
+
for sub_directory in directory['contents']:
|
285 |
+
if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
|
286 |
+
return sub_directory
|
287 |
+
return None
|
288 |
+
|
289 |
+
def get_film_id(self, title):
|
290 |
+
"""Generate a film ID based on the title."""
|
291 |
+
return title.replace(" ", "_").lower()
|
292 |
+
|
293 |
+
def prefetch_metadata(self):
|
294 |
+
"""Prefetch metadata for all items in the file structure."""
|
295 |
+
for item in self.file_structure:
|
296 |
+
if 'contents' in item:
|
297 |
+
for sub_item in item['contents']:
|
298 |
+
original_title = sub_item['path'].split('/')[-1]
|
299 |
+
media_type = 'series' if item['path'].startswith('tv') else 'movie'
|
300 |
+
title = original_title
|
301 |
+
year = None
|
302 |
+
|
303 |
+
# Extract year from the title if available
|
304 |
+
match = re.search(r'\((\d{4})\)', original_title)
|
305 |
+
if match:
|
306 |
+
year_str = match.group(1)
|
307 |
+
if year_str.isdigit() and len(year_str) == 4:
|
308 |
+
title = original_title[:match.start()].strip()
|
309 |
+
year = int(year_str)
|
310 |
+
else:
|
311 |
+
parts = original_title.rsplit(' ', 1)
|
312 |
+
if len(parts) > 1 and parts[-1].isdigit() and len(parts[-1]) == 4:
|
313 |
+
title = parts[0].strip()
|
314 |
+
year = int(parts[-1])
|
315 |
+
|
316 |
+
fetch_and_cache_json(original_title, title, media_type, year)
|
317 |
+
|
318 |
+
def bytes_to_human_readable(self, num, suffix="B"):
|
319 |
+
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
|
320 |
+
if abs(num) < 1024.0:
|
321 |
+
return f"{num:3.1f} {unit}{suffix}"
|
322 |
+
num /= 1024.0
|
323 |
+
return f"{num:.1f} Y{suffix}"
|
324 |
|
325 |
+
def encode_episodeid(self, title, season, episode):
|
326 |
+
return f"{title}_{season}_{episode}"
|
327 |
|
328 |
+
def get_all_tv_shows(self):
|
329 |
+
"""Get all TV shows from the indexed cache structure JSON file."""
|
330 |
+
tv_shows = {}
|
331 |
+
for directory in self.file_structure:
|
332 |
+
if directory['type'] == 'directory' and directory['path'] == 'tv':
|
333 |
+
for sub_directory in directory['contents']:
|
334 |
+
if sub_directory['type'] == 'directory':
|
335 |
+
show_title = sub_directory['path'].split('/')[-1]
|
336 |
+
tv_shows[show_title] = []
|
337 |
+
for season_directory in sub_directory['contents']:
|
338 |
+
if season_directory['type'] == 'directory':
|
339 |
+
season = season_directory['path'].split('/')[-1]
|
340 |
+
for episode in season_directory['contents']:
|
341 |
+
if episode['type'] == 'file':
|
342 |
+
tv_shows[show_title].append({
|
343 |
+
"season": season,
|
344 |
+
"episode": episode['path'].split('/')[-1],
|
345 |
+
"path": episode['path']
|
346 |
+
})
|
347 |
+
return tv_shows
|
348 |
|
349 |
+
def get_all_films(self):
|
350 |
+
"""Get all films from the indexed cache structure JSON file."""
|
351 |
+
films = []
|
352 |
+
for directory in self.file_structure:
|
353 |
+
if directory['type'] == 'directory' and directory['path'] == 'films':
|
354 |
+
for sub_directory in directory['contents']:
|
355 |
+
if sub_directory['type'] == 'directory':
|
356 |
+
films.append(sub_directory['path'])
|
357 |
+
return films
|
358 |
|
359 |
+
def start_prefetching(self):
|
360 |
+
"""Start the metadata prefetching in a separate thread."""
|
361 |
+
self.prefetch_metadata()
|
old.app.py
CHANGED
@@ -2,246 +2,70 @@ from flask import Flask, jsonify, request, send_from_directory
|
|
2 |
from flask_cors import CORS
|
3 |
import os
|
4 |
import json
|
5 |
-
import
|
6 |
import urllib.parse
|
7 |
-
from hf_scrapper import download_film, download_episode, get_system_proxies, get_download_progress
|
8 |
-
from indexer import indexer
|
9 |
-
from tvdb import fetch_and_cache_json
|
10 |
-
import re
|
11 |
-
import logging
|
12 |
from LoadBalancer import LoadBalancer
|
|
|
13 |
|
14 |
-
logging.basicConfig(level=logging.INFO)
|
15 |
-
load_balancer = LoadBalancer()
|
16 |
-
|
17 |
-
# Start polling in a separate thread
|
18 |
-
polling_thread = threading.Thread(target=load_balancer.start_polling)
|
19 |
-
polling_thread.start()
|
20 |
|
21 |
app = Flask(__name__)
|
22 |
CORS(app)
|
23 |
|
|
|
24 |
# Constants and Configuration
|
25 |
CACHE_DIR = os.getenv("CACHE_DIR")
|
26 |
INDEX_FILE = os.getenv("INDEX_FILE")
|
27 |
TOKEN = os.getenv("TOKEN")
|
28 |
-
FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
|
29 |
-
TV_STORE_JSON_PATH = os.path.join(CACHE_DIR, "tv_store.json")
|
30 |
-
INSTANCE_REGISTER_JSON_PATH = os.path.join(CACHE_DIR, "instance_register.json")
|
31 |
REPO = os.getenv("REPO")
|
32 |
-
download_threads = {}
|
33 |
-
|
34 |
-
# Ensure CACHE_DIR exists
|
35 |
-
if not os.path.exists(CACHE_DIR):
|
36 |
-
os.makedirs(CACHE_DIR)
|
37 |
-
|
38 |
-
for path in [FILM_STORE_JSON_PATH, TV_STORE_JSON_PATH]:
|
39 |
-
if not os.path.exists(path):
|
40 |
-
with open(path, 'w') as json_file:
|
41 |
-
json.dump({}, json_file)
|
42 |
-
|
43 |
-
# Index the file structure
|
44 |
-
indexer()
|
45 |
|
46 |
-
|
47 |
-
if not os.path.exists(INDEX_FILE):
|
48 |
-
raise FileNotFoundError(f"{INDEX_FILE} not found. Please make sure the file exists.")
|
49 |
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
# Function Definitions
|
54 |
-
def load_json(file_path):
|
55 |
-
"""Load JSON data from a file."""
|
56 |
-
with open(file_path, 'r') as file:
|
57 |
-
return json.load(file)
|
58 |
-
|
59 |
-
def find_movie_path(json_data, title):
|
60 |
-
"""Find the path of the movie in the JSON data based on the title."""
|
61 |
-
for directory in json_data:
|
62 |
-
if directory['type'] == 'directory' and directory['path'] == 'films':
|
63 |
-
for sub_directory in directory['contents']:
|
64 |
-
if sub_directory['type'] == 'directory':
|
65 |
-
for item in sub_directory['contents']:
|
66 |
-
if item['type'] == 'file' and title.lower() in item['path'].lower():
|
67 |
-
return item['path']
|
68 |
-
return None
|
69 |
-
|
70 |
-
def find_tv_path(json_data, title):
|
71 |
-
"""Find the path of the TV show in the JSON data based on the title."""
|
72 |
-
for directory in json_data:
|
73 |
-
if directory['type'] == 'directory' and directory['path'] == 'tv':
|
74 |
-
for sub_directory in directory['contents']:
|
75 |
-
if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
|
76 |
-
return sub_directory['path']
|
77 |
-
return None
|
78 |
-
|
79 |
-
def get_tv_structure(json_data,title):
|
80 |
-
"""Find the path of the TV show in the JSON data based on the title."""
|
81 |
-
for directory in json_data:
|
82 |
-
if directory['type'] == 'directory' and directory['path'] == 'tv':
|
83 |
-
for sub_directory in directory['contents']:
|
84 |
-
if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
|
85 |
-
return sub_directory
|
86 |
-
return None
|
87 |
-
|
88 |
-
def get_film_id(title):
|
89 |
-
"""Generate a film ID based on the title."""
|
90 |
-
return title.replace(" ", "_").lower()
|
91 |
-
|
92 |
-
def prefetch_metadata():
|
93 |
-
"""Prefetch metadata for all items in the file structure."""
|
94 |
-
for item in file_structure:
|
95 |
-
if 'contents' in item:
|
96 |
-
for sub_item in item['contents']:
|
97 |
-
original_title = sub_item['path'].split('/')[-1]
|
98 |
-
media_type = 'series' if item['path'].startswith('tv') else 'movie'
|
99 |
-
title = original_title
|
100 |
-
year = None
|
101 |
-
|
102 |
-
# Extract year from the title if available
|
103 |
-
match = re.search(r'\((\d{4})\)', original_title)
|
104 |
-
if match:
|
105 |
-
year_str = match.group(1)
|
106 |
-
if year_str.isdigit() and len(year_str) == 4:
|
107 |
-
title = original_title[:match.start()].strip()
|
108 |
-
year = int(year_str)
|
109 |
-
else:
|
110 |
-
parts = original_title.rsplit(' ', 1)
|
111 |
-
if len(parts) > 1 and parts[-1].isdigit() and len(parts[-1]) == 4:
|
112 |
-
title = parts[0].strip()
|
113 |
-
year = int(parts[-1])
|
114 |
-
|
115 |
-
fetch_and_cache_json(original_title, title, media_type, year)
|
116 |
-
|
117 |
-
def bytes_to_human_readable(num, suffix="B"):
|
118 |
-
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
|
119 |
-
if abs(num) < 1024.0:
|
120 |
-
return f"{num:3.1f} {unit}{suffix}"
|
121 |
-
num /= 1024.0
|
122 |
-
return f"{num:.1f} Y{suffix}"
|
123 |
-
|
124 |
-
def encode_episodeid(title,season,episode):
|
125 |
-
return f"{title}_{season}_{episode}"
|
126 |
-
|
127 |
-
def is_valid_url(url):
|
128 |
-
# Simple URL validation (could be more complex if needed)
|
129 |
-
regex = re.compile(
|
130 |
-
r'^(?:http|ftp)s?://' # http:// or https://
|
131 |
-
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
|
132 |
-
r'localhost|' # localhost...
|
133 |
-
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
|
134 |
-
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
|
135 |
-
r'(?::\d+)?' # optional port
|
136 |
-
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
137 |
-
return re.match(regex, url) is not None
|
138 |
-
|
139 |
-
def get_all_tv_shows(indexed_cache):
|
140 |
-
"""Get all TV shows from the indexed cache structure JSON file."""
|
141 |
-
tv_shows = {}
|
142 |
-
for directory in indexed_cache:
|
143 |
-
if directory['type'] == 'directory' and directory['path'] == 'tv':
|
144 |
-
for sub_directory in directory['contents']:
|
145 |
-
if sub_directory['type'] == 'directory':
|
146 |
-
show_title = sub_directory['path'].split('/')[-1]
|
147 |
-
tv_shows[show_title] = []
|
148 |
-
for season_directory in sub_directory['contents']:
|
149 |
-
if season_directory['type'] == 'directory':
|
150 |
-
season = season_directory['path'].split('/')[-1]
|
151 |
-
for episode in season_directory['contents']:
|
152 |
-
if episode['type'] == 'file':
|
153 |
-
tv_shows[show_title].append({
|
154 |
-
"season": season,
|
155 |
-
"episode": episode['path'].split('/')[-1],
|
156 |
-
"path": episode['path']
|
157 |
-
})
|
158 |
-
return tv_shows
|
159 |
-
|
160 |
-
def get_all_films(indexed_cache):
|
161 |
-
"""Get all films from the indexed cache structure JSON file."""
|
162 |
-
films = []
|
163 |
-
for directory in indexed_cache:
|
164 |
-
if directory['type'] == 'directory' and directory['path'] == 'films':
|
165 |
-
for sub_directory in directory['contents']:
|
166 |
-
if sub_directory['type'] == 'directory':
|
167 |
-
films.append(sub_directory['path'])
|
168 |
-
return films
|
169 |
-
|
170 |
-
def start_prefetching():
|
171 |
-
"""Start the metadata prefetching in a separate thread."""
|
172 |
-
prefetch_metadata()
|
173 |
-
|
174 |
-
# Start prefetching metadata
|
175 |
-
thread = threading.Thread(target=start_prefetching)
|
176 |
-
thread.daemon = True
|
177 |
-
thread.start()
|
178 |
|
179 |
# API Endpoints
|
180 |
-
|
181 |
-
|
182 |
-
def get_movie_api():
|
183 |
"""Endpoint to get the movie by title."""
|
184 |
-
title = request.args.get('title')
|
185 |
if not title:
|
186 |
return jsonify({"error": "Title parameter is required"}), 400
|
187 |
|
188 |
-
# Load the film store JSON
|
189 |
-
with open(FILM_STORE_JSON_PATH, 'r') as json_file:
|
190 |
-
film_store_data = json.load(json_file)
|
191 |
-
|
192 |
# Check if the film is already cached
|
193 |
-
if title in
|
194 |
-
|
195 |
-
|
196 |
-
|
197 |
-
|
198 |
-
movie_path = find_movie_path(file_structure, title)
|
199 |
|
200 |
if not movie_path:
|
201 |
return jsonify({"error": "Movie not found"}), 404
|
202 |
|
203 |
-
|
204 |
-
|
205 |
-
|
206 |
-
|
207 |
-
|
208 |
-
# Start the download in a separate thread if not already downloading
|
209 |
-
if film_id not in download_threads or not download_threads[film_id].is_alive():
|
210 |
-
thread = threading.Thread(target=download_film, args=(file_url, TOKEN, cache_path, proxies, film_id, title))
|
211 |
-
download_threads[film_id] = thread
|
212 |
-
thread.start()
|
213 |
-
|
214 |
-
return jsonify({"status": "Download started", "film_id": film_id})
|
215 |
|
216 |
-
@app.route('/api/tv', methods=['GET'])
|
217 |
-
def get_tv_show_api():
|
218 |
"""Endpoint to get the TV show by title, season, and episode."""
|
219 |
-
title = request.args.get('title')
|
220 |
-
season = request.args.get('season')
|
221 |
-
episode = request.args.get('episode')
|
222 |
-
|
223 |
if not title or not season or not episode:
|
224 |
return jsonify({"error": "Title, season, and episode parameters are required"}), 400
|
225 |
|
226 |
-
# Load the TV store JSON
|
227 |
-
with open(TV_STORE_JSON_PATH, 'r') as json_file:
|
228 |
-
tv_store_data = json.load(json_file)
|
229 |
-
|
230 |
# Check if the episode is already cached
|
231 |
-
if title in
|
232 |
-
for ep in
|
233 |
if episode in ep:
|
234 |
-
|
235 |
-
|
236 |
-
return send_from_directory(os.path.dirname(cache_path), os.path.basename(cache_path))
|
237 |
|
238 |
-
tv_path = find_tv_path(
|
239 |
|
240 |
if not tv_path:
|
241 |
return jsonify({"error": "TV show not found"}), 404
|
242 |
|
243 |
episode_path = None
|
244 |
-
for directory in file_structure:
|
245 |
if directory['type'] == 'directory' and directory['path'] == 'tv':
|
246 |
for sub_directory in directory['contents']:
|
247 |
if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
|
@@ -255,25 +79,26 @@ def get_tv_show_api():
|
|
255 |
if not episode_path:
|
256 |
return jsonify({"error": "Episode not found"}), 404
|
257 |
|
258 |
-
|
259 |
-
|
260 |
-
|
261 |
-
|
262 |
-
|
263 |
-
# Start the download in a separate thread if not already downloading
|
264 |
-
if episode_id not in download_threads or not download_threads[episode_id].is_alive():
|
265 |
-
thread = threading.Thread(target=download_episode, args=(file_url, TOKEN, cache_path, proxies, episode_id, title))
|
266 |
-
download_threads[episode_id] = thread
|
267 |
-
thread.start()
|
268 |
-
|
269 |
-
return jsonify({"status": "Download started", "episode_id": episode_id})
|
270 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
271 |
|
272 |
-
@app.route('/api/
|
273 |
-
def
|
274 |
-
"""Endpoint to get the
|
275 |
-
|
276 |
-
|
|
|
|
|
277 |
|
278 |
@app.route('/api/cache/size', methods=['GET'])
|
279 |
def get_cache_size_api():
|
@@ -282,7 +107,7 @@ def get_cache_size_api():
|
|
282 |
for f in filenames:
|
283 |
fp = os.path.join(dirpath, f)
|
284 |
total_size += os.path.getsize(fp)
|
285 |
-
readable_size = bytes_to_human_readable(total_size)
|
286 |
return jsonify({"cache_size": readable_size})
|
287 |
|
288 |
@app.route('/api/cache/clear', methods=['POST'])
|
@@ -296,48 +121,18 @@ def clear_cache_api():
|
|
296 |
@app.route('/api/tv/store', methods=['GET'])
|
297 |
def get_tv_store_api():
|
298 |
"""Endpoint to get the TV store JSON."""
|
299 |
-
|
300 |
-
|
301 |
-
tv_store_data = json.load(json_file)
|
302 |
-
return jsonify(tv_store_data)
|
303 |
-
return jsonify({}), 404
|
304 |
|
305 |
@app.route('/api/film/store', methods=['GET'])
|
306 |
def get_film_store_api():
|
307 |
"""Endpoint to get the film store JSON."""
|
308 |
-
|
309 |
-
with open(FILM_STORE_JSON_PATH, 'r') as json_file:
|
310 |
-
tv_store_data = json.load(json_file)
|
311 |
-
return jsonify(tv_store_data)
|
312 |
-
return jsonify({}), 404
|
313 |
-
|
314 |
-
#################################################
|
315 |
-
# No change needed
|
316 |
|
317 |
-
@app.route('/api/filmid', methods=['GET'])
|
318 |
-
def get_film_id_by_title_api():
|
319 |
-
"""Endpoint to get the film ID by providing the movie title."""
|
320 |
-
title = request.args.get('title')
|
321 |
-
if not title:
|
322 |
-
return jsonify({"error": "Title parameter is required"}), 400
|
323 |
-
film_id = get_film_id(title)
|
324 |
-
return jsonify({"film_id": film_id})
|
325 |
-
|
326 |
-
@app.route('/api/episodeid', methods=['GET'])
|
327 |
-
def get_episode_id_api():
|
328 |
-
"""Endpoint to get the episode ID by providing the TV show title, season, and episode."""
|
329 |
-
title = request.args.get('title')
|
330 |
-
season = request.args.get('season')
|
331 |
-
episode = request.args.get('episode')
|
332 |
-
if not title or not season or not episode:
|
333 |
-
return jsonify({"error": "Title, season, and episode parameters are required"}), 400
|
334 |
-
episode_id = encode_episodeid(title,season,episode)
|
335 |
-
return jsonify({"episode_id": episode_id})
|
336 |
|
337 |
-
@app.route('/api/film/metadata', methods=['GET'])
|
338 |
-
def get_film_metadata_api():
|
339 |
"""Endpoint to get the film metadata by title."""
|
340 |
-
title = request.args.get('title')
|
341 |
if not title:
|
342 |
return jsonify({'error': 'No title provided'}), 400
|
343 |
|
@@ -350,10 +145,9 @@ def get_film_metadata_api():
|
|
350 |
|
351 |
return jsonify({'error': 'Metadata not found'}), 404
|
352 |
|
353 |
-
@app.route('/api/tv/metadata', methods=['GET'])
|
354 |
-
def get_tv_metadata_api():
|
355 |
"""Endpoint to get the TV show metadata by title."""
|
356 |
-
title = request.args.get('title')
|
357 |
if not title:
|
358 |
return jsonify({'error': 'No title provided'}), 400
|
359 |
|
@@ -364,7 +158,7 @@ def get_tv_metadata_api():
|
|
364 |
data = json.load(f)
|
365 |
|
366 |
# Add the file structure to the metadata
|
367 |
-
tv_structure_data = get_tv_structure(
|
368 |
if tv_structure_data:
|
369 |
data['file_structure'] = tv_structure_data
|
370 |
|
@@ -375,14 +169,21 @@ def get_tv_metadata_api():
|
|
375 |
|
376 |
@app.route("/api/film/all")
|
377 |
def get_all_films_api():
|
378 |
-
return get_all_films(
|
379 |
|
380 |
@app.route("/api/tv/all")
|
381 |
def get_all_tvshows_api():
|
382 |
-
return get_all_tv_shows(
|
383 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
384 |
#############################################################
|
385 |
-
#
|
386 |
@app.route('/api/register', methods=['POST'])
|
387 |
def register_instance():
|
388 |
try:
|
@@ -391,7 +192,7 @@ def register_instance():
|
|
391 |
return jsonify({"error": "No URL provided"}), 400
|
392 |
|
393 |
url = data["url"]
|
394 |
-
if not is_valid_url(url):
|
395 |
return jsonify({"error": "Invalid URL"}), 400
|
396 |
|
397 |
# Register the instance
|
@@ -403,16 +204,13 @@ def register_instance():
|
|
403 |
except Exception as e:
|
404 |
logging.error(f"Error registering instance: {e}")
|
405 |
return jsonify({"error": "Failed to register instance"}), 500
|
|
|
406 |
#############################################################
|
407 |
# Routes
|
408 |
@app.route('/')
|
409 |
def index():
|
410 |
return f"Load Balancer is Running {load_balancer.version}"
|
411 |
|
412 |
-
@app.route('/api/instances',methods=["GET"])
|
413 |
-
def get_instances():
|
414 |
-
return load_balancer.instances
|
415 |
-
|
416 |
# Main entry point
|
417 |
if __name__ == "__main__":
|
418 |
app.run(debug=True, host="0.0.0.0", port=7860)
|
|
|
2 |
from flask_cors import CORS
|
3 |
import os
|
4 |
import json
|
5 |
+
from threading import Thread
|
6 |
import urllib.parse
|
|
|
|
|
|
|
|
|
|
|
7 |
from LoadBalancer import LoadBalancer
|
8 |
+
import logging
|
9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
|
11 |
app = Flask(__name__)
|
12 |
CORS(app)
|
13 |
|
14 |
+
logging.basicConfig(level=logging.INFO)
|
15 |
# Constants and Configuration
|
16 |
CACHE_DIR = os.getenv("CACHE_DIR")
|
17 |
INDEX_FILE = os.getenv("INDEX_FILE")
|
18 |
TOKEN = os.getenv("TOKEN")
|
|
|
|
|
|
|
19 |
REPO = os.getenv("REPO")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
+
load_balancer = LoadBalancer(cache_dir=CACHE_DIR, index_file=INDEX_FILE, token=TOKEN, repo=REPO)
|
|
|
|
|
22 |
|
23 |
+
# Start polling in a separate thread
|
24 |
+
polling_thread = Thread(target=load_balancer.start_polling)
|
25 |
+
polling_thread.start()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
26 |
|
27 |
# API Endpoints
|
28 |
+
@app.route('/api/film/<title>', methods=['GET'])
|
29 |
+
def get_movie_api(title):
|
|
|
30 |
"""Endpoint to get the movie by title."""
|
|
|
31 |
if not title:
|
32 |
return jsonify({"error": "Title parameter is required"}), 400
|
33 |
|
|
|
|
|
|
|
|
|
34 |
# Check if the film is already cached
|
35 |
+
if title in load_balancer.FILM_STORE:
|
36 |
+
url = load_balancer.FILM_STORE[title]
|
37 |
+
return jsonify({"url":url})
|
38 |
+
|
39 |
+
movie_path = load_balancer.find_movie_path(title)
|
|
|
40 |
|
41 |
if not movie_path:
|
42 |
return jsonify({"error": "Movie not found"}), 404
|
43 |
|
44 |
+
# Start the download in a instance
|
45 |
+
response = load_balancer.download_film_to_best_instance(title=title)
|
46 |
+
if response:
|
47 |
+
return jsonify(response)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
48 |
|
49 |
+
@app.route('/api/tv/<title>/<season>/<episode>', methods=['GET'])
|
50 |
+
def get_tv_show_api(title, season, episode):
|
51 |
"""Endpoint to get the TV show by title, season, and episode."""
|
|
|
|
|
|
|
|
|
52 |
if not title or not season or not episode:
|
53 |
return jsonify({"error": "Title, season, and episode parameters are required"}), 400
|
54 |
|
|
|
|
|
|
|
|
|
55 |
# Check if the episode is already cached
|
56 |
+
if title in load_balancer.TV_STORE and season in load_balancer.TV_STORE[title]:
|
57 |
+
for ep in load_balancer.TV_STORE[title][season]:
|
58 |
if episode in ep:
|
59 |
+
url = load_balancer.TV_STORE[title][season][ep]
|
60 |
+
return jsonify({"url":url})
|
|
|
61 |
|
62 |
+
tv_path = load_balancer.find_tv_path(title)
|
63 |
|
64 |
if not tv_path:
|
65 |
return jsonify({"error": "TV show not found"}), 404
|
66 |
|
67 |
episode_path = None
|
68 |
+
for directory in load_balancer.file_structure:
|
69 |
if directory['type'] == 'directory' and directory['path'] == 'tv':
|
70 |
for sub_directory in directory['contents']:
|
71 |
if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
|
|
|
79 |
if not episode_path:
|
80 |
return jsonify({"error": "Episode not found"}), 404
|
81 |
|
82 |
+
# Start the download in a instance
|
83 |
+
response = load_balancer.download_episode_to_best_instance(title=title, season=season, episode=episode)
|
84 |
+
if response:
|
85 |
+
return jsonify(response)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
86 |
|
87 |
+
@app.route('/api/filmid/<title>', methods=['GET'])
|
88 |
+
def get_film_id_by_title_api(title):
|
89 |
+
"""Endpoint to get the film ID by providing the movie title."""
|
90 |
+
if not title:
|
91 |
+
return jsonify({"error": "Title parameter is required"}), 400
|
92 |
+
film_id = load_balancer.get_film_id(title)
|
93 |
+
return jsonify({"film_id": film_id})
|
94 |
|
95 |
+
@app.route('/api/episodeid/<title>/<season>/<episode>', methods=['GET'])
|
96 |
+
def get_episode_id_api(title,season,episode):
|
97 |
+
"""Endpoint to get the episode ID by providing the TV show title, season, and episode."""
|
98 |
+
if not title or not season or not episode:
|
99 |
+
return jsonify({"error": "Title, season, and episode parameters are required"}), 400
|
100 |
+
episode_id = load_balancer.encode_episodeid(title,season,episode)
|
101 |
+
return jsonify({"episode_id": episode_id})
|
102 |
|
103 |
@app.route('/api/cache/size', methods=['GET'])
|
104 |
def get_cache_size_api():
|
|
|
107 |
for f in filenames:
|
108 |
fp = os.path.join(dirpath, f)
|
109 |
total_size += os.path.getsize(fp)
|
110 |
+
readable_size = load_balancer.bytes_to_human_readable(total_size)
|
111 |
return jsonify({"cache_size": readable_size})
|
112 |
|
113 |
@app.route('/api/cache/clear', methods=['POST'])
|
|
|
121 |
@app.route('/api/tv/store', methods=['GET'])
|
122 |
def get_tv_store_api():
|
123 |
"""Endpoint to get the TV store JSON."""
|
124 |
+
return jsonify(load_balancer.TV_STORE)
|
125 |
+
|
|
|
|
|
|
|
126 |
|
127 |
@app.route('/api/film/store', methods=['GET'])
|
128 |
def get_film_store_api():
|
129 |
"""Endpoint to get the film store JSON."""
|
130 |
+
return jsonify(load_balancer.FILM_STORE)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
131 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
132 |
|
133 |
+
@app.route('/api/film/metadata/<title>', methods=['GET'])
|
134 |
+
def get_film_metadata_api(title):
|
135 |
"""Endpoint to get the film metadata by title."""
|
|
|
136 |
if not title:
|
137 |
return jsonify({'error': 'No title provided'}), 400
|
138 |
|
|
|
145 |
|
146 |
return jsonify({'error': 'Metadata not found'}), 404
|
147 |
|
148 |
+
@app.route('/api/tv/metadata/<title>', methods=['GET'])
|
149 |
+
def get_tv_metadata_api(title):
|
150 |
"""Endpoint to get the TV show metadata by title."""
|
|
|
151 |
if not title:
|
152 |
return jsonify({'error': 'No title provided'}), 400
|
153 |
|
|
|
158 |
data = json.load(f)
|
159 |
|
160 |
# Add the file structure to the metadata
|
161 |
+
tv_structure_data = load_balancer.get_tv_structure(title)
|
162 |
if tv_structure_data:
|
163 |
data['file_structure'] = tv_structure_data
|
164 |
|
|
|
169 |
|
170 |
@app.route("/api/film/all")
|
171 |
def get_all_films_api():
|
172 |
+
return load_balancer.get_all_films()
|
173 |
|
174 |
@app.route("/api/tv/all")
|
175 |
def get_all_tvshows_api():
|
176 |
+
return load_balancer.get_all_tv_shows()
|
177 |
|
178 |
+
@app.route('/api/instances',methods=["GET"])
|
179 |
+
def get_instances():
|
180 |
+
return load_balancer.instances
|
181 |
+
|
182 |
+
@app.route('/api/instances/health',methods=["GET"])
|
183 |
+
def get_instances_health():
|
184 |
+
return load_balancer.instances_health
|
185 |
#############################################################
|
186 |
+
# This API is only for instances
|
187 |
@app.route('/api/register', methods=['POST'])
|
188 |
def register_instance():
|
189 |
try:
|
|
|
192 |
return jsonify({"error": "No URL provided"}), 400
|
193 |
|
194 |
url = data["url"]
|
195 |
+
if not load_balancer.is_valid_url(url):
|
196 |
return jsonify({"error": "Invalid URL"}), 400
|
197 |
|
198 |
# Register the instance
|
|
|
204 |
except Exception as e:
|
205 |
logging.error(f"Error registering instance: {e}")
|
206 |
return jsonify({"error": "Failed to register instance"}), 500
|
207 |
+
|
208 |
#############################################################
|
209 |
# Routes
|
210 |
@app.route('/')
|
211 |
def index():
|
212 |
return f"Load Balancer is Running {load_balancer.version}"
|
213 |
|
|
|
|
|
|
|
|
|
214 |
# Main entry point
|
215 |
if __name__ == "__main__":
|
216 |
app.run(debug=True, host="0.0.0.0", port=7860)
|
utils.py
ADDED
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import re
|
2 |
+
|
3 |
+
def is_valid_url(url):
|
4 |
+
"""
|
5 |
+
Validates the URL.
|
6 |
+
|
7 |
+
Args:
|
8 |
+
url (str): The URL to validate.
|
9 |
+
|
10 |
+
Returns:
|
11 |
+
bool: True if the URL is valid, False otherwise.
|
12 |
+
"""
|
13 |
+
regex = re.compile(
|
14 |
+
r'^(?:http|ftp)s?://' # http:// or https://
|
15 |
+
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
|
16 |
+
r'localhost|' # localhost...
|
17 |
+
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
|
18 |
+
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
|
19 |
+
r'(?::\d+)?' # optional port
|
20 |
+
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
21 |
+
return re.match(regex, url) is not None
|
22 |
+
|
23 |
+
def convert_to_gb(space_str):
|
24 |
+
"""
|
25 |
+
Converts a space string like '50 GB' or '3.33 GB' to a float representing the number of GB.
|
26 |
+
|
27 |
+
Args:
|
28 |
+
space_str (str): The space string to convert.
|
29 |
+
|
30 |
+
Returns:
|
31 |
+
float: The space in GB.
|
32 |
+
"""
|
33 |
+
return float(space_str.split()[0])
|
34 |
+
|
35 |
+
def bytes_to_human_readable(num, suffix="B"):
|
36 |
+
"""
|
37 |
+
Converts bytes to a human-readable format.
|
38 |
+
|
39 |
+
Args:
|
40 |
+
num (int): The number of bytes.
|
41 |
+
suffix (str): The suffix to use (default is 'B').
|
42 |
+
|
43 |
+
Returns:
|
44 |
+
str: The human-readable string.
|
45 |
+
"""
|
46 |
+
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
|
47 |
+
if abs(num) < 1024.0:
|
48 |
+
return f"{num:3.1f} {unit}{suffix}"
|
49 |
+
num /= 1024.0
|
50 |
+
return f"{num:.1f} Y{suffix}"
|
51 |
+
|
52 |
+
def encode_episodeid(title, season, episode):
|
53 |
+
"""
|
54 |
+
Encodes the episode ID based on title, season, and episode.
|
55 |
+
|
56 |
+
Args:
|
57 |
+
title (str): The title of the TV show.
|
58 |
+
season (str): The season of the TV show.
|
59 |
+
episode (str): The episode number.
|
60 |
+
|
61 |
+
Returns:
|
62 |
+
str: The encoded episode ID.
|
63 |
+
"""
|
64 |
+
return f"{title}_{season}_{episode}"
|