ChandimaPrabath commited on
Commit
151773c
1 Parent(s): 815f274

0.0.2.7 V Beta

Browse files
Files changed (8) hide show
  1. LoadBalancer.py +43 -68
  2. TODO.md +1 -1
  3. app.py +4 -3
  4. hf_scrapper.py +1 -177
  5. indexer.py +0 -1
  6. old.LoadBalancer.py +308 -20
  7. old.app.py +65 -267
  8. utils.py +64 -0
LoadBalancer.py CHANGED
@@ -1,25 +1,21 @@
1
  import os
2
  import json
3
- import urllib.request
4
- import concurrent.futures
5
- from tqdm import tqdm
6
  from indexer import indexer
7
  import re
8
  from tvdb import fetch_and_cache_json
9
- from threading import Event
10
  import time
11
  import logging
12
- from threading import Thread, Event
13
  from api import InstancesAPI
14
 
15
-
16
  CACHE_DIR = os.getenv("CACHE_DIR")
17
 
18
  download_progress = {}
19
 
20
  class LoadBalancer:
21
  def __init__(self, cache_dir, index_file, token, repo, polling_interval=4, max_retries=3, initial_delay=1):
22
- self.version = "0.0.2.6 V Beta"
23
  self.instances = []
24
  self.instances_health = {}
25
  self.polling_interval = polling_interval
@@ -34,24 +30,52 @@ class LoadBalancer:
34
  self.FILM_STORE = {}
35
  self.TV_STORE = {}
36
  self.file_structure = None
 
37
 
38
  # Ensure CACHE_DIR exists
39
  if not os.path.exists(self.CACHE_DIR):
40
  os.makedirs(self.CACHE_DIR)
41
 
42
- # Index the file structure
43
  indexer()
44
 
45
  # Load the file structure JSON
 
 
 
 
 
 
 
 
 
 
 
 
46
  if not os.path.exists(self.INDEX_FILE):
47
  raise FileNotFoundError(f"{self.INDEX_FILE} not found. Please make sure the file exists.")
48
 
49
  with open(self.INDEX_FILE, 'r') as f:
50
  self.file_structure = json.load(f)
 
51
 
52
- prefetch_thread = Thread(target=self.start_prefetching)
53
- prefetch_thread.daemon = True
54
- prefetch_thread.start()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
55
 
56
  def register_instance(self, instance_url):
57
  if instance_url not in self.instances:
@@ -124,38 +148,9 @@ class LoadBalancer:
124
  logging.info("Stopping polling.")
125
  self.stop_event.set()
126
 
127
- ######################################################################
128
- @staticmethod
129
- def get_system_proxies():
130
- """
131
- Retrieves the system's HTTP and HTTPS proxies.
132
-
133
- Returns:
134
- dict: A dictionary containing the proxies.
135
- """
136
- try:
137
- proxies = urllib.request.getproxies()
138
- print("System proxies:", proxies)
139
- return {
140
- "http": proxies.get("http"),
141
- "https": proxies.get("http")
142
- }
143
- except Exception as e:
144
- print(f"Error getting system proxies: {e}")
145
- return {}
146
-
147
- @staticmethod
148
- def is_valid_url(url):
149
- # Simple URL validation (could be more complex if needed)
150
- regex = re.compile(
151
- r'^(?:http|ftp)s?://' # http:// or https://
152
- r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
153
- r'localhost|' # localhost...
154
- r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
155
- r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
156
- r'(?::\d+)?' # optional port
157
- r'(?:/?|[/?]\S+)$', re.IGNORECASE)
158
- return re.match(regex, url) is not None
159
 
160
  #################################################################
161
 
@@ -183,8 +178,8 @@ class LoadBalancer:
183
 
184
  # Calculate free space for each instance
185
  for instance_url, space_info in self.instances_health.items():
186
- total_space = self._convert_to_gb(space_info['total'])
187
- used_space = self._convert_to_gb(space_info['used'])
188
  free_space = total_space - used_space
189
 
190
  if free_space > max_free_space:
@@ -227,8 +222,8 @@ class LoadBalancer:
227
 
228
  # Calculate free space for each instance
229
  for instance_url, space_info in self.instances_health.items():
230
- total_space = self._convert_to_gb(space_info['total'])
231
- used_space = self._convert_to_gb(space_info['used'])
232
  free_space = total_space - used_space
233
 
234
  if free_space > max_free_space:
@@ -251,12 +246,6 @@ class LoadBalancer:
251
  logging.error("No suitable instance found for downloading the film.")
252
  return {"error": "No suitable instance found for downloading the film."}
253
 
254
- def _convert_to_gb(self, space_str):
255
- """
256
- Converts a space string like '50 GB' or '3.33 GB' to a float representing the number of GB.
257
- """
258
- return float(space_str.split()[0])
259
-
260
  #################################################################
261
  def find_movie_path(self, title):
262
  """Find the path of the movie in the JSON data based on the title."""
@@ -316,16 +305,6 @@ class LoadBalancer:
316
 
317
  fetch_and_cache_json(original_title, title, media_type, year)
318
 
319
- def bytes_to_human_readable(self, num, suffix="B"):
320
- for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
321
- if abs(num) < 1024.0:
322
- return f"{num:3.1f} {unit}{suffix}"
323
- num /= 1024.0
324
- return f"{num:.1f} Y{suffix}"
325
-
326
- def encode_episodeid(self, title, season, episode):
327
- return f"{title}_{season}_{episode}"
328
-
329
  def get_all_tv_shows(self):
330
  """Get all TV shows from the indexed cache structure JSON file."""
331
  tv_shows = {}
@@ -355,8 +334,4 @@ class LoadBalancer:
355
  for sub_directory in directory['contents']:
356
  if sub_directory['type'] == 'directory':
357
  films.append(sub_directory['path'])
358
- return films
359
-
360
- def start_prefetching(self):
361
- """Start the metadata prefetching in a separate thread."""
362
- self.prefetch_metadata()
 
1
  import os
2
  import json
 
 
 
3
  from indexer import indexer
4
  import re
5
  from tvdb import fetch_and_cache_json
6
+ from threading import Event, Thread
7
  import time
8
  import logging
9
+ from utils import convert_to_gb
10
  from api import InstancesAPI
11
 
 
12
  CACHE_DIR = os.getenv("CACHE_DIR")
13
 
14
  download_progress = {}
15
 
16
  class LoadBalancer:
17
  def __init__(self, cache_dir, index_file, token, repo, polling_interval=4, max_retries=3, initial_delay=1):
18
+ self.version = "0.0.2.7 V Beta"
19
  self.instances = []
20
  self.instances_health = {}
21
  self.polling_interval = polling_interval
 
30
  self.FILM_STORE = {}
31
  self.TV_STORE = {}
32
  self.file_structure = None
33
+ self.index_file_last_modified = None
34
 
35
  # Ensure CACHE_DIR exists
36
  if not os.path.exists(self.CACHE_DIR):
37
  os.makedirs(self.CACHE_DIR)
38
 
39
+ # Index the file structure initially
40
  indexer()
41
 
42
  # Load the file structure JSON
43
+ self.load_file_structure()
44
+
45
+ # Start polling and file checking in separate threads
46
+ polling_thread = Thread(target=self.start_polling)
47
+ polling_thread.daemon = True
48
+ polling_thread.start()
49
+
50
+ file_checking_thread = Thread(target=self.check_file_updates)
51
+ file_checking_thread.daemon = True
52
+ file_checking_thread.start()
53
+
54
+ def load_file_structure(self):
55
  if not os.path.exists(self.INDEX_FILE):
56
  raise FileNotFoundError(f"{self.INDEX_FILE} not found. Please make sure the file exists.")
57
 
58
  with open(self.INDEX_FILE, 'r') as f:
59
  self.file_structure = json.load(f)
60
+ logging.info("File structure loaded successfully.")
61
 
62
+ def check_file_updates(self):
63
+ while not self.stop_event.is_set():
64
+ if self.index_file_last_modified != os.path.getmtime(self.INDEX_FILE):
65
+ logging.info(f"{self.INDEX_FILE} has been updated. Re-indexing...")
66
+ indexer() # Re-run the indexer
67
+ self.load_file_structure() # Reload the file structure
68
+ self.index_file_last_modified = os.path.getmtime(self.INDEX_FILE)
69
+
70
+ # Restart prefetching thread
71
+ if hasattr(self, 'prefetch_thread') and self.prefetch_thread.is_alive():
72
+ self.prefetch_thread.join()
73
+
74
+ self.prefetch_thread = Thread(target=self.start_prefetching)
75
+ self.prefetch_thread.daemon = True
76
+ self.prefetch_thread.start()
77
+
78
+ time.sleep(120) # Check every 2 minutes
79
 
80
  def register_instance(self, instance_url):
81
  if instance_url not in self.instances:
 
148
  logging.info("Stopping polling.")
149
  self.stop_event.set()
150
 
151
+ def start_prefetching(self):
152
+ """Start the metadata prefetching in a separate thread."""
153
+ self.prefetch_metadata()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
154
 
155
  #################################################################
156
 
 
178
 
179
  # Calculate free space for each instance
180
  for instance_url, space_info in self.instances_health.items():
181
+ total_space = convert_to_gb(space_info['total'])
182
+ used_space = convert_to_gb(space_info['used'])
183
  free_space = total_space - used_space
184
 
185
  if free_space > max_free_space:
 
222
 
223
  # Calculate free space for each instance
224
  for instance_url, space_info in self.instances_health.items():
225
+ total_space = convert_to_gb(space_info['total'])
226
+ used_space = convert_to_gb(space_info['used'])
227
  free_space = total_space - used_space
228
 
229
  if free_space > max_free_space:
 
246
  logging.error("No suitable instance found for downloading the film.")
247
  return {"error": "No suitable instance found for downloading the film."}
248
 
 
 
 
 
 
 
249
  #################################################################
250
  def find_movie_path(self, title):
251
  """Find the path of the movie in the JSON data based on the title."""
 
305
 
306
  fetch_and_cache_json(original_title, title, media_type, year)
307
 
 
 
 
 
 
 
 
 
 
 
308
  def get_all_tv_shows(self):
309
  """Get all TV shows from the indexed cache structure JSON file."""
310
  tv_shows = {}
 
334
  for sub_directory in directory['contents']:
335
  if sub_directory['type'] == 'directory':
336
  films.append(sub_directory['path'])
337
+ return films
 
 
 
 
TODO.md CHANGED
@@ -1 +1 @@
1
- # implement a better way to collect store jsons
 
1
+ # implement a better way clear instance cache and refresh
app.py CHANGED
@@ -1,5 +1,6 @@
1
  from flask import Flask, jsonify, request, send_from_directory
2
  from flask_cors import CORS
 
3
  import os
4
  import json
5
  from threading import Thread
@@ -97,7 +98,7 @@ def get_episode_id_api(title,season,episode):
97
  """Endpoint to get the episode ID by providing the TV show title, season, and episode."""
98
  if not title or not season or not episode:
99
  return jsonify({"error": "Title, season, and episode parameters are required"}), 400
100
- episode_id = load_balancer.encode_episodeid(title,season,episode)
101
  return jsonify({"episode_id": episode_id})
102
 
103
  @app.route('/api/cache/size', methods=['GET'])
@@ -107,7 +108,7 @@ def get_cache_size_api():
107
  for f in filenames:
108
  fp = os.path.join(dirpath, f)
109
  total_size += os.path.getsize(fp)
110
- readable_size = load_balancer.bytes_to_human_readable(total_size)
111
  return jsonify({"cache_size": readable_size})
112
 
113
  @app.route('/api/cache/clear', methods=['POST'])
@@ -192,7 +193,7 @@ def register_instance():
192
  return jsonify({"error": "No URL provided"}), 400
193
 
194
  url = data["url"]
195
- if not load_balancer.is_valid_url(url):
196
  return jsonify({"error": "Invalid URL"}), 400
197
 
198
  # Register the instance
 
1
  from flask import Flask, jsonify, request, send_from_directory
2
  from flask_cors import CORS
3
+ from utils import is_valid_url, bytes_to_human_readable, encode_episodeid
4
  import os
5
  import json
6
  from threading import Thread
 
98
  """Endpoint to get the episode ID by providing the TV show title, season, and episode."""
99
  if not title or not season or not episode:
100
  return jsonify({"error": "Title, season, and episode parameters are required"}), 400
101
+ episode_id = encode_episodeid(title,season,episode)
102
  return jsonify({"episode_id": episode_id})
103
 
104
  @app.route('/api/cache/size', methods=['GET'])
 
108
  for f in filenames:
109
  fp = os.path.join(dirpath, f)
110
  total_size += os.path.getsize(fp)
111
+ readable_size = bytes_to_human_readable(total_size)
112
  return jsonify({"cache_size": readable_size})
113
 
114
  @app.route('/api/cache/clear', methods=['POST'])
 
193
  return jsonify({"error": "No URL provided"}), 400
194
 
195
  url = data["url"]
196
+ if not is_valid_url(url):
197
  return jsonify({"error": "Invalid URL"}), 400
198
 
199
  # Register the instance
hf_scrapper.py CHANGED
@@ -29,178 +29,6 @@ def get_system_proxies():
29
  print(f"Error getting system proxies: {e}")
30
  return {}
31
 
32
- def download_film(file_url, token, cache_path, proxies, film_id, title, chunk_size=100 * 1024 * 1024):
33
- """
34
- Downloads a file from the specified URL and saves it to the cache path.
35
- Tracks the download progress.
36
-
37
- Args:
38
- file_url (str): The URL of the file to download.
39
- token (str): The authorization token for the request.
40
- cache_path (str): The path to save the downloaded file.
41
- proxies (dict): Proxies for the request.
42
- film_id (str): Unique identifier for the film download.
43
- title (str): The title of the film.
44
- chunk_size (int): Size of each chunk to download.
45
- """
46
- print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
47
- headers = {'Authorization': f'Bearer {token}'}
48
- try:
49
- response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
50
- response.raise_for_status()
51
-
52
- total_size = int(response.headers.get('content-length', 0))
53
- download_progress[film_id] = {"total": total_size, "downloaded": 0, "status": "Downloading", "start_time": time.time()}
54
-
55
- os.makedirs(os.path.dirname(cache_path), exist_ok=True)
56
- with open(cache_path, 'wb') as file, tqdm(total=total_size, unit='B', unit_scale=True, desc=cache_path) as pbar:
57
- for data in response.iter_content(chunk_size=chunk_size):
58
- file.write(data)
59
- pbar.update(len(data))
60
- download_progress[film_id]["downloaded"] += len(data)
61
-
62
- print(f'File cached to {cache_path} successfully.')
63
- update_film_store_json(title, cache_path)
64
- download_progress[film_id]["status"] = "Completed"
65
- except RequestException as e:
66
- print(f"Error downloading file: {e}")
67
- download_progress[film_id]["status"] = "Failed"
68
- except IOError as e:
69
- print(f"Error writing file {cache_path}: {e}")
70
- download_progress[film_id]["status"] = "Failed"
71
- finally:
72
- if download_progress[film_id]["status"] != "Downloading":
73
- download_progress[film_id]["end_time"] = time.time()
74
-
75
- def get_download_progress(id):
76
- """
77
- Gets the download progress for a specific film.
78
-
79
- Args:
80
- film_id (str): The unique identifier for the film download.
81
-
82
- Returns:
83
- dict: A dictionary containing the total size, downloaded size, progress percentage, status, and ETA.
84
- """
85
- if id in download_progress:
86
- total = download_progress[id]["total"]
87
- downloaded = download_progress[id]["downloaded"]
88
- status = download_progress[id].get("status", "In Progress")
89
- progress = (downloaded / total) * 100 if total > 0 else 0
90
-
91
- eta = None
92
- if status == "Downloading" and downloaded > 0:
93
- elapsed_time = time.time() - download_progress[id]["start_time"]
94
- estimated_total_time = elapsed_time * (total / downloaded)
95
- eta = estimated_total_time - elapsed_time
96
- elif status == "Completed":
97
- eta = 0
98
-
99
- return {"total": total, "downloaded": downloaded, "progress": progress, "status": status, "eta": eta}
100
- return {"total": 0, "downloaded": 0, "progress": 0, "status": "Not Found", "eta": None}
101
-
102
- def update_film_store_json(title, cache_path):
103
- """
104
- Updates the film store JSON with the new file.
105
-
106
- Args:
107
- title (str): The title of the film.
108
- cache_path (str): The local path where the file is saved.
109
- """
110
- FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
111
-
112
- film_store_data = {}
113
- if os.path.exists(FILM_STORE_JSON_PATH):
114
- with open(FILM_STORE_JSON_PATH, 'r') as json_file:
115
- film_store_data = json.load(json_file)
116
-
117
- film_store_data[title] = cache_path
118
-
119
- with open(FILM_STORE_JSON_PATH, 'w') as json_file:
120
- json.dump(film_store_data, json_file, indent=2)
121
- print(f'Film store updated with {title}.')
122
-
123
-
124
- ###############################################################################
125
- def download_episode(file_url, token, cache_path, proxies, episode_id, title, chunk_size=100 * 1024 * 1024):
126
- """
127
- Downloads a file from the specified URL and saves it to the cache path.
128
- Tracks the download progress.
129
-
130
- Args:
131
- file_url (str): The URL of the file to download.
132
- token (str): The authorization token for the request.
133
- cache_path (str): The path to save the downloaded file.
134
- proxies (dict): Proxies for the request.
135
- film_id (str): Unique identifier for the film download.
136
- title (str): The title of the film.
137
- chunk_size (int): Size of each chunk to download.
138
- """
139
- print(f"Downloading file from URL: {file_url} to {cache_path} with proxies: {proxies}")
140
- headers = {'Authorization': f'Bearer {token}'}
141
- try:
142
- response = requests.get(file_url, headers=headers, proxies=proxies, stream=True)
143
- response.raise_for_status()
144
-
145
- total_size = int(response.headers.get('content-length', 0))
146
- download_progress[episode_id] = {"total": total_size, "downloaded": 0, "status": "Downloading", "start_time": time.time()}
147
-
148
- os.makedirs(os.path.dirname(cache_path), exist_ok=True)
149
- with open(cache_path, 'wb') as file, tqdm(total=total_size, unit='B', unit_scale=True, desc=cache_path) as pbar:
150
- for data in response.iter_content(chunk_size=chunk_size):
151
- file.write(data)
152
- pbar.update(len(data))
153
- download_progress[episode_id]["downloaded"] += len(data)
154
-
155
- print(f'File cached to {cache_path} successfully.')
156
- update_tv_store_json(title, cache_path)
157
- download_progress[episode_id]["status"] = "Completed"
158
- except RequestException as e:
159
- print(f"Error downloading file: {e}")
160
- download_progress[episode_id]["status"] = "Failed"
161
- except IOError as e:
162
- print(f"Error writing file {cache_path}: {e}")
163
- download_progress[episode_id]["status"] = "Failed"
164
- finally:
165
- if download_progress[episode_id]["status"] != "Downloading":
166
- download_progress[episode_id]["end_time"] = time.time()
167
-
168
-
169
- def update_tv_store_json(title, cache_path):
170
- """
171
- Updates the TV store JSON with the new file, organizing by title, season, and episode.
172
-
173
- Args:
174
- title (str): The title of the TV show.
175
- cache_path (str): The local path where the file is saved.
176
- """
177
- TV_STORE_JSON_PATH = os.path.join(CACHE_DIR, "tv_store.json")
178
-
179
- tv_store_data = {}
180
- if os.path.exists(TV_STORE_JSON_PATH):
181
- with open(TV_STORE_JSON_PATH, 'r') as json_file:
182
- tv_store_data = json.load(json_file)
183
-
184
- # Extract season and episode information from the cache_path
185
- season_part = os.path.basename(os.path.dirname(cache_path)) # Extracts 'Season 1'
186
- episode_part = os.path.basename(cache_path) # Extracts 'Grand Blue Dreaming - S01E01 - Deep Blue HDTV-720p.mp4'
187
-
188
- # Create the structure if not already present
189
- if title not in tv_store_data:
190
- tv_store_data[title] = {}
191
-
192
- if season_part not in tv_store_data[title]:
193
- tv_store_data[title][season_part] = {}
194
-
195
- # Assuming episode_part is unique for each episode within a season
196
- tv_store_data[title][season_part][episode_part] = cache_path
197
-
198
- with open(TV_STORE_JSON_PATH, 'w') as json_file:
199
- json.dump(tv_store_data, json_file, indent=2)
200
-
201
- print(f'TV store updated with {title}, {season_part}, {episode_part}.')
202
-
203
- ###############################################################################
204
  def get_file_structure(repo, token, path="", proxies=None):
205
  """
206
  Fetches the file structure of a specified Hugging Face repository.
@@ -242,8 +70,4 @@ def write_file_structure_to_json(file_structure, file_path):
242
 
243
  if __name__ == "__main__":
244
  file_url = "https://huggingface.co/Unicone-Studio/jellyfin_media/resolve/main/films/Funky%20Monkey%202004/Funky%20Monkey%20(2004)%20Web-dl%201080p.mp4"
245
- token = os.getenv("TOKEN")
246
- cache_path = os.path.join(CACHE_DIR, "films/Funky Monkey 2004/Funky Monkey (2004) Web-dl 1080p.mp4")
247
- proxies = get_system_proxies()
248
- film_id = "funky_monkey_2004" # Unique identifier for the film download
249
- download_film(file_url, token, cache_path, proxies=proxies, film_id=film_id)
 
29
  print(f"Error getting system proxies: {e}")
30
  return {}
31
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
32
  def get_file_structure(repo, token, path="", proxies=None):
33
  """
34
  Fetches the file structure of a specified Hugging Face repository.
 
70
 
71
  if __name__ == "__main__":
72
  file_url = "https://huggingface.co/Unicone-Studio/jellyfin_media/resolve/main/films/Funky%20Monkey%202004/Funky%20Monkey%20(2004)%20Web-dl%201080p.mp4"
73
+ token = os.getenv("TOKEN")
 
 
 
 
indexer.py CHANGED
@@ -24,7 +24,6 @@ def indexer():
24
  token = os.getenv("TOKEN")
25
  repo = os.getenv("REPO")
26
  output_path = os.getenv("INDEX_FILE")
27
-
28
  proxies = get_system_proxies()
29
  full_structure = index_repository(token, repo, "", proxies)
30
  write_file_structure_to_json(full_structure, output_path)
 
24
  token = os.getenv("TOKEN")
25
  repo = os.getenv("REPO")
26
  output_path = os.getenv("INDEX_FILE")
 
27
  proxies = get_system_proxies()
28
  full_structure = index_repository(token, repo, "", proxies)
29
  write_file_structure_to_json(full_structure, output_path)
old.LoadBalancer.py CHANGED
@@ -1,17 +1,56 @@
 
 
 
 
 
 
 
 
1
  import time
2
  import logging
3
- from threading import Thread, Event, Timer
4
  from api import InstancesAPI
5
 
 
 
 
 
 
6
  class LoadBalancer:
7
- def __init__(self, polling_interval=10, max_retries=3, initial_delay=1):
8
- self.version = "0.0.1 V Beta"
9
  self.instances = []
 
10
  self.polling_interval = polling_interval
11
  self.max_retries = max_retries
12
  self.initial_delay = initial_delay
13
  self.stop_event = Event()
14
  self.instances_api = InstancesAPI(self.instances)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
 
16
  def register_instance(self, instance_url):
17
  if instance_url not in self.instances:
@@ -23,28 +62,55 @@ class LoadBalancer:
23
  def remove_instance(self, instance_url):
24
  if instance_url in self.instances:
25
  self.instances.remove(instance_url)
 
26
  logging.info(f"Removed instance {instance_url}")
27
  else:
28
  logging.info(f"Instance {instance_url} not found for removal.")
29
 
30
  def get_reports(self):
31
  reports = self.instances_api.fetch_reports()
 
 
 
 
 
32
  for instance_url in self.instances[:]: # Copy list to avoid modification during iteration
33
  if instance_url in reports:
34
  report = reports[instance_url]
35
  logging.info(f"Report from {instance_url}: {report}")
36
- self.process_report(instance_url, report)
37
  else:
38
  logging.error(f"Failed to get report from {instance_url}. Removing instance.")
39
  self.remove_instance(instance_url)
40
 
41
- def process_report(self, instance_url, report):
42
- # Process the report (film_store, tv_store, cache_size) here
 
 
 
 
 
 
43
  logging.info(f"Processing report from {instance_url}")
44
- # Example: Print the film_store and tv_store
45
- logging.info(f"Film Store: {report.get('film_store')}")
46
- logging.info(f"TV Store: {report.get('tv_store')}")
47
- logging.info(f"Cache Size: {report.get('cache_size')}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
 
49
  def start_polling(self):
50
  logging.info("Starting polling.")
@@ -57,17 +123,239 @@ class LoadBalancer:
57
  logging.info("Stopping polling.")
58
  self.stop_event.set()
59
 
60
- if __name__ == "__main__":
61
- logging.basicConfig(level=logging.INFO)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
62
 
63
- load_balancer = LoadBalancer()
 
64
 
65
- # Example registration (in real use, handle this via an API endpoint)
66
- load_balancer.register_instance("http://localhost:5000")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
 
68
- # Start polling in a separate thread
69
- polling_thread = Thread(target=load_balancer.start_polling)
70
- polling_thread.start()
 
 
 
 
 
 
71
 
72
- # Stop polling after some time for demonstration purposes
73
- Timer(300, load_balancer.stop_polling).start() # Stop after 5 minutes
 
 
1
+ import os
2
+ import json
3
+ import urllib.request
4
+ from tqdm import tqdm
5
+ from indexer import indexer
6
+ import re
7
+ from tvdb import fetch_and_cache_json
8
+ from threading import Event
9
  import time
10
  import logging
11
+ from threading import Thread, Event
12
  from api import InstancesAPI
13
 
14
+
15
+ CACHE_DIR = os.getenv("CACHE_DIR")
16
+
17
+ download_progress = {}
18
+
19
  class LoadBalancer:
20
+ def __init__(self, cache_dir, index_file, token, repo, polling_interval=4, max_retries=3, initial_delay=1):
21
+ self.version = "0.0.2.6 V Beta"
22
  self.instances = []
23
+ self.instances_health = {}
24
  self.polling_interval = polling_interval
25
  self.max_retries = max_retries
26
  self.initial_delay = initial_delay
27
  self.stop_event = Event()
28
  self.instances_api = InstancesAPI(self.instances)
29
+ self.CACHE_DIR = cache_dir
30
+ self.INDEX_FILE = index_file
31
+ self.TOKEN = token
32
+ self.REPO = repo
33
+ self.FILM_STORE = {}
34
+ self.TV_STORE = {}
35
+ self.file_structure = None
36
+
37
+ # Ensure CACHE_DIR exists
38
+ if not os.path.exists(self.CACHE_DIR):
39
+ os.makedirs(self.CACHE_DIR)
40
+
41
+ # Index the file structure
42
+ indexer()
43
+
44
+ # Load the file structure JSON
45
+ if not os.path.exists(self.INDEX_FILE):
46
+ raise FileNotFoundError(f"{self.INDEX_FILE} not found. Please make sure the file exists.")
47
+
48
+ with open(self.INDEX_FILE, 'r') as f:
49
+ self.file_structure = json.load(f)
50
+
51
+ prefetch_thread = Thread(target=self.start_prefetching)
52
+ prefetch_thread.daemon = True
53
+ prefetch_thread.start()
54
 
55
  def register_instance(self, instance_url):
56
  if instance_url not in self.instances:
 
62
  def remove_instance(self, instance_url):
63
  if instance_url in self.instances:
64
  self.instances.remove(instance_url)
65
+ self.instances_health.pop(instance_url, None)
66
  logging.info(f"Removed instance {instance_url}")
67
  else:
68
  logging.info(f"Instance {instance_url} not found for removal.")
69
 
70
  def get_reports(self):
71
  reports = self.instances_api.fetch_reports()
72
+
73
+ # Initialize temporary JSON data holders
74
+ temp_film_store = {}
75
+ temp_tv_store = {}
76
+
77
  for instance_url in self.instances[:]: # Copy list to avoid modification during iteration
78
  if instance_url in reports:
79
  report = reports[instance_url]
80
  logging.info(f"Report from {instance_url}: {report}")
81
+ self.process_report(instance_url, report, temp_film_store, temp_tv_store)
82
  else:
83
  logging.error(f"Failed to get report from {instance_url}. Removing instance.")
84
  self.remove_instance(instance_url)
85
 
86
+ self.FILM_STORE = temp_film_store
87
+ self.TV_STORE = temp_tv_store
88
+
89
+ def process_report(self, instance_url, report, temp_film_store, temp_tv_store):
90
+ film_store = report.get('film_store', {})
91
+ tv_store = report.get('tv_store', {})
92
+ cache_size = report.get('cache_size')
93
+
94
  logging.info(f"Processing report from {instance_url}")
95
+
96
+ # Update temporary film store
97
+ for title, path in film_store.items():
98
+ url = f"{instance_url}/api/film/{title.replace(' ', '%20')}"
99
+ temp_film_store[title] = url
100
+
101
+ # Update temporary TV store
102
+ for title, seasons in tv_store.items():
103
+ if title not in temp_tv_store:
104
+ temp_tv_store[title] = {}
105
+ for season, episodes in seasons.items():
106
+ if season not in temp_tv_store[title]:
107
+ temp_tv_store[title][season] = {}
108
+ for episode, path in episodes.items():
109
+ url = f"{instance_url}/api/tv/{title.replace(' ', '%20')}/{season.replace(' ', '%20')}/{episode.replace(' ', '%20')}"
110
+ temp_tv_store[title][season][episode] = url
111
+
112
+ logging.info("Film and TV Stores processed successfully.")
113
+ self.update_instances_health(instance=instance_url, cache_size=cache_size)
114
 
115
  def start_polling(self):
116
  logging.info("Starting polling.")
 
123
  logging.info("Stopping polling.")
124
  self.stop_event.set()
125
 
126
+ ######################################################################
127
+ @staticmethod
128
+ def get_system_proxies():
129
+ """
130
+ Retrieves the system's HTTP and HTTPS proxies.
131
+
132
+ Returns:
133
+ dict: A dictionary containing the proxies.
134
+ """
135
+ try:
136
+ proxies = urllib.request.getproxies()
137
+ print("System proxies:", proxies)
138
+ return {
139
+ "http": proxies.get("http"),
140
+ "https": proxies.get("http")
141
+ }
142
+ except Exception as e:
143
+ print(f"Error getting system proxies: {e}")
144
+ return {}
145
+
146
+ @staticmethod
147
+ def is_valid_url(url):
148
+ # Simple URL validation (could be more complex if needed)
149
+ regex = re.compile(
150
+ r'^(?:http|ftp)s?://' # http:// or https://
151
+ r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
152
+ r'localhost|' # localhost...
153
+ r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
154
+ r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
155
+ r'(?::\d+)?' # optional port
156
+ r'(?:/?|[/?]\S+)$', re.IGNORECASE)
157
+ return re.match(regex, url) is not None
158
+
159
+ #################################################################
160
+
161
+ def update_instances_health(self, instance, cache_size):
162
+ self.instances_health[instance] = {"used":cache_size["cache_size"],
163
+ "total": "50 GB"}
164
+ logging.info(f"Updated instance {instance} with cache size {cache_size}")
165
+
166
+
167
+ def download_film_to_best_instance(self, title):
168
+ """
169
+ Downloads a film to the first instance that has more free space on the self.instance_health list variable.
170
+ The instance_health looks like this:
171
+ {
172
+ "https://unicone-studio-instance1.hf.space": {
173
+ "total": "50 GB",
174
+ "used": "3.33 GB"
175
+ }
176
+ }
177
+ Args:
178
+ title (str): The title of the film.
179
+ """
180
+ best_instance = None
181
+ max_free_space = -1
182
+
183
+ # Calculate free space for each instance
184
+ for instance_url, space_info in self.instances_health.items():
185
+ total_space = self._convert_to_gb(space_info['total'])
186
+ used_space = self._convert_to_gb(space_info['used'])
187
+ free_space = total_space - used_space
188
+
189
+ if free_space > max_free_space:
190
+ max_free_space = free_space
191
+ best_instance = instance_url
192
+
193
+ if best_instance:
194
+ result = self.instances_api.download_film(best_instance, title)
195
+ film_id = result["film_id"]
196
+ status = result["status"]
197
+ progress_url = f'{best_instance}/api/progress/{film_id}'
198
+ response = {
199
+ "film_id":film_id,
200
+ "status":status,
201
+ "progress_url":progress_url
202
+ }
203
+
204
+ return response
205
+ else:
206
+ logging.error("No suitable instance found for downloading the film.")
207
+ return {"error": "No suitable instance found for downloading the film."}
208
+
209
+ def download_episode_to_best_instance(self, title, season, episode):
210
+ """
211
+ Downloads a episode to the first instance that has more free space on the self.instance_health list variable.
212
+ The instance_health looks like this:
213
+ {
214
+ "https://unicone-studio-instance1.hf.space": {
215
+ "total": "50 GB",
216
+ "used": "3.33 GB"
217
+ }
218
+ }
219
+ Args:
220
+ title (str): The title of the Tv show.
221
+ season (str): The season of the Tv show.
222
+ episode (str): The title of the Tv show.
223
+ """
224
+ best_instance = None
225
+ max_free_space = -1
226
+
227
+ # Calculate free space for each instance
228
+ for instance_url, space_info in self.instances_health.items():
229
+ total_space = self._convert_to_gb(space_info['total'])
230
+ used_space = self._convert_to_gb(space_info['used'])
231
+ free_space = total_space - used_space
232
+
233
+ if free_space > max_free_space:
234
+ max_free_space = free_space
235
+ best_instance = instance_url
236
+
237
+ if best_instance:
238
+ result = self.instances_api.download_episode(best_instance, title, season, episode)
239
+ episode_id = result["episode_id"]
240
+ status = result["status"]
241
+ progress_url = f'{best_instance}/api/progress/{episode_id}'
242
+ response = {
243
+ "episode_id":episode_id,
244
+ "status":status,
245
+ "progress_url":progress_url
246
+ }
247
+
248
+ return response
249
+ else:
250
+ logging.error("No suitable instance found for downloading the film.")
251
+ return {"error": "No suitable instance found for downloading the film."}
252
+
253
+ def _convert_to_gb(self, space_str):
254
+ """
255
+ Converts a space string like '50 GB' or '3.33 GB' to a float representing the number of GB.
256
+ """
257
+ return float(space_str.split()[0])
258
+
259
+ #################################################################
260
+ def find_movie_path(self, title):
261
+ """Find the path of the movie in the JSON data based on the title."""
262
+ for directory in self.file_structure:
263
+ if directory['type'] == 'directory' and directory['path'] == 'films':
264
+ for sub_directory in directory['contents']:
265
+ if sub_directory['type'] == 'directory':
266
+ for item in sub_directory['contents']:
267
+ if item['type'] == 'file' and title.lower() in item['path'].lower():
268
+ return item['path']
269
+ return None
270
+
271
+ def find_tv_path(self, title):
272
+ """Find the path of the TV show in the JSON data based on the title."""
273
+ for directory in self.file_structure:
274
+ if directory['type'] == 'directory' and directory['path'] == 'tv':
275
+ for sub_directory in directory['contents']:
276
+ if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
277
+ return sub_directory['path']
278
+ return None
279
+
280
+ def get_tv_structure(self, title):
281
+ """Find the path of the TV show in the JSON data based on the title."""
282
+ for directory in self.file_structure:
283
+ if directory['type'] == 'directory' and directory['path'] == 'tv':
284
+ for sub_directory in directory['contents']:
285
+ if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
286
+ return sub_directory
287
+ return None
288
+
289
+ def get_film_id(self, title):
290
+ """Generate a film ID based on the title."""
291
+ return title.replace(" ", "_").lower()
292
+
293
+ def prefetch_metadata(self):
294
+ """Prefetch metadata for all items in the file structure."""
295
+ for item in self.file_structure:
296
+ if 'contents' in item:
297
+ for sub_item in item['contents']:
298
+ original_title = sub_item['path'].split('/')[-1]
299
+ media_type = 'series' if item['path'].startswith('tv') else 'movie'
300
+ title = original_title
301
+ year = None
302
+
303
+ # Extract year from the title if available
304
+ match = re.search(r'\((\d{4})\)', original_title)
305
+ if match:
306
+ year_str = match.group(1)
307
+ if year_str.isdigit() and len(year_str) == 4:
308
+ title = original_title[:match.start()].strip()
309
+ year = int(year_str)
310
+ else:
311
+ parts = original_title.rsplit(' ', 1)
312
+ if len(parts) > 1 and parts[-1].isdigit() and len(parts[-1]) == 4:
313
+ title = parts[0].strip()
314
+ year = int(parts[-1])
315
+
316
+ fetch_and_cache_json(original_title, title, media_type, year)
317
+
318
+ def bytes_to_human_readable(self, num, suffix="B"):
319
+ for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
320
+ if abs(num) < 1024.0:
321
+ return f"{num:3.1f} {unit}{suffix}"
322
+ num /= 1024.0
323
+ return f"{num:.1f} Y{suffix}"
324
 
325
+ def encode_episodeid(self, title, season, episode):
326
+ return f"{title}_{season}_{episode}"
327
 
328
+ def get_all_tv_shows(self):
329
+ """Get all TV shows from the indexed cache structure JSON file."""
330
+ tv_shows = {}
331
+ for directory in self.file_structure:
332
+ if directory['type'] == 'directory' and directory['path'] == 'tv':
333
+ for sub_directory in directory['contents']:
334
+ if sub_directory['type'] == 'directory':
335
+ show_title = sub_directory['path'].split('/')[-1]
336
+ tv_shows[show_title] = []
337
+ for season_directory in sub_directory['contents']:
338
+ if season_directory['type'] == 'directory':
339
+ season = season_directory['path'].split('/')[-1]
340
+ for episode in season_directory['contents']:
341
+ if episode['type'] == 'file':
342
+ tv_shows[show_title].append({
343
+ "season": season,
344
+ "episode": episode['path'].split('/')[-1],
345
+ "path": episode['path']
346
+ })
347
+ return tv_shows
348
 
349
+ def get_all_films(self):
350
+ """Get all films from the indexed cache structure JSON file."""
351
+ films = []
352
+ for directory in self.file_structure:
353
+ if directory['type'] == 'directory' and directory['path'] == 'films':
354
+ for sub_directory in directory['contents']:
355
+ if sub_directory['type'] == 'directory':
356
+ films.append(sub_directory['path'])
357
+ return films
358
 
359
+ def start_prefetching(self):
360
+ """Start the metadata prefetching in a separate thread."""
361
+ self.prefetch_metadata()
old.app.py CHANGED
@@ -2,246 +2,70 @@ from flask import Flask, jsonify, request, send_from_directory
2
  from flask_cors import CORS
3
  import os
4
  import json
5
- import threading
6
  import urllib.parse
7
- from hf_scrapper import download_film, download_episode, get_system_proxies, get_download_progress
8
- from indexer import indexer
9
- from tvdb import fetch_and_cache_json
10
- import re
11
- import logging
12
  from LoadBalancer import LoadBalancer
 
13
 
14
- logging.basicConfig(level=logging.INFO)
15
- load_balancer = LoadBalancer()
16
-
17
- # Start polling in a separate thread
18
- polling_thread = threading.Thread(target=load_balancer.start_polling)
19
- polling_thread.start()
20
 
21
  app = Flask(__name__)
22
  CORS(app)
23
 
 
24
  # Constants and Configuration
25
  CACHE_DIR = os.getenv("CACHE_DIR")
26
  INDEX_FILE = os.getenv("INDEX_FILE")
27
  TOKEN = os.getenv("TOKEN")
28
- FILM_STORE_JSON_PATH = os.path.join(CACHE_DIR, "film_store.json")
29
- TV_STORE_JSON_PATH = os.path.join(CACHE_DIR, "tv_store.json")
30
- INSTANCE_REGISTER_JSON_PATH = os.path.join(CACHE_DIR, "instance_register.json")
31
  REPO = os.getenv("REPO")
32
- download_threads = {}
33
-
34
- # Ensure CACHE_DIR exists
35
- if not os.path.exists(CACHE_DIR):
36
- os.makedirs(CACHE_DIR)
37
-
38
- for path in [FILM_STORE_JSON_PATH, TV_STORE_JSON_PATH]:
39
- if not os.path.exists(path):
40
- with open(path, 'w') as json_file:
41
- json.dump({}, json_file)
42
-
43
- # Index the file structure
44
- indexer()
45
 
46
- # Load the file structure JSON
47
- if not os.path.exists(INDEX_FILE):
48
- raise FileNotFoundError(f"{INDEX_FILE} not found. Please make sure the file exists.")
49
 
50
- with open(INDEX_FILE, 'r') as f:
51
- file_structure = json.load(f)
52
-
53
- # Function Definitions
54
- def load_json(file_path):
55
- """Load JSON data from a file."""
56
- with open(file_path, 'r') as file:
57
- return json.load(file)
58
-
59
- def find_movie_path(json_data, title):
60
- """Find the path of the movie in the JSON data based on the title."""
61
- for directory in json_data:
62
- if directory['type'] == 'directory' and directory['path'] == 'films':
63
- for sub_directory in directory['contents']:
64
- if sub_directory['type'] == 'directory':
65
- for item in sub_directory['contents']:
66
- if item['type'] == 'file' and title.lower() in item['path'].lower():
67
- return item['path']
68
- return None
69
-
70
- def find_tv_path(json_data, title):
71
- """Find the path of the TV show in the JSON data based on the title."""
72
- for directory in json_data:
73
- if directory['type'] == 'directory' and directory['path'] == 'tv':
74
- for sub_directory in directory['contents']:
75
- if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
76
- return sub_directory['path']
77
- return None
78
-
79
- def get_tv_structure(json_data,title):
80
- """Find the path of the TV show in the JSON data based on the title."""
81
- for directory in json_data:
82
- if directory['type'] == 'directory' and directory['path'] == 'tv':
83
- for sub_directory in directory['contents']:
84
- if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
85
- return sub_directory
86
- return None
87
-
88
- def get_film_id(title):
89
- """Generate a film ID based on the title."""
90
- return title.replace(" ", "_").lower()
91
-
92
- def prefetch_metadata():
93
- """Prefetch metadata for all items in the file structure."""
94
- for item in file_structure:
95
- if 'contents' in item:
96
- for sub_item in item['contents']:
97
- original_title = sub_item['path'].split('/')[-1]
98
- media_type = 'series' if item['path'].startswith('tv') else 'movie'
99
- title = original_title
100
- year = None
101
-
102
- # Extract year from the title if available
103
- match = re.search(r'\((\d{4})\)', original_title)
104
- if match:
105
- year_str = match.group(1)
106
- if year_str.isdigit() and len(year_str) == 4:
107
- title = original_title[:match.start()].strip()
108
- year = int(year_str)
109
- else:
110
- parts = original_title.rsplit(' ', 1)
111
- if len(parts) > 1 and parts[-1].isdigit() and len(parts[-1]) == 4:
112
- title = parts[0].strip()
113
- year = int(parts[-1])
114
-
115
- fetch_and_cache_json(original_title, title, media_type, year)
116
-
117
- def bytes_to_human_readable(num, suffix="B"):
118
- for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
119
- if abs(num) < 1024.0:
120
- return f"{num:3.1f} {unit}{suffix}"
121
- num /= 1024.0
122
- return f"{num:.1f} Y{suffix}"
123
-
124
- def encode_episodeid(title,season,episode):
125
- return f"{title}_{season}_{episode}"
126
-
127
- def is_valid_url(url):
128
- # Simple URL validation (could be more complex if needed)
129
- regex = re.compile(
130
- r'^(?:http|ftp)s?://' # http:// or https://
131
- r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
132
- r'localhost|' # localhost...
133
- r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
134
- r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
135
- r'(?::\d+)?' # optional port
136
- r'(?:/?|[/?]\S+)$', re.IGNORECASE)
137
- return re.match(regex, url) is not None
138
-
139
- def get_all_tv_shows(indexed_cache):
140
- """Get all TV shows from the indexed cache structure JSON file."""
141
- tv_shows = {}
142
- for directory in indexed_cache:
143
- if directory['type'] == 'directory' and directory['path'] == 'tv':
144
- for sub_directory in directory['contents']:
145
- if sub_directory['type'] == 'directory':
146
- show_title = sub_directory['path'].split('/')[-1]
147
- tv_shows[show_title] = []
148
- for season_directory in sub_directory['contents']:
149
- if season_directory['type'] == 'directory':
150
- season = season_directory['path'].split('/')[-1]
151
- for episode in season_directory['contents']:
152
- if episode['type'] == 'file':
153
- tv_shows[show_title].append({
154
- "season": season,
155
- "episode": episode['path'].split('/')[-1],
156
- "path": episode['path']
157
- })
158
- return tv_shows
159
-
160
- def get_all_films(indexed_cache):
161
- """Get all films from the indexed cache structure JSON file."""
162
- films = []
163
- for directory in indexed_cache:
164
- if directory['type'] == 'directory' and directory['path'] == 'films':
165
- for sub_directory in directory['contents']:
166
- if sub_directory['type'] == 'directory':
167
- films.append(sub_directory['path'])
168
- return films
169
-
170
- def start_prefetching():
171
- """Start the metadata prefetching in a separate thread."""
172
- prefetch_metadata()
173
-
174
- # Start prefetching metadata
175
- thread = threading.Thread(target=start_prefetching)
176
- thread.daemon = True
177
- thread.start()
178
 
179
  # API Endpoints
180
-
181
- @app.route('/api/film', methods=['GET'])
182
- def get_movie_api():
183
  """Endpoint to get the movie by title."""
184
- title = request.args.get('title')
185
  if not title:
186
  return jsonify({"error": "Title parameter is required"}), 400
187
 
188
- # Load the film store JSON
189
- with open(FILM_STORE_JSON_PATH, 'r') as json_file:
190
- film_store_data = json.load(json_file)
191
-
192
  # Check if the film is already cached
193
- if title in film_store_data:
194
- cache_path = film_store_data[title]
195
- if os.path.exists(cache_path):
196
- return send_from_directory(os.path.dirname(cache_path), os.path.basename(cache_path))
197
-
198
- movie_path = find_movie_path(file_structure, title)
199
 
200
  if not movie_path:
201
  return jsonify({"error": "Movie not found"}), 404
202
 
203
- cache_path = os.path.join(CACHE_DIR, movie_path)
204
- file_url = f"https://huggingface.co/{REPO}/resolve/main/{movie_path}"
205
- proxies = get_system_proxies()
206
- film_id = get_film_id(title)
207
-
208
- # Start the download in a separate thread if not already downloading
209
- if film_id not in download_threads or not download_threads[film_id].is_alive():
210
- thread = threading.Thread(target=download_film, args=(file_url, TOKEN, cache_path, proxies, film_id, title))
211
- download_threads[film_id] = thread
212
- thread.start()
213
-
214
- return jsonify({"status": "Download started", "film_id": film_id})
215
 
216
- @app.route('/api/tv', methods=['GET'])
217
- def get_tv_show_api():
218
  """Endpoint to get the TV show by title, season, and episode."""
219
- title = request.args.get('title')
220
- season = request.args.get('season')
221
- episode = request.args.get('episode')
222
-
223
  if not title or not season or not episode:
224
  return jsonify({"error": "Title, season, and episode parameters are required"}), 400
225
 
226
- # Load the TV store JSON
227
- with open(TV_STORE_JSON_PATH, 'r') as json_file:
228
- tv_store_data = json.load(json_file)
229
-
230
  # Check if the episode is already cached
231
- if title in tv_store_data and season in tv_store_data[title]:
232
- for ep in tv_store_data[title][season]:
233
  if episode in ep:
234
- cache_path = tv_store_data[title][season][ep]
235
- if os.path.exists(cache_path):
236
- return send_from_directory(os.path.dirname(cache_path), os.path.basename(cache_path))
237
 
238
- tv_path = find_tv_path(file_structure, title)
239
 
240
  if not tv_path:
241
  return jsonify({"error": "TV show not found"}), 404
242
 
243
  episode_path = None
244
- for directory in file_structure:
245
  if directory['type'] == 'directory' and directory['path'] == 'tv':
246
  for sub_directory in directory['contents']:
247
  if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
@@ -255,25 +79,26 @@ def get_tv_show_api():
255
  if not episode_path:
256
  return jsonify({"error": "Episode not found"}), 404
257
 
258
- cache_path = os.path.join(CACHE_DIR, episode_path)
259
- file_url = f"https://huggingface.co/{REPO}/resolve/main/{episode_path}"
260
- proxies = get_system_proxies()
261
- episode_id = encode_episodeid(title,season,episode)
262
-
263
- # Start the download in a separate thread if not already downloading
264
- if episode_id not in download_threads or not download_threads[episode_id].is_alive():
265
- thread = threading.Thread(target=download_episode, args=(file_url, TOKEN, cache_path, proxies, episode_id, title))
266
- download_threads[episode_id] = thread
267
- thread.start()
268
-
269
- return jsonify({"status": "Download started", "episode_id": episode_id})
270
 
 
 
 
 
 
 
 
271
 
272
- @app.route('/api/progress/<id>', methods=['GET'])
273
- def get_progress_api(id):
274
- """Endpoint to get the download progress of a movie or TV show episode."""
275
- progress = get_download_progress(id)
276
- return jsonify({"id": id, "progress": progress})
 
 
277
 
278
  @app.route('/api/cache/size', methods=['GET'])
279
  def get_cache_size_api():
@@ -282,7 +107,7 @@ def get_cache_size_api():
282
  for f in filenames:
283
  fp = os.path.join(dirpath, f)
284
  total_size += os.path.getsize(fp)
285
- readable_size = bytes_to_human_readable(total_size)
286
  return jsonify({"cache_size": readable_size})
287
 
288
  @app.route('/api/cache/clear', methods=['POST'])
@@ -296,48 +121,18 @@ def clear_cache_api():
296
  @app.route('/api/tv/store', methods=['GET'])
297
  def get_tv_store_api():
298
  """Endpoint to get the TV store JSON."""
299
- if os.path.exists(TV_STORE_JSON_PATH):
300
- with open(TV_STORE_JSON_PATH, 'r') as json_file:
301
- tv_store_data = json.load(json_file)
302
- return jsonify(tv_store_data)
303
- return jsonify({}), 404
304
 
305
  @app.route('/api/film/store', methods=['GET'])
306
  def get_film_store_api():
307
  """Endpoint to get the film store JSON."""
308
- if os.path.exists(FILM_STORE_JSON_PATH):
309
- with open(FILM_STORE_JSON_PATH, 'r') as json_file:
310
- tv_store_data = json.load(json_file)
311
- return jsonify(tv_store_data)
312
- return jsonify({}), 404
313
-
314
- #################################################
315
- # No change needed
316
 
317
- @app.route('/api/filmid', methods=['GET'])
318
- def get_film_id_by_title_api():
319
- """Endpoint to get the film ID by providing the movie title."""
320
- title = request.args.get('title')
321
- if not title:
322
- return jsonify({"error": "Title parameter is required"}), 400
323
- film_id = get_film_id(title)
324
- return jsonify({"film_id": film_id})
325
-
326
- @app.route('/api/episodeid', methods=['GET'])
327
- def get_episode_id_api():
328
- """Endpoint to get the episode ID by providing the TV show title, season, and episode."""
329
- title = request.args.get('title')
330
- season = request.args.get('season')
331
- episode = request.args.get('episode')
332
- if not title or not season or not episode:
333
- return jsonify({"error": "Title, season, and episode parameters are required"}), 400
334
- episode_id = encode_episodeid(title,season,episode)
335
- return jsonify({"episode_id": episode_id})
336
 
337
- @app.route('/api/film/metadata', methods=['GET'])
338
- def get_film_metadata_api():
339
  """Endpoint to get the film metadata by title."""
340
- title = request.args.get('title')
341
  if not title:
342
  return jsonify({'error': 'No title provided'}), 400
343
 
@@ -350,10 +145,9 @@ def get_film_metadata_api():
350
 
351
  return jsonify({'error': 'Metadata not found'}), 404
352
 
353
- @app.route('/api/tv/metadata', methods=['GET'])
354
- def get_tv_metadata_api():
355
  """Endpoint to get the TV show metadata by title."""
356
- title = request.args.get('title')
357
  if not title:
358
  return jsonify({'error': 'No title provided'}), 400
359
 
@@ -364,7 +158,7 @@ def get_tv_metadata_api():
364
  data = json.load(f)
365
 
366
  # Add the file structure to the metadata
367
- tv_structure_data = get_tv_structure(file_structure, title)
368
  if tv_structure_data:
369
  data['file_structure'] = tv_structure_data
370
 
@@ -375,14 +169,21 @@ def get_tv_metadata_api():
375
 
376
  @app.route("/api/film/all")
377
  def get_all_films_api():
378
- return get_all_films(file_structure)
379
 
380
  @app.route("/api/tv/all")
381
  def get_all_tvshows_api():
382
- return get_all_tv_shows(file_structure)
383
 
 
 
 
 
 
 
 
384
  #############################################################
385
- # unique api's
386
  @app.route('/api/register', methods=['POST'])
387
  def register_instance():
388
  try:
@@ -391,7 +192,7 @@ def register_instance():
391
  return jsonify({"error": "No URL provided"}), 400
392
 
393
  url = data["url"]
394
- if not is_valid_url(url):
395
  return jsonify({"error": "Invalid URL"}), 400
396
 
397
  # Register the instance
@@ -403,16 +204,13 @@ def register_instance():
403
  except Exception as e:
404
  logging.error(f"Error registering instance: {e}")
405
  return jsonify({"error": "Failed to register instance"}), 500
 
406
  #############################################################
407
  # Routes
408
  @app.route('/')
409
  def index():
410
  return f"Load Balancer is Running {load_balancer.version}"
411
 
412
- @app.route('/api/instances',methods=["GET"])
413
- def get_instances():
414
- return load_balancer.instances
415
-
416
  # Main entry point
417
  if __name__ == "__main__":
418
  app.run(debug=True, host="0.0.0.0", port=7860)
 
2
  from flask_cors import CORS
3
  import os
4
  import json
5
+ from threading import Thread
6
  import urllib.parse
 
 
 
 
 
7
  from LoadBalancer import LoadBalancer
8
+ import logging
9
 
 
 
 
 
 
 
10
 
11
  app = Flask(__name__)
12
  CORS(app)
13
 
14
+ logging.basicConfig(level=logging.INFO)
15
  # Constants and Configuration
16
  CACHE_DIR = os.getenv("CACHE_DIR")
17
  INDEX_FILE = os.getenv("INDEX_FILE")
18
  TOKEN = os.getenv("TOKEN")
 
 
 
19
  REPO = os.getenv("REPO")
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
+ load_balancer = LoadBalancer(cache_dir=CACHE_DIR, index_file=INDEX_FILE, token=TOKEN, repo=REPO)
 
 
22
 
23
+ # Start polling in a separate thread
24
+ polling_thread = Thread(target=load_balancer.start_polling)
25
+ polling_thread.start()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
 
27
  # API Endpoints
28
+ @app.route('/api/film/<title>', methods=['GET'])
29
+ def get_movie_api(title):
 
30
  """Endpoint to get the movie by title."""
 
31
  if not title:
32
  return jsonify({"error": "Title parameter is required"}), 400
33
 
 
 
 
 
34
  # Check if the film is already cached
35
+ if title in load_balancer.FILM_STORE:
36
+ url = load_balancer.FILM_STORE[title]
37
+ return jsonify({"url":url})
38
+
39
+ movie_path = load_balancer.find_movie_path(title)
 
40
 
41
  if not movie_path:
42
  return jsonify({"error": "Movie not found"}), 404
43
 
44
+ # Start the download in a instance
45
+ response = load_balancer.download_film_to_best_instance(title=title)
46
+ if response:
47
+ return jsonify(response)
 
 
 
 
 
 
 
 
48
 
49
+ @app.route('/api/tv/<title>/<season>/<episode>', methods=['GET'])
50
+ def get_tv_show_api(title, season, episode):
51
  """Endpoint to get the TV show by title, season, and episode."""
 
 
 
 
52
  if not title or not season or not episode:
53
  return jsonify({"error": "Title, season, and episode parameters are required"}), 400
54
 
 
 
 
 
55
  # Check if the episode is already cached
56
+ if title in load_balancer.TV_STORE and season in load_balancer.TV_STORE[title]:
57
+ for ep in load_balancer.TV_STORE[title][season]:
58
  if episode in ep:
59
+ url = load_balancer.TV_STORE[title][season][ep]
60
+ return jsonify({"url":url})
 
61
 
62
+ tv_path = load_balancer.find_tv_path(title)
63
 
64
  if not tv_path:
65
  return jsonify({"error": "TV show not found"}), 404
66
 
67
  episode_path = None
68
+ for directory in load_balancer.file_structure:
69
  if directory['type'] == 'directory' and directory['path'] == 'tv':
70
  for sub_directory in directory['contents']:
71
  if sub_directory['type'] == 'directory' and title.lower() in sub_directory['path'].lower():
 
79
  if not episode_path:
80
  return jsonify({"error": "Episode not found"}), 404
81
 
82
+ # Start the download in a instance
83
+ response = load_balancer.download_episode_to_best_instance(title=title, season=season, episode=episode)
84
+ if response:
85
+ return jsonify(response)
 
 
 
 
 
 
 
 
86
 
87
+ @app.route('/api/filmid/<title>', methods=['GET'])
88
+ def get_film_id_by_title_api(title):
89
+ """Endpoint to get the film ID by providing the movie title."""
90
+ if not title:
91
+ return jsonify({"error": "Title parameter is required"}), 400
92
+ film_id = load_balancer.get_film_id(title)
93
+ return jsonify({"film_id": film_id})
94
 
95
+ @app.route('/api/episodeid/<title>/<season>/<episode>', methods=['GET'])
96
+ def get_episode_id_api(title,season,episode):
97
+ """Endpoint to get the episode ID by providing the TV show title, season, and episode."""
98
+ if not title or not season or not episode:
99
+ return jsonify({"error": "Title, season, and episode parameters are required"}), 400
100
+ episode_id = load_balancer.encode_episodeid(title,season,episode)
101
+ return jsonify({"episode_id": episode_id})
102
 
103
  @app.route('/api/cache/size', methods=['GET'])
104
  def get_cache_size_api():
 
107
  for f in filenames:
108
  fp = os.path.join(dirpath, f)
109
  total_size += os.path.getsize(fp)
110
+ readable_size = load_balancer.bytes_to_human_readable(total_size)
111
  return jsonify({"cache_size": readable_size})
112
 
113
  @app.route('/api/cache/clear', methods=['POST'])
 
121
  @app.route('/api/tv/store', methods=['GET'])
122
  def get_tv_store_api():
123
  """Endpoint to get the TV store JSON."""
124
+ return jsonify(load_balancer.TV_STORE)
125
+
 
 
 
126
 
127
  @app.route('/api/film/store', methods=['GET'])
128
  def get_film_store_api():
129
  """Endpoint to get the film store JSON."""
130
+ return jsonify(load_balancer.FILM_STORE)
 
 
 
 
 
 
 
131
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
132
 
133
+ @app.route('/api/film/metadata/<title>', methods=['GET'])
134
+ def get_film_metadata_api(title):
135
  """Endpoint to get the film metadata by title."""
 
136
  if not title:
137
  return jsonify({'error': 'No title provided'}), 400
138
 
 
145
 
146
  return jsonify({'error': 'Metadata not found'}), 404
147
 
148
+ @app.route('/api/tv/metadata/<title>', methods=['GET'])
149
+ def get_tv_metadata_api(title):
150
  """Endpoint to get the TV show metadata by title."""
 
151
  if not title:
152
  return jsonify({'error': 'No title provided'}), 400
153
 
 
158
  data = json.load(f)
159
 
160
  # Add the file structure to the metadata
161
+ tv_structure_data = load_balancer.get_tv_structure(title)
162
  if tv_structure_data:
163
  data['file_structure'] = tv_structure_data
164
 
 
169
 
170
  @app.route("/api/film/all")
171
  def get_all_films_api():
172
+ return load_balancer.get_all_films()
173
 
174
  @app.route("/api/tv/all")
175
  def get_all_tvshows_api():
176
+ return load_balancer.get_all_tv_shows()
177
 
178
+ @app.route('/api/instances',methods=["GET"])
179
+ def get_instances():
180
+ return load_balancer.instances
181
+
182
+ @app.route('/api/instances/health',methods=["GET"])
183
+ def get_instances_health():
184
+ return load_balancer.instances_health
185
  #############################################################
186
+ # This API is only for instances
187
  @app.route('/api/register', methods=['POST'])
188
  def register_instance():
189
  try:
 
192
  return jsonify({"error": "No URL provided"}), 400
193
 
194
  url = data["url"]
195
+ if not load_balancer.is_valid_url(url):
196
  return jsonify({"error": "Invalid URL"}), 400
197
 
198
  # Register the instance
 
204
  except Exception as e:
205
  logging.error(f"Error registering instance: {e}")
206
  return jsonify({"error": "Failed to register instance"}), 500
207
+
208
  #############################################################
209
  # Routes
210
  @app.route('/')
211
  def index():
212
  return f"Load Balancer is Running {load_balancer.version}"
213
 
 
 
 
 
214
  # Main entry point
215
  if __name__ == "__main__":
216
  app.run(debug=True, host="0.0.0.0", port=7860)
utils.py ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import re
2
+
3
+ def is_valid_url(url):
4
+ """
5
+ Validates the URL.
6
+
7
+ Args:
8
+ url (str): The URL to validate.
9
+
10
+ Returns:
11
+ bool: True if the URL is valid, False otherwise.
12
+ """
13
+ regex = re.compile(
14
+ r'^(?:http|ftp)s?://' # http:// or https://
15
+ r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
16
+ r'localhost|' # localhost...
17
+ r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
18
+ r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
19
+ r'(?::\d+)?' # optional port
20
+ r'(?:/?|[/?]\S+)$', re.IGNORECASE)
21
+ return re.match(regex, url) is not None
22
+
23
+ def convert_to_gb(space_str):
24
+ """
25
+ Converts a space string like '50 GB' or '3.33 GB' to a float representing the number of GB.
26
+
27
+ Args:
28
+ space_str (str): The space string to convert.
29
+
30
+ Returns:
31
+ float: The space in GB.
32
+ """
33
+ return float(space_str.split()[0])
34
+
35
+ def bytes_to_human_readable(num, suffix="B"):
36
+ """
37
+ Converts bytes to a human-readable format.
38
+
39
+ Args:
40
+ num (int): The number of bytes.
41
+ suffix (str): The suffix to use (default is 'B').
42
+
43
+ Returns:
44
+ str: The human-readable string.
45
+ """
46
+ for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
47
+ if abs(num) < 1024.0:
48
+ return f"{num:3.1f} {unit}{suffix}"
49
+ num /= 1024.0
50
+ return f"{num:.1f} Y{suffix}"
51
+
52
+ def encode_episodeid(title, season, episode):
53
+ """
54
+ Encodes the episode ID based on title, season, and episode.
55
+
56
+ Args:
57
+ title (str): The title of the TV show.
58
+ season (str): The season of the TV show.
59
+ episode (str): The episode number.
60
+
61
+ Returns:
62
+ str: The encoded episode ID.
63
+ """
64
+ return f"{title}_{season}_{episode}"