sc_ma commited on
Commit
a0d1776
1 Parent(s): 09305ff

edit fundamental logic of passing openai key.
support user's key input.

app.py CHANGED
@@ -1,6 +1,8 @@
1
  import gradio as gr
2
  import os
3
- from auto_backgrounds import generate_backgrounds, fake_generate_backgrounds
 
 
4
 
5
  openai_key = os.getenv("OPENAI_API_KEY")
6
  access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
@@ -16,7 +18,12 @@ if openai_key is None:
16
  IS_OPENAI_API_KEY_AVAILABLE = False
17
  else:
18
  # todo: check if this key is available or not
19
- IS_OPENAI_API_KEY_AVAILABLE = True
 
 
 
 
 
20
 
21
 
22
 
@@ -24,11 +31,19 @@ def clear_inputs(text1, text2):
24
  return "", ""
25
 
26
 
27
- def wrapped_generate_backgrounds(title, description, openai_key = None, cache_mode = True):
 
 
28
  # if `cache_mode` is True, then follow the following steps:
29
  # check if "title"+"description" have been generated before
30
  # if so, download from the cloud storage, return it
31
  # if not, generate the result.
 
 
 
 
 
 
32
  if cache_mode:
33
  from utils.storage import list_all_files, hash_name, download_file, upload_file
34
  # check if "title"+"description" have been generated before
@@ -41,21 +56,26 @@ def wrapped_generate_backgrounds(title, description, openai_key = None, cache_mo
41
  else:
42
  # generate the result.
43
  # output = fake_generate_backgrounds(title, description, openai_key)
44
- output = generate_backgrounds(title, description, openai_key) #todo: change the output of this function to hashed title
45
  upload_file(file_name)
46
  return output
47
  else:
48
  # output = fake_generate_backgrounds(title, description, openai_key)
49
- output = generate_backgrounds(title, description, openai_key) #todo: change the output of this function to hashed title
50
  return output
51
 
52
 
 
 
 
 
 
53
 
54
- with gr.Blocks() as demo:
55
  gr.Markdown('''
56
  # Auto-Draft: 文献整理辅助工具-限量免费使用
57
 
58
- 本Demo提供对[Auto-Draft](https://github.com/CCCBora/auto-draft)的auto_backgrounds功能的测试。通过输入一个领域的名称(比如Deep Reinforcement Learning),即可自动对这个领域的相关文献进行归纳总结.
59
 
60
  ***2023-04-30 Update***: 如果有更多想法和建议欢迎加入群里交流, 群号: ***249738228***.
61
 
@@ -66,26 +86,26 @@ with gr.Blocks() as demo:
66
  输入一个领域的名称(比如Deep Reinforcement Learning), 点击Submit, 等待大概十分钟, 下载output.zip,在Overleaf上编译浏览.
67
  ''')
68
  with gr.Row():
69
- with gr.Column():
70
- # key = gr.Textbox(value=openai_key, lines=1, max_lines=1, label="OpenAI Key", visible=not IS_OPENAI_API_KEY_AVAILABLE)
71
- key = gr.Textbox(value=openai_key, lines=1, max_lines=1, label="OpenAI Key", visible=False)
72
  title = gr.Textbox(value="Deep Reinforcement Learning", lines=1, max_lines=1, label="Title")
73
  description = gr.Textbox(lines=5, label="Description (Optional)")
74
 
75
  with gr.Row():
76
  clear_button = gr.Button("Clear")
77
  submit_button = gr.Button("Submit")
78
- with gr.Column():
79
  style_mapping = {True: "color:white;background-color:green", False: "color:white;background-color:red"} #todo: to match website's style
80
- availablity_mapping = {True: "AVAILABLE", False: "NOT AVAILABLE"}
81
  gr.Markdown(f'''## Huggingface Space Status
82
  当`OpenAI API`显示AVAILABLE的时候这个Space可以直接使用.
83
- 当`OpenAI API`显示UNAVAILABLE的时候这个Space可以通过在左侧输入OPENAI KEY来使用 (暂时不支持).
84
- `OpenAI API`: <span style="{style_mapping[IS_OPENAI_API_KEY_AVAILABLE]}">{availablity_mapping[IS_OPENAI_API_KEY_AVAILABLE]}</span>. `Cache`: <span style="{style_mapping[IS_CACHE_AVAILABLE]}">{availablity_mapping[IS_CACHE_AVAILABLE]}</span>.''')
85
  file_output = gr.File(label="Output")
86
 
87
  clear_button.click(fn=clear_inputs, inputs=[title, description], outputs=[title, description])
88
- submit_button.click(fn=wrapped_generate_backgrounds, inputs=[title, description, key], outputs=file_output)
89
 
90
  demo.queue(concurrency_count=1, max_size=5, api_open=False)
91
  demo.launch()
 
1
  import gradio as gr
2
  import os
3
+ import openai
4
+ from auto_backgrounds import generate_backgrounds, fake_generator
5
+ from auto_draft import generate_draft
6
 
7
  openai_key = os.getenv("OPENAI_API_KEY")
8
  access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
 
18
  IS_OPENAI_API_KEY_AVAILABLE = False
19
  else:
20
  # todo: check if this key is available or not
21
+ openai.api_key = openai_key
22
+ try:
23
+ openai.Model.list()
24
+ IS_OPENAI_API_KEY_AVAILABLE = True
25
+ except Exception as e:
26
+ IS_OPENAI_API_KEY_AVAILABLE = False
27
 
28
 
29
 
 
31
  return "", ""
32
 
33
 
34
+ def wrapped_generator(title, description, openai_key = None,
35
+ template = "ICLR2022",
36
+ cache_mode = IS_CACHE_AVAILABLE, generator=None):
37
  # if `cache_mode` is True, then follow the following steps:
38
  # check if "title"+"description" have been generated before
39
  # if so, download from the cloud storage, return it
40
  # if not, generate the result.
41
+ if generator is None:
42
+ generator = generate_backgrounds
43
+ if openai_key is not None:
44
+ openai.api_key = openai_key
45
+ openai.Model.list()
46
+
47
  if cache_mode:
48
  from utils.storage import list_all_files, hash_name, download_file, upload_file
49
  # check if "title"+"description" have been generated before
 
56
  else:
57
  # generate the result.
58
  # output = fake_generate_backgrounds(title, description, openai_key)
59
+ output = generate_backgrounds(title, description, template, "gpt-4")
60
  upload_file(file_name)
61
  return output
62
  else:
63
  # output = fake_generate_backgrounds(title, description, openai_key)
64
+ output = generate_backgrounds(title, description, template, "gpt-4")
65
  return output
66
 
67
 
68
+ theme = gr.themes.Monochrome(font=gr.themes.GoogleFont("Questrial")).set(
69
+ background_fill_primary='#F6F6F6',
70
+ button_primary_background_fill="#281A39",
71
+ input_background_fill='#E5E4E2'
72
+ )
73
 
74
+ with gr.Blocks(theme=theme) as demo:
75
  gr.Markdown('''
76
  # Auto-Draft: 文献整理辅助工具-限量免费使用
77
 
78
+ 本Demo提供对[Auto-Draft](https://github.com/CCCBora/auto-draft)的auto_backgrounds功能的测试。通过输入一个领域的名称(比如Deep Reinforcement Learning),即可自动对这个领域的相关文献进行归纳总结.
79
 
80
  ***2023-04-30 Update***: 如果有更多想法和建议欢迎加入群里交流, 群号: ***249738228***.
81
 
 
86
  输入一个领域的名称(比如Deep Reinforcement Learning), 点击Submit, 等待大概十分钟, 下载output.zip,在Overleaf上编译浏览.
87
  ''')
88
  with gr.Row():
89
+ with gr.Column(scale=2):
90
+ key = gr.Textbox(value=openai_key, lines=1, max_lines=1, label="OpenAI Key", visible=not IS_OPENAI_API_KEY_AVAILABLE)
91
+ # key = gr.Textbox(value=openai_key, lines=1, max_lines=1, label="OpenAI Key", visible=False)
92
  title = gr.Textbox(value="Deep Reinforcement Learning", lines=1, max_lines=1, label="Title")
93
  description = gr.Textbox(lines=5, label="Description (Optional)")
94
 
95
  with gr.Row():
96
  clear_button = gr.Button("Clear")
97
  submit_button = gr.Button("Submit")
98
+ with gr.Column(scale=1):
99
  style_mapping = {True: "color:white;background-color:green", False: "color:white;background-color:red"} #todo: to match website's style
100
+ availability_mapping = {True: "AVAILABLE", False: "NOT AVAILABLE"}
101
  gr.Markdown(f'''## Huggingface Space Status
102
  当`OpenAI API`显示AVAILABLE的时候这个Space可以直接使用.
103
+ 当`OpenAI API`显示NOT AVAILABLE的时候这个Space可以通过在左侧输入OPENAI KEY来使用.
104
+ `OpenAI API`: <span style="{style_mapping[IS_OPENAI_API_KEY_AVAILABLE]}">{availability_mapping[IS_OPENAI_API_KEY_AVAILABLE]}</span>. `Cache`: <span style="{style_mapping[IS_CACHE_AVAILABLE]}">{availability_mapping[IS_CACHE_AVAILABLE]}</span>.''')
105
  file_output = gr.File(label="Output")
106
 
107
  clear_button.click(fn=clear_inputs, inputs=[title, description], outputs=[title, description])
108
+ submit_button.click(fn=wrapped_generator, inputs=[title, description, key], outputs=file_output)
109
 
110
  demo.queue(concurrency_count=1, max_size=5, api_open=False)
111
  demo.launch()
auto_backgrounds.py CHANGED
@@ -1,29 +1,12 @@
1
  from utils.references import References
2
- from utils.prompts import generate_bg_keywords_prompts, generate_bg_summary_prompts
3
- from utils.gpt_interaction import get_responses, extract_responses, extract_keywords, extract_json
4
- from utils.tex_processing import replace_title
5
- import datetime
6
- import shutil
7
- import time
8
  import logging
9
- import os
10
 
11
  TOTAL_TOKENS = 0
12
  TOTAL_PROMPTS_TOKENS = 0
13
  TOTAL_COMPLETION_TOKENS = 0
14
 
15
-
16
- def hash_name(title, description):
17
- '''
18
- For same title and description, it should return the same value.
19
- '''
20
- name = title + description
21
- name = name.lower()
22
- md5 = hashlib.md5()
23
- md5.update(name.encode('utf-8'))
24
- hashed_string = md5.hexdigest()
25
- return hashed_string
26
-
27
  def log_usage(usage, generating_target, print_out=True):
28
  global TOTAL_TOKENS
29
  global TOTAL_PROMPTS_TOKENS
@@ -43,70 +26,19 @@ def log_usage(usage, generating_target, print_out=True):
43
  print(message)
44
  logging.info(message)
45
 
46
- def make_archive(source, destination):
47
- base = os.path.basename(destination)
48
- name = base.split('.')[0]
49
- format = base.split('.')[1]
50
- archive_from = os.path.dirname(source)
51
- archive_to = os.path.basename(source.strip(os.sep))
52
- shutil.make_archive(name, format, archive_from, archive_to)
53
- shutil.move('%s.%s'%(name,format), destination)
54
- return destination
55
-
56
- def pipeline(paper, section, save_to_path, model, openai_key=None):
57
- """
58
- The main pipeline of generating a section.
59
- 1. Generate prompts.
60
- 2. Get responses from AI assistant.
61
- 3. Extract the section text.
62
- 4. Save the text to .tex file.
63
- :return usage
64
- """
65
- print(f"Generating {section}...")
66
- prompts = generate_bg_summary_prompts(paper, section)
67
- gpt_response, usage = get_responses(prompts, model)
68
- output = extract_responses(gpt_response)
69
- paper["body"][section] = output
70
- tex_file = save_to_path + f"{section}.tex"
71
- if section == "abstract":
72
- with open(tex_file, "w") as f:
73
- f.write(r"\begin{abstract}")
74
- with open(tex_file, "a") as f:
75
- f.write(output)
76
- with open(tex_file, "a") as f:
77
- f.write(r"\end{abstract}")
78
- else:
79
- with open(tex_file, "w") as f:
80
- f.write(f"\section{{{section.upper()}}}\n")
81
- with open(tex_file, "a") as f:
82
- f.write(output)
83
- time.sleep(5)
84
- print(f"{section} has been generated. Saved to {tex_file}.")
85
- return usage
86
-
87
-
88
-
89
- def generate_backgrounds(title, description="", template="ICLR2022", model="gpt-4", openai_key=None):
90
  paper = {}
91
  paper_body = {}
92
 
93
  # Create a copy in the outputs folder.
94
- now = datetime.datetime.now()
95
- target_name = now.strftime("outputs_%Y%m%d_%H%M%S")
96
- source_folder = f"latex_templates/{template}"
97
- destination_folder = f"outputs/{target_name}"
98
- shutil.copytree(source_folder, destination_folder)
99
-
100
- bibtex_path = destination_folder + "/ref.bib"
101
- save_to_path = destination_folder +"/"
102
- replace_title(save_to_path, "A Survey on " + title)
103
- logging.basicConfig( level=logging.INFO, filename=save_to_path+"generation.log")
104
 
105
  # Generate keywords and references
106
  print("Initialize the paper information ...")
107
- prompts = generate_bg_keywords_prompts(title, description)
108
- gpt_response, usage = get_responses(prompts, model)
109
- keywords = extract_keywords(gpt_response)
110
  log_usage(usage, "keywords")
111
 
112
  ref = References(load_papers = "")
@@ -123,28 +55,23 @@ def generate_backgrounds(title, description="", template="ICLR2022", model="gpt-
123
 
124
  for section in ["introduction", "related works", "backgrounds"]:
125
  try:
126
- usage = pipeline(paper, section, save_to_path, model=model)
 
127
  log_usage(usage, section)
128
  except Exception as e:
129
  print(f"Failed to generate {section} due to the error: {e}")
130
- print(f"The paper {title} has been generated. Saved to {save_to_path}.")
131
  # shutil.make_archive("output.zip", 'zip', save_to_path)
132
- return make_archive(destination_folder, "output.zip")
 
 
 
133
 
134
 
135
- def fake_generate_backgrounds(title, description, openai_key = None):
136
  """
137
  This function is used to test the whole pipeline without calling OpenAI API.
138
  """
139
- filename = hash_name(title, description) + ".zip"
 
140
  return make_archive("sample-output.pdf", filename)
141
-
142
-
143
- if __name__ == "__main__":
144
- title = "Reinforcement Learning"
145
- description = ""
146
- template = "Summary"
147
- model = "gpt-4"
148
- # model = "gpt-3.5-turbo"
149
-
150
- generate_backgrounds(title, description, template, model)
 
1
  from utils.references import References
2
+ from utils.file_operations import hash_name, make_archive, copy_templates
3
+ from section_generator import section_generation_bg, keywords_generation
 
 
 
 
4
  import logging
 
5
 
6
  TOTAL_TOKENS = 0
7
  TOTAL_PROMPTS_TOKENS = 0
8
  TOTAL_COMPLETION_TOKENS = 0
9
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  def log_usage(usage, generating_target, print_out=True):
11
  global TOTAL_TOKENS
12
  global TOTAL_PROMPTS_TOKENS
 
26
  print(message)
27
  logging.info(message)
28
 
29
+ def generate_backgrounds(title, description="", template="ICLR2022", model="gpt-4"):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  paper = {}
31
  paper_body = {}
32
 
33
  # Create a copy in the outputs folder.
34
+ bibtex_path, destination_folder = copy_templates(template, title)
35
+ logging.basicConfig(level=logging.INFO, filename=destination_folder + "/generation.log")
 
 
 
 
 
 
 
 
36
 
37
  # Generate keywords and references
38
  print("Initialize the paper information ...")
39
+ input_dict = {"title": title, "description": description}
40
+ keywords, usage = keywords_generation(input_dict, model="gpt-3.5-turbo")
41
+ print(f"keywords: {keywords}")
42
  log_usage(usage, "keywords")
43
 
44
  ref = References(load_papers = "")
 
55
 
56
  for section in ["introduction", "related works", "backgrounds"]:
57
  try:
58
+ # usage = pipeline(paper, section, destination_folder, model=model)
59
+ usage = section_generation_bg(paper, section, destination_folder, model=model)
60
  log_usage(usage, section)
61
  except Exception as e:
62
  print(f"Failed to generate {section} due to the error: {e}")
63
+ print(f"The paper {title} has been generated. Saved to {destination_folder}.")
64
  # shutil.make_archive("output.zip", 'zip', save_to_path)
65
+
66
+ input_dict = {"title": title, "description": description, "generator": "generate_backgrounds"}
67
+ filename = hash_name(input_dict) + ".zip"
68
+ return make_archive(destination_folder, filename)
69
 
70
 
71
+ def fake_generator(title, description="", template="ICLR2022", model="gpt-4"):
72
  """
73
  This function is used to test the whole pipeline without calling OpenAI API.
74
  """
75
+ input_dict = {"title": title, "description": description, "generator": "generate_backgrounds"}
76
+ filename = hash_name(input_dict) + ".zip"
77
  return make_archive("sample-output.pdf", filename)
 
 
 
 
 
 
 
 
 
 
auto_draft.py CHANGED
@@ -88,6 +88,7 @@ def generate_draft(title, description="", template="ICLR2022", model="gpt-4"):
88
  paper_body = {}
89
 
90
  # Create a copy in the outputs folder.
 
91
  now = datetime.datetime.now()
92
  target_name = now.strftime("outputs_%Y%m%d_%H%M%S")
93
  source_folder = f"latex_templates/{template}"
@@ -105,16 +106,15 @@ def generate_draft(title, description="", template="ICLR2022", model="gpt-4"):
105
  gpt_response, usage = get_responses(prompts, model)
106
  keywords = extract_keywords(gpt_response)
107
  log_usage(usage, "keywords")
108
-
109
- ref = References(load_papers = "")
110
- ref.collect_papers(keywords, method="arxiv")
111
  all_paper_ids = ref.to_bibtex(bibtex_path) #todo: this will used to check if all citations are in this list
112
 
113
  print(f"The paper information has been initialized. References are saved to {bibtex_path}.")
114
 
115
  paper["title"] = title
116
  paper["description"] = description
117
- paper["references"] = ref.to_prompts() # to_prompts(top_papers)
118
  paper["body"] = paper_body
119
  paper["bibtex"] = bibtex_path
120
 
 
88
  paper_body = {}
89
 
90
  # Create a copy in the outputs folder.
91
+ # todo: use copy_templates function instead.
92
  now = datetime.datetime.now()
93
  target_name = now.strftime("outputs_%Y%m%d_%H%M%S")
94
  source_folder = f"latex_templates/{template}"
 
106
  gpt_response, usage = get_responses(prompts, model)
107
  keywords = extract_keywords(gpt_response)
108
  log_usage(usage, "keywords")
109
+ ref = References(load_papers = "") #todo: allow users to upload bibfile.
110
+ ref.collect_papers(keywords, method="arxiv") #todo: add more methods to find related papers
 
111
  all_paper_ids = ref.to_bibtex(bibtex_path) #todo: this will used to check if all citations are in this list
112
 
113
  print(f"The paper information has been initialized. References are saved to {bibtex_path}.")
114
 
115
  paper["title"] = title
116
  paper["description"] = description
117
+ paper["references"] = ref.to_prompts() #todo: see if this prompts can be compressed.
118
  paper["body"] = paper_body
119
  paper["bibtex"] = bibtex_path
120
 
output.zip CHANGED
Binary files a/output.zip and b/output.zip differ
 
section_generator.py ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from utils.prompts import generate_paper_prompts, generate_keywords_prompts, generate_experiments_prompts, generate_bg_summary_prompts
2
+ from utils.gpt_interaction import get_responses, extract_responses, extract_keywords, extract_json
3
+ import time
4
+ import os
5
+
6
+ # three GPT-based content generator:
7
+ # 1. section_generation: used to generate main content of the paper
8
+ # 2. keywords_generation: used to generate a json output {key1: output1, key2: output2} for multiple purpose.
9
+ # 3. figure_generation: used to generate sample figures.
10
+ # all generator should return the token usage.
11
+
12
+
13
+ def section_generation_bg(paper, section, save_to_path, model):
14
+ """
15
+ The main pipeline of generating a section.
16
+ 1. Generate prompts.
17
+ 2. Get responses from AI assistant.
18
+ 3. Extract the section text.
19
+ 4. Save the text to .tex file.
20
+ :return usage
21
+ """
22
+ print(f"Generating {section}...")
23
+ prompts = generate_bg_summary_prompts(paper, section)
24
+ gpt_response, usage = get_responses(prompts, model)
25
+ output = extract_responses(gpt_response)
26
+ paper["body"][section] = output
27
+ tex_file = os.path.join(save_to_path, f"{section}.tex")
28
+ # tex_file = save_to_path + f"/{section}.tex"
29
+ if section == "abstract":
30
+ with open(tex_file, "w") as f:
31
+ f.write(r"\begin{abstract}")
32
+ with open(tex_file, "a") as f:
33
+ f.write(output)
34
+ with open(tex_file, "a") as f:
35
+ f.write(r"\end{abstract}")
36
+ else:
37
+ with open(tex_file, "w") as f:
38
+ f.write(f"\section{{{section.upper()}}}\n")
39
+ with open(tex_file, "a") as f:
40
+ f.write(output)
41
+ time.sleep(5)
42
+ print(f"{section} has been generated. Saved to {tex_file}.")
43
+ return usage
44
+
45
+
46
+ def keywords_generation(input_dict, model):
47
+ title = input_dict.get("title")
48
+ description = input_dict.get("description", "")
49
+ if title is not None:
50
+ prompts = generate_keywords_prompts(title, description)
51
+ gpt_response, usage = get_responses(prompts, model)
52
+ keywords = extract_keywords(gpt_response)
53
+ return keywords, usage
54
+ else:
55
+ raise ValueError("`input_dict` must include the key 'title'.")
56
+
57
+ def figures_generation():
58
+ pass
utils/file_operations.py ADDED
@@ -0,0 +1,45 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import hashlib
2
+ import os, shutil
3
+ import datetime
4
+ from utils.tex_processing import replace_title
5
+
6
+ def hash_name(input_dict):
7
+ '''
8
+ input_dict= {"title": title, "description": description}
9
+
10
+ For same input_dict, it should return the same value.
11
+ '''
12
+ name = str(input_dict)
13
+ name = name.lower()
14
+ md5 = hashlib.md5()
15
+ md5.update(name.encode('utf-8'))
16
+ hashed_string = md5.hexdigest()
17
+ return hashed_string
18
+
19
+
20
+
21
+ def make_archive(source, destination):
22
+ base = os.path.basename(destination)
23
+ name = base.split('.')[0]
24
+ format = base.split('.')[1]
25
+ archive_from = os.path.dirname(source)
26
+ archive_to = os.path.basename(source.strip(os.sep))
27
+ shutil.make_archive(name, format, archive_from, archive_to)
28
+ shutil.move('%s.%s'%(name,format), destination)
29
+ return destination
30
+
31
+ def copy_templates(template, title):
32
+ # Create a copy in the outputs folder.
33
+ # 1. create a folder "outputs_%Y%m%d_%H%M%S" (destination_folder)
34
+ # 2. copy all contents in "latex_templates/{template}" to that folder
35
+ # 3. return (bibtex_path, destination_folder)
36
+ now = datetime.datetime.now()
37
+ target_name = now.strftime("outputs_%Y%m%d_%H%M%S")
38
+ source_folder = f"latex_templates/{template}"
39
+ destination_folder = f"outputs/{target_name}"
40
+ shutil.copytree(source_folder, destination_folder)
41
+ bibtex_path = os.path.join(destination_folder, "ref.bib")
42
+ # bibtex_path = destination_folder + "/ref.bib"
43
+ replace_title(destination_folder, title)
44
+ return bibtex_path, destination_folder
45
+
utils/gpt_interaction.py CHANGED
@@ -1,12 +1,10 @@
1
  import openai
2
  import re
3
- import os
4
  import json
5
  import logging
 
6
  log = logging.getLogger(__name__)
7
 
8
- # todo: 将api_key通过函数传入; 需要改很多地方
9
- # openai.api_key = os.environ['OPENAI_API_KEY']
10
 
11
  def extract_responses(assistant_message):
12
  # pattern = re.compile(r"f\.write\(r'{1,3}(.*?)'{0,3}\){0,1}$", re.DOTALL)
@@ -19,9 +17,10 @@ def extract_responses(assistant_message):
19
  log.info(f"assistant_message: {assistant_message}")
20
  return assistant_message
21
 
 
22
  def extract_keywords(assistant_message, default_keywords=None):
23
  if default_keywords is None:
24
- default_keywords = {"machine learning":5}
25
 
26
  try:
27
  keywords = json.loads(assistant_message)
@@ -31,6 +30,7 @@ def extract_keywords(assistant_message, default_keywords=None):
31
  return default_keywords
32
  return keywords
33
 
 
34
  def extract_section_name(assistant_message, default_section_name=""):
35
  try:
36
  keywords = json.loads(assistant_message)
@@ -55,7 +55,7 @@ def extract_json(assistant_message, default_output=None):
55
  return dict.keys()
56
 
57
 
58
- def get_responses(user_message, model="gpt-4", temperature=0.4, openai_key = None):
59
  if openai.api_key is None and openai_key is None:
60
  raise ValueError("OpenAI API key must be provided.")
61
  if openai_key is not None:
@@ -79,8 +79,8 @@ def get_responses(user_message, model="gpt-4", temperature=0.4, openai_key = Non
79
 
80
  if __name__ == "__main__":
81
  test_strings = [r"f.write(r'hello world')", r"f.write(r'''hello world''')", r"f.write(r'''hello world",
82
- r"f.write(r'''hello world'", r'f.write(r"hello world")', r'f.write(r"""hello world""")',
83
- r'f.write(r"""hello world"', r'f.write(r"""hello world']
84
  for input_string in test_strings:
85
  print("input_string: ", input_string)
86
  pattern = re.compile(r"f\.write\(r['\"]{1,3}(.*?)['\"]{0,3}\){0,1}$", re.DOTALL)
@@ -90,4 +90,4 @@ if __name__ == "__main__":
90
  extracted_string = match.group(1)
91
  print("Extracted string:", extracted_string)
92
  else:
93
- print("No match found")
 
1
  import openai
2
  import re
 
3
  import json
4
  import logging
5
+
6
  log = logging.getLogger(__name__)
7
 
 
 
8
 
9
  def extract_responses(assistant_message):
10
  # pattern = re.compile(r"f\.write\(r'{1,3}(.*?)'{0,3}\){0,1}$", re.DOTALL)
 
17
  log.info(f"assistant_message: {assistant_message}")
18
  return assistant_message
19
 
20
+
21
  def extract_keywords(assistant_message, default_keywords=None):
22
  if default_keywords is None:
23
+ default_keywords = {"machine learning": 5}
24
 
25
  try:
26
  keywords = json.loads(assistant_message)
 
30
  return default_keywords
31
  return keywords
32
 
33
+
34
  def extract_section_name(assistant_message, default_section_name=""):
35
  try:
36
  keywords = json.loads(assistant_message)
 
55
  return dict.keys()
56
 
57
 
58
+ def get_responses(user_message, model="gpt-4", temperature=0.4, openai_key=None):
59
  if openai.api_key is None and openai_key is None:
60
  raise ValueError("OpenAI API key must be provided.")
61
  if openai_key is not None:
 
79
 
80
  if __name__ == "__main__":
81
  test_strings = [r"f.write(r'hello world')", r"f.write(r'''hello world''')", r"f.write(r'''hello world",
82
+ r"f.write(r'''hello world'", r'f.write(r"hello world")', r'f.write(r"""hello world""")',
83
+ r'f.write(r"""hello world"', r'f.write(r"""hello world']
84
  for input_string in test_strings:
85
  print("input_string: ", input_string)
86
  pattern = re.compile(r"f\.write\(r['\"]{1,3}(.*?)['\"]{0,3}\){0,1}$", re.DOTALL)
 
90
  extracted_string = match.group(1)
91
  print("Extracted string:", extracted_string)
92
  else:
93
+ print("No match found")
utils/prompts.py CHANGED
@@ -16,8 +16,8 @@ BG_INSTRUCTIONS = {"introduction": "Please include four paragraph: Establishing
16
 
17
  def generate_keywords_prompts(title, description="", num_refs=5):
18
  prompts = f"I am writing a machine learning paper with the title '{title}'. {description}\n" \
19
- f"Please generate three to five keywords. For each keyword, rate it from 1 to {num_refs}; the larger number means more important." \
20
- r"Response in a dictionary format like {\"keyword1\":1, \"keyword2\":3}."
21
  return prompts
22
 
23
  def generate_rename_prompts(paper_info, section):
@@ -72,15 +72,6 @@ def generate_paper_prompts(paper_info, section):
72
  return prompts
73
 
74
 
75
-
76
-
77
- def generate_bg_keywords_prompts(title, description="", num_refs=5):
78
- prompts = f"I am writing a survey on the topic '{title}'. {description}\n" \
79
- f"Please generate three to five keywords. For each keyword, rate it from 1 to {num_refs}; the larger number means more important." \
80
- r"Response in a dictionary format like {keyword1:1, keyword2:3}."
81
- return prompts
82
-
83
-
84
  def generate_bg_summary_prompts(paper_info, section):
85
  title = paper_info["title"]
86
  description = paper_info["description"]
 
16
 
17
  def generate_keywords_prompts(title, description="", num_refs=5):
18
  prompts = f"I am writing a machine learning paper with the title '{title}'. {description}\n" \
19
+ f"Generate three to five keywords. For each keyword, rate it from 1 to {num_refs}; the larger number means more important." \
20
+ r"Your response must be in JSON format like {\"keyword1\":1, \"keyword2\":3}."
21
  return prompts
22
 
23
  def generate_rename_prompts(paper_info, section):
 
72
  return prompts
73
 
74
 
 
 
 
 
 
 
 
 
 
75
  def generate_bg_summary_prompts(paper_info, section):
76
  title = paper_info["title"]
77
  description = paper_info["description"]
utils/storage.py CHANGED
@@ -1,50 +1,45 @@
 
 
 
 
 
1
  import os
2
  import boto3
3
- import hashlib
4
 
5
- access_key_id = os.environ['AWS_ACCESS_KEY_ID']
6
- secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
7
  bucket_name = "hf-storage"
8
 
9
- session = boto3.Session(
10
- aws_access_key_id=access_key_id,
11
- aws_secret_access_key=secret_access_key,
12
- )
13
-
14
- s3 = session.resource('s3')
15
- bucket = s3.Bucket(bucket_name)
16
-
17
- def upload_file(file_name, target_name=None):
18
- if target_name is None:
19
- target_name = file_name
20
- try:
21
- s3.meta.client.upload_file(Filename=file_name, Bucket=bucket_name, Key=target_name)
22
- print(f"The file {file_name} has been uploaded!")
23
- except:
24
- print("Uploading failed!")
25
-
26
- def list_all_files():
27
- return [obj.key for obj in bucket.objects.all()]
28
-
29
- def download_file(file_name):
30
- ''' Download `file_name` from the bucket. todo:check existence before downloading!
31
- Bucket (str) – The name of the bucket to download from.
32
- Key (str) The name of the key to download from.
33
- Filename (str) – The path to the file to download to.
34
- '''
35
- try:
36
- s3.meta.client.download_file(Bucket=bucket_name, Key=file_name, Filename=file_name)
37
- print(f"The file {file_name} has been downloaded!")
38
- except:
39
- print("Uploading failed!")
40
-
41
- def hash_name(title, description):
42
- '''
43
- For same title and description, it should return the same value.
44
- '''
45
- name = title + description
46
- name = name.lower()
47
- md5 = hashlib.md5()
48
- md5.update(name.encode('utf-8'))
49
- hashed_string = md5.hexdigest()
50
- return hashed_string
 
1
+ # This script `storage.py` is used to handle the cloud storage.
2
+ # `upload_file`:
3
+ # `list_all_files`:
4
+ # `download_file`:
5
+
6
  import os
7
  import boto3
 
8
 
9
+ access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
10
+ secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
11
  bucket_name = "hf-storage"
12
 
13
+ if (access_key_id is not None) and (secret_access_key is not None):
14
+ session = boto3.Session(
15
+ aws_access_key_id=access_key_id,
16
+ aws_secret_access_key=secret_access_key,
17
+ )
18
+
19
+ s3 = session.resource('s3')
20
+ bucket = s3.Bucket(bucket_name)
21
+
22
+
23
+ def upload_file(file_name, target_name=None):
24
+ if target_name is None:
25
+ target_name = file_name
26
+ try:
27
+ s3.meta.client.upload_file(Filename=file_name, Bucket=bucket_name, Key=target_name)
28
+ print(f"The file {file_name} has been uploaded!")
29
+ except:
30
+ print("Uploading failed!")
31
+
32
+ def list_all_files():
33
+ return [obj.key for obj in bucket.objects.all()]
34
+
35
+ def download_file(file_name):
36
+ ''' Download `file_name` from the bucket. todo:check existence before downloading!
37
+ Bucket (str) – The name of the bucket to download from.
38
+ Key (str) – The name of the key to download from.
39
+ Filename (str) – The path to the file to download to.
40
+ '''
41
+ try:
42
+ s3.meta.client.download_file(Bucket=bucket_name, Key=file_name, Filename=file_name)
43
+ print(f"The file {file_name} has been downloaded!")
44
+ except:
45
+ print("Uploading failed!")
 
 
 
 
 
 
 
 
 
utils/tex_processing.py CHANGED
@@ -1,7 +1,11 @@
 
 
1
  def replace_title(save_to_path, title):
2
  # Define input and output file names
3
- input_file_name = save_to_path + "template.tex"
4
- output_file_name = save_to_path + "main.tex"
 
 
5
 
6
  # Open the input file and read its content
7
  with open(input_file_name, 'r') as infile:
@@ -15,6 +19,9 @@ def replace_title(save_to_path, title):
15
  outfile.write(content)
16
 
17
 
18
- # return all string in \cite{...}
 
 
 
19
 
20
  # replace citations
 
1
+ import os
2
+
3
  def replace_title(save_to_path, title):
4
  # Define input and output file names
5
+ # input_file_name = save_to_path + "/template.tex"
6
+ # output_file_name = save_to_path + "/main.tex"
7
+ input_file_name = os.path.join(save_to_path, "template.tex")
8
+ output_file_name = os.path.join(save_to_path , "main.tex")
9
 
10
  # Open the input file and read its content
11
  with open(input_file_name, 'r') as infile:
 
19
  outfile.write(content)
20
 
21
 
22
+ # return all string in \cite{...}.
23
+
24
+ # check if citations are in bibtex.
25
+
26
 
27
  # replace citations