Spaces:
Running
Running
sc_ma
commited on
Commit
•
a0d1776
1
Parent(s):
09305ff
bug fix.
Browse filesedit fundamental logic of passing openai key.
support user's key input.
- app.py +35 -15
- auto_backgrounds.py +18 -91
- auto_draft.py +4 -4
- output.zip +0 -0
- section_generator.py +58 -0
- utils/file_operations.py +45 -0
- utils/gpt_interaction.py +8 -8
- utils/prompts.py +2 -11
- utils/storage.py +40 -45
- utils/tex_processing.py +10 -3
app.py
CHANGED
@@ -1,6 +1,8 @@
|
|
1 |
import gradio as gr
|
2 |
import os
|
3 |
-
|
|
|
|
|
4 |
|
5 |
openai_key = os.getenv("OPENAI_API_KEY")
|
6 |
access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
|
@@ -16,7 +18,12 @@ if openai_key is None:
|
|
16 |
IS_OPENAI_API_KEY_AVAILABLE = False
|
17 |
else:
|
18 |
# todo: check if this key is available or not
|
19 |
-
|
|
|
|
|
|
|
|
|
|
|
20 |
|
21 |
|
22 |
|
@@ -24,11 +31,19 @@ def clear_inputs(text1, text2):
|
|
24 |
return "", ""
|
25 |
|
26 |
|
27 |
-
def
|
|
|
|
|
28 |
# if `cache_mode` is True, then follow the following steps:
|
29 |
# check if "title"+"description" have been generated before
|
30 |
# if so, download from the cloud storage, return it
|
31 |
# if not, generate the result.
|
|
|
|
|
|
|
|
|
|
|
|
|
32 |
if cache_mode:
|
33 |
from utils.storage import list_all_files, hash_name, download_file, upload_file
|
34 |
# check if "title"+"description" have been generated before
|
@@ -41,21 +56,26 @@ def wrapped_generate_backgrounds(title, description, openai_key = None, cache_mo
|
|
41 |
else:
|
42 |
# generate the result.
|
43 |
# output = fake_generate_backgrounds(title, description, openai_key)
|
44 |
-
output = generate_backgrounds(title, description,
|
45 |
upload_file(file_name)
|
46 |
return output
|
47 |
else:
|
48 |
# output = fake_generate_backgrounds(title, description, openai_key)
|
49 |
-
output = generate_backgrounds(title, description,
|
50 |
return output
|
51 |
|
52 |
|
|
|
|
|
|
|
|
|
|
|
53 |
|
54 |
-
with gr.Blocks() as demo:
|
55 |
gr.Markdown('''
|
56 |
# Auto-Draft: 文献整理辅助工具-限量免费使用
|
57 |
|
58 |
-
本Demo提供对[Auto-Draft](https://github.com/CCCBora/auto-draft)的auto_backgrounds功能的测试。通过输入一个领域的名称(比如Deep Reinforcement Learning),即可自动对这个领域的相关文献进行归纳总结.
|
59 |
|
60 |
***2023-04-30 Update***: 如果有更多想法和建议欢迎加入群里交流, 群号: ***249738228***.
|
61 |
|
@@ -66,26 +86,26 @@ with gr.Blocks() as demo:
|
|
66 |
输入一个领域的名称(比如Deep Reinforcement Learning), 点击Submit, 等待大概十分钟, 下载output.zip,在Overleaf上编译浏览.
|
67 |
''')
|
68 |
with gr.Row():
|
69 |
-
with gr.Column():
|
70 |
-
|
71 |
-
key = gr.Textbox(value=openai_key, lines=1, max_lines=1, label="OpenAI Key", visible=False)
|
72 |
title = gr.Textbox(value="Deep Reinforcement Learning", lines=1, max_lines=1, label="Title")
|
73 |
description = gr.Textbox(lines=5, label="Description (Optional)")
|
74 |
|
75 |
with gr.Row():
|
76 |
clear_button = gr.Button("Clear")
|
77 |
submit_button = gr.Button("Submit")
|
78 |
-
with gr.Column():
|
79 |
style_mapping = {True: "color:white;background-color:green", False: "color:white;background-color:red"} #todo: to match website's style
|
80 |
-
|
81 |
gr.Markdown(f'''## Huggingface Space Status
|
82 |
当`OpenAI API`显示AVAILABLE的时候这个Space可以直接使用.
|
83 |
-
当`OpenAI API`显示
|
84 |
-
`OpenAI API`: <span style="{style_mapping[IS_OPENAI_API_KEY_AVAILABLE]}">{
|
85 |
file_output = gr.File(label="Output")
|
86 |
|
87 |
clear_button.click(fn=clear_inputs, inputs=[title, description], outputs=[title, description])
|
88 |
-
submit_button.click(fn=
|
89 |
|
90 |
demo.queue(concurrency_count=1, max_size=5, api_open=False)
|
91 |
demo.launch()
|
|
|
1 |
import gradio as gr
|
2 |
import os
|
3 |
+
import openai
|
4 |
+
from auto_backgrounds import generate_backgrounds, fake_generator
|
5 |
+
from auto_draft import generate_draft
|
6 |
|
7 |
openai_key = os.getenv("OPENAI_API_KEY")
|
8 |
access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
|
|
|
18 |
IS_OPENAI_API_KEY_AVAILABLE = False
|
19 |
else:
|
20 |
# todo: check if this key is available or not
|
21 |
+
openai.api_key = openai_key
|
22 |
+
try:
|
23 |
+
openai.Model.list()
|
24 |
+
IS_OPENAI_API_KEY_AVAILABLE = True
|
25 |
+
except Exception as e:
|
26 |
+
IS_OPENAI_API_KEY_AVAILABLE = False
|
27 |
|
28 |
|
29 |
|
|
|
31 |
return "", ""
|
32 |
|
33 |
|
34 |
+
def wrapped_generator(title, description, openai_key = None,
|
35 |
+
template = "ICLR2022",
|
36 |
+
cache_mode = IS_CACHE_AVAILABLE, generator=None):
|
37 |
# if `cache_mode` is True, then follow the following steps:
|
38 |
# check if "title"+"description" have been generated before
|
39 |
# if so, download from the cloud storage, return it
|
40 |
# if not, generate the result.
|
41 |
+
if generator is None:
|
42 |
+
generator = generate_backgrounds
|
43 |
+
if openai_key is not None:
|
44 |
+
openai.api_key = openai_key
|
45 |
+
openai.Model.list()
|
46 |
+
|
47 |
if cache_mode:
|
48 |
from utils.storage import list_all_files, hash_name, download_file, upload_file
|
49 |
# check if "title"+"description" have been generated before
|
|
|
56 |
else:
|
57 |
# generate the result.
|
58 |
# output = fake_generate_backgrounds(title, description, openai_key)
|
59 |
+
output = generate_backgrounds(title, description, template, "gpt-4")
|
60 |
upload_file(file_name)
|
61 |
return output
|
62 |
else:
|
63 |
# output = fake_generate_backgrounds(title, description, openai_key)
|
64 |
+
output = generate_backgrounds(title, description, template, "gpt-4")
|
65 |
return output
|
66 |
|
67 |
|
68 |
+
theme = gr.themes.Monochrome(font=gr.themes.GoogleFont("Questrial")).set(
|
69 |
+
background_fill_primary='#F6F6F6',
|
70 |
+
button_primary_background_fill="#281A39",
|
71 |
+
input_background_fill='#E5E4E2'
|
72 |
+
)
|
73 |
|
74 |
+
with gr.Blocks(theme=theme) as demo:
|
75 |
gr.Markdown('''
|
76 |
# Auto-Draft: 文献整理辅助工具-限量免费使用
|
77 |
|
78 |
+
本Demo提供对[Auto-Draft](https://github.com/CCCBora/auto-draft)的auto_backgrounds功能的测试。通过输入一个领域的名称(比如Deep Reinforcement Learning),即可自动对这个领域的相关文献进行归纳总结.
|
79 |
|
80 |
***2023-04-30 Update***: 如果有更多想法和建议欢迎加入群里交流, 群号: ***249738228***.
|
81 |
|
|
|
86 |
输入一个领域的名称(比如Deep Reinforcement Learning), 点击Submit, 等待大概十分钟, 下载output.zip,在Overleaf上编译浏览.
|
87 |
''')
|
88 |
with gr.Row():
|
89 |
+
with gr.Column(scale=2):
|
90 |
+
key = gr.Textbox(value=openai_key, lines=1, max_lines=1, label="OpenAI Key", visible=not IS_OPENAI_API_KEY_AVAILABLE)
|
91 |
+
# key = gr.Textbox(value=openai_key, lines=1, max_lines=1, label="OpenAI Key", visible=False)
|
92 |
title = gr.Textbox(value="Deep Reinforcement Learning", lines=1, max_lines=1, label="Title")
|
93 |
description = gr.Textbox(lines=5, label="Description (Optional)")
|
94 |
|
95 |
with gr.Row():
|
96 |
clear_button = gr.Button("Clear")
|
97 |
submit_button = gr.Button("Submit")
|
98 |
+
with gr.Column(scale=1):
|
99 |
style_mapping = {True: "color:white;background-color:green", False: "color:white;background-color:red"} #todo: to match website's style
|
100 |
+
availability_mapping = {True: "AVAILABLE", False: "NOT AVAILABLE"}
|
101 |
gr.Markdown(f'''## Huggingface Space Status
|
102 |
当`OpenAI API`显示AVAILABLE的时候这个Space可以直接使用.
|
103 |
+
当`OpenAI API`显示NOT AVAILABLE的时候这个Space可以通过在左侧输入OPENAI KEY来使用.
|
104 |
+
`OpenAI API`: <span style="{style_mapping[IS_OPENAI_API_KEY_AVAILABLE]}">{availability_mapping[IS_OPENAI_API_KEY_AVAILABLE]}</span>. `Cache`: <span style="{style_mapping[IS_CACHE_AVAILABLE]}">{availability_mapping[IS_CACHE_AVAILABLE]}</span>.''')
|
105 |
file_output = gr.File(label="Output")
|
106 |
|
107 |
clear_button.click(fn=clear_inputs, inputs=[title, description], outputs=[title, description])
|
108 |
+
submit_button.click(fn=wrapped_generator, inputs=[title, description, key], outputs=file_output)
|
109 |
|
110 |
demo.queue(concurrency_count=1, max_size=5, api_open=False)
|
111 |
demo.launch()
|
auto_backgrounds.py
CHANGED
@@ -1,29 +1,12 @@
|
|
1 |
from utils.references import References
|
2 |
-
from utils.
|
3 |
-
from
|
4 |
-
from utils.tex_processing import replace_title
|
5 |
-
import datetime
|
6 |
-
import shutil
|
7 |
-
import time
|
8 |
import logging
|
9 |
-
import os
|
10 |
|
11 |
TOTAL_TOKENS = 0
|
12 |
TOTAL_PROMPTS_TOKENS = 0
|
13 |
TOTAL_COMPLETION_TOKENS = 0
|
14 |
|
15 |
-
|
16 |
-
def hash_name(title, description):
|
17 |
-
'''
|
18 |
-
For same title and description, it should return the same value.
|
19 |
-
'''
|
20 |
-
name = title + description
|
21 |
-
name = name.lower()
|
22 |
-
md5 = hashlib.md5()
|
23 |
-
md5.update(name.encode('utf-8'))
|
24 |
-
hashed_string = md5.hexdigest()
|
25 |
-
return hashed_string
|
26 |
-
|
27 |
def log_usage(usage, generating_target, print_out=True):
|
28 |
global TOTAL_TOKENS
|
29 |
global TOTAL_PROMPTS_TOKENS
|
@@ -43,70 +26,19 @@ def log_usage(usage, generating_target, print_out=True):
|
|
43 |
print(message)
|
44 |
logging.info(message)
|
45 |
|
46 |
-
def
|
47 |
-
base = os.path.basename(destination)
|
48 |
-
name = base.split('.')[0]
|
49 |
-
format = base.split('.')[1]
|
50 |
-
archive_from = os.path.dirname(source)
|
51 |
-
archive_to = os.path.basename(source.strip(os.sep))
|
52 |
-
shutil.make_archive(name, format, archive_from, archive_to)
|
53 |
-
shutil.move('%s.%s'%(name,format), destination)
|
54 |
-
return destination
|
55 |
-
|
56 |
-
def pipeline(paper, section, save_to_path, model, openai_key=None):
|
57 |
-
"""
|
58 |
-
The main pipeline of generating a section.
|
59 |
-
1. Generate prompts.
|
60 |
-
2. Get responses from AI assistant.
|
61 |
-
3. Extract the section text.
|
62 |
-
4. Save the text to .tex file.
|
63 |
-
:return usage
|
64 |
-
"""
|
65 |
-
print(f"Generating {section}...")
|
66 |
-
prompts = generate_bg_summary_prompts(paper, section)
|
67 |
-
gpt_response, usage = get_responses(prompts, model)
|
68 |
-
output = extract_responses(gpt_response)
|
69 |
-
paper["body"][section] = output
|
70 |
-
tex_file = save_to_path + f"{section}.tex"
|
71 |
-
if section == "abstract":
|
72 |
-
with open(tex_file, "w") as f:
|
73 |
-
f.write(r"\begin{abstract}")
|
74 |
-
with open(tex_file, "a") as f:
|
75 |
-
f.write(output)
|
76 |
-
with open(tex_file, "a") as f:
|
77 |
-
f.write(r"\end{abstract}")
|
78 |
-
else:
|
79 |
-
with open(tex_file, "w") as f:
|
80 |
-
f.write(f"\section{{{section.upper()}}}\n")
|
81 |
-
with open(tex_file, "a") as f:
|
82 |
-
f.write(output)
|
83 |
-
time.sleep(5)
|
84 |
-
print(f"{section} has been generated. Saved to {tex_file}.")
|
85 |
-
return usage
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
def generate_backgrounds(title, description="", template="ICLR2022", model="gpt-4", openai_key=None):
|
90 |
paper = {}
|
91 |
paper_body = {}
|
92 |
|
93 |
# Create a copy in the outputs folder.
|
94 |
-
|
95 |
-
|
96 |
-
source_folder = f"latex_templates/{template}"
|
97 |
-
destination_folder = f"outputs/{target_name}"
|
98 |
-
shutil.copytree(source_folder, destination_folder)
|
99 |
-
|
100 |
-
bibtex_path = destination_folder + "/ref.bib"
|
101 |
-
save_to_path = destination_folder +"/"
|
102 |
-
replace_title(save_to_path, "A Survey on " + title)
|
103 |
-
logging.basicConfig( level=logging.INFO, filename=save_to_path+"generation.log")
|
104 |
|
105 |
# Generate keywords and references
|
106 |
print("Initialize the paper information ...")
|
107 |
-
|
108 |
-
|
109 |
-
keywords
|
110 |
log_usage(usage, "keywords")
|
111 |
|
112 |
ref = References(load_papers = "")
|
@@ -123,28 +55,23 @@ def generate_backgrounds(title, description="", template="ICLR2022", model="gpt-
|
|
123 |
|
124 |
for section in ["introduction", "related works", "backgrounds"]:
|
125 |
try:
|
126 |
-
usage = pipeline(paper, section,
|
|
|
127 |
log_usage(usage, section)
|
128 |
except Exception as e:
|
129 |
print(f"Failed to generate {section} due to the error: {e}")
|
130 |
-
print(f"The paper {title} has been generated. Saved to {
|
131 |
# shutil.make_archive("output.zip", 'zip', save_to_path)
|
132 |
-
|
|
|
|
|
|
|
133 |
|
134 |
|
135 |
-
def
|
136 |
"""
|
137 |
This function is used to test the whole pipeline without calling OpenAI API.
|
138 |
"""
|
139 |
-
|
|
|
140 |
return make_archive("sample-output.pdf", filename)
|
141 |
-
|
142 |
-
|
143 |
-
if __name__ == "__main__":
|
144 |
-
title = "Reinforcement Learning"
|
145 |
-
description = ""
|
146 |
-
template = "Summary"
|
147 |
-
model = "gpt-4"
|
148 |
-
# model = "gpt-3.5-turbo"
|
149 |
-
|
150 |
-
generate_backgrounds(title, description, template, model)
|
|
|
1 |
from utils.references import References
|
2 |
+
from utils.file_operations import hash_name, make_archive, copy_templates
|
3 |
+
from section_generator import section_generation_bg, keywords_generation
|
|
|
|
|
|
|
|
|
4 |
import logging
|
|
|
5 |
|
6 |
TOTAL_TOKENS = 0
|
7 |
TOTAL_PROMPTS_TOKENS = 0
|
8 |
TOTAL_COMPLETION_TOKENS = 0
|
9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
def log_usage(usage, generating_target, print_out=True):
|
11 |
global TOTAL_TOKENS
|
12 |
global TOTAL_PROMPTS_TOKENS
|
|
|
26 |
print(message)
|
27 |
logging.info(message)
|
28 |
|
29 |
+
def generate_backgrounds(title, description="", template="ICLR2022", model="gpt-4"):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
paper = {}
|
31 |
paper_body = {}
|
32 |
|
33 |
# Create a copy in the outputs folder.
|
34 |
+
bibtex_path, destination_folder = copy_templates(template, title)
|
35 |
+
logging.basicConfig(level=logging.INFO, filename=destination_folder + "/generation.log")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
36 |
|
37 |
# Generate keywords and references
|
38 |
print("Initialize the paper information ...")
|
39 |
+
input_dict = {"title": title, "description": description}
|
40 |
+
keywords, usage = keywords_generation(input_dict, model="gpt-3.5-turbo")
|
41 |
+
print(f"keywords: {keywords}")
|
42 |
log_usage(usage, "keywords")
|
43 |
|
44 |
ref = References(load_papers = "")
|
|
|
55 |
|
56 |
for section in ["introduction", "related works", "backgrounds"]:
|
57 |
try:
|
58 |
+
# usage = pipeline(paper, section, destination_folder, model=model)
|
59 |
+
usage = section_generation_bg(paper, section, destination_folder, model=model)
|
60 |
log_usage(usage, section)
|
61 |
except Exception as e:
|
62 |
print(f"Failed to generate {section} due to the error: {e}")
|
63 |
+
print(f"The paper {title} has been generated. Saved to {destination_folder}.")
|
64 |
# shutil.make_archive("output.zip", 'zip', save_to_path)
|
65 |
+
|
66 |
+
input_dict = {"title": title, "description": description, "generator": "generate_backgrounds"}
|
67 |
+
filename = hash_name(input_dict) + ".zip"
|
68 |
+
return make_archive(destination_folder, filename)
|
69 |
|
70 |
|
71 |
+
def fake_generator(title, description="", template="ICLR2022", model="gpt-4"):
|
72 |
"""
|
73 |
This function is used to test the whole pipeline without calling OpenAI API.
|
74 |
"""
|
75 |
+
input_dict = {"title": title, "description": description, "generator": "generate_backgrounds"}
|
76 |
+
filename = hash_name(input_dict) + ".zip"
|
77 |
return make_archive("sample-output.pdf", filename)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
auto_draft.py
CHANGED
@@ -88,6 +88,7 @@ def generate_draft(title, description="", template="ICLR2022", model="gpt-4"):
|
|
88 |
paper_body = {}
|
89 |
|
90 |
# Create a copy in the outputs folder.
|
|
|
91 |
now = datetime.datetime.now()
|
92 |
target_name = now.strftime("outputs_%Y%m%d_%H%M%S")
|
93 |
source_folder = f"latex_templates/{template}"
|
@@ -105,16 +106,15 @@ def generate_draft(title, description="", template="ICLR2022", model="gpt-4"):
|
|
105 |
gpt_response, usage = get_responses(prompts, model)
|
106 |
keywords = extract_keywords(gpt_response)
|
107 |
log_usage(usage, "keywords")
|
108 |
-
|
109 |
-
ref
|
110 |
-
ref.collect_papers(keywords, method="arxiv")
|
111 |
all_paper_ids = ref.to_bibtex(bibtex_path) #todo: this will used to check if all citations are in this list
|
112 |
|
113 |
print(f"The paper information has been initialized. References are saved to {bibtex_path}.")
|
114 |
|
115 |
paper["title"] = title
|
116 |
paper["description"] = description
|
117 |
-
paper["references"] = ref.to_prompts() #
|
118 |
paper["body"] = paper_body
|
119 |
paper["bibtex"] = bibtex_path
|
120 |
|
|
|
88 |
paper_body = {}
|
89 |
|
90 |
# Create a copy in the outputs folder.
|
91 |
+
# todo: use copy_templates function instead.
|
92 |
now = datetime.datetime.now()
|
93 |
target_name = now.strftime("outputs_%Y%m%d_%H%M%S")
|
94 |
source_folder = f"latex_templates/{template}"
|
|
|
106 |
gpt_response, usage = get_responses(prompts, model)
|
107 |
keywords = extract_keywords(gpt_response)
|
108 |
log_usage(usage, "keywords")
|
109 |
+
ref = References(load_papers = "") #todo: allow users to upload bibfile.
|
110 |
+
ref.collect_papers(keywords, method="arxiv") #todo: add more methods to find related papers
|
|
|
111 |
all_paper_ids = ref.to_bibtex(bibtex_path) #todo: this will used to check if all citations are in this list
|
112 |
|
113 |
print(f"The paper information has been initialized. References are saved to {bibtex_path}.")
|
114 |
|
115 |
paper["title"] = title
|
116 |
paper["description"] = description
|
117 |
+
paper["references"] = ref.to_prompts() #todo: see if this prompts can be compressed.
|
118 |
paper["body"] = paper_body
|
119 |
paper["bibtex"] = bibtex_path
|
120 |
|
output.zip
CHANGED
Binary files a/output.zip and b/output.zip differ
|
|
section_generator.py
ADDED
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from utils.prompts import generate_paper_prompts, generate_keywords_prompts, generate_experiments_prompts, generate_bg_summary_prompts
|
2 |
+
from utils.gpt_interaction import get_responses, extract_responses, extract_keywords, extract_json
|
3 |
+
import time
|
4 |
+
import os
|
5 |
+
|
6 |
+
# three GPT-based content generator:
|
7 |
+
# 1. section_generation: used to generate main content of the paper
|
8 |
+
# 2. keywords_generation: used to generate a json output {key1: output1, key2: output2} for multiple purpose.
|
9 |
+
# 3. figure_generation: used to generate sample figures.
|
10 |
+
# all generator should return the token usage.
|
11 |
+
|
12 |
+
|
13 |
+
def section_generation_bg(paper, section, save_to_path, model):
|
14 |
+
"""
|
15 |
+
The main pipeline of generating a section.
|
16 |
+
1. Generate prompts.
|
17 |
+
2. Get responses from AI assistant.
|
18 |
+
3. Extract the section text.
|
19 |
+
4. Save the text to .tex file.
|
20 |
+
:return usage
|
21 |
+
"""
|
22 |
+
print(f"Generating {section}...")
|
23 |
+
prompts = generate_bg_summary_prompts(paper, section)
|
24 |
+
gpt_response, usage = get_responses(prompts, model)
|
25 |
+
output = extract_responses(gpt_response)
|
26 |
+
paper["body"][section] = output
|
27 |
+
tex_file = os.path.join(save_to_path, f"{section}.tex")
|
28 |
+
# tex_file = save_to_path + f"/{section}.tex"
|
29 |
+
if section == "abstract":
|
30 |
+
with open(tex_file, "w") as f:
|
31 |
+
f.write(r"\begin{abstract}")
|
32 |
+
with open(tex_file, "a") as f:
|
33 |
+
f.write(output)
|
34 |
+
with open(tex_file, "a") as f:
|
35 |
+
f.write(r"\end{abstract}")
|
36 |
+
else:
|
37 |
+
with open(tex_file, "w") as f:
|
38 |
+
f.write(f"\section{{{section.upper()}}}\n")
|
39 |
+
with open(tex_file, "a") as f:
|
40 |
+
f.write(output)
|
41 |
+
time.sleep(5)
|
42 |
+
print(f"{section} has been generated. Saved to {tex_file}.")
|
43 |
+
return usage
|
44 |
+
|
45 |
+
|
46 |
+
def keywords_generation(input_dict, model):
|
47 |
+
title = input_dict.get("title")
|
48 |
+
description = input_dict.get("description", "")
|
49 |
+
if title is not None:
|
50 |
+
prompts = generate_keywords_prompts(title, description)
|
51 |
+
gpt_response, usage = get_responses(prompts, model)
|
52 |
+
keywords = extract_keywords(gpt_response)
|
53 |
+
return keywords, usage
|
54 |
+
else:
|
55 |
+
raise ValueError("`input_dict` must include the key 'title'.")
|
56 |
+
|
57 |
+
def figures_generation():
|
58 |
+
pass
|
utils/file_operations.py
ADDED
@@ -0,0 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import hashlib
|
2 |
+
import os, shutil
|
3 |
+
import datetime
|
4 |
+
from utils.tex_processing import replace_title
|
5 |
+
|
6 |
+
def hash_name(input_dict):
|
7 |
+
'''
|
8 |
+
input_dict= {"title": title, "description": description}
|
9 |
+
|
10 |
+
For same input_dict, it should return the same value.
|
11 |
+
'''
|
12 |
+
name = str(input_dict)
|
13 |
+
name = name.lower()
|
14 |
+
md5 = hashlib.md5()
|
15 |
+
md5.update(name.encode('utf-8'))
|
16 |
+
hashed_string = md5.hexdigest()
|
17 |
+
return hashed_string
|
18 |
+
|
19 |
+
|
20 |
+
|
21 |
+
def make_archive(source, destination):
|
22 |
+
base = os.path.basename(destination)
|
23 |
+
name = base.split('.')[0]
|
24 |
+
format = base.split('.')[1]
|
25 |
+
archive_from = os.path.dirname(source)
|
26 |
+
archive_to = os.path.basename(source.strip(os.sep))
|
27 |
+
shutil.make_archive(name, format, archive_from, archive_to)
|
28 |
+
shutil.move('%s.%s'%(name,format), destination)
|
29 |
+
return destination
|
30 |
+
|
31 |
+
def copy_templates(template, title):
|
32 |
+
# Create a copy in the outputs folder.
|
33 |
+
# 1. create a folder "outputs_%Y%m%d_%H%M%S" (destination_folder)
|
34 |
+
# 2. copy all contents in "latex_templates/{template}" to that folder
|
35 |
+
# 3. return (bibtex_path, destination_folder)
|
36 |
+
now = datetime.datetime.now()
|
37 |
+
target_name = now.strftime("outputs_%Y%m%d_%H%M%S")
|
38 |
+
source_folder = f"latex_templates/{template}"
|
39 |
+
destination_folder = f"outputs/{target_name}"
|
40 |
+
shutil.copytree(source_folder, destination_folder)
|
41 |
+
bibtex_path = os.path.join(destination_folder, "ref.bib")
|
42 |
+
# bibtex_path = destination_folder + "/ref.bib"
|
43 |
+
replace_title(destination_folder, title)
|
44 |
+
return bibtex_path, destination_folder
|
45 |
+
|
utils/gpt_interaction.py
CHANGED
@@ -1,12 +1,10 @@
|
|
1 |
import openai
|
2 |
import re
|
3 |
-
import os
|
4 |
import json
|
5 |
import logging
|
|
|
6 |
log = logging.getLogger(__name__)
|
7 |
|
8 |
-
# todo: 将api_key通过函数传入; 需要改很多地方
|
9 |
-
# openai.api_key = os.environ['OPENAI_API_KEY']
|
10 |
|
11 |
def extract_responses(assistant_message):
|
12 |
# pattern = re.compile(r"f\.write\(r'{1,3}(.*?)'{0,3}\){0,1}$", re.DOTALL)
|
@@ -19,9 +17,10 @@ def extract_responses(assistant_message):
|
|
19 |
log.info(f"assistant_message: {assistant_message}")
|
20 |
return assistant_message
|
21 |
|
|
|
22 |
def extract_keywords(assistant_message, default_keywords=None):
|
23 |
if default_keywords is None:
|
24 |
-
default_keywords = {"machine learning":5}
|
25 |
|
26 |
try:
|
27 |
keywords = json.loads(assistant_message)
|
@@ -31,6 +30,7 @@ def extract_keywords(assistant_message, default_keywords=None):
|
|
31 |
return default_keywords
|
32 |
return keywords
|
33 |
|
|
|
34 |
def extract_section_name(assistant_message, default_section_name=""):
|
35 |
try:
|
36 |
keywords = json.loads(assistant_message)
|
@@ -55,7 +55,7 @@ def extract_json(assistant_message, default_output=None):
|
|
55 |
return dict.keys()
|
56 |
|
57 |
|
58 |
-
def get_responses(user_message, model="gpt-4", temperature=0.4, openai_key
|
59 |
if openai.api_key is None and openai_key is None:
|
60 |
raise ValueError("OpenAI API key must be provided.")
|
61 |
if openai_key is not None:
|
@@ -79,8 +79,8 @@ def get_responses(user_message, model="gpt-4", temperature=0.4, openai_key = Non
|
|
79 |
|
80 |
if __name__ == "__main__":
|
81 |
test_strings = [r"f.write(r'hello world')", r"f.write(r'''hello world''')", r"f.write(r'''hello world",
|
82 |
-
|
83 |
-
|
84 |
for input_string in test_strings:
|
85 |
print("input_string: ", input_string)
|
86 |
pattern = re.compile(r"f\.write\(r['\"]{1,3}(.*?)['\"]{0,3}\){0,1}$", re.DOTALL)
|
@@ -90,4 +90,4 @@ if __name__ == "__main__":
|
|
90 |
extracted_string = match.group(1)
|
91 |
print("Extracted string:", extracted_string)
|
92 |
else:
|
93 |
-
print("No match found")
|
|
|
1 |
import openai
|
2 |
import re
|
|
|
3 |
import json
|
4 |
import logging
|
5 |
+
|
6 |
log = logging.getLogger(__name__)
|
7 |
|
|
|
|
|
8 |
|
9 |
def extract_responses(assistant_message):
|
10 |
# pattern = re.compile(r"f\.write\(r'{1,3}(.*?)'{0,3}\){0,1}$", re.DOTALL)
|
|
|
17 |
log.info(f"assistant_message: {assistant_message}")
|
18 |
return assistant_message
|
19 |
|
20 |
+
|
21 |
def extract_keywords(assistant_message, default_keywords=None):
|
22 |
if default_keywords is None:
|
23 |
+
default_keywords = {"machine learning": 5}
|
24 |
|
25 |
try:
|
26 |
keywords = json.loads(assistant_message)
|
|
|
30 |
return default_keywords
|
31 |
return keywords
|
32 |
|
33 |
+
|
34 |
def extract_section_name(assistant_message, default_section_name=""):
|
35 |
try:
|
36 |
keywords = json.loads(assistant_message)
|
|
|
55 |
return dict.keys()
|
56 |
|
57 |
|
58 |
+
def get_responses(user_message, model="gpt-4", temperature=0.4, openai_key=None):
|
59 |
if openai.api_key is None and openai_key is None:
|
60 |
raise ValueError("OpenAI API key must be provided.")
|
61 |
if openai_key is not None:
|
|
|
79 |
|
80 |
if __name__ == "__main__":
|
81 |
test_strings = [r"f.write(r'hello world')", r"f.write(r'''hello world''')", r"f.write(r'''hello world",
|
82 |
+
r"f.write(r'''hello world'", r'f.write(r"hello world")', r'f.write(r"""hello world""")',
|
83 |
+
r'f.write(r"""hello world"', r'f.write(r"""hello world']
|
84 |
for input_string in test_strings:
|
85 |
print("input_string: ", input_string)
|
86 |
pattern = re.compile(r"f\.write\(r['\"]{1,3}(.*?)['\"]{0,3}\){0,1}$", re.DOTALL)
|
|
|
90 |
extracted_string = match.group(1)
|
91 |
print("Extracted string:", extracted_string)
|
92 |
else:
|
93 |
+
print("No match found")
|
utils/prompts.py
CHANGED
@@ -16,8 +16,8 @@ BG_INSTRUCTIONS = {"introduction": "Please include four paragraph: Establishing
|
|
16 |
|
17 |
def generate_keywords_prompts(title, description="", num_refs=5):
|
18 |
prompts = f"I am writing a machine learning paper with the title '{title}'. {description}\n" \
|
19 |
-
f"
|
20 |
-
r"
|
21 |
return prompts
|
22 |
|
23 |
def generate_rename_prompts(paper_info, section):
|
@@ -72,15 +72,6 @@ def generate_paper_prompts(paper_info, section):
|
|
72 |
return prompts
|
73 |
|
74 |
|
75 |
-
|
76 |
-
|
77 |
-
def generate_bg_keywords_prompts(title, description="", num_refs=5):
|
78 |
-
prompts = f"I am writing a survey on the topic '{title}'. {description}\n" \
|
79 |
-
f"Please generate three to five keywords. For each keyword, rate it from 1 to {num_refs}; the larger number means more important." \
|
80 |
-
r"Response in a dictionary format like {keyword1:1, keyword2:3}."
|
81 |
-
return prompts
|
82 |
-
|
83 |
-
|
84 |
def generate_bg_summary_prompts(paper_info, section):
|
85 |
title = paper_info["title"]
|
86 |
description = paper_info["description"]
|
|
|
16 |
|
17 |
def generate_keywords_prompts(title, description="", num_refs=5):
|
18 |
prompts = f"I am writing a machine learning paper with the title '{title}'. {description}\n" \
|
19 |
+
f"Generate three to five keywords. For each keyword, rate it from 1 to {num_refs}; the larger number means more important." \
|
20 |
+
r"Your response must be in JSON format like {\"keyword1\":1, \"keyword2\":3}."
|
21 |
return prompts
|
22 |
|
23 |
def generate_rename_prompts(paper_info, section):
|
|
|
72 |
return prompts
|
73 |
|
74 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
75 |
def generate_bg_summary_prompts(paper_info, section):
|
76 |
title = paper_info["title"]
|
77 |
description = paper_info["description"]
|
utils/storage.py
CHANGED
@@ -1,50 +1,45 @@
|
|
|
|
|
|
|
|
|
|
|
|
1 |
import os
|
2 |
import boto3
|
3 |
-
import hashlib
|
4 |
|
5 |
-
access_key_id = os.
|
6 |
-
secret_access_key = os.
|
7 |
bucket_name = "hf-storage"
|
8 |
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
37 |
-
|
38 |
-
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
'''
|
43 |
-
For same title and description, it should return the same value.
|
44 |
-
'''
|
45 |
-
name = title + description
|
46 |
-
name = name.lower()
|
47 |
-
md5 = hashlib.md5()
|
48 |
-
md5.update(name.encode('utf-8'))
|
49 |
-
hashed_string = md5.hexdigest()
|
50 |
-
return hashed_string
|
|
|
1 |
+
# This script `storage.py` is used to handle the cloud storage.
|
2 |
+
# `upload_file`:
|
3 |
+
# `list_all_files`:
|
4 |
+
# `download_file`:
|
5 |
+
|
6 |
import os
|
7 |
import boto3
|
|
|
8 |
|
9 |
+
access_key_id = os.getenv('AWS_ACCESS_KEY_ID')
|
10 |
+
secret_access_key = os.getenv('AWS_SECRET_ACCESS_KEY')
|
11 |
bucket_name = "hf-storage"
|
12 |
|
13 |
+
if (access_key_id is not None) and (secret_access_key is not None):
|
14 |
+
session = boto3.Session(
|
15 |
+
aws_access_key_id=access_key_id,
|
16 |
+
aws_secret_access_key=secret_access_key,
|
17 |
+
)
|
18 |
+
|
19 |
+
s3 = session.resource('s3')
|
20 |
+
bucket = s3.Bucket(bucket_name)
|
21 |
+
|
22 |
+
|
23 |
+
def upload_file(file_name, target_name=None):
|
24 |
+
if target_name is None:
|
25 |
+
target_name = file_name
|
26 |
+
try:
|
27 |
+
s3.meta.client.upload_file(Filename=file_name, Bucket=bucket_name, Key=target_name)
|
28 |
+
print(f"The file {file_name} has been uploaded!")
|
29 |
+
except:
|
30 |
+
print("Uploading failed!")
|
31 |
+
|
32 |
+
def list_all_files():
|
33 |
+
return [obj.key for obj in bucket.objects.all()]
|
34 |
+
|
35 |
+
def download_file(file_name):
|
36 |
+
''' Download `file_name` from the bucket. todo:check existence before downloading!
|
37 |
+
Bucket (str) – The name of the bucket to download from.
|
38 |
+
Key (str) – The name of the key to download from.
|
39 |
+
Filename (str) – The path to the file to download to.
|
40 |
+
'''
|
41 |
+
try:
|
42 |
+
s3.meta.client.download_file(Bucket=bucket_name, Key=file_name, Filename=file_name)
|
43 |
+
print(f"The file {file_name} has been downloaded!")
|
44 |
+
except:
|
45 |
+
print("Uploading failed!")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
utils/tex_processing.py
CHANGED
@@ -1,7 +1,11 @@
|
|
|
|
|
|
1 |
def replace_title(save_to_path, title):
|
2 |
# Define input and output file names
|
3 |
-
input_file_name =
|
4 |
-
output_file_name = save_to_path + "main.tex"
|
|
|
|
|
5 |
|
6 |
# Open the input file and read its content
|
7 |
with open(input_file_name, 'r') as infile:
|
@@ -15,6 +19,9 @@ def replace_title(save_to_path, title):
|
|
15 |
outfile.write(content)
|
16 |
|
17 |
|
18 |
-
# return all string in \cite{...}
|
|
|
|
|
|
|
19 |
|
20 |
# replace citations
|
|
|
1 |
+
import os
|
2 |
+
|
3 |
def replace_title(save_to_path, title):
|
4 |
# Define input and output file names
|
5 |
+
# input_file_name = save_to_path + "/template.tex"
|
6 |
+
# output_file_name = save_to_path + "/main.tex"
|
7 |
+
input_file_name = os.path.join(save_to_path, "template.tex")
|
8 |
+
output_file_name = os.path.join(save_to_path , "main.tex")
|
9 |
|
10 |
# Open the input file and read its content
|
11 |
with open(input_file_name, 'r') as infile:
|
|
|
19 |
outfile.write(content)
|
20 |
|
21 |
|
22 |
+
# return all string in \cite{...}.
|
23 |
+
|
24 |
+
# check if citations are in bibtex.
|
25 |
+
|
26 |
|
27 |
# replace citations
|