space working with chatgpt 4

.gitignore

@@ -1 +1,2 @@
-__pycache__/
+__pycache__/
+.venv/

anonymize_file_clear.py

@@ -0,0 +1,61 @@
+import argparse
+import json
+import re
+import uuid
+from pathlib import Path
+import gensim
+from concrete.ml.common.serialization.loaders import load
+
+def load_models():
+    base_dir = Path(__file__).parent
+    embeddings_model = gensim.models.FastText.load(str(base_dir / "embedded_model.model"))
+    with open(base_dir / "cml_xgboost.model", "r") as model_file:
+        fhe_ner_detection = load(file=model_file)
+    return embeddings_model, fhe_ner_detection
+
+def anonymize_text(text, embeddings_model, fhe_ner_detection):
+    token_pattern = r"(\b[\w\.\/\-@]+\b|[\s,.!?;:'\"-]+)"
+    tokens = re.findall(token_pattern, text)
+    uuid_map = {}
+    processed_tokens = []
+
+    for token in tokens:
+        if token.strip() and re.match(r"\w+", token):  # If the token is a word
+            x = embeddings_model.wv[token][None]
+            prediction_proba = fhe_ner_detection.predict_proba(x)
+            probability = prediction_proba[0][1]
+            prediction = probability >= 0.5
+            if prediction:
+                if token not in uuid_map:
+                    uuid_map[token] = str(uuid.uuid4())[:8]
+                processed_tokens.append(uuid_map[token])
+            else:
+                processed_tokens.append(token)
+        else:
+            processed_tokens.append(token)  # Preserve punctuation and spaces as is
+
+    return uuid_map
+
+def main():
+    parser = argparse.ArgumentParser(description="Anonymize named entities in a text file and save the mapping to a JSON file.")
+    parser.add_argument("file_path", type=str, help="The path to the file to be processed.")
+    args = parser.parse_args()
+
+    embeddings_model, fhe_ner_detection = load_models()
+
+    # Read the input file
+    with open(args.file_path, 'r', encoding='utf-8') as file:
+        text = file.read()
+
+    # Anonymize the text
+    uuid_map = anonymize_text(text, embeddings_model, fhe_ner_detection)
+
+    # Save the UUID mapping to a JSON file
+    mapping_path = Path(args.file_path).stem + "_uuid_mapping.json"
+    with open(mapping_path, 'w', encoding='utf-8') as file:
+        json.dump(uuid_map, file, indent=4, sort_keys=True)
+
+    print(f"UUID mapping saved to {mapping_path}")
+
+if __name__ == "__main__":
+    main()

app.py

@@ -1,13 +1,19 @@
 """A Gradio app for anonymizing text data using FHE."""
 
 import gradio as gr
-import re
 from fhe_anonymizer import FHEAnonymizer
 import pandas as pd
-
+from openai import OpenAI
+import os
+import json
+import re
 
 anonymizer = FHEAnonymizer()
 
+client = OpenAI(
+    api_key=os.environ.get("openaikey"),
+)
+
 
 def deidentify_text(input_text):
     anonymized_text, identified_words_with_prob = anonymizer(input_text)
@@ -22,10 +28,62 @@ def deidentify_text(input_text):
     return anonymized_text, identified_df
 
 
+def query_chatgpt(anonymized_query):
+
+    with open("files/anonymized_document.txt", "r") as file:
+        anonymized_document = file.read()
+    with open("files/chatgpt_prompt.txt", "r") as file:
+        prompt = file.read()
+
+    # Prepare prompt
+    full_prompt = (
+        prompt + "\n" 
+    )
+    query = "Document content:\n```\n" + anonymized_document + "\n\n```" + "Query:\n```\n" + anonymized_query + "\n```"
+    print(full_prompt)
+
+    completion = client.chat.completions.create(
+        model="gpt-4-1106-preview",  # Replace with "gpt-4" if available
+        messages=[
+            {"role": "system", "content": prompt},
+            {"role": "user", "content": query},
+        ],
+    )
+    anonymized_response = completion.choices[0].message.content
+    with open("original_document_uuid_mapping.json", "r") as file:
+        uuid_map = json.load(file)
+    inverse_uuid_map = {v: k for k, v in uuid_map.items()}  # TODO load the inverse mapping from disk for efficiency
+
+    # Pattern to identify words and non-words (including punctuation, spaces, etc.)
+    token_pattern = r"(\b[\w\.\/\-@]+\b|[\s,.!?;:'\"-]+)"
+    tokens = re.findall(token_pattern, anonymized_response)
+    processed_tokens = []
+
+    print(tokens)
+    for token in tokens:
+        # Directly append non-word tokens or whitespace to processed_tokens
+        if not token.strip() or not re.match(r"\w+", token):
+            processed_tokens.append(token)
+            continue
+        print(token)
+        if token in inverse_uuid_map:
+            processed_tokens.append(inverse_uuid_map[token])
+        else:
+            processed_tokens.append(token)
+    deanonymized_response = "".join(processed_tokens)
+    return anonymized_response, deanonymized_response
+
+
 # Default demo text from the file
 with open("demo_text.txt", "r") as file:
     default_demo_text = file.read()
 
+with open("files/original_document.txt", "r") as file:
+    original_document = file.read()
+
+with open("files/anonymized_document.txt", "r") as file:
+    anonymized_document = file.read()
+
 demo = gr.Blocks()
 
 with demo:
@@ -50,33 +108,23 @@ with demo:
     with gr.Accordion("What is Encrypted Anonymization?", open=False):
         gr.Markdown(
             """
-            Encrypted Anonymization leverages Fully Homomorphic Encryption (FHE) to protect sensitive information during data processing. This approach allows for the anonymization of text data, such as personal identifiers, while ensuring that the data remains encrypted throughout the entire process. It enables organizations to utilize sensitive data for analytics and machine learning without compromising individual privacy or security.
+            Encrypted Anonymization leverages Fully Homomorphic Encryption (FHE) to protect sensitive information during data processing. This approach allows for the anonymization of text data, such as personal identifiers, while ensuring that the data remains encrypted throughout the entire process.
             """
         )
 
-    with gr.Accordion("Why is privacy important in data processing?", open=False):
-        gr.Markdown(
-            """
-            Privacy in data processing is critical to protect individuals' personal information from unauthorized access and potential misuse. With the increasing amount of personal data being collected and analyzed, the risks associated with data breaches and identity theft have also risen. By implementing privacy-preserving techniques, such as encrypted anonymization, organizations can safeguard sensitive information, build trust with their customers, and comply with stringent data protection regulations.
-            """
-        )
+    with gr.Accordion("Original Document", open=False):
+        gr.Markdown(original_document)
 
-    with gr.Accordion(
-        "How does Fully Homomorphic Encryption enhance data privacy?", open=False
-    ):
-        gr.Markdown(
-            """
-            Fully Homomorphic Encryption (FHE) enhances data privacy by enabling computations on encrypted data without needing to decrypt it first. This revolutionary technology ensures that sensitive data can be processed and analyzed securely, without exposing it to potential threats. FHE is a game-changer for privacy-preserving computations, allowing for the secure analysis of encrypted data, which is particularly beneficial in sectors like finance, healthcare, and beyond.
-            """
-        )
+    with gr.Accordion("Anonymized Document", open=False):
+        gr.Markdown(anonymized_document)
 
-    gr.Markdown(
-        """
-        <p align="center">
-            <img src="file/images/banner.png">
-        </p>
-        """
-    )
+    # gr.Markdown(
+    #     """
+    #     <p align="center">
+    #         <img src="file/images/banner.png">
+    #     </p>
+    #     """
+    # )
 
     with gr.Row():
         input_text = gr.Textbox(
@@ -88,7 +136,7 @@ with demo:
 
         anonymized_text_output = gr.Textbox(label="Anonymized Text", lines=13)
 
-    identified_words_output = gr.Dataframe(label="Identified Words")
+    identified_words_output = gr.Dataframe(label="Identified Words", visible=False)
 
     submit_button = gr.Button("Anonymize")
 
@@ -98,6 +146,16 @@ with demo:
         outputs=[anonymized_text_output, identified_words_output],
     )
 
+    with gr.Row():
+        chatgpt_response_anonymized = gr.Textbox(label="ChatGPT Anonymized Response", lines=13)
+        chatgpt_response_deanonymized = gr.Textbox(label="ChatGPT Deanonymized Response", lines=13)
+
+    chatgpt_button = gr.Button("Query ChatGPT")
+    chatgpt_button.click(
+        query_chatgpt,
+        inputs=[anonymized_text_output],
+        outputs=[chatgpt_response_anonymized, chatgpt_response_deanonymized],
+    )
 
 # Launch the app
 demo.launch(share=False)

demo_text.txt

@@ -1,10 +1 @@
-Hello, my name is David Johnson and I live in Maine.
-My credit card number is 4095-2609-9393-4932 and my crypto wallet id is 16Yeky6GMjeNkAiNcBY7ZhrLoMSgg1BoyZ.
-
-On September 18 I visited microsoft.com and sent an email to [email protected],  from the IP 192.168.0.1.
-
-My passport: 191280342 and my phone number: (212) 555-1234.
-
-This is a valid International Bank Account Number: IL150120690000003111111 . Can you please check the status on bank account 954567876544?
-
-Kate's social security number is 078-05-1126.  Her driver license? it is 1234567A.
+Who lives in Maine?

fhe_anonymizer.py

@@ -3,6 +3,8 @@ import re
 from concrete.ml.deployment import FHEModelClient, FHEModelServer
 from pathlib import Path
 from concrete.ml.common.serialization.loaders import load
+import uuid
+import json
 
 base_dir = Path(__file__).parent
 
@@ -17,6 +19,9 @@ class FHEAnonymizer:
         with open(base_dir / "cml_xgboost.model", "r") as model_file:
             self.fhe_ner_detection = load(file=model_file)
 
+        with open(base_dir / "original_document_uuid_mapping.json", 'r') as file:
+            self.uuid_map = json.load(file)
+
         path_to_model = (base_dir / "deployment").resolve()
         self.client = FHEModelClient(path_to_model)
         self.server = FHEModelServer(path_to_model)
@@ -30,57 +35,33 @@ class FHEAnonymizer:
         return y
 
     def __call__(self, text: str):
-        text = self.preprocess_sentences(text)
-        identified_words_with_prob = []  # tuples of (word, probability)
-        new_text = []
+        # Pattern to identify words and non-words (including punctuation, spaces, etc.)
+        token_pattern = r"(\b[\w\.\/\-@]+\b|[\s,.!?;:'\"-]+)"
+        tokens = re.findall(token_pattern, text)
+        identified_words_with_prob = []
+        processed_tokens = []
+
+        print(tokens)
+        for token in tokens:
+            # Directly append non-word tokens or whitespace to processed_tokens
+            if not token.strip() or not re.match(r"\w+", token):
+                processed_tokens.append(token)
+                continue
 
-        for word in text.split():
             # Prediction for each word
-            x = self.embeddings_model.wv[word][None]
+            x = self.embeddings_model.wv[token][None]
             prediction_proba = self.fhe_ner_detection.predict_proba(x)
-            # prediction = self.fhe_inference(x).argmax(1)[0]
-            # print(word, prediction)
             probability = prediction_proba[0][1]
-            prediction = probability >= 0.5
-            if prediction == 1:
-                identified_words_with_prob.append((word, probability))
-                new_text.append("<REMOVED>")
-            else:
-                new_text.append(word)
-
-        # Joining the modified text
-        modified_text = " ".join(new_text)
-
-        return modified_text, identified_words_with_prob
-
-    def preprocess_sentences(self, sentence, verbose=False):
-        """Preprocess the sentence."""
-
-        sentence = re.sub(r"\n+", " ", sentence)
-        if verbose:
-            print(sentence)
 
-        sentence = re.sub(" +", " ", sentence)
-        if verbose:
-            print(sentence)
-
-        sentence = re.sub(r"'s\b", " s", sentence)
-        if verbose:
-            print(sentence)
-
-        sentence = re.sub(r"\s([,.!?;:])", r"\1", sentence)
-        if verbose:
-            print(sentence)
-
-        pattern = r"(?<!\w)[{}]|[{}](?!\w)".format(
-            re.escape(self.punctuation_list), re.escape(self.punctuation_list)
-        )
-        sentence = re.sub(pattern, "", sentence)
-        if verbose:
-            print(sentence)
-
-        sentence = re.sub(r"\s([,.!?;:])", r"\1", sentence)
-        if verbose:
-            print(sentence)
+            if probability >= 0.5:
+                identified_words_with_prob.append((token, probability))
+                # Use the existing UUID if available, otherwise generate a new one
+                tmp_uuid = self.uuid_map.get(token, str(uuid.uuid4())[:8])
+                processed_tokens.append(tmp_uuid)
+                self.uuid_map[token] = tmp_uuid
+            else:
+                processed_tokens.append(token)
 
-        return sentence
+        # Reconstruct the sentence
+        reconstructed_sentence = ''.join(processed_tokens)
+        return reconstructed_sentence, identified_words_with_prob

files/anonymized_document.txt

@@ -1,54 +1,10 @@
-ca7a17e0 297a7a2c and 47bae4dd f1118b85 for "207c2210 b0572c9c."
-6bd00984 15642039
+Hello, 0a182475 name is 84e24eb8 0ea0e35c and 9f2963af live in 5d0a593d.
+6d7a71e0 credit card number is 1155d2a5 and 0a182475 d8e7c7fc c627b841 id is be0eddce.
 
-5437eb6f document 259323e3 a 31ab92f5 8eaa0b83 cf94131c and 17f2c576 2a2c7407 plan for cd947192 0540cbbb., a leading 1d557b0d 6d25d852 2fa3434e in developing 65a5ebb9 bd3fdeea for the digital market. 6b80462c document e218c683 3d0b62ab data and ee796684 74629446 that are based on 3d59250b scenarios and dcf1fdbe b0ea476c, ensuring no real 2361ac41 or 857b9636 1bdb9e71 data are 4d87a9cd.
-b8859a2c 553cc220
+On b926f540 ee1fa38f 9f2963af visited 1e63e774 and sent an email to f70cf334,  from the IP 2d66aefe.
 
-58e3af47 171d6281., 8033ec9c in 8b700cc6 by 1e307c1e c7c0231f fd67020c and f6e600b4 41ad12d1, has d0a0127a rapid growth in the daf3dc5f 048e01b9, focusing on areas aa85b69f as 4c23ac66 76bb2672, cloud computing, and e16dd9ad 516c4333. c8ddf3e1 033d58f5 b907bfb3 in 5742554b 2a08bcbe and 2a503bc4 across the 5b5218fd, the b15d6bc2 ae901e0a to 30d358b7 how 14bd9782 0e9ce93c in the digital 3bbeab83.
-a28dbb08 9f84ce23
-90112484 5c1705ba
+6d7a71e0 passport: 53b52085 and 0a182475 phone number: 81133b16 08aeb9f5.
 
-a9c81aad 819cc7fe. has bbde0c42 53aefe16 revenue through various a48d5930, including software 91daf2e0, 78064bc8 services, and ac852356 sales. In the 478060c5 e9a560ef d59684ea, the 12b4539d reported the 7b3f3216 eee94704 figures:
+3bbcaf02 is a valid 11e27c41 5a60bcfa 3730179e 97008100: ab18e6bb . Can 773b08cb f4953628 check the 8ff83dc3 on bank account e1a23ada?
 
-    63d7b6bc Subscriptions: 6090defb million
-    4cd9e789 a1f51933: 019ac660 million
-    3196fc94 Sales: 50311924 million
-
-496b82c2 bdf223ed: da54a11a million
-61307cb8 and f4e35ce3
-
-0a5f5ee5 3a0429d8's 904bce1e are ce79e4ff 8d5303a8 with research and 0b4a7271 RD, sales and 0807fdda, and c1d7820d 4c0101b2. c5894e7a a0e3c060 are the 2b81915c figures for the 2042d5df f2ddfe31 6cee2393:
-
-    RD: 2bfddef5 million
-    Sales and 8e7609fc: 682daa09 million
-    1eb46a02 93908764: c4a6681a million
-
-9cc85bb0 9b15cd0b: 20c5b8db million
-
-30d55f53 ef4684e2 for 274f506b 7153d60a: 92063576 million
-26fa65d5 299f9a34 3e07531f
-
-To ensure f48418ed growth and 4e52397d, d93931c1 bf8dd379. needs to focus on several key areas. 8b71b2c4 plan b8d6f955 77273f70 eff17325 6e32e65a, using bd178c9b data and 56935895 for 94fe5b9b 5c729e9f.
-b56f5f8e into f17b897f f0458470
-
-fbe11266 the global digital c5bc7803 79c37224, emerging markets present f6ea9661 9c3eaa4d for growth. c1d933f6 plans to 7612d3f9 e64531b7 in e3a6050a bd2a43c4 as 375af421, 5dc6aed3, and 60fd1ca9 7675408f, 2bfce010 to 71336309 revenue by b40729e6 over the 807f7c70 16671a96 933e82e7.
-430afebf e77cc3ff
-
-cfdd4e98 841ff829 in RD is 206d362e for maintaining a 13bb2340 bbf9e6e6. 9132698f 1c52e182 16d64f13 million 8d9c99ec developing new d1e42c0a 62e93c73 and cloud f6017e35, projecting a 313b7853 8b793da0 in software 8719a9b3 revenue by 1bfef750.
-820123a4 and 12093506
-
-To 14785122 1ce8ed84 product d21e9b14 and 3cafa2cb 8b2d9344 market 9a9ea085, 3a1c4899 will 2c8df8f2 7bef6ad5 8def87a9 and 8c2453f7. eb9d3366 02bf0ddf has set 99e4cfd8 a fund of a1dbd949 million for 1168967a 180c6196 in 68b2be65 and f3d42218 aa61a94d, expecting to enhance 5c96deb8 offering and drive a ba5f69cf revenue growth.
-f7531366 8367b98b
-
-Identifying and a1c193cd dd578067 is 45ba7c0c for the d9f1babd of 6b2c4206's growth. 734bf200 60ed7498 16a3f35e:
-
-    a860d0af 911f97f6: 9a030bf5 ce10476f and adaptation to 76f34e5f 703994f9 are b1a68f0d.
-    2cb167d2 53ba4fff: edfc1496 ahead in f0426ec0 to meet 3443e779 market 0c1ac39b.
-    6c72c52f e2d3bfae: fc5d928d 71902961 to global 1277dd76, 3374e1d9 in new markets.
-
-d0c05d81
-
-20c10c38 document ac328034 a 55d3f414 80a154f5 7b580e92 and 8c5457ab b75d19ad plan for af820f2a 83382bae., using 7dc8aae7 851e65f7 and figures. It is designed to bd7f1439 how the 6f55f229 can fb8a773e the 5f837d8d of the global digital market, 42f6054b ba70b81c for growth while c9a6eef7 potential 2252cf74. cd4c1afa data and scenarios d9d31530 71a5d9f4 are 09d6c906 3bfc963f and 5352f583 6ef4d7c7 be d69e06ae c65365d9 of 828e04dd real 003af3df or b26c90f6.
-7275fe97
-
-7ab379b9 3e911aa4 figures, 41df9da7, and scenarios 24ded709 in 765fb783 document are b14afae8 351fbe3c and created for 62b7490d e6f0cc19 only. 7917e19d do 9b4f4c24 50c52f98 real a92adfc3, c25f65bc, or 17e4168d 3224758c data. 0de09cb5 document is 631d727a as a 25114a3c d7470130 in df9c5fed 6ac1a4e7 and 5430f574 planning.
+7ab7335c's social security number is 11e90148.  67e5e61e driver license? it is c6652117.

files/chatgpt_prompt.txt

@@ -0,0 +1,9 @@
+Task: Analyze an anonymized document containing sensitive information, where personal details are replaced with unique identifiers (alphanumeric). You're to answer queries based on this document, using these identifiers to reference specific data.
+
+Details:
+
+- Sensitive information includes: names, locations, credit card numbers, email addresses, IP addresses, passport details, phone numbers, bank accounts, social security numbers, and driver's licenses.
+- Each piece of information is represented by a unique identifier, maintaining privacy while discussing document content.
+- Your role is to interpret the document's anonymized content and accurately respond to queries using the identifiers.
+- Consistency in identifiers is crucial for connecting the text with the queries correctly.
+- You must not discuss the anonymized nature of the text and use the identifiers as if they were real words for a smooth chat with users.

files/original_document.txt

@@ -1,54 +1,10 @@
-Financial Analysis and Strategy Development for "TechInnovate Inc."
-Executive Summary
+Hello, my name is David Johnson and I live in Maine.
+My credit card number is 4095-2609-9393-4932 and my crypto wallet id is 16Yeky6GMjeNkAiNcBY7ZhrLoMSgg1BoyZ.
 
-This document provides a comprehensive financial analysis and strategic development plan for TechInnovate Inc., a leading technology firm specializing in developing innovative solutions for the digital market. The document contains synthesized data and proposed strategies that are based on hypothetical scenarios and fictional characters, ensuring no real individuals or actual company data are represented.
-Company Overview
+On September 18 I visited microsoft.com and sent an email to test@presidio.site,  from the IP 192.168.0.1.
 
-TechInnovate Inc., established in 2010 by founders Alex Johnson and Jamie Smith, has seen rapid growth in the technology sector, focusing on areas such as AI-driven analytics, cloud computing, and IoT solutions. With its headquarters in Silicon Valley and offices across the globe, the company aims to revolutionize how businesses operate in the digital age.
-Financial Analysis
-Revenue Streams
+My passport: 191280342 and my phone number: (212) 555-1234.
 
-TechInnovate Inc. has diversified its revenue through various channels, including software subscriptions, consulting services, and hardware sales. In the fiscal year 2023, the company reported the following fictional figures:
+This is a valid International Bank Account Number: IL150120690000003111111 . Can you please check the status on bank account 954567876544?
 
-    Software Subscriptions: $150 million
-    Consulting Services: $50 million
-    Hardware Sales: $30 million
-
-Total Revenue: $230 million
-Costs and Expenses
-
-The company's costs are primarily associated with research and development (R&D), sales and marketing, and administrative expenses. The following are the fabricated figures for the fiscal year 2023:
-
-    R&D: $70 million
-    Sales and Marketing: $50 million
-    Administrative Expenses: $30 million
-
-Total Expenses: $150 million
-
-Net Profit for FY 2023: $80 million
-Strategic Development Plan
-
-To ensure continued growth and profitability, TechInnovate Inc. needs to focus on several key areas. The plan below outlines strategic initiatives, using fictitious data and assumptions for illustration purposes.
-Expansion into Emerging Markets
-
-With the global digital transformation accelerating, emerging markets present significant opportunities for growth. TechInnovate plans to establish operations in countries such as India, Brazil, and South Africa, aiming to increase revenue by 20% over the next three years.
-Product Innovation
-
-Continued investment in R&D is crucial for maintaining a competitive edge. TechInnovate allocates $100 million towards developing new AI algorithms and cloud solutions, projecting a 30% increase in software subscription revenue by 2025.
-Partnerships and Acquisitions
-
-To diversify its product portfolio and strengthen its market position, TechInnovate will seek strategic partnerships and acquisitions. The company has set aside a fund of $200 million for acquiring startups in AI and IoT sectors, expecting to enhance its offering and drive a 25% revenue growth.
-Risk Management
-
-Identifying and mitigating risks is essential for the sustainability of TechInnovate's growth. Potential risks include:
-
-    Market Competition: Continuous analysis and adaptation to competitive strategies are required.
-    Technological Changes: Staying ahead in innovation to meet evolving market demands.
-    Regulatory Compliance: Ensuring adherence to global regulations, especially in new markets.
-
-Conclusion
-
-This document presents a fictional financial analysis and strategic development plan for TechInnovate Inc., using made-up names and figures. It is designed to showcase how the company can navigate the complexities of the global digital market, leveraging opportunities for growth while managing potential risks. All data and scenarios mentioned herein are entirely fabricated and should not be considered reflective of any real individuals or entities.
-Disclaimer
-
-The financial figures, names, and scenarios presented in this document are purely fictional and created for illustrative purposes only. They do not represent real individuals, companies, or actual financial data. This document is intended as a hypothetical exercise in financial analysis and strategic planning.
+Kate's social security number is 078-05-1126.  Her driver license? it is 1234567A.

original_document_uuid_mapping.json

@@ -0,0 +1,34 @@
+{
+    "078-05-1126": "11e90148",
+    "1234567A": "c6652117",
+    "16Yeky6GMjeNkAiNcBY7ZhrLoMSgg1BoyZ": "be0eddce",
+    "18": "ee1fa38f",
+    "191280342": "53b52085",
+    "192.168.0.1": "2d66aefe",
+    "212": "81133b16",
+    "4095-2609-9393-4932": "1155d2a5",
+    "555-1234": "08aeb9f5",
+    "954567876544": "e1a23ada",
+    "Account": "3730179e",
+    "Bank": "5a60bcfa",
+    "David": "84e24eb8",
+    "Her": "67e5e61e",
+    "I": "9f2963af",
+    "IL150120690000003111111": "ab18e6bb",
+    "International": "11e27c41",
+    "Johnson": "0ea0e35c",
+    "Kate": "7ab7335c",
+    "Maine": "5d0a593d",
+    "My": "6d7a71e0",
+    "Number": "97008100",
+    "September": "b926f540",
+    "This": "3bbcaf02",
+    "crypto": "d8e7c7fc",
+    "microsoft.com": "1e63e774",
+    "my": "0a182475",
+    "please": "f4953628",
+    "status": "8ff83dc3",
+    "[email protected]": "f70cf334",
+    "wallet": "c627b841",
+    "you": "773b08cb"
+}

requirements.txt

@@ -1,3 +1,4 @@
 concrete-ml==1.5.0rc0
 gensim==4.3.2
-gradio==3.40.1
+gradio==3.40.1
+openai==1.13.3

utils_demo.py

@@ -1,274 +0,0 @@
-
-import logging
-import re
-import string
-from flair.data import Sentence
-from flair.models import SequenceTagger
-from presidio_analyzer import AnalyzerEngine
-from presidio_anonymizer import AnonymizerEngine
-
-
-
-entity_label_to_code_map = {'<PERSON>': 0,
- '<O>': 1,
- '<MISC>-<NRP>': 2,
- '<NUMBER>': 3,
- '<PER>-<LOCATION>': 4,
- '<LOC>': 5,
- '<MISC>': 6, # Miscellaneous: doesn't fall into the more common categories of PERSON, LOCATION, ORGANIZATION,
- '<DATE_TIME>': 7,
- '<LOCATION>': 8,
- '<PRONOUNS>': 9,
- '<IN_PAN>': 10,
- '<MISC>-<DATE_TIME>': 11,
- '<ORG>': 12,
- '<MISC>-<IN_PAN>': 13,
- '<MISC>-<LOCATION>': 14,
- '<PER>': 15,
- '<MISC>-<PERSON>': 16,
- '<LOC>-<PERSON>': 17,
- '<PHONE_NUMBER>': 18,
- '<LOC>-<DATE_TIME>': 19,
- '<LOC>-<NRP>': 20,
- '<NRP>': 21,
- '<ORG>-<PERSON>': 22,
- '<PER>-<NRP>': 23,
- '<ORG>-<LOCATION>': 24,
- '<PER>-<DATE_TIME>': 25,
- '<PER>-<IN_PAN>': 26,
- '<ORG>-<IN_PAN>': 27,
- '<ORG>-<NRP>': 28,
- '<US_DRIVER_LICENSE>': 29,
- '<KEY <EMAIL_ADDRESS>': 30,
- '<US_BANK_NUMBER>': 33,
- '<IN_AADHAAR>': 34,
- '<CRYPTO>': 35,
- '<IP_ADDRESS>': 36,
- '<EMAIL_ADDRESS>': 35,
- '<US_PASSPORT>': 36,
- '<US_SSN>': 37,
- '<MISC>-<URL>': 38}
-
-
-pronoun_list = [
-    'I', 'i', 'me', 'my', 'mine', 'myself', 'you', 'your', 'yours', "I'm", "I am",\
-    'yourself', 'yourselves', 'he', 'him', 'his', 'himself', 'she', "i'm", \
-    'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', \
-    'their', 'theirs', 'themselves', 'we', 'us', 'our', 'ours', 'ourselves' \
-    'Me', 'My', 'Mine', 'Myself', 'You', 'Your', 'Yours', 'Yourself', 'Yourselves', \
-    'He', 'Him', 'His', 'Himself', 'She', 'Her', 'Hers', 'Herself', 'It', 'Its', 'Itself', \
-    'They', 'Them', 'Their', 'Theirs', 'Themselves', 'We', 'Us', 'Our', 'Ours', 'Ourselves',
-    "Lady", "Madam", "Mr.", "Mister", "Sir", "Miss", "Ms.", "Mrs.", "Mr"
-]
-
-
-privacy_category_codes = {'<PRIVATE>': 1, '<NON_PRIVATE>': 2, '<OTHER>': 3}
-
-punctuation_list = list(string.punctuation)
-punctuation_list.remove('%')
-punctuation_list.remove('$')
-punctuation_list = ''.join(punctuation_list)
-
-def get_word_boundaries(sentence):
-    """ Find the start and end positions of each word in a sentence."""
-    return [(match.start(), match.end()) for match in re.finditer(r'[^\s]+', sentence)]
-
-
-def fuse_ner_labels(flair_ner, presidio_ner, text_type="<PRIVATE>"):
-    """Merges The NER labels from 'Flair' and 'Presidio' for a given text.
-
-    We add take into account custom cases and predefined rules for entity classification.
-    """
-    merged_ner = []
-
-    # Sanity check
-    assert len(flair_ner) == len(presidio_ner)
-
-    for i, ((w1, n1), (w2, n2)) in enumerate(zip(presidio_ner, flair_ner)):
-        
-        assert w1 == w2
-
-        if w1.lower() in pronoun_list:
-            common_ner = "<PRONOUNS>"  
-        # elif w1 in ['A+', 'A-', 'B+', 'B-', 'AB+', 'AB-', 'O+', 'O-']:
-        #     common_ner = "<PRIVATE>"
-        elif n1 == "<O>" and n2 == "<O>":
-            if w1.lower() in ["am", "'m"] and (i - 1) >= 0 and  presidio_ner[i - 1][0].lower() == 'i':
-                common_ner = "<PRONOUNS>"  
-                
-            elif bool(re.match(r'(?<!\S)[\$€]?(?:\d{1,3}(?:[ ,.]\d{3})*|\d+)(?:\.\d+)?%?', w1)):
-                common_ner = "<NUMBER>"
-            else: 
-                common_ner = '<O>'  
-        elif n1 in n2:
-            common_ner = n2  
-        elif n1 == '<O>' and n2 != '<O>':
-            common_ner = n2
-        elif n2 == '<O>'  and n1 != '<O>':
-            common_ner = f"<{n1}>"
-        else:
-             common_ner = f"<{n1}>-{n2}"        
-        try:
-            common_binary_label = 0 if common_ner =="<O>" else 1
-
-        except:
-            print(f"ERROR: common_binary_label = 0 if common_ner =='<O>' else 1 | {w1=}, {w2=}, {n1=}, {n2=}")
-
-        if common_ner not in entity_label_to_code_map.keys():
-            common_multi_label = len(entity_label_to_code_map)
-            if common_ner not in entity_label_to_code_map.keys():
-                print("NOT in KEY", common_ner)
-            entity_label_to_code_map[common_ner] = common_multi_label
-        else:
-            common_multi_label = entity_label_to_code_map[common_ner]
-
-        is_private = text_type if common_ner != '<O>' else '<OTHER>'
-        
-        merged_ner.append([w1, common_ner, is_private, privacy_category_codes[is_private], common_binary_label, common_multi_label])
-
-    return merged_ner
-
-analyzer = AnalyzerEngine()
-anonymizer = AnonymizerEngine()
-
-
-def apply_presidio_model(sentence, verbose=True):
-    """Get Presidio predictions."""
-
-    if verbose: print(f"{sentence=}")
-    # anonymized_text looks like: ['<PERSON>', 'went', 'to', 'Pitier', 'Hospital', ...]
-
-    anonymized_text = anonymizer.anonymize(text=sentence, analyzer_results=analyzer.analyze(text=sentence, language='en'))
-    anonymized_text = anonymized_text.__dict__['text'].split()
-    anonymized_text = ' '.join(anonymized_text)
-    next_word_to_concate = None
-    
-    if verbose: print(f"{anonymized_text=}")
-    if verbose: print(f"{anonymized_text.split('<')=}")
-    
-    start_index, label = 0, []
-    previous_label = None
-
-    for i, before_split in enumerate(anonymized_text.split('<')):
-
-        if verbose: 
-            print(f"\nSubseq_{i}: {before_split=}")
-
-        if i == 0:  
-            assert len(before_split) == len(sentence[start_index: len(before_split)])
-            start_index = len(before_split)
-            label.extend([(s, '<O>') for s in before_split.split()])
-        else:
-            after_split = before_split.split(">")
-            if verbose: 
-                print(f" -----> ", after_split)
-                print(sentence[start_index:])
-                print(sentence[start_index:].find(after_split[-1]))
-            
-            start2_index = start_index + sentence[start_index:].find(after_split[-1])
-            end2_index = start2_index + len(after_split[-1])
-            
-            if verbose:
-                print(f"Sanity check: '[{sentence[start2_index: end2_index]}]' VS '[{after_split[-1]}]'")
-                print(f"Hidden part: sentence[{start2_index}: {end2_index}] = {sentence[start2_index: end2_index]}")
-
-            assert sentence[start2_index: end2_index] == after_split[-1]
-
-            start2_index = start2_index if start2_index != start_index else len(sentence)
-
-            for j, anonimyzed_word in enumerate((sentence[start_index: start2_index]).split()):
-                if next_word_to_concate != None and j == 0:
-                    label.append((f"{next_word_to_concate}{anonimyzed_word}", f"<{after_split[0]}>"))
-                    next_word_to_concate = None
-                else:
-                    label.append((anonimyzed_word, f"<{after_split[0]}>"))
-
-                previous_label = f"<{after_split[0]}>"
-
-            if len(sentence[start2_index: end2_index]) >= 1 and after_split[-1][-1] != ' ' and i != len(anonymized_text.split('<')) - 1:
-                if verbose: print("Is there a space after?", after_split, after_split[-1][-1], i, len(anonymized_text.split('<')))
-
-                for j, anonimyzed_word in enumerate((after_split[-1]).split()[:-1]):
-                    label.append((anonimyzed_word, "<O>"))
-                
-                next_word_to_concate = (after_split[-1]).split()[-1]
-
-            elif len(sentence[start2_index: end2_index]) >= 1 and after_split[-1][0] != ' ' and i != len(anonymized_text.split('<')) - 1:
-                if verbose: print("Is there a space before?", after_split, after_split[-1][0], i, len(anonymized_text.split('<')))
-
-                label[-1] = (f"{label[-1][0]}{after_split[-1].split()[0]}", previous_label)
-
-                for j, anonimyzed_word in enumerate((after_split[-1]).split()[1:]):
-                    label.append((anonimyzed_word, "<O>"))
-
-            else:
-                for j, anonimyzed_word in enumerate((after_split[-1]).split()):
-                    label.append((anonimyzed_word, "<O>"))
-
-            start_index = end2_index
-
-    return label
-
-
-def apply_flair_model(original_sentence):
-    """Get Flair predictions."""
-
-    logging.getLogger('flair').setLevel(logging.WARNING)
-
-    tagger = SequenceTagger.load("flair/ner-english-large")
-    flair_sentence = Sentence(original_sentence)
-    tagger.predict(flair_sentence)
-
-    word_boundaries = get_word_boundaries(original_sentence)
-
-    ner = [[i_token.form, \
-            b_token.get_label().value, \
-            i_token.get_label().score, \
-            i_token.start_position, \
-            i_token.end_position] for b_token in flair_sentence.get_spans("ner") for i_token in b_token]
-
-    ner_labels, ner_index = [], 0
-
-    for start, end in word_boundaries:
-        word_from_text = original_sentence[start:end] 
-        if ner_index < len(ner):
-            form, label, _, s, e = ner[ner_index]
-            
-            if (s, e) == (start, end) and word_from_text == form:
-                ner_labels.append((word_from_text, label))
-                ner_index += 1
-            else:
-                ner_labels.append((word_from_text, "<O>"))
-        else:
-            ner_labels.append((word_from_text, "<O>"))
-
-    assert len(ner_labels) == len(word_boundaries)
-
-    return ner_labels
-
-
-def preprocess_sentences(sentence, verbose=False):
-    """Preprocess the sentence."""
-
-    # Removing Extra Newlines:
-    sentence = re.sub(r'\n+', ' ', sentence)
-    if verbose: print(sentence)
-    
-    # Collapsing Multiple Spaces:
-    sentence = re.sub(' +', ' ', sentence)
-    if verbose: print(sentence)
-
-    # Handling Apostrophes in Possessives:
-    sentence = re.sub(r"'s\b", " s", sentence)
-    if verbose: print(sentence)
-
-    # Removing Spaces Before Punctuation:
-    sentence = re.sub(r'\s([,.!?;:])', r'\1', sentence)
-    if verbose: print(sentence)
-
-    # Pattern for Matching Leading or Trailing Punctuation:
-    pattern = r'(?<!\w)[{}]|[{}](?!\w)'.format(re.escape(punctuation_list), re.escape(punctuation_list))
-    sentence = re.sub(pattern, '', sentence)
-    if verbose: print(sentence)
-
-    return sentence