Initial commit v1.0

.dockerignore

@@ -0,0 +1,15 @@
+*.ipynb
+*checkpoint.py
+*.pyc
+*.csv
+*.parquet
+*.pem
+*.pkl
+*.env
+*.zip
+test/*
+nnet_model/*
+deprecated_models/*
+.ipynb_checkpoints/*
+orchestration/*
+.vscode/*

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+.zip filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text

.github/workflows/check_file_size.yml

@@ -0,0 +1,16 @@
+name: Check file size
+on:               # or directly `on: [push]` to run the action on every push on any branch
+  pull_request:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check large files
+        uses: ActionsDesk/[email protected]
+        with:
+          filesizelimit: 10485760 # this is 10MB so we can sync to HF Spaces

.github/workflows/sync_to_hf.yml

@@ -0,0 +1,20 @@
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push https://seanpedrickcase:[email protected]/spaces/seanpedrickcase/address_matcher main

.gitignore

@@ -0,0 +1,14 @@
+*.ipynb
+*checkpoint.py
+*.pyc
+*.csv
+*.parquet
+*.pem
+*.pkl
+*.env
+*.zip
+test/*
+deprecated_models/*
+.ipynb_checkpoints/*
+orchestration/*
+.vscode/*

Dockerfile

@@ -0,0 +1,43 @@
+FROM public.ecr.aws/docker/library/python:3.11.8-slim-bookworm
+# FROM public.ecr.aws/docker/library/python:3.10.13-slim
+
+WORKDIR /src
+
+COPY requirements.txt .
+
+RUN pip install -r requirements.txt
+
+# Set up a new user named "user" with user ID 1000
+#RUN useradd -m -u 1000 user
+
+# Change ownership of /home/user directory
+#RUN chown -R user:user /home/user
+
+# Create the temp files directory and set its permissions
+#RUN mkdir -p /home/user/tmp && chown -R user:user /home/user/tmp
+
+# Switch to the "user" user
+#USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH \
+    PYTHONPATH=$HOME/app \
+	PYTHONUNBUFFERED=1 \
+	GRADIO_ALLOW_FLAGGING=never \
+	GRADIO_NUM_PORTS=1 \
+	GRADIO_SERVER_NAME=0.0.0.0 \
+	GRADIO_SERVER_PORT=7861 \
+	GRADIO_THEME=huggingface \
+	#GRADIO_TEMP_DIR=$HOME/tmp \
+	#GRADIO_ROOT_PATH=/address-match \
+	SYSTEM=spaces
+ 
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+#COPY --chown=user . $HOME/app
+COPY . $HOME/app
+
+CMD ["python", "app.py"]

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,24 @@
+---
+title: Address Matching
+emoji: 🌍
+colorFrom: purple
+colorTo: gray
+sdk: gradio
+sdk_version: 4.20.1
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Introduction 
+Fuzzy matching a dataset with an LLPG dataset in the LPI format (with columns SaoText, SaoStartNumber etc.). Address columns are concatenated together to form a single string address. Important details are extracted by regex (e.g. flat, house numbers, postcodes). Addresses may be 'standardised' in a number of ways; e.g. variations of words used for 'ground floor' such as 'grd' or 'grnd' are replaced with 'ground floor' to give a more consistent address wording. This has been found to increase match rates.
+
+Then the two datasets are compared with fuzzy matching. The closest fuzzy matches are selected, and then a post hoc test compares flat/property numbers to ensure a 'full match'.
+
+If the LLPG reference file is in the standard LPI format, the neural net model should then initialise. This will break down the addresses to match into a list of sub address fields in the LLPG LPI format. It will then do exact or fuzzy comparisons of each address to the LLPG dataset to find closest matches. The neural net is capable of blocking on postcode and on street name, which is where most of the new matches are found according to testing.
+
+The final files will appear in the relevant output boxes, which you can download.
+
+

app.py

@@ -0,0 +1,447 @@
+# Load in packages, variables for fuzzy matching
+import os
+from datetime import datetime
+from pathlib import Path
+import time
+import copy
+import gradio as gr
+import re
+#import polars as pl
+
+from tools.constants import *
+from tools.matcher_funcs import load_matcher_data, run_match_batch, combine_two_matches, create_match_summary
+from tools.gradio import initial_data_load
+from tools.aws_functions import load_data_from_aws
+from tools.preparation import prepare_search_address_string, prepare_search_address, prepare_ref_address, remove_non_postal, check_no_number_addresses
+from tools.standardise import standardise_wrapper_func
+
+import warnings
+warnings.filterwarnings("ignore", 'This pattern is interpreted as a regular expression')
+warnings.filterwarnings("ignore", 'Downcasting behavior')
+warnings.filterwarnings("ignore", 'A value is trying to be set on a copy of a slice from a DataFrame')
+warnings.filterwarnings("ignore")
+
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+# Base folder is where the code file is stored
+base_folder = Path(os.getcwd())
+input_folder = base_folder/"Input/"
+output_folder = base_folder/"Output/"
+diagnostics_folder = base_folder/"Diagnostics/"
+prep_folder = base_folder/"Helper functions/"
+
+def create_simple_batch_ranges(df:PandasDataFrame, ref_df:PandasDataFrame, batch_size:int, ref_batch_size:int):
+    #print("Search df batch size: ", batch_size)
+    #print("ref_df df batch size: ", ref_batch_size)
+
+    total_rows = df.shape[0]
+    ref_total_rows = ref_df.shape[0]
+
+    # Creating bottom and top limits for search data
+    search_ranges = []
+    for start in range(0, total_rows, batch_size):
+        end = min(start + batch_size - 1, total_rows - 1)  # Adjusted to get the top limit
+        search_ranges.append((start, end))
+
+    # Creating bottom and top limits for reference data
+    ref_ranges = []
+    for start in range(0, ref_total_rows, ref_batch_size):
+        end = min(start + ref_batch_size - 1, ref_total_rows - 1)  # Adjusted to get the top limit
+        ref_ranges.append((start, end))
+
+    # Create DataFrame with combinations of search_range and ref_range
+    result_data = []
+    for search_range in search_ranges:
+        for ref_range in ref_ranges:
+            result_data.append((search_range, ref_range))
+
+    range_df = pd.DataFrame(result_data, columns=['search_range', 'ref_range'])
+
+    return range_df
+
+
+def create_batch_ranges(df:PandasDataFrame, ref_df:PandasDataFrame, batch_size:int, ref_batch_size:int, search_postcode_col:str, ref_postcode_col:str):
+    '''
+    Create batches of address indexes for search and reference dataframes based on shortened postcodes.
+    '''
+
+    # If df sizes are smaller than the batch size limits, no need to run through everything
+    if len(df) < batch_size and len(ref_df) < ref_batch_size:
+        print("Dataframe sizes are smaller than maximum batch sizes, no need to split data.")
+        lengths_df = pd.DataFrame(data={'search_range':[df.index.tolist()], 'ref_range':[ref_df.index.tolist()], 'batch_length':len(df), 'ref_length':len(ref_df)})
+        return lengths_df
+    
+    #df.index = df[search_postcode_col]
+
+    df['index'] = df.index
+    ref_df['index'] = ref_df.index    
+
+    # Remove the last character of postcode
+    df['postcode_minus_last_character'] = df[search_postcode_col].str.lower().str.strip().str.replace("\s+", "", regex=True).str[:-1]
+    ref_df['postcode_minus_last_character'] = ref_df[ref_postcode_col].str.lower().str.strip().str.replace("\s+", "", regex=True).str[:-1]
+
+    unique_postcodes = df['postcode_minus_last_character'][df['postcode_minus_last_character'].str.len()>=4].unique().tolist()
+
+    df = df.set_index('postcode_minus_last_character')
+    ref_df = ref_df.set_index('postcode_minus_last_character')
+
+    df = df.sort_index()
+    ref_df = ref_df.sort_index()
+
+    #df.to_csv("batch_search_df.csv")
+
+    # Overall batch variables
+    batch_indexes = []
+    ref_indexes = []
+    batch_lengths = []
+    ref_lengths = []
+
+    # Current batch variables for loop
+    current_batch = []
+    current_ref_batch = []
+    current_batch_length = []
+    current_ref_length = []
+
+    unique_postcodes_iterator = unique_postcodes.copy()
+
+    while unique_postcodes_iterator:
+        
+        unique_postcodes_loop = unique_postcodes_iterator.copy()
+
+        #print("Current loop postcodes: ", unique_postcodes_loop)
+
+        for current_postcode in unique_postcodes_loop:
+            
+
+
+            if len(current_batch) >= batch_size or len(current_ref_batch) >= ref_batch_size:
+                print("Batch length reached - breaking")
+                break
+            
+            try:
+                current_postcode_search_data_add = df.loc[[current_postcode]]#[df['postcode_minus_last_character'].isin(current_postcode)]
+                current_postcode_ref_data_add = ref_df.loc[[current_postcode]]#[ref_df['postcode_minus_last_character'].isin(current_postcode)]
+
+                #print(current_postcode_search_data_add)
+
+                if not current_postcode_search_data_add.empty:
+                    current_batch.extend(current_postcode_search_data_add['index'])
+
+                if not current_postcode_ref_data_add.empty:
+                    current_ref_batch.extend(current_postcode_ref_data_add['index'])  
+
+            except:
+                #print("postcode not found: ", current_postcode)
+                pass
+
+            unique_postcodes_iterator.remove(current_postcode)
+
+        # Append the batch data to the master lists and reset lists
+        batch_indexes.append(current_batch)
+        ref_indexes.append(current_ref_batch)
+
+        current_batch_length = len(current_batch)
+        current_ref_length = len(current_ref_batch)
+
+        batch_lengths.append(current_batch_length)
+        ref_lengths.append(current_ref_length)
+
+        current_batch = []
+        current_ref_batch = []
+        current_batch_length = []
+        current_ref_length = []
+        
+    # Create df to store lengths
+    lengths_df = pd.DataFrame(data={'search_range':batch_indexes, 'ref_range':ref_indexes, 'batch_length':batch_lengths, 'ref_length':ref_lengths})
+    
+    return lengths_df
+
+
+def run_matcher(in_text, in_file, in_ref, data_state:PandasDataFrame, results_data_state:PandasDataFrame, ref_data_state:PandasDataFrame, in_colnames:List[str], in_refcol:List[str], in_joincol:List[str], in_existing:List[str], in_api:str, in_api_key:str, InitMatch:MatcherClass = InitMatch, progress=gr.Progress()):  
+    '''
+    Split search and reference data into batches. Loop and run through the match script.
+    '''
+
+    overall_tic = time.perf_counter()
+
+    # Load in initial data. This will filter to relevant addresses in the search and reference datasets that can potentially be matched, and will pull in API data if asked for.
+    InitMatch = load_matcher_data(in_text, in_file, in_ref, data_state, results_data_state, ref_data_state, in_colnames, in_refcol, in_joincol, in_existing, InitMatch, in_api, in_api_key)
+
+    if InitMatch.search_df.empty or InitMatch.ref_df.empty:
+        out_message = "Nothing to match!"
+        print(out_message)
+        return out_message, [InitMatch.results_orig_df_name, InitMatch.match_outputs_name]
+    
+    # Run initial address preparation and standardisation processes   
+    # Prepare address format
+
+    # Polars implementation not yet finalised
+    #InitMatch.search_df = pl.from_pandas(InitMatch.search_df)
+    #InitMatch.ref_df = pl.from_pandas(InitMatch.ref_df)
+
+    
+    # Prepare all search addresses
+    if type(InitMatch.search_df) == str:
+        InitMatch.search_df_cleaned, InitMatch.search_df_key_field, InitMatch.search_address_cols = prepare_search_address_string(InitMatch.search_df)
+    else: 
+        InitMatch.search_df_cleaned = prepare_search_address(InitMatch.search_df, InitMatch.search_address_cols, InitMatch.search_postcode_col, InitMatch.search_df_key_field)
+
+        # Remove addresses that are not postal addresses
+    InitMatch.search_df_cleaned = remove_non_postal(InitMatch.search_df_cleaned, "full_address")
+
+    # Remove addresses that have no numbers in from consideration
+    InitMatch.search_df_cleaned = check_no_number_addresses(InitMatch.search_df_cleaned, "full_address")
+
+    # Initial preparation of reference addresses
+    InitMatch.ref_df_cleaned = prepare_ref_address(InitMatch.ref_df, InitMatch.ref_address_cols, InitMatch.new_join_col)
+    
+
+    # Sort dataframes by postcode - will allow for more efficient matching process if using multiple batches
+    #InitMatch.search_df_cleaned = InitMatch.search_df_cleaned.sort_values(by="postcode")
+    #InitMatch.ref_df_cleaned = InitMatch.ref_df_cleaned.sort_values(by="Postcode")
+
+    # Polars implementation - not finalised
+    #InitMatch.search_df_cleaned = InitMatch.search_df_cleaned.to_pandas()
+    #InitMatch.ref_df_cleaned = InitMatch.ref_df_cleaned.to_pandas()
+
+    # Standardise addresses    
+    # Standardise - minimal
+
+
+    tic = time.perf_counter()
+    InitMatch.search_df_after_stand, InitMatch.ref_df_after_stand = standardise_wrapper_func(
+        InitMatch.search_df_cleaned.copy(),
+        InitMatch.ref_df_cleaned.copy(),
+        standardise = False,
+        filter_to_lambeth_pcodes=filter_to_lambeth_pcodes,
+        match_task="fuzzy") # InitMatch.search_df_after_stand_series, InitMatch.ref_df_after_stand_series
+
+    toc = time.perf_counter()
+    print(f"Performed the minimal standardisation step in {toc - tic:0.1f} seconds")
+
+    # Standardise - full
+    tic = time.perf_counter()
+    InitMatch.search_df_after_full_stand, InitMatch.ref_df_after_full_stand = standardise_wrapper_func(
+        InitMatch.search_df_cleaned.copy(),
+        InitMatch.ref_df_cleaned.copy(),
+        standardise = True,
+        filter_to_lambeth_pcodes=filter_to_lambeth_pcodes,
+        match_task="fuzzy") # , InitMatch.search_df_after_stand_series_full_stand, InitMatch.ref_df_after_stand_series_full_stand
+
+    toc = time.perf_counter()
+    print(f"Performed the full standardisation step in {toc - tic:0.1f} seconds")
+
+    # Determine length of search df to create batches to send through the functions.
+    #try:
+    range_df = create_batch_ranges(InitMatch.search_df_cleaned.copy(), InitMatch.ref_df_cleaned.copy(), batch_size, ref_batch_size, "postcode", "Postcode")
+    #except:
+    #    range_df = create_simple_batch_ranges(InitMatch.search_df_cleaned, InitMatch.ref_df_cleaned, batch_size, #ref_batch_size)
+
+    print("Batches to run in this session: ", range_df)
+
+    OutputMatch = copy.copy(InitMatch)
+
+    n = 0
+    number_of_batches = range_df.shape[0]
+
+    for row in progress.tqdm(range(0,len(range_df)), desc= "Running through batches", unit="batches", total=number_of_batches):
+        print("Running batch ", str(n+1))
+
+        search_range = range_df.iloc[row]['search_range']
+        ref_range = range_df.iloc[row]['ref_range']
+
+        #print("search_range: ", search_range)
+        #pd.DataFrame(search_range).to_csv("search_range.csv")
+        #print("ref_range: ", ref_range)
+        
+        BatchMatch = copy.copy(InitMatch)
+
+        # Subset the search and reference dfs based on current batch ranges
+        # BatchMatch.search_df = BatchMatch.search_df.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.search_df_not_matched = BatchMatch.search_df.copy()
+        # BatchMatch.search_df_cleaned = BatchMatch.search_df_cleaned.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.ref_df = BatchMatch.ref_df.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.ref_df_cleaned = BatchMatch.ref_df_cleaned.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
+
+
+        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand_series.iloc[search_range[0]:search_range[1] + 1]
+        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand_series.iloc[ref_range[0]:ref_range[1] + 1]
+        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_stand_series_full_stand.iloc[search_range[0]:search_range[1] + 1]
+        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_stand_series_full_stand.iloc[ref_range[0]:ref_range[1] + 1]
+
+        # BatchMatch.search_df_after_stand = BatchMatch.search_df_after_stand.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.ref_df_after_stand = BatchMatch.ref_df_after_stand.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.search_df_after_full_stand = BatchMatch.search_df_after_full_stand.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.ref_df_after_full_stand = BatchMatch.ref_df_after_full_stand.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
+
+        BatchMatch.search_df = BatchMatch.search_df[BatchMatch.search_df.index.isin(search_range)].reset_index(drop=True)
+        BatchMatch.search_df_not_matched = BatchMatch.search_df.copy()
+        BatchMatch.search_df_cleaned = BatchMatch.search_df_cleaned[BatchMatch.search_df_cleaned.index.isin(search_range)].reset_index(drop=True)
+
+        BatchMatch.ref_df = BatchMatch.ref_df[BatchMatch.ref_df.index.isin(ref_range)].reset_index(drop=True)
+        BatchMatch.ref_df_cleaned = BatchMatch.ref_df_cleaned[BatchMatch.ref_df_cleaned.index.isin(ref_range)].reset_index(drop=True)
+
+        # Dataframes after standardisation process
+        BatchMatch.search_df_after_stand = BatchMatch.search_df_after_stand[BatchMatch.search_df_after_stand.index.isin(search_range)].reset_index(drop=True)
+        BatchMatch.search_df_after_full_stand = BatchMatch.search_df_after_full_stand[BatchMatch.search_df_after_full_stand.index.isin(search_range)].reset_index(drop=True)
+
+        ### Create lookup lists for fuzzy matches
+        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand.copy().set_index('postcode_search')['search_address_stand']
+        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_full_stand.copy().set_index('postcode_search')['search_address_stand']
+        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand_series.sort_index()
+        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_stand_series_full_stand.sort_index()
+
+        #BatchMatch.search_df_after_stand.reset_index(inplace=True, drop = True)
+        #BatchMatch.search_df_after_full_stand.reset_index(inplace=True, drop = True)
+
+        BatchMatch.ref_df_after_stand = BatchMatch.ref_df_after_stand[BatchMatch.ref_df_after_stand.index.isin(ref_range)].reset_index(drop=True)
+        BatchMatch.ref_df_after_full_stand = BatchMatch.ref_df_after_full_stand[BatchMatch.ref_df_after_full_stand.index.isin(ref_range)].reset_index(drop=True)
+
+        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand.copy().set_index('postcode_search')['ref_address_stand']
+        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_full_stand.copy().set_index('postcode_search')['ref_address_stand']
+        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand_series.sort_index()
+        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_stand_series_full_stand.sort_index()
+
+        # BatchMatch.ref_df_after_stand.reset_index(inplace=True, drop=True)
+        # BatchMatch.ref_df_after_full_stand.reset_index(inplace=True, drop=True)
+
+        # Match the data, unless the search or reference dataframes are empty
+        if BatchMatch.search_df.empty or BatchMatch.ref_df.empty:
+            out_message = "Nothing to match for batch: " + str(n)
+            print(out_message)
+            BatchMatch_out = BatchMatch
+            BatchMatch_out.results_on_orig_df = pd.DataFrame(data={"index":BatchMatch.search_df.index,
+                                                                   "Excluded from search":False,
+                                                                    "Matched with reference address":False})
+        else:
+            summary_of_summaries, BatchMatch_out = run_match_batch(BatchMatch, n, number_of_batches)
+        
+        OutputMatch = combine_two_matches(OutputMatch, BatchMatch_out, "All up to and including batch " + str(n+1))
+
+        n += 1
+
+    if in_api==True:
+        OutputMatch.results_on_orig_df['Matched with reference address'] = OutputMatch.results_on_orig_df['Matched with reference address'].replace({1:True, 0:False})
+        OutputMatch.results_on_orig_df['Excluded from search'] = OutputMatch.results_on_orig_df['Excluded from search'].replace('nan', False).fillna(False)
+
+    # Remove any duplicates from reference df, prioritise successful matches
+    OutputMatch.results_on_orig_df = OutputMatch.results_on_orig_df.sort_values(by=["index", "Matched with reference address"], ascending=[True,False]).drop_duplicates(subset="index")
+
+
+    overall_toc = time.perf_counter()
+    time_out = f"The overall match (all batches) took {overall_toc - overall_tic:0.1f} seconds"
+
+    print(OutputMatch.output_summary)
+
+    if OutputMatch.output_summary == "":
+        OutputMatch.output_summary = "No matches were found."
+
+    fuzzy_not_std_output = OutputMatch.match_results_output.copy()
+    fuzzy_not_std_output_mask = ~(fuzzy_not_std_output["match_method"].str.contains("Fuzzy match")) | (fuzzy_not_std_output["standardised_address"] == True)
+    fuzzy_not_std_output.loc[fuzzy_not_std_output_mask, "full_match"] = False
+    fuzzy_not_std_summary = create_match_summary(fuzzy_not_std_output, "Fuzzy not standardised")
+
+    fuzzy_std_output = OutputMatch.match_results_output.copy()
+    fuzzy_std_output_mask = fuzzy_std_output["match_method"].str.contains("Fuzzy match")
+    fuzzy_std_output.loc[fuzzy_std_output_mask == False, "full_match"] = False
+    fuzzy_std_summary = create_match_summary(fuzzy_std_output, "Fuzzy standardised")
+
+    nnet_std_output = OutputMatch.match_results_output.copy()
+    nnet_std_summary = create_match_summary(nnet_std_output, "Neural net standardised")
+
+    final_summary = fuzzy_not_std_summary + "\n" + fuzzy_std_summary + "\n" + nnet_std_summary + "\n" + time_out
+
+    return final_summary, [OutputMatch.results_orig_df_name, OutputMatch.match_outputs_name]
+
+# Create the gradio interface
+
+block = gr.Blocks(theme = gr.themes.Base())
+
+with block:
+
+    data_state = gr.State(pd.DataFrame())
+    ref_data_state = gr.State(pd.DataFrame())
+    results_data_state = gr.State(pd.DataFrame())
+    ref_results_data_state =gr.State(pd.DataFrame())
+
+    gr.Markdown(
+    """
+    # Address matcher
+    Match single or multiple addresses to the reference address file of your choice. Fuzzy matching should work on any address columns as long as you specify the postcode column at the end. The neural network component only activates with the in-house neural network model - contact me for details if you have access to AddressBase already.The neural network component works with LLPG files in the LPI format.
+    
+    The tool can accept csv, xlsx (with one sheet), and parquet files. You
+     need to specify the address columns of the file to match specifically in the address column area with postcode at the end. 
+    
+    Use the 'New Column' button to create a new cell for each column name. After you have chosen a reference file, an address match file, and specified its address columns (plus postcode), you can press 'Match addresses' to run the tool.
+    """)
+
+    with gr.Tab("Match addresses"):
+    
+        with gr.Accordion("I have multiple addresses", open = True):
+            in_file = gr.File(label="Input addresses from file", file_count= "multiple")
+            in_colnames = gr.Dropdown(choices=[], multiselect=True, label="Select columns that make up the address. Make sure postcode is at the end")
+            in_existing = gr.Dropdown(choices=[], multiselect=False, label="Select columns that indicate existing matches.")
+        
+        with gr.Accordion("I only have a single address", open = False):
+            in_text = gr.Textbox(label="Input a single address as text")
+        
+        gr.Markdown(
+        """
+        ## Choose reference file
+        Fuzzy matching will work on any address format, but the neural network will only work with the LLPG LPI format, e.g. with columns SaoText, SaoStartNumber etc.. This joins on the UPRN column. If any of these are different for you, 
+        open 'Custom reference file format or join columns' below.
+        """)
+        
+        in_ref = gr.File(label="Input reference addresses from file", file_count= "multiple")
+
+        with gr.Accordion("Use Addressbase API instead of reference file", open = False):
+            in_api = gr.Dropdown(label="Choose API type", multiselect=False, value=None, choices=["Postcode", "UPRN"]) #choices=["Address", "Postcode", "UPRN"])
+            in_api_key = gr.Textbox(label="Addressbase API key")
+        
+        with gr.Accordion("Custom reference file format or join columns (i.e. not LLPG LPI format)", open = False):
+            in_refcol = gr.Dropdown(choices=[], multiselect=True, label="Select columns that make up the reference address. Make sure postcode is at the end")
+            in_joincol = gr.Dropdown(choices=[], multiselect=True, label="Select columns you want to join on to the search dataset")
+        
+        match_btn = gr.Button("Match addresses")
+        
+        with gr.Row():
+            output_summary = gr.Textbox(label="Output summary")
+            output_file = gr.File(label="Output file")
+
+    with gr.Tab(label="Advanced options"):
+        with gr.Accordion(label = "AWS data access", open = False):
+                aws_password_box = gr.Textbox(label="Password for AWS data access (ask the Data team if you don't have this)")
+                with gr.Row():
+                    in_aws_file = gr.Dropdown(label="Choose keyword file to load from AWS (only valid for API Gateway app)", choices=["None", "Lambeth address data example file"])
+                    load_aws_data_button = gr.Button(value="Load keyword data from AWS", variant="secondary")
+                    
+                aws_log_box = gr.Textbox(label="AWS data load status")
+
+    
+    ### Loading AWS data ###
+    load_aws_data_button.click(fn=load_data_from_aws, inputs=[in_aws_file, aws_password_box], outputs=[in_ref, aws_log_box])
+    
+
+    # Updates to components
+    in_file.change(fn = initial_data_load, inputs=[in_file], outputs=[output_summary, in_colnames, in_existing, data_state, results_data_state])
+    in_ref.change(fn = initial_data_load, inputs=[in_ref], outputs=[output_summary, in_refcol, in_joincol, ref_data_state, ref_results_data_state])      
+
+    match_btn.click(fn = run_matcher, inputs=[in_text, in_file, in_ref, data_state, results_data_state, ref_data_state, in_colnames, in_refcol, in_joincol, in_existing, in_api, in_api_key],
+                    outputs=[output_summary, output_file], api_name="address")
+    
+# Simple run for HF spaces or local on your computer
+#block.queue().launch(debug=True) # root_path="/address-match", debug=True, server_name="0.0.0.0",
+
+# Simple run for AWS server
+block.queue().launch(ssl_verify=False) # root_path="/address-match", debug=True, server_name="0.0.0.0", server_port=7861
+
+# Download OpenSSL from here: 
+# Running on local server with https: https://discuss.huggingface.co/t/how-to-run-gradio-with-0-0-0-0-and-https/38003 or https://dev.to/rajshirolkar/fastapi-over-https-for-development-on-windows-2p7d
+#block.queue().launch(ssl_verify=False, share=False, debug=False, server_name="0.0.0.0",server_port=443,
+#                     ssl_certfile="cert.pem", ssl_keyfile="key.pem") # port 443 for https. Certificates currently not valid
+
+# Running on local server without https
+#block.queue().launch(server_name="0.0.0.0", server_port=7861, ssl_verify=False)
+

requirements.txt

@@ -0,0 +1,13 @@
+#fuzzywuzzy==0.18.0
+numpy==1.26.2
+pandas==2.2.1
+rapidfuzz==3.8.1
+torch==2.2.1
+recordlinkage==0.16
+pyap==0.3.1
+pytest==7.4.3
+pyarrow==14.0.1
+openpyxl==3.1.2
+gradio==4.20.1
+boto3==1.34.63
+polars==0.20.19

tools/addressbase_api_funcs.py

@@ -0,0 +1,197 @@
+# %%
+import urllib
+from datetime import datetime
+import pandas as pd
+import time
+import requests
+
+today_rev = datetime.now().strftime("%Y%m%d")
+
+
+# url = 'https://api.os.uk/search/places/v1/uprn?%s'
+# params = urllib.parse.urlencode({'uprn':<UPRN>,'dataset':'LPI', 'key':os.environ["ADDRESSBASE_API_KEY"]})
+
+# Places API
+# Technical guide: https://osdatahub.os.uk/docs/places/technicalSpecification
+
+
+def places_api_query(query, api_key, query_type):
+
+    def make_api_call(url):
+        max_retries = 3
+        retries = 0
+
+        while retries < max_retries:
+            try:
+                response = requests.get(url)
+                if response.status_code == 200:
+                    # If successful response, return the response
+                    return response
+                elif response.status_code == 429:
+                    # If rate limited, wait for 5 seconds before retrying
+                    print("Rate limited. Retrying in 5 seconds...")
+                    time.sleep(3)
+                    retries += 1
+                else:
+                    # For other errors, return the response
+                    return response
+            except Exception as e:
+                print("Error:", str(e))
+                retries += 1
+
+        # If maximum retries reached, return None
+        return None
+
+    if api_key:
+
+        overall_tic = time.perf_counter()
+
+        #filter_code_lsc = "LOGICAL_STATUS_CODE:1"
+        filter_code_lpi_lsc ="LPI_LOGICAL_STATUS_CODE:1"
+        concat_results = []
+
+        if query_type == "Address":
+            url = 'https://api.os.uk/search/places/v1/find?%s'
+            params = urllib.parse.urlencode({'query':query,
+                                            'dataset':'LPI',
+                                            'key':api_key,
+                                            "maxresults" : 20,
+                                            'minmatch':0.70, # This includes partial matches
+                                            'matchprecision':2,
+                                            'fq':filter_code_lpi_lsc,
+                                            'lr':'EN'})
+            
+            try:
+                request_text = url % params
+                #print(request_text)
+                response = make_api_call(request_text)
+            except Exception as e:
+                print(str(e))
+        
+
+            if response is not None:
+                if response.status_code == 200:
+                    # Process the response
+                    print("Successful response")
+                    #print("Successful response:", response.json())
+                else:
+                    print("Error:", response.status_code)
+
+            else:
+                print("Maximum retries reached. Error occurred.")
+                return pd.DataFrame() # Return blank dataframe
+            
+            # Load JSON response
+            response_data = response.json()
+
+            # Extract 'results' part
+            try:
+                results = response_data['results']
+                concat_results.extend(results)
+
+            except Exception as e:
+                print(str(e))
+                return pd.DataFrame() # Return blank dataframe
+
+        # If querying postcode, need to use pagination and postcode API
+        elif query_type == "Postcode":
+
+            max_results_requested = 100
+            remaining_calls = 1
+            totalresults = max_results_requested
+            call_number = 1
+
+            while remaining_calls > 0 and call_number <= 10:
+
+                offset = (call_number-1) * max_results_requested
+
+                #print("Remaining to query:", remaining_calls)
+                
+
+                url = 'https://api.os.uk/search/places/v1/postcode?%s'
+                params = urllib.parse.urlencode({'postcode':query,
+                                                'dataset':'LPI',
+                                                'key':api_key,
+                                                "maxresults" : max_results_requested,
+                                                'offset':offset,
+                                                #'fq':filter_code_lsc,
+                                                'fq':filter_code_lpi_lsc,
+                                                'lr':'EN'})
+
+                try:
+                    request_text = url % params
+                    #print(request_text)
+                    response = make_api_call(request_text)
+                except Exception as e:
+                    print(str(e))
+        
+                if response is not None:
+                    if response.status_code == 200:
+                        totalresults = response.json()['header']['totalresults']
+
+                        print("Successful response")
+                        print("Total results:", totalresults)
+
+                        remaining_calls = totalresults - (max_results_requested * call_number)
+
+                        call_number += 1
+
+                        # Concat results together
+                        try:
+                            results = response.json()['results']
+                            concat_results.extend(results)
+                        except Exception as e:
+                            print("Result concat failed with error: ", str(e))
+                            concat_results.append({"invalid_request":True, "POSTCODE_LOCATOR": query})
+
+                    else:
+                        print("Error:", response.status_code, "For postcode: ", query, " With query: ", request_text)
+                        concat_results.append({"invalid_request":True, "POSTCODE_LOCATOR": query})
+                        return pd.DataFrame(data={"invalid_request":[True], "POSTCODE_LOCATOR": [query]},index=[0]) # Return blank dataframe
+                else:
+                    print("Maximum retries reached. Error occurred.")
+                    return pd.DataFrame() # Return blank dataframe
+    
+    else:
+        print("No API key provided.")
+        return pd.DataFrame() # Return blank dataframe
+    
+    #print('RESPONSE:', concat_results)
+    
+    
+    # Convert 'results' to DataFrame
+
+    # Check if 'LPI' sub-branch exists in the JSON response
+    #print(concat_results)
+
+    if 'LPI' in concat_results[-1]:
+        #print("LPI in result columns")
+        df = pd.json_normalize(concat_results)
+        df.rename(columns=lambda x: x.replace('LPI.', ''), inplace=True)
+    else:
+        # Normalize the entire JSON data if 'LPI' sub-branch doesn't exist
+        df = pd.json_normalize(concat_results)
+
+        
+    # Ensure df is a DataFrame, even if it has a single row    
+    if isinstance(df, pd.Series):
+        print("This is a series!")
+        df = df.to_frame().T  # Convert the Series to a DataFrame with a single row
+    # if isinstance(df, pd.DataFrame):
+    #     print("This is a dataframe!")
+    # else:
+    #     print("This is not a dataframe!")
+    #     return pd.DataFrame() # Return blank dataframe
+
+
+    print(df)
+    #print(df.columns)
+    #df.to_csv(query + ".csv")
+
+    
+
+    overall_toc = time.perf_counter()
+    time_out = f"The API call took {overall_toc - overall_tic:0.1f} seconds"
+    print(time_out)
+
+    return df

tools/aws_functions.py

@@ -0,0 +1,166 @@
+from typing import Type
+import pandas as pd
+import boto3
+import tempfile
+import os
+
+PandasDataFrame = Type[pd.DataFrame]
+
+bucket_name = 'address-matcher-data'
+
+try:
+    session = boto3.Session(profile_name="default")
+except Exception as e:
+    print(e)
+
+# sts = session.client("sts")
+# Create a Session with the IAM role ARN
+# aws_role = os.environ['AWS_ROLE_DATA_TEXT_SEARCH']
+# response = sts.assume_role(
+#     RoleArn=aws_role,
+#     RoleSessionName="ecs-test-session"
+# )
+# print(response)
+
+
+def get_assumed_role_info():
+    sts = boto3.client('sts')
+    response = sts.get_caller_identity()
+
+    # Extract ARN of the assumed role
+    assumed_role_arn = response['Arn']
+    
+    # Extract the name of the assumed role from the ARN
+    assumed_role_name = assumed_role_arn.split('/')[-1]
+    
+    return assumed_role_arn, assumed_role_name
+
+try:
+    assumed_role_arn, assumed_role_name = get_assumed_role_info()
+
+    print("Assumed Role ARN:", assumed_role_arn)
+    print("Assumed Role Name:", assumed_role_name)
+except Exception as e:
+    print(e)
+
+# Download direct from S3 - requires login credentials
+def download_file_from_s3(bucket_name, key, local_file_path):
+
+    s3 = boto3.client('s3')
+    s3.download_file(bucket_name, key, local_file_path)
+    print(f"File downloaded from S3: s3://{bucket_name}/{key} to {local_file_path}")
+                         
+#download_file_from_s3(bucket_name, object_key, local_file_loc)
+
+def download_folder_from_s3(bucket_name, s3_folder, local_folder):
+    """
+    Download all files from an S3 folder to a local folder.
+    """
+    s3 = boto3.client('s3')
+
+    # List objects in the specified S3 folder
+    response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
+
+    # Download each object
+    for obj in response.get('Contents', []):
+        # Extract object key and construct local file path
+        object_key = obj['Key']
+        local_file_path = os.path.join(local_folder, os.path.relpath(object_key, s3_folder))
+
+        # Create directories if necessary
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+        # Download the object
+        try:
+            s3.download_file(bucket_name, object_key, local_file_path)
+            print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
+        except Exception as e:
+            print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
+
+
+def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
+    """
+    Download specific files from an S3 folder to a local folder.
+    """
+    s3 = boto3.client('s3')
+
+    print("Trying to download file: ", filenames)
+
+    if filenames == '*':
+        # List all objects in the S3 folder
+        print("Trying to download all files in AWS folder: ", s3_folder)
+        response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
+
+        print("Found files in AWS folder: ", response.get('Contents', []))
+
+        filenames = [obj['Key'].split('/')[-1] for obj in response.get('Contents', [])]
+
+        print("Found filenames in AWS folder: ", filenames)
+
+    for filename in filenames:
+        object_key = os.path.join(s3_folder, filename)
+        local_file_path = os.path.join(local_folder, filename)
+
+        # Create directories if necessary
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+        # Download the object
+        try:
+            s3.download_file(bucket_name, object_key, local_file_path)
+            print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
+        except Exception as e:
+            print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
+
+
+
+def load_data_from_aws(in_aws_keyword_file, aws_password="", bucket_name=bucket_name):
+
+    temp_dir = tempfile.mkdtemp()
+    local_address_stub = temp_dir + '/address-match/'
+    files = []
+
+    if not 'LAMBETH_ADDRESS_PASSWORD' in os.environ:
+        out_message = "Can't verify password for dataset access. Do you have a valid AWS connection? Data not loaded."
+        return files, out_message
+    
+    if aws_password:
+        if "Lambeth address data example file" in in_aws_keyword_file and aws_password == os.environ['LAMBETH_ADDRESS_PASSWORD']:
+
+            s3_folder_stub = 'example-data/lambeth-address-data/latest/'
+
+            local_folder_path = local_address_stub                
+
+            # Check if folder exists
+            if not os.path.exists(local_folder_path):
+                print(f"Folder {local_folder_path} does not exist! Making folder.")
+
+                os.mkdir(local_folder_path)
+
+            # Check if folder is empty
+            if len(os.listdir(local_folder_path)) == 0:
+                print(f"Folder {local_folder_path} is empty")
+                # Download data
+                download_files_from_s3(bucket_name, s3_folder_stub, local_folder_path, filenames='*')
+
+                print("AWS data downloaded")
+
+            else:
+                print(f"Folder {local_folder_path} is not empty")
+
+            #files = os.listdir(local_folder_stub)
+            #print(files)
+
+            files = [os.path.join(local_folder_path, f) for f in os.listdir(local_folder_path) if os.path.isfile(os.path.join(local_folder_path, f))]
+
+            out_message = "Data successfully loaded from AWS"
+            print(out_message)
+
+        else:
+            out_message = "Data not loaded from AWS"
+            print(out_message)
+    else:
+        out_message = "No password provided. Please ask the data team for access if you need this."
+        print(out_message)
+
+    return files, out_message
+

tools/constants.py

@@ -0,0 +1,435 @@
+import os
+import pandas as pd
+import pickle
+import torch
+import zipfile
+from typing import List, Union, Type, Dict
+from pydantic import BaseModel
+
+from .pytorch_models import *
+
+PandasDataFrame = Type[pd.DataFrame]
+PandasSeries = Type[pd.Series]
+
+# +
+''' Fuzzywuzzy/Rapidfuzz scorer to use. Options are: ratio, partial_ratio, token_sort_ratio, partial_token_sort_ratio,
+token_set_ratio, partial_token_set_ratio, QRatio, UQRatio, WRatio (default), UWRatio
+details here: https://stackoverflow.com/questions/31806695/when-to-use-which-fuzz-function-to-compare-2-strings'''
+
+fuzzy_scorer_used = "token_set_ratio"
+
+# +
+fuzzy_match_limit = 85
+
+fuzzy_search_addr_limit = 20
+
+filter_to_lambeth_pcodes= True 
+# -
+
+standardise = False
+
+# +
+if standardise == True:
+    std = "_std"
+if standardise == False:
+    std = "_not_std"
+    
+dataset_name = "data" + std
+    
+suffix_used = dataset_name + "_" + fuzzy_scorer_used
+
+# https://stackoverflow.com/questions/59221557/tensorflow-v2-replacement-for-tf-contrib-predictor-from-saved-model
+
+ROOT_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))
+print(ROOT_DIR)
+
+# Uncomment these lines for the tensorflow model
+#model_type = "tf"
+#model_stub = "addr_model_out_lon"
+#model_version = "00000001"
+#file_step_suffix = "550" # I add a suffix to output files to be able to separate comparisons of test data from the same model with different steps e.g. '350' indicates a model that has been through 350,000 steps of training
+
+# Uncomment these lines for the pytorch model
+model_type = "lstm"
+model_stub = "pytorch/lstm"
+model_version = ""
+file_step_suffix = ""
+data_sample_size = 476887
+N_EPOCHS = 10
+max_predict_len = 12000
+
+word_to_index = {}
+cat_to_idx = {}
+vocab = []
+device = "cpu"
+
+global labels_list
+labels_list = []
+
+model_dir_name = os.path.join(ROOT_DIR, "nnet_model" , model_stub , model_version)
+print(model_dir_name)
+
+model_path = os.path.join(model_dir_name, "saved_model.zip")
+print("model path: ")
+print(model_path)
+
+if os.path.exists(model_path):
+
+    os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # Better to go without GPU to avoid 'out of memory' issues
+    device = "cpu"
+        
+
+
+    ## The labels_list object defines the structure of the prediction outputs. It must be the same as what the model was originally trained on
+    
+
+    
+    ''' Load pre-trained model '''
+    
+
+
+    with zipfile.ZipFile(model_path,"r") as zip_ref:
+        zip_ref.extractall(model_dir_name)
+        
+    # if model_stub == "addr_model_out_lon":
+
+        #import tensorflow as tf
+
+        #tf.config.list_physical_devices('GPU')
+            
+        #     # Number of labels in total (+1 for the blank category)
+        #     n_labels = len(labels_list) + 1
+
+        #     # Allowable characters for the encoded representation
+        #     vocab = list(string.digits + string.ascii_lowercase + string.punctuation + string.whitespace)
+            
+        #     #print("Loading TF model")
+            
+        #     exported_model = tf.saved_model.load(model_dir_name)
+    
+        #     labels_list = [
+        #     'SaoText',  # 1
+        #     'SaoStartNumber',  # 2
+        #     'SaoStartSuffix',  # 3
+        #     'SaoEndNumber',  # 4
+        #     'SaoEndSuffix',  # 5
+        #     'PaoText',  # 6
+        #     'PaoStartNumber',  # 7
+        #     'PaoStartSuffix',  # 8
+        #     'PaoEndNumber',  # 9
+        #     'PaoEndSuffix',  # 10
+        #     'Street',  # 11
+        #     'PostTown',  # 12
+        #     'AdministrativeArea', #13
+        #     'Postcode'  # 14
+        #     ]
+     
+    if "pytorch" in model_stub:
+        
+        labels_list = [
+        'SaoText',  # 1
+        'SaoStartNumber',  # 2
+        'SaoStartSuffix',  # 3
+        'SaoEndNumber',  # 4
+        'SaoEndSuffix',  # 5
+        'PaoText',  # 6
+        'PaoStartNumber',  # 7
+        'PaoStartSuffix',  # 8
+        'PaoEndNumber',  # 9
+        'PaoEndSuffix',  # 10
+        'Street',  # 11
+        'PostTown',  # 12
+        'AdministrativeArea', #13
+        'Postcode',  # 14
+        'IGNORE'
+        ]
+        
+    #labels_list.to_csv("labels_list.csv", index = None)    
+            
+        if (model_type == "transformer") | (model_type == "gru") | (model_type == "lstm") :   
+            # Load vocab and word_to_index
+            with open(model_dir_name + "vocab.txt", "r") as f:
+                vocab = eval(f.read())
+            with open(model_dir_name + "/word_to_index.txt", "r") as f:
+                word_to_index = eval(f.read())
+            with open(model_dir_name + "/cat_to_idx.txt", "r") as f:
+                cat_to_idx = eval(f.read())
+
+            VOCAB_SIZE = len(word_to_index)
+            OUTPUT_DIM = len(cat_to_idx) + 1 # Number of classes/categories
+            EMBEDDING_DIM = 48
+            DROPOUT = 0.1
+            PAD_TOKEN = 0
+        
+        
+            if model_type == "transformer":
+                NHEAD = 4
+                NUM_ENCODER_LAYERS = 1
+    
+                exported_model = TransformerClassifier(VOCAB_SIZE, EMBEDDING_DIM, NHEAD, NUM_ENCODER_LAYERS, OUTPUT_DIM, DROPOUT, PAD_TOKEN)
+    
+            elif model_type == "gru":
+                N_LAYERS = 3
+                HIDDEN_DIM = 128
+                exported_model = TextClassifier(VOCAB_SIZE, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM, N_LAYERS, DROPOUT, PAD_TOKEN)
+    
+            elif model_type == "lstm":
+                N_LAYERS = 3
+                HIDDEN_DIM = 128
+    
+                exported_model = LSTMTextClassifier(VOCAB_SIZE, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM, N_LAYERS, DROPOUT, PAD_TOKEN)
+        
+        
+            exported_model.load_state_dict(torch.load(model_dir_name + "output_model_" + str(data_sample_size) +\
+               "_" + str(N_EPOCHS) + "_" + model_type + ".pth", map_location=torch.device('cpu')))
+            exported_model.eval()
+
+            device='cpu'
+            #device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+            exported_model.to(device)
+
+
+    else:
+        exported_model = [] #tf.keras.models.load_model(model_dir_name, compile=False)
+        # Compile the model with a loss function and an optimizer
+        #exported_model.compile(loss='categorical_crossentropy', optimizer='adam', metrics = ['categorical_crossentropy'])
+
+else: exported_model = []
+
+#if exported_model:
+#        exported_model = exported_model
+#else: exported_model = []
+
+
+
+# +
+# Address matcher will try to match <batch_size> records in one go to avoid exceeding memory limits.
+batch_size = 10000
+ref_batch_size = 150000
+
+### Fuzzy match method
+
+''' https://recordlinkage.readthedocs.io/en/latest/ref_df-compare.html#recordlinkage.compare.String
+ The Python Record Linkage Toolkit uses the jellyfish package for the Jaro, Jaro-Winkler, Levenshtein and Damerau- Levenshtein algorithms.
+ Options are [‘jaro’, ‘jarowinkler’, ‘levenshtein’, ‘damerau_levenshtein’, ‘qgram’, ‘cosine’, ‘smith_waterman’, ‘lcs’]
+
+ Comparison of some of the Jellyfish string comparison methods: https://manpages.debian.org/testing/python-jellyfish-doc/jellyfish.3.en.html '''
+
+
+fuzzy_method = "jarowinkler"
+
+# Required overall match score for all columns to count as a match
+score_cut_off = 98.7 # 97.5
+# I set a higher score cut off for nnet street blocking based on empirical data. Under this match value I was seeing errors. This value was (.99238), but set here to .995 to be maximally stringent. It is set in 'recordlinkage_funcs.py', score_based_match function
+score_cut_off_nnet_street = 99.5 # 99.238
+# If there are no numbers in the address, then the matcher needs to get a perfect score (otherwise too many issues).
+no_number_fuzzy_match_limit = 100
+
+# Reference data 'official' column names
+ref_address_cols = ["Organisation", "SaoStartNumber", "SaoStartSuffix", "SaoEndNumber", "SaoEndSuffix",
+       "SaoText", "PaoStartNumber", "PaoStartSuffix", "PaoEndNumber",
+       "PaoEndSuffix", "PaoText", "Street", "PostTown", "Postcode"]
+
+# Create a list of matching variables. Text columns will be fuzzy matched.
+matching_variables = ref_address_cols
+text_columns = ["Organisation", "PaoText", "Street", "PostTown", "Postcode"]
+
+# Modify relative importance of columns (weights) for the recordlinkage part of the match. Modify weighting for scores - Town and AdministrativeArea are not very important as we have postcode. Street number and name are important
+Organisation_weight = 0.1 # Organisation weight is very low just to resolve tie breakers for very similar addresses
+PaoStartNumber_weight = 2
+SaoStartNumber_weight = 2
+Street_weight = 2
+PostTown_weight = 0
+Postcode_weight = 0.5
+AdministrativeArea_weight = 0
+# -
+
+weight_vals = [1] * len(ref_address_cols)
+weight_keys = ref_address_cols
+weights = {weight_keys[i]: weight_vals[i] for i in range(len(weight_keys))} 
+
+# +
+# Modify weighting for scores - Town and AdministrativeArea are not very important as we have postcode. Street number and name are important
+
+weights["Organisation"] = Organisation_weight
+weights["SaoStartNumber"] = SaoStartNumber_weight
+weights["PaoStartNumber"] = PaoStartNumber_weight
+weights["Street"] = Street_weight
+weights["PostTown"] = PostTown_weight
+weights["Postcode"] = Postcode_weight
+
+# Creating Pydantic basemodel class
+
+
+class MatcherClass(BaseModel):
+    # Fuzzy/general attributes
+    fuzzy_scorer_used: str
+    fuzzy_match_limit: int
+    fuzzy_search_addr_limit: int
+    filter_to_lambeth_pcodes: bool
+    standardise: bool
+    suffix_used: str
+
+    # Neural net attributes
+    matching_variables: List[str]
+    model_dir_name: str
+    file_step_suffix: str
+    exported_model: List
+
+    fuzzy_method: str
+    score_cut_off: float
+    text_columns: List[str]
+    weights: dict
+    model_type: str
+    labels_list: List[str]
+        
+    # These are variables that are added on later
+    # Pytorch optional variables
+    word_to_index: dict
+    cat_to_idx: dict
+    device: str
+    vocab: List[str]
+        
+    # Join data
+    file_name: str
+    ref_name: str
+    search_df: pd.DataFrame
+    excluded_df: pd.DataFrame
+    pre_filter_search_df: pd.DataFrame
+    search_address_cols: List[str]
+    search_postcode_col: List[str]
+    search_df_key_field: str
+    ref_df: pd.DataFrame
+    ref_pre_filter: pd.DataFrame
+    ref_address_cols: List[str]
+    new_join_col: List[str]
+    #in_joincol_list: List[str]
+    existing_match_cols: List[str]
+    standard_llpg_format: List[str]
+        
+    # Results attributes
+    match_results_output: pd.DataFrame
+    predict_df_nnet: pd.DataFrame
+        
+    # Other attributes generated during training
+    compare_all_candidates: List[str]
+    diag_shortlist: List[str]
+    diag_best_match: List[str]
+        
+    results_on_orig_df: pd.DataFrame
+            
+    summary: str
+    output_summary: str
+    match_outputs_name: str
+    results_orig_df_name: str
+
+    search_df_after_stand: pd.DataFrame
+    ref_df_after_stand: pd.DataFrame
+    search_df_after_full_stand: pd.DataFrame
+    ref_df_after_full_stand: pd.DataFrame
+
+    search_df_after_stand_series: pd.Series
+    ref_df_after_stand_series: pd.Series
+    search_df_after_stand_series_full_stand: pd.Series
+    ref_df_after_stand_series_full_stand: pd.Series
+
+        
+    # Abort flag if the matcher couldn't even get the results of the first match
+    abort_flag: bool
+
+    # This is to allow for Pandas DataFrame types as an argument
+    class Config:
+        # Allow for custom types such as Pandas DataFrames in the class
+        arbitrary_types_allowed = True
+        extra = 'allow'
+        # Disable protected namespaces to avoid conflicts
+        protected_namespaces = ()
+
+
+
+# Creating an instance of MatcherClass
+InitMatch = MatcherClass(
+
+    # Fuzzy/general attributes
+    fuzzy_scorer_used = fuzzy_scorer_used,
+    fuzzy_match_limit = fuzzy_match_limit,
+    fuzzy_search_addr_limit = fuzzy_search_addr_limit,
+    filter_to_lambeth_pcodes = filter_to_lambeth_pcodes,
+    standardise = standardise,
+    suffix_used = suffix_used,
+
+    # Neural net attributes
+    matching_variables = matching_variables,
+    model_dir_name = model_dir_name,
+    file_step_suffix = file_step_suffix,
+
+    exported_model = [exported_model],
+
+    fuzzy_method = fuzzy_method,
+    score_cut_off = score_cut_off,
+    text_columns = text_columns,
+    weights = weights,
+    model_type = model_type,
+    labels_list = labels_list,
+    
+
+    # These are variables that are added on later
+    # Pytorch optional variables
+    word_to_index = word_to_index,
+    cat_to_idx = cat_to_idx,
+    device = device,
+    vocab = vocab,
+    
+    # Join data
+    file_name = '',
+    ref_name = '',
+    df_name = '',
+    search_df = pd.DataFrame(),
+    excluded_df = pd.DataFrame(),
+    pre_filter_search_df = pd.DataFrame(),
+    search_df_not_matched = pd.DataFrame(),
+    search_df_cleaned = pd.DataFrame(),
+    search_address_cols = [],
+    search_postcode_col = [],
+    search_df_key_field = 'index',
+
+    ref_df = pd.DataFrame(),
+    ref_df_cleaned = pd.DataFrame(),
+    ref_pre_filter = pd.DataFrame(),
+    ref_address_cols = [],
+    new_join_col = [],
+    #in_joincol_list = [],
+    existing_match_cols = [],
+    standard_llpg_format = [],
+
+    
+    # Results attributes
+    match_results_output = pd.DataFrame(),
+    predict_df_nnet = pd.DataFrame(),
+    
+    # Other attributes generated during training
+    compare_all_candidates = [],
+    diag_shortlist = [],
+    diag_best_match = [],
+    
+    results_on_orig_df = pd.DataFrame(),
+    summary = "",
+    output_summary = "",
+    
+    match_outputs_name = "",
+    results_orig_df_name = "",
+
+    # Post dataset preparation variables
+    search_df_after_stand = pd.DataFrame(),
+    ref_df_after_stand = pd.DataFrame(),
+    search_df_after_stand_series = pd.Series(),
+    ref_df_after_stand_series = pd.Series(),
+
+    search_df_after_full_stand = pd.DataFrame(),
+    ref_df_after_full_stand = pd.DataFrame(),
+    search_df_after_stand_series_full_stand = pd.Series(),
+    ref_df_after_stand_series_full_stand = pd.Series(),
+
+    # Abort flag if the matcher couldn't even get the results of the first match
+    abort_flag = False
+)

tools/fuzzy_match.py

@@ -0,0 +1,437 @@
+import pandas as pd
+import numpy as np
+from typing import Dict, List, Tuple, Type
+from datetime import datetime
+from rapidfuzz import fuzz, process
+import gradio as gr
+
+PandasDataFrame = Type[pd.DataFrame]
+PandasSeries = Type[pd.Series]
+MatchedResults = Dict[str,Tuple[str,int]]
+array = List[str]
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+from tools.constants import no_number_fuzzy_match_limit, fuzzy_match_limit
+
+def string_match_array(to_match:array, choices:array,
+                      index_name:str, matched_name:str) -> PandasDataFrame:
+    
+    temp = {name: process.extractOne(name,choices) 
+            for name in to_match}
+    
+    return _create_frame(matched_results=temp, index_name=index_name,
+                        matched_name=matched_name)
+
+# Fuzzy match algorithm
+def create_fuzzy_matched_col(df:PandasDataFrame, orig_match_address_series:PandasSeries, pred_match_address_series:PandasSeries, fuzzy_method:"WRatio", match_score=95):
+
+    results = []
+
+    for orig_index, orig_string in df[orig_match_address_series].items():
+        
+        predict_string = df[pred_match_address_series][orig_index] 
+        
+        if (orig_string == '') and (predict_string == ''):
+            results.append(np.nan)
+            
+        else:
+            fuzz_score = process.extract(orig_string, [predict_string], scorer= getattr(fuzz, fuzzy_method))
+            results.append(fuzz_score[0][1])
+
+    new_result_col_score = (orig_match_address_series + "_fuzz_score")
+    new_result_col_match = (orig_match_address_series + "_fuzz_match") 
+
+    df[new_result_col_score] = results
+    df[new_result_col_match] = df[new_result_col_score] >= match_score
+    #df[new_result_col_match][df[new_result_col_score].isna()] = np.nan
+    df.loc[df[new_result_col_score].isna(), new_result_col_match] = np.nan
+    
+    return df
+
+def string_match_by_post_code_multiple(match_address_series:PandasSeries, reference_address_series:PandasSeries,
+                              search_limit=100, scorer_name="token_set_ratio", progress=gr.Progress())-> MatchedResults:
+    '''
+    Matches by Series values; for example idx is post code and 
+    values address. Search field is reduced by comparing same post codes address reference_address_series.
+    
+    Default scorer is fuzz.Wratio. This tries to weight the different algorithms
+    to give the best score.
+    Choice of ratio type seems to make a big difference. Looking at this link:
+    https://chairnerd.seatgeek.com/fuzzywuzzy-fuzzy-string-matching-in-python/
+    and this one: 
+    https://stackoverflow.com/questions/31806695/when-to-use-which-fuzz-function-to-compare-2-strings    
+
+    '''
+
+    def do_one_match(reference_addresses: pd.Series, scorer: callable, search_limit: int, postcode_match: str, search_addresses: pd.Series) -> MatchedResults:
+
+        def _prepare_results(search_addresses, reference_addresses, matched, postcode_match):
+
+            # Create a list to store the results
+            results = []
+
+            # Iterate through the matched dataframe and store results in the list
+            for i, search_address in enumerate(search_addresses):
+                for j, reference_address in enumerate(reference_addresses):
+                    score = matched[i][j]
+                    results.append((postcode_match, search_address, reference_address, score))
+
+            # Create a dataframe from the results list
+            matched_out = pd.DataFrame(results, columns=['postcode_search', 'fuzzy_match_search_address', 'fuzzy_match_reference_address', 'fuzzy_score'])
+
+            return matched_out
+
+        try:
+            if isinstance(reference_addresses, str):  # reference_addresses can be a str-> 1 address per postcode
+                matched = process.cdist(search_addresses.values, [reference_addresses], scorer=scorer, score_cutoff=fuzzy_match_limit, workers=-1)
+
+                # Transform results into a dataframe
+                matched_out = _prepare_results(search_addresses, reference_addresses, matched, postcode_match)
+
+            else:  # 1+ addresses
+                matched = process.cdist(search_addresses.values, reference_addresses.values, scorer=scorer, score_cutoff=fuzzy_match_limit, workers=-1) 
+
+                # Transform results into a dataframe
+                matched_out = _prepare_results(search_addresses, reference_addresses, matched, postcode_match)
+
+            # Sort the matched results by score in descending order
+            matched_out = matched_out.sort_values(by='fuzzy_score', ascending=False)
+
+            # Keep only the top search_limit number of results - doesn't work anymore when working with multiple results
+            #matched_out = matched_out.head(search_limit)
+
+        except KeyError:
+            matched_out = pd.DataFrame()
+
+        return matched_out
+ 
+    def apply_fuzzy_matching(postcode_match:str, search_addresses:PandasSeries, reference_addresses:PandasSeries, scorer:callable, search_limit:int)-> tuple:
+        
+        try:
+            matched = do_one_match(reference_addresses, scorer, search_limit, postcode_match, search_addresses)
+            return matched
+        except KeyError:
+            matched = pd.DataFrame() #[("NA", 0)] # for _ in range(1, search_limit + 1)]
+            return matched
+
+    print("Fuzzy match column length: ", len(match_address_series))
+    print("Fuzzy Reference column length: ", len(reference_address_series))
+
+    match_address_series = match_address_series.rename_axis('postcode_search')
+    match_address_df = pd.DataFrame(match_address_series.reset_index())
+    match_address_df['index'] = list(range(0,len(match_address_df)))
+
+    reference_address_series = reference_address_series.rename_axis('postcode_search')
+    reference_address_df = pd.DataFrame(reference_address_series.reset_index())
+    reference_address_df['index'] = list(range(0,len(reference_address_df)))
+
+    
+    # Apply the match functions to each address
+    scorer = getattr(fuzz, scorer_name)                  
+    results = {}
+    #counter = 0
+
+    index_list = []
+    match_list = []
+    search_addresses_list = []
+    reference_addresses_list = []
+
+    unique_postcodes = pd.unique(match_address_df['postcode_search'])
+
+    for postcode_match in progress.tqdm(unique_postcodes, desc="Fuzzy matching", unit="fuzzy matched postcodes"):
+
+        postcode_match_list = [postcode_match]
+        search_indexes = pd.Series()
+        search_addresses = pd.Series()
+        reference_addresses = pd.Series()
+
+        try:
+            search_indexes = match_address_df.loc[match_address_df["postcode_search"].isin(postcode_match_list), "index"]
+            search_addresses = match_address_df.loc[match_address_df["postcode_search"].isin(postcode_match_list), "search_address_stand"]
+            reference_addresses = reference_address_df.loc[reference_address_df["postcode_search"].isin(postcode_match_list), "ref_address_stand"]
+
+            if isinstance(reference_addresses, str):  # reference_addresses can be a str-> 1 address per postcode
+                reference_addresses = pd.Series(reference_addresses)
+        except KeyError:
+            reference_addresses = pd.Series("NA")
+
+        matched = apply_fuzzy_matching(postcode_match, search_addresses, reference_addresses, scorer, search_limit)
+
+        # Write to output lists
+        match_list.extend([matched])
+        index_list.extend(search_indexes.tolist())
+        search_addresses_list.extend(search_addresses.tolist())
+        reference_addresses_list.extend(reference_addresses.tolist())
+
+    out_frame = pd.concat(match_list)
+
+    return out_frame
+
+def _create_fuzzy_match_results_output(results, search_df_after_stand, ref_df_cleaned, ref_df_after_stand, fuzzy_match_limit, search_df_cleaned, search_df_key_field, new_join_col, standardise, blocker_col):
+
+        ## Diagnostics
+
+        diag_shortlist, diag_best_match = refine_export_results(results_df=results,\
+                                      matched_df = search_df_after_stand, ref_list_df = ref_df_after_stand,
+                                      fuzzy_match_limit = fuzzy_match_limit, blocker_col=blocker_col)
+        
+        ## Fuzzy search results
+
+        match_results_cols = ['search_orig_address','reference_orig_address', 'ref_index',
+        'full_match',
+        'full_number_match',
+        'flat_number_match',
+        'room_number_match',
+        'block_number_match',
+        'unit_number_match',
+        'property_number_match',
+        'close_postcode_match',
+        'house_court_name_match',
+        'fuzzy_score_match',
+        "fuzzy_score",
+        "wratio_score",
+        'property_number_search', 'property_number_reference',  
+        'flat_number_search', 'flat_number_reference', 
+        'room_number_search', 'room_number_reference',
+        'unit_number_search', 'unit_number_reference',
+        'block_number_search', 'block_number_reference',
+        'house_court_name_search', 'house_court_name_reference',
+        "search_mod_address", 'reference_mod_address','Postcode']
+
+        # Join results data onto the original housing list to create the full output
+        search_df_cleaned_join_cols = [search_df_key_field, "full_address","postcode"]
+
+        match_results_output = search_df_cleaned[search_df_cleaned_join_cols].merge(
+            diag_best_match[match_results_cols], how = "left", left_on = "full_address", right_on = "search_orig_address")
+        
+        match_results_output = match_results_output.drop(["postcode", "search_orig_address"], axis = 1).rename(columns={"full_address":"search_orig_address"})
+        
+        # Join UPRN back onto the data from reference data
+        joined_ref_cols = ["fulladdress", "Reference file"]
+        joined_ref_cols.extend(new_join_col)
+
+        match_results_output = pd.merge(match_results_output,ref_df_cleaned[joined_ref_cols].drop_duplicates("fulladdress"), how = "left", left_on = "reference_orig_address",right_on = "fulladdress").drop("fulladdress", axis = 1)
+
+        # Convert long keys to string to avoid data loss
+        match_results_output[search_df_key_field] = match_results_output[search_df_key_field].astype("str")
+        match_results_output[new_join_col] = match_results_output[new_join_col].astype("string")
+        match_results_output["standardised_address"] = standardise
+    
+        match_results_output = match_results_output.sort_values(search_df_key_field, ascending = True)
+                
+        return match_results_output, diag_shortlist, diag_best_match
+
+def create_diag_shortlist(results_df, matched_col, fuzzy_match_limit, blocker_col, fuzzy_col="fuzzy_score", search_mod_address = "search_mod_address", resolve_tie_breaks=True, no_number_fuzzy_match_limit=no_number_fuzzy_match_limit):
+    '''
+    Create a shortlist of the best matches from a list of suggested matches
+    '''
+
+    ## Calculate highest fuzzy score from all candidates, keep all candidates with matching highest fuzzy score
+    results_max_fuzzy_score = results_df.groupby(matched_col)[fuzzy_col].max().reset_index().rename(columns={fuzzy_col: "max_fuzzy_score"}).drop_duplicates(subset=matched_col)
+
+    results_df = pd.merge(results_df, results_max_fuzzy_score, how = "left", on = matched_col)
+
+    diag_shortlist = results_df[(results_df[fuzzy_col] == results_df["max_fuzzy_score"])]
+
+    # Fuzzy match limit for records with no numbers in it is 0.95 or the provided fuzzy_match_limit, whichever is higher
+    #diag_shortlist["fuzzy_score_match"] = diag_shortlist[fuzzy_col] >= fuzzy_match_limit
+    diag_shortlist.loc[diag_shortlist[fuzzy_col] >= fuzzy_match_limit, "fuzzy_score_match"] = True
+
+    ### Count number of numbers in search string
+    # Using .loc
+    diag_shortlist.loc[:, "number_count_search_string"] = diag_shortlist.loc[:, search_mod_address].str.count(r'\d')
+    diag_shortlist.loc[:, "no_numbers_in_search_string"] = (diag_shortlist.loc[:, "number_count_search_string"] == 0)
+
+
+    # Replace fuzzy_score_match values for addresses with no numbers in them
+    diag_shortlist.loc[(diag_shortlist["no_numbers_in_search_string"]==True) & (diag_shortlist[fuzzy_col] >= no_number_fuzzy_match_limit), "fuzzy_score_match"] = True
+    diag_shortlist.loc[(diag_shortlist["no_numbers_in_search_string"]==True) & (diag_shortlist[fuzzy_col] < no_number_fuzzy_match_limit), "fuzzy_score_match"] = False
+
+    # If blocking on street, don't match addresses with 0 numbers in. There are too many options and the matches are rarely good
+    if blocker_col == "Street":
+        diag_shortlist.loc[(diag_shortlist["no_numbers_in_search_string"]==True), "fuzzy_score_match"] = False
+                            
+    diag_shortlist = diag_shortlist.fillna("").infer_objects(copy=False).drop(["number_count_search_string", "no_numbers_in_search_string"], axis = 1)
+
+    # Following considers full matches to be those that match on property number and flat number, and the postcode is relatively close.
+    #print(diag_shortlist.columns) 
+    diag_shortlist["property_number_match"] = (diag_shortlist["property_number_search"] == diag_shortlist["property_number_reference"])
+    diag_shortlist["flat_number_match"] = (diag_shortlist['flat_number_search'] == diag_shortlist['flat_number_reference'])
+    diag_shortlist["room_number_match"] = (diag_shortlist['room_number_search'] == diag_shortlist['room_number_reference'])
+    diag_shortlist["block_number_match"] = (diag_shortlist['block_number_search'] == diag_shortlist['block_number_reference'])
+    diag_shortlist["unit_number_match"] = (diag_shortlist['unit_number_search'] == diag_shortlist['unit_number_reference'])
+    diag_shortlist["house_court_name_match"] = (diag_shortlist['house_court_name_search'] == diag_shortlist['house_court_name_reference'])
+
+    # Full number match is currently considered only a match between property number and flat number
+                            
+    diag_shortlist['full_number_match'] = (diag_shortlist["property_number_match"] == True) &\
+        (diag_shortlist["flat_number_match"] == True) &\
+        (diag_shortlist["room_number_match"] == True) &\
+        (diag_shortlist["block_number_match"] == True) &\
+        (diag_shortlist["unit_number_match"] == True) &\
+        (diag_shortlist["house_court_name_match"] == True)
+
+    
+    ### Postcodes need to be close together, so all the characters should match apart from the last two 
+    diag_shortlist['close_postcode_match'] = diag_shortlist['postcode'].str.lower().str.replace(" ","").str[:-2] == diag_shortlist['Postcode'].str.lower().str.replace(" ","").str[:-2]
+        
+    
+    diag_shortlist["full_match"] = (diag_shortlist["fuzzy_score_match"] == True) &\
+        (diag_shortlist['full_number_match'] == True) &\
+        (diag_shortlist['close_postcode_match'] == True)
+    
+    diag_shortlist = diag_shortlist.rename(columns = {"reference_list_address":"reference_mod_address"})
+
+    ### Dealing with tie breaks ##
+    # Do a backup simple Wratio search on the open text to act as a tie breaker when the fuzzy scores are identical
+    # fuzz.WRatio
+    if resolve_tie_breaks == True:
+        def compare_strings_wratio(row, scorer = fuzz.ratio, fuzzy_col = fuzzy_col):
+            search_score = process.cdist([row[search_mod_address]], [row["reference_mod_address"]], scorer=scorer)
+            return search_score[0][0]
+
+        diag_shortlist_dups = diag_shortlist[diag_shortlist['full_number_match'] == True]
+        diag_shortlist_dups = diag_shortlist_dups.loc[diag_shortlist_dups.duplicated(subset= [search_mod_address, 'full_number_match', "room_number_search", fuzzy_col], keep=False)]
+
+        if not diag_shortlist_dups.empty:
+            diag_shortlist_dups["wratio_score"] = diag_shortlist_dups.apply(compare_strings_wratio, axis=1)
+                                
+            diag_shortlist = diag_shortlist.merge(diag_shortlist_dups[["wratio_score"]], left_index=True, right_index=True, how = "left")
+
+    if 'wratio_score' not in diag_shortlist.columns:
+        diag_shortlist['wratio_score'] = '' 
+
+    # Order by best score
+    diag_shortlist = diag_shortlist.sort_values([
+        search_mod_address, 'full_match', 'full_number_match', fuzzy_col, "wratio_score"],
+        ascending = [True, False, False, False, False])          
+
+    return diag_shortlist
+
+def refine_export_results(results_df:PandasDataFrame, 
+                           matched_df:PandasDataFrame,
+                           ref_list_df:PandasDataFrame,
+                           matched_col="fuzzy_match_search_address",
+                           ref_list_col="fuzzy_match_reference_address",
+                           final_matched_address_col="search_address_stand",
+                           final_ref_address_col="ref_address_stand",
+                           orig_matched_address_col = "full_address",
+                           orig_ref_address_col = "fulladdress",
+                           fuzzy_match_limit=fuzzy_match_limit,
+                           blocker_col="Postcode") -> PandasDataFrame:
+    '''
+    This function takes a result file from the fuzzy search, then refines the 'matched results' according
+    the score limit specified by the user and exports results list, matched and unmatched files.
+    '''
+       
+    # Rename score column
+    results_df = results_df.rename(columns = {"score":"fuzzy_score"})
+          
+    # Remove empty addresses
+    results_df = results_df[results_df[matched_col] !=0 ]
+
+    ### Join property number and flat/room number etc. onto results_df
+    ref_list_df["ref_index"] = ref_list_df.index
+    ref_join_cols = ["ref_index", final_ref_address_col, "property_number","flat_number","room_number","block_number", "unit_number", 'house_court_name', orig_ref_address_col,"Postcode"]
+    ref_list_df = ref_list_df[ref_join_cols].rename(columns={orig_ref_address_col: "reference_orig_address", final_ref_address_col:'reference_list_address'})
+
+    results_df = results_df.merge(ref_list_df, how = "left", left_on = ref_list_col, right_on = "reference_list_address")
+
+
+    ### Join on relevant details from the standardised match dataframe
+    matched_df_cols = [final_matched_address_col,"property_number","flat_number","room_number", "block_number", "unit_number", 'house_court_name', orig_matched_address_col, "postcode"]
+    matched_df = matched_df[matched_df_cols].rename(columns={orig_matched_address_col:"search_orig_address",final_matched_address_col:'search_mod_address'})
+    
+    results_df = results_df.merge(matched_df, how = "left", left_on = matched_col, right_on = "search_mod_address", suffixes=("_reference", "_search"))
+    
+    # Choose your best matches from the list of options
+    diag_shortlist = create_diag_shortlist(results_df, matched_col, fuzzy_match_limit, blocker_col)
+
+    ### Create matched results output ###
+    # Columns for the output match_results file in order
+    match_results_cols = ['search_orig_address','reference_orig_address', 'ref_index',
+        'full_match',
+        'full_number_match',
+        'flat_number_match',
+        'room_number_match',
+        'block_number_match',
+        'unit_number_match',
+        'house_court_name_match',
+        'property_number_match',
+        'close_postcode_match',
+        'fuzzy_score_match',
+        "fuzzy_score",
+        "wratio_score",
+        'property_number_search', 'property_number_reference',  
+        'flat_number_search', 'flat_number_reference', 
+        'room_number_search', 'room_number_reference',
+        'block_number_search', 'block_number_reference',
+        'unit_number_search', 'unit_number_reference',
+        'house_court_name_search', 'house_court_name_reference',
+        "search_mod_address", 'reference_mod_address', 'postcode','Postcode']
+
+    diag_shortlist = diag_shortlist[match_results_cols]
+
+    # Choose best match from the shortlist that has been ordered according to score descending
+    diag_best_match = diag_shortlist[match_results_cols].drop_duplicates("search_mod_address")
+   
+    return diag_shortlist, diag_best_match
+
+def join_to_orig_df(match_results_output:PandasDataFrame, search_df:PandasDataFrame, search_df_key_field:str, new_join_col:List[str]) -> PandasDataFrame:
+    ''' 
+    Following the fuzzy match, join the match results back to the original search dataframe to create a results dataframe.
+    '''
+    match_results_output_success = match_results_output[match_results_output["full_match"]==True]
+
+    # If you're joining to the original df on index you will need to recreate the index again 
+
+    match_results_output_success = match_results_output_success.rename(columns={
+                                        "reference_orig_address":"Reference matched address",
+                                        "full_match":"Matched with reference address",
+                                        'uprn':'UPRN'                                                                             
+                                     })
+    
+    ref_df_after_stand_cols = ["ref_index", "Reference matched address","Matched with reference address", "Reference file", search_df_key_field]
+    ref_df_after_stand_cols.extend(new_join_col)
+ 
+    
+    if (search_df_key_field == "index"):
+        # Check index is int
+        print("Search df key field is index")
+        #match_results_output_success[search_df_key_field] = match_results_output_success[search_df_key_field].astype(float).astype(int)     
+        results_for_orig_df_join = search_df.merge(match_results_output_success[ref_df_after_stand_cols], on = search_df_key_field, how = "left", suffixes = ('', '_y'))  
+    else:
+        results_for_orig_df_join = search_df.merge(match_results_output_success[ref_df_after_stand_cols],how = "left", on = search_df_key_field, suffixes = ('', '_y'))
+
+    # If the join columns already exist in the search_df, then use the new column to fill in the NAs in the original column, then delete the new column
+
+    if "Reference matched address_y" in results_for_orig_df_join.columns: 
+        results_for_orig_df_join['Reference matched address'] = results_for_orig_df_join['Reference matched address'].fillna(results_for_orig_df_join['Reference matched address_y']).infer_objects(copy=False)
+
+    if "Matched with reference address_y" in results_for_orig_df_join.columns: 
+        results_for_orig_df_join['Matched with reference address'] = pd.Series(np.where(results_for_orig_df_join['Matched with reference address_y'].notna(), results_for_orig_df_join['Matched with reference address_y'], results_for_orig_df_join['Matched with reference address']))
+
+        #results_for_orig_df_join['Matched with reference address'] = results_for_orig_df_join['Matched with reference address'].fillna(results_for_orig_df_join['Matched with reference address_y']).infer_objects(copy=False)
+
+    if "Reference file_y" in results_for_orig_df_join.columns: 
+        results_for_orig_df_join['Reference file'] = results_for_orig_df_join['Reference file'].fillna(results_for_orig_df_join['Reference file_y']).infer_objects(copy=False)
+
+    if "UPRN_y" in results_for_orig_df_join.columns: 
+        results_for_orig_df_join['UPRN'] = results_for_orig_df_join['UPRN'].fillna(results_for_orig_df_join['UPRN_y']).infer_objects(copy=False)
+
+    # Drop columns that aren't useful
+    results_for_orig_df_join = results_for_orig_df_join.drop(['Reference matched address_y', 'Matched with reference address_y', 'Reference file_y', 'search_df_key_field_y', 'UPRN_y', 'index_y', "full_address_search","postcode_search", "full_address_1", "full_address_2", "full_address",
+                                   "address_stand", "property_number","prop_number" "flat_number" "apart_number" "first_sec_number" "room_number"], axis = 1, errors = "ignore")
+
+    # Replace blanks with NA, fix UPRNs
+    results_for_orig_df_join = results_for_orig_df_join.replace(r'^\s*$', np.nan, regex=True)   
+
+    results_for_orig_df_join[new_join_col] = results_for_orig_df_join[new_join_col].astype(str).replace(".0","", regex=False).replace("nan","", regex=False)
+    
+    # Replace cells with only 'nan' with blank
+    results_for_orig_df_join = results_for_orig_df_join.replace(r'^nan$', "", regex=True)
+  
+    
+    return results_for_orig_df_join

tools/gradio.py

@@ -0,0 +1,63 @@
+import gradio as gr
+import pandas as pd
+
+def detect_file_type(filename):
+    """Detect the file type based on its extension."""
+    if (filename.endswith('.csv')) | (filename.endswith('.csv.gz')) | (filename.endswith('.zip')):
+        return 'csv'
+    elif filename.endswith('.xlsx'):
+        return 'xlsx'
+    elif filename.endswith('.parquet'):
+        return 'parquet'
+    else:
+        raise ValueError("Unsupported file type.")
+
+def read_file(filename):
+    """Read the file based on its detected type."""
+    file_type = detect_file_type(filename)
+    
+    if file_type == 'csv':
+        return pd.read_csv(filename, low_memory=False)
+    elif file_type == 'xlsx':
+        return pd.read_excel(filename)
+    elif file_type == 'parquet':
+        return pd.read_parquet(filename)
+
+
+def initial_data_load(in_file):
+    new_choices = []
+    concat_choices = []
+    output_message = ""
+    results_df = pd.DataFrame()
+    df = pd.DataFrame()
+    
+    file_list = [string.name for string in in_file]
+
+    data_file_names = [string for string in file_list if "results_on_orig" not in string.lower()]
+    if data_file_names:
+        df = read_file(data_file_names[0])
+    else:
+        error_message = "No data file found."
+        return error_message, gr.Dropdown(choices=concat_choices), gr.Dropdown(choices=concat_choices), df, results_df
+
+    results_file_names = [string for string in file_list if "results_on_orig" in string.lower()]
+    if results_file_names:
+        results_df = read_file(results_file_names[0])
+    
+    new_choices = list(df.columns)
+    concat_choices.extend(new_choices)
+
+    output_message = "Data successfully loaded"  
+        
+    return output_message, gr.Dropdown(choices=concat_choices), gr.Dropdown(choices=concat_choices), df, results_df
+
+
+def dummy_function(in_colnames):
+    """
+    A dummy function that exists just so that dropdown updates work correctly.
+    """
+    return None    
+
+
+def clear_inputs(in_file, in_ref, in_text):
+    return gr.File.update(value=[]), gr.File.update(value=[]), gr.Textbox.update(value='')

tools/matcher_funcs.py

@@ -0,0 +1,1300 @@
+import os
+import numpy as np
+import pandas as pd
+
+from typing import Dict, List, Tuple, Type
+import time
+import re
+import math
+from datetime import datetime
+import copy
+import gradio as gr
+
+PandasDataFrame = Type[pd.DataFrame]
+PandasSeries = Type[pd.Series]
+MatchedResults = Dict[str,Tuple[str,int]]
+array = List[str]
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+today_month_rev = datetime.now().strftime("%Y%m")
+
+# Constants
+run_fuzzy_match = True
+run_nnet_match = True
+run_standardise = True
+
+from tools.preparation import prepare_search_address_string, prepare_search_address,  prepare_ref_address, check_no_number_addresses, extract_street_name, remove_non_postal 
+from tools.standardise import standardise_wrapper_func
+from tools.fuzzy_match import string_match_by_post_code_multiple, _create_fuzzy_match_results_output, join_to_orig_df
+
+# Neural network functions
+### Predict function for imported model
+from tools.model_predict import full_predict_func, full_predict_torch, post_predict_clean
+from tools.recordlinkage_funcs import score_based_match, check_matches_against_fuzzy
+from tools.gradio import initial_data_load
+
+# API functions
+from tools.addressbase_api_funcs import places_api_query
+
+# Maximum number of neural net predictions in a single batch
+from tools.constants import max_predict_len, MatcherClass
+
+
+# Load in data functions
+
+def detect_file_type(filename):
+    """Detect the file type based on its extension."""
+    if (filename.endswith('.csv')) | (filename.endswith('.csv.gz')) | (filename.endswith('.zip')):
+        return 'csv'
+    elif filename.endswith('.xlsx'):
+        return 'xlsx'
+    elif filename.endswith('.parquet'):
+        return 'parquet'
+    else:
+        raise ValueError("Unsupported file type.")
+
+def read_file(filename):
+    """Read the file based on its detected type."""
+    file_type = detect_file_type(filename)
+    
+    if file_type == 'csv':
+        return pd.read_csv(filename, low_memory=False)
+    elif file_type == 'xlsx':
+        return pd.read_excel(filename)
+    elif file_type == 'parquet':
+        return pd.read_parquet(filename)
+
+def get_file_name(in_name):
+    # Corrected regex pattern
+    match = re.search(r'\\(?!.*\\)(.*)', in_name)
+    if match:
+        matched_result = match.group(1)
+    else:
+        matched_result = None
+    
+    return matched_result
+
+def filter_not_matched(
+    matched_results: pd.DataFrame, 
+    search_df: pd.DataFrame, 
+    key_col: str
+) -> pd.DataFrame:
+    """Filters search_df to only rows with key_col not in matched_results"""
+    
+    # Validate inputs
+    if not isinstance(matched_results, pd.DataFrame):
+        raise TypeError("not_matched_results must be a Pandas DataFrame")
+        
+    if not isinstance(search_df, pd.DataFrame):
+        raise TypeError("search_df must be a Pandas DataFrame")
+        
+    if not isinstance(key_col, str):
+        raise TypeError("key_col must be a string")
+        
+    if key_col not in matched_results.columns:
+        raise ValueError(f"{key_col} not a column in matched_results")
+        
+    matched_results_success = matched_results[matched_results["full_match"]==True]
+
+    # Filter search_df
+    #print(search_df.columns)
+    #print(key_col)
+    
+    matched = search_df[key_col].astype(str).isin(matched_results_success[key_col].astype(str))#.drop(['level_0','index'], axis = 1, errors="ignore").reset_index() # 
+    
+    return search_df.iloc[np.where(~matched)[0]] # search_df[~matched] 
+
+def run_all_api_calls(in_api_key:str, Matcher:MatcherClass, query_type:str, progress=gr.Progress()):
+    if in_api_key == "":
+        print ("No API key provided, please provide one to continue")
+        return Matcher
+    else:
+        # Call the API
+        #Matcher.ref_df = pd.DataFrame()
+
+        # Check if the ref_df file already exists
+        def check_and_create_api_folder():
+            # Check if the environmental variable is available
+            file_path = os.environ.get('ADDRESSBASE_API_OUT')  # Replace 'YOUR_ENV_VARIABLE_NAME' with the name of your environmental variable
+
+            if file_path is None:
+                # Environmental variable is not set
+                print("API output environmental variable not set.")
+                # Create the 'api/' folder if it doesn't already exist
+                api_folder_path = 'api/'
+                if not os.path.exists(api_folder_path):
+                    os.makedirs(api_folder_path)
+                print(f"'{api_folder_path}' folder created.")
+            else:
+                # Environmental variable is set
+                api_folder_path = file_path
+                print(f"Environmental variable found: {api_folder_path}")
+            
+            return api_folder_path
+
+        api_output_folder = check_and_create_api_folder()
+
+        # Check if the file exists
+        print("Matcher file name: ", Matcher.file_name)
+        search_file_name_without_extension = re.sub(r'\.[^.]+$', '', Matcher.file_name)
+        #print("Search file name without extension: ", search_file_name_without_extension)
+        api_ref_save_loc = api_output_folder + search_file_name_without_extension + "_api_" + today_month_rev + "_" + query_type + "_ckpt"
+        print("API reference save location: ", api_ref_save_loc)
+
+        # Allow for csv, parquet and gzipped csv files
+        if os.path.isfile(api_ref_save_loc + ".csv"):
+            print("API reference CSV file found")
+            Matcher.ref_df = pd.read_csv(api_ref_save_loc + ".csv")
+        elif os.path.isfile(api_ref_save_loc + ".parquet"):
+            print("API reference Parquet file found")
+            Matcher.ref_df = pd.read_parquet(api_ref_save_loc + ".parquet")
+        elif os.path.isfile(api_ref_save_loc + ".csv.gz"):
+            print("API reference gzipped CSV file found")
+            Matcher.ref_df = pd.read_csv(api_ref_save_loc + ".csv.gz", compression='gzip')
+        else:
+            print("API reference file not found, querying API for reference data.")
+
+
+        def conduct_api_loop(in_query, in_api_key, query_type, i, api_ref_save_loc, loop_list, api_search_index):
+            ref_addresses = places_api_query(in_query, in_api_key, query_type)
+
+            ref_addresses['Address_row_number'] = api_search_index[i]
+        
+            loop_list.append(ref_addresses)
+
+            if (i + 1) % 500 == 0:
+                print("Saving api call checkpoint for query:", str(i + 1))
+                
+                pd.concat(loop_list).to_parquet(api_ref_save_loc + ".parquet", index=False)
+
+            return loop_list
+        
+        def check_postcode(postcode):
+            # Remove spaces on the ends or in the middle of the postcode, and any symbols
+            cleaned_postcode = re.sub(r'[^\w\s]|[\s]', '', postcode)
+            # Ensure that the postcode meets the specified format
+            postcode_pattern = r'\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?[0-9][A-Z]{2}|GIR0AA|GIR0A{2}|[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?[0-9]{1}?)\b'
+            match = re.match(postcode_pattern, cleaned_postcode)
+            if match and len(cleaned_postcode) in (6, 7):
+                return cleaned_postcode  # Return the matched postcode string
+            else:
+                return None  # Return None if no match is found
+
+        if query_type == "Address":
+            save_file = True
+            # Do an API call for each unique address
+
+            if not Matcher.ref_df.empty:
+                api_search_df = Matcher.search_df.copy().drop(list(set(Matcher.ref_df["Address_row_number"])))
+
+            else:
+                print("Matcher ref_df data empty")
+                api_search_df = Matcher.search_df.copy()
+            
+            i = 0
+            loop_df = Matcher.ref_df
+            loop_list = [Matcher.ref_df]
+
+            for address in progress.tqdm(api_search_df['full_address_postcode'], desc= "Making API calls", unit="addresses", total=len(api_search_df['full_address_postcode'])):
+                print("Query number: " + str(i+1), "with address: ", address)
+
+                api_search_index = api_search_df.index
+
+                loop_list = conduct_api_loop(address, in_api_key, query_type, i, api_ref_save_loc, loop_list, api_search_index)
+
+                i += 1
+
+            loop_df = pd.concat(loop_list)
+            Matcher.ref_df = loop_df.drop_duplicates(keep='first', ignore_index=True)
+        
+
+        elif query_type == "Postcode":
+            save_file = True
+            # Do an API call for each unique postcode. Each API call can only return 100 results maximum :/
+
+            if not Matcher.ref_df.empty:
+                print("Excluding postcodes that already exist in API call data.")
+
+                # Retain original index values after filtering
+                Matcher.search_df["index_keep"] = Matcher.search_df.index
+                
+                if 'invalid_request' in Matcher.ref_df.columns and 'Address_row_number' in Matcher.ref_df.columns:
+                    print("Joining on invalid_request column")
+                    Matcher.search_df = Matcher.search_df.merge(Matcher.ref_df[['Address_row_number', 'invalid_request']].drop_duplicates(subset="Address_row_number"), left_on = Matcher.search_df_key_field, right_on='Address_row_number', how='left')
+
+                elif not 'invalid_request' in Matcher.search_df.columns:
+                    Matcher.search_df['invalid_request'] = False
+
+                postcode_col = Matcher.search_postcode_col[0]
+                
+                # Check ref_df df against cleaned and non-cleaned postcodes
+                Matcher.search_df[postcode_col] = Matcher.search_df[postcode_col].astype(str)
+                search_df_cleaned_pcodes = Matcher.search_df[postcode_col].apply(check_postcode)
+                ref_df_cleaned_pcodes = Matcher.ref_df['POSTCODE_LOCATOR'].dropna().apply(check_postcode)
+
+                api_search_df = Matcher.search_df.copy().loc[
+                    ~Matcher.search_df[postcode_col].isin(Matcher.ref_df['POSTCODE_LOCATOR']) &
+                    ~(Matcher.search_df['invalid_request']==True) &
+                    ~(search_df_cleaned_pcodes.isin(ref_df_cleaned_pcodes)), :]
+                
+                #api_search_index = api_search_df["index_keep"]
+                #api_search_df.index = api_search_index
+                
+                print("Remaining invalid request count: ", Matcher.search_df['invalid_request'].value_counts())
+                
+            else:
+                print("Matcher ref_df data empty")
+                api_search_df = Matcher.search_df.copy()
+                api_search_index = api_search_df.index
+                api_search_df['index_keep'] = api_search_index
+
+                postcode_col = Matcher.search_postcode_col[0]
+
+            unique_pcodes = api_search_df.loc[:, ["index_keep", postcode_col]].drop_duplicates(subset=[postcode_col], keep='first')
+            print("Unique postcodes: ", unique_pcodes[postcode_col])
+
+            # Apply the function to each postcode in the Series
+            unique_pcodes["cleaned_unique_postcodes"] = unique_pcodes[postcode_col].apply(check_postcode)
+
+            # Filter out the postcodes that comply with the specified format
+            valid_unique_postcodes = unique_pcodes.dropna(subset=["cleaned_unique_postcodes"])
+
+            valid_postcode_search_index = valid_unique_postcodes['index_keep']
+            valid_postcode_search_index_list = valid_postcode_search_index.tolist()
+
+            if not valid_unique_postcodes.empty:
+
+                print("Unique valid postcodes: ", valid_unique_postcodes)
+                print("Number of unique valid postcodes: ", len(valid_unique_postcodes))
+
+                tic = time.perf_counter()
+
+                i = 0
+                loop_df = Matcher.ref_df
+                loop_list = [Matcher.ref_df]
+
+                for pcode in progress.tqdm(valid_unique_postcodes["cleaned_unique_postcodes"], desc= "Making API calls", unit="unique postcodes", total=len(valid_unique_postcodes["cleaned_unique_postcodes"])):
+                    #api_search_index = api_search_df.index
+                    
+                    print("Query number: " + str(i+1), " with postcode: ", pcode, " and index: ", valid_postcode_search_index_list[i])
+
+                    loop_list = conduct_api_loop(pcode, in_api_key, query_type, i, api_ref_save_loc, loop_list, valid_postcode_search_index_list)
+                    
+                    i += 1
+                    
+                loop_df = pd.concat(loop_list)
+                Matcher.ref_df = loop_df.drop_duplicates(keep='first', ignore_index=True)
+
+                toc = time.perf_counter()
+                print("API call time in seconds: ", toc-tic)
+            else:
+                print("No valid postcodes found.")
+
+        elif query_type == "UPRN":
+            save_file = True
+            # Do an API call for each unique address
+
+            if not Matcher.ref_df.empty:
+                api_search_df = Matcher.search_df.copy().drop(list(set(Matcher.ref_df["Address_row_number"])))
+
+            else:
+                print("Matcher ref_df data empty")
+                api_search_df = Matcher.search_df.copy()
+            
+            i = 0
+            loop_df = Matcher.ref_df
+            loop_list = [Matcher.ref_df]
+            uprn_check_col = 'ADR_UPRN'
+
+            for uprn in progress.tqdm(api_search_df[uprn_check_col], desc= "Making API calls", unit="UPRNs", total=len(api_search_df[uprn_check_col])):
+                print("Query number: " + str(i+1), "with address: ", uprn)
+
+                api_search_index = api_search_df.index
+
+                loop_list = conduct_api_loop(uprn, in_api_key, query_type, i, api_ref_save_loc, loop_list, api_search_index)
+
+                i += 1
+
+            loop_df = pd.concat(loop_list)
+            Matcher.ref_df = loop_df.drop_duplicates(keep='first', ignore_index=True)
+
+        else:
+            print("Reference file loaded from file, no API calls made.")
+            save_file = False
+
+        # Post API call processing
+
+        Matcher.ref_name = "API"
+        #Matcher.ref_df = Matcher.ref_df.reset_index(drop=True)
+        Matcher.ref_df['Reference file'] = Matcher.ref_name
+
+        if query_type == "Postcode":
+            #print(Matcher.ref_df.columns)
+
+            cols_of_interest = ["ADDRESS",	"ORGANISATION",	"SAO_TEXT", "SAO_START_NUMBER", "SAO_START_SUFFIX", "SAO_END_NUMBER", "SAO_END_SUFFIX", "PAO_TEXT",	"PAO_START_NUMBER", "PAO_START_SUFFIX", "PAO_END_NUMBER", "PAO_END_SUFFIX", "STREET_DESCRIPTION", "TOWN_NAME"	,"ADMINISTRATIVE_AREA", "LOCALITY_NAME", "POSTCODE_LOCATOR", "UPRN", "PARENT_UPRN",	"USRN",	"LPI_KEY",	"RPC",	"LOGICAL_STATUS_CODE",	"CLASSIFICATION_CODE",	"LOCAL_CUSTODIAN_CODE",	"COUNTRY_CODE",	"POSTAL_ADDRESS_CODE",	"BLPU_STATE_CODE",	"LAST_UPDATE_DATE",	"ENTRY_DATE",	"STREET_STATE_CODE",	"STREET_CLASSIFICATION_CODE",	"LPI_LOGICAL_STATUS_CODE", "invalid_request",	"Address_row_number",	"Reference file"]
+
+            try:
+                # Attempt to select only the columns of interest
+                Matcher.ref_df = Matcher.ref_df[cols_of_interest]
+            except KeyError as e:
+                missing_columns = [col for col in e.args[0][1:-1].split(", ") if col not in cols_of_interest]
+                # Handle the missing columns gracefully
+                print(f"Some columns are missing: {missing_columns}")
+
+            #if "LOCAL_CUSTODIAN_CODE" in Matcher.ref_df.columns:
+                # These are items that are 'owned' by Ordnance Survey like telephone boxes, bus shelters
+                # Matcher.ref_df = Matcher.ref_df.loc[Matcher.ref_df["LOCAL_CUSTODIAN_CODE"] != 7655,:] 
+
+        if save_file:
+            print("Saving reference file to: " + api_ref_save_loc[:-5] + ".parquet")
+            Matcher.ref_df.to_parquet(api_ref_save_loc + ".parquet", index=False) # Save checkpoint as well
+            Matcher.ref_df.to_parquet(api_ref_save_loc[:-5] + ".parquet", index=False)
+
+        if Matcher.ref_df.empty:
+            print ("No reference data found with API")
+            return Matcher
+                        
+    return Matcher
+
+def check_ref_data_exists(Matcher:MatcherClass, ref_data_state:PandasDataFrame, in_ref:List[str], in_refcol:List[str], in_api:List[str], in_api_key:str, query_type:str, progress=gr.Progress()):
+        '''
+        Check for reference address data, do some preprocessing, and load in from the Addressbase API if required.
+        '''
+        
+        # Check if reference data loaded, bring in if already there
+        if not ref_data_state.empty:
+            Matcher.ref_df = ref_data_state
+            Matcher.ref_name = get_file_name(in_ref[0].name)
+            Matcher.ref_df["Reference file"] = Matcher.ref_name
+
+        # Otherwise check for file name and load in. If nothing found, fail
+        else:
+            Matcher.ref_df = pd.DataFrame()
+            
+            if not in_ref:
+                if in_api==False:
+                    print ("No reference file provided, please provide one to continue")
+                    return Matcher
+                # Check if api call required and api key is provided
+                else:
+                    Matcher = run_all_api_calls(in_api_key, Matcher, query_type)
+
+            else:
+                Matcher.ref_name = get_file_name(in_ref[0].name)
+
+                # Concatenate all in reference files together
+                for ref_file in in_ref:
+                    #print(ref_file.name)
+                    temp_ref_file = read_file(ref_file.name) 
+
+                    file_name_out = get_file_name(ref_file.name)
+                    temp_ref_file["Reference file"] = file_name_out
+                    
+                    Matcher.ref_df = pd.concat([Matcher.ref_df, temp_ref_file])              
+
+        # For the neural net model to work, the llpg columns have to be in the LPI format (e.g. with columns SaoText, SaoStartNumber etc. Here we check if we have that format.
+
+        if 'Address_LPI' in Matcher.ref_df.columns:
+            Matcher.ref_df = Matcher.ref_df.rename(columns={
+            "Name_LPI": "PaoText",    
+            "Num_LPI": "PaoStartNumber",
+            "Num_Suffix_LPI":"PaoStartSuffix",
+            "Number End_LPI":"PaoEndNumber",
+            "Number_End_Suffix_LPI":"PaoEndSuffix",
+
+            "Secondary_Name_LPI":"SaoText",
+            "Secondary_Num_LPI":"SaoStartNumber",
+            "Secondary_Num_Suffix_LPI":"SaoStartSuffix",
+            "Secondary_Num_End_LPI":"SaoEndNumber",
+            "Secondary_Num_End_Suffix_LPI":"SaoEndSuffix",
+            "Postcode_LPI":"Postcode",
+            "Postal_Town_LPI":"PostTown",
+            "UPRN_BLPU": "UPRN"
+        })
+            
+        #print("Matcher reference file: ", Matcher.ref_df['Reference file'])
+            
+        # Check if the source is the Addressbase places API
+        if Matcher.ref_df.iloc[0]['Reference file'] == 'API' or '_api_' in Matcher.ref_df.iloc[0]['Reference file']:
+            Matcher.ref_df = Matcher.ref_df.rename(columns={
+            "ORGANISATION_NAME": "Organisation",
+            "ORGANISATION": "Organisation",
+            "PAO_TEXT": "PaoText",    
+            "PAO_START_NUMBER": "PaoStartNumber",
+            "PAO_START_SUFFIX":"PaoStartSuffix",
+            "PAO_END_NUMBER":"PaoEndNumber",
+            "PAO_END_SUFFIX":"PaoEndSuffix",
+            "STREET_DESCRIPTION":"Street",
+            
+            "SAO_TEXT":"SaoText",
+            "SAO_START_NUMBER":"SaoStartNumber",
+            "SAO_START_SUFFIX":"SaoStartSuffix",
+            "SAO_END_NUMBER":"SaoEndNumber",
+            "SAO_END_SUFFIX":"SaoEndSuffix",
+            
+            "POSTCODE_LOCATOR":"Postcode",
+            "TOWN_NAME":"PostTown",
+            "LOCALITY_NAME":"LocalityName",
+            "ADMINISTRATIVE_AREA":"AdministrativeArea"
+        }, errors="ignore")
+    
+        # Check ref_df file format
+        # If standard format, or it's an API call
+        if 'SaoText' in Matcher.ref_df.columns or in_api:
+            Matcher.standard_llpg_format = True
+            Matcher.ref_address_cols = ["Organisation", "SaoStartNumber", "SaoStartSuffix", "SaoEndNumber", "SaoEndSuffix", "SaoText", "PaoStartNumber", "PaoStartSuffix", "PaoEndNumber",
+            "PaoEndSuffix", "PaoText", "Street", "PostTown", "Postcode"]
+            # Add columns from the list if they don't exist
+            for col in Matcher.ref_address_cols:
+                if col not in Matcher.ref_df:
+                    Matcher.ref_df[col] = np.nan
+        else: 
+            Matcher.standard_llpg_format = False
+            Matcher.ref_address_cols = in_refcol
+            Matcher.ref_df = Matcher.ref_df.rename(columns={Matcher.ref_address_cols[-1]:"Postcode"})
+            Matcher.ref_address_cols[-1] = "Postcode"
+
+
+        # Reset index for ref_df as multiple files may have been combined with identical indices
+        Matcher.ref_df = Matcher.ref_df.reset_index() #.drop(["index","level_0"], axis = 1, errors="ignore").reset_index().drop(["index","level_0"], axis = 1, errors="ignore")
+        Matcher.ref_df.index.name = 'index'
+
+        return Matcher
+
+def check_match_data_filter(Matcher, data_state, results_data_state, in_file, in_text, in_colnames, in_joincol, in_existing, in_api):
+        # Assign join field if not known
+        if not Matcher.search_df_key_field:
+                Matcher.search_df_key_field = "index"
+
+        # Set search address cols as entered column names
+        #print("In colnames in check match data: ", in_colnames)
+        Matcher.search_address_cols = in_colnames
+
+        # Check if data loaded already and bring it in
+        if not data_state.empty:
+            
+            Matcher.search_df = data_state
+
+            
+
+            Matcher.search_df['index'] = Matcher.search_df.index
+
+        else:        
+            Matcher.search_df = pd.DataFrame()       
+
+        # If someone has just entered open text, just load this instead
+        if in_text:
+            Matcher.search_df, Matcher.search_df_key_field, Matcher.search_address_cols, Matcher.search_postcode_col = prepare_search_address_string(in_text) 
+
+        # If two matcher files are loaded in, the algorithm will combine them together
+        if Matcher.search_df.empty and in_file:
+            output_message, drop1, drop2, Matcher.search_df, results_data_state = initial_data_load(in_file)
+
+            file_list = [string.name for string in in_file]
+            data_file_names = [string for string in file_list if "results_on_orig" not in string.lower()]
+            
+            #print("Data file names: ", data_file_names)
+            Matcher.file_name = get_file_name(data_file_names[0])
+            
+            # search_df makes column to use as index
+            Matcher.search_df['index'] = Matcher.search_df.index
+
+
+        # Join previously created results file onto search_df if previous results file exists
+        if not results_data_state.empty:
+
+            print("Joining on previous results file")
+            Matcher.results_on_orig_df = results_data_state.copy()
+            Matcher.search_df = Matcher.search_df.merge(results_data_state, on = "index", how = "left") 
+
+        # If no join on column suggested, assume the user wants the UPRN
+        # print("in_joincol: ", in_joincol)
+
+        if not in_joincol:
+            Matcher.new_join_col = ['UPRN']
+            #Matcher.new_join_col = Matcher.new_join_col#[0]
+            
+        else:  
+            Matcher.new_join_col = in_joincol
+            #Matcher.new_join_col = Matcher.new_join_col
+
+        # Extract the column names from the input data
+        print("In colnames: ", in_colnames)
+
+        if len(in_colnames) > 1:
+            Matcher.search_postcode_col = [in_colnames[-1]]
+
+            print("Postcode col: ", Matcher.search_postcode_col)
+            
+        elif len(in_colnames) == 1:
+            Matcher.search_df['full_address_postcode'] = Matcher.search_df[in_colnames[0]]
+            Matcher.search_postcode_col = ['full_address_postcode']
+            Matcher.search_address_cols.append('full_address_postcode')
+
+        # Check for column that indicates there are existing matches. The code will then search this column for entries, and will remove them from the data to be searched
+        Matcher.existing_match_cols = in_existing
+
+        if in_existing:
+            if "Matched with reference address" in Matcher.search_df.columns:
+                Matcher.search_df.loc[~Matcher.search_df[in_existing].isna(), "Matched with reference address"] = True
+            else: Matcher.search_df["Matched with reference address"] = ~Matcher.search_df[in_existing].isna()
+              
+        print("Shape of search_df before filtering is: ", Matcher.search_df.shape)
+
+        ### Filter addresses to those with length > 0
+        zero_length_search_df = Matcher.search_df.copy()[Matcher.search_address_cols]
+        zero_length_search_df = zero_length_search_df.fillna('').infer_objects(copy=False)
+        Matcher.search_df["address_cols_joined"] = zero_length_search_df.astype(str).sum(axis=1).str.strip()
+
+        length_more_than_0 = Matcher.search_df["address_cols_joined"].str.len() > 0
+    
+ 
+        ### Filter addresses to match to postcode areas present in both search_df and ref_df_cleaned only (postcode without the last three characters). Only run if API call is false. When the API is called, relevant addresses and postcodes should be brought in by the API.
+        if not in_api:
+            if Matcher.filter_to_lambeth_pcodes == True:
+                Matcher.search_df["postcode_search_area"] = Matcher.search_df[Matcher.search_postcode_col[0]].str.strip().str.upper().str.replace(" ", "").str[:-2]
+                Matcher.ref_df["postcode_search_area"] = Matcher.ref_df["Postcode"].str.strip().str.upper().str.replace(" ", "").str[:-2]
+                
+                unique_ref_pcode_area = (Matcher.ref_df["postcode_search_area"][Matcher.ref_df["postcode_search_area"].str.len() > 3]).unique()
+                postcode_found_in_search = Matcher.search_df["postcode_search_area"].isin(unique_ref_pcode_area)
+
+                Matcher.search_df["Excluded from search"] = "Included in search"
+                Matcher.search_df.loc[~(postcode_found_in_search), "Excluded from search"] = "Postcode area not found"
+                Matcher.search_df.loc[~(length_more_than_0), "Excluded from search"] = "Address length 0"
+                Matcher.pre_filter_search_df = Matcher.search_df.copy()#.drop(["index", "level_0"], axis = 1, errors = "ignore").reset_index()
+                Matcher.pre_filter_search_df = Matcher.pre_filter_search_df.drop("address_cols_joined", axis = 1)
+
+                Matcher.excluded_df = Matcher.search_df.copy()[~(postcode_found_in_search) | ~(length_more_than_0)]
+                Matcher.search_df = Matcher.search_df[(postcode_found_in_search) & (length_more_than_0)]
+
+                
+                # Exclude records that have already been matched separately, i.e. if 'Matched with reference address' column exists, and has trues in it
+                if "Matched with reference address" in Matcher.search_df.columns:
+                    previously_matched = Matcher.pre_filter_search_df["Matched with reference address"] == True 
+                    Matcher.pre_filter_search_df.loc[previously_matched, "Excluded from search"] = "Previously matched"
+                    
+                    Matcher.excluded_df = Matcher.search_df.copy()[~(postcode_found_in_search) | ~(length_more_than_0) | (previously_matched)]
+                    Matcher.search_df = Matcher.search_df[(postcode_found_in_search) & (length_more_than_0) & ~(previously_matched)]
+
+                else:
+                    Matcher.excluded_df = Matcher.search_df.copy()[~(postcode_found_in_search) | ~(length_more_than_0)]
+                    Matcher.search_df = Matcher.search_df[(postcode_found_in_search) & (length_more_than_0)]
+
+                print("Shape of ref_df before filtering is: ", Matcher.ref_df.shape)   
+
+                unique_search_pcode_area = (Matcher.search_df["postcode_search_area"]).unique()
+                postcode_found_in_ref = Matcher.ref_df["postcode_search_area"].isin(unique_search_pcode_area)
+                Matcher.ref_df = Matcher.ref_df[postcode_found_in_ref]
+
+                Matcher.pre_filter_search_df = Matcher.pre_filter_search_df.drop("postcode_search_area", axis = 1)
+                Matcher.search_df = Matcher.search_df.drop("postcode_search_area", axis = 1)
+                Matcher.ref_df = Matcher.ref_df.drop("postcode_search_area", axis = 1)
+                Matcher.excluded_df = Matcher.excluded_df.drop("postcode_search_area", axis = 1)
+            else:
+                Matcher.pre_filter_search_df = Matcher.search_df.copy()
+                Matcher.search_df.loc[~(length_more_than_0), "Excluded from search"] = "Address length 0"
+                
+                Matcher.excluded_df = Matcher.search_df[~(length_more_than_0)]
+                Matcher.search_df = Matcher.search_df[length_more_than_0]
+ 
+
+        Matcher.search_df = Matcher.search_df.drop("address_cols_joined", axis = 1, errors="ignore")
+        Matcher.excluded_df = Matcher.excluded_df.drop("address_cols_joined", axis = 1, errors="ignore")
+
+        Matcher.search_df_not_matched = Matcher.search_df
+
+
+        # If this is for an API call, we need to convert the search_df address columns to one column now. This is so the API call can be made and the reference dataframe created.
+        if in_api:
+
+            if in_file:
+                output_message, drop1, drop2, df, results_data_state = initial_data_load(in_file)
+
+                file_list = [string.name for string in in_file]
+                data_file_names = [string for string in file_list if "results_on_orig" not in string.lower()]
+            
+                Matcher.file_name = get_file_name(data_file_names[0])
+
+            else:
+                if in_text:
+                    Matcher.file_name = in_text
+                else:
+                    Matcher.file_name = "API call"
+
+            # Exclude records that have already been matched separately, i.e. if 'Matched with reference address' column exists, and has trues in it
+            if in_existing:
+                print("Checking for previously matched records")
+                Matcher.pre_filter_search_df = Matcher.search_df.copy()
+                previously_matched = ~Matcher.pre_filter_search_df[in_existing].isnull()
+                Matcher.pre_filter_search_df.loc[previously_matched, "Excluded from search"] = "Previously matched"
+                
+                Matcher.excluded_df = Matcher.search_df.copy()[~(length_more_than_0) | (previously_matched)]
+                Matcher.search_df = Matcher.search_df[(length_more_than_0) & ~(previously_matched)]
+
+            if type(Matcher.search_df) == str: search_df_cleaned, search_df_key_field, search_address_cols = prepare_search_address_string(Matcher.search_df)
+            else: search_df_cleaned = prepare_search_address(Matcher.search_df, Matcher.search_address_cols, Matcher.search_postcode_col, Matcher.search_df_key_field)
+
+
+            Matcher.search_df['full_address_postcode'] = search_df_cleaned["full_address"]
+            #Matcher.search_df = Matcher.search_df.reset_index(drop=True)
+            #Matcher.search_df.index.name = 'index'
+
+        return Matcher
+
+def load_matcher_data(in_text, in_file, in_ref, data_state, results_data_state, ref_data_state, in_colnames, in_refcol, in_joincol, in_existing, Matcher, in_api, in_api_key):
+        '''
+        Load in user inputs from the Gradio interface. Convert all input types (single address, or csv input) into standardised data format that can be used downstream for the fuzzy matching.
+        '''
+        today_rev = datetime.now().strftime("%Y%m%d")
+
+        # Abort flag for if it's not even possible to attempt the first stage of the match for some reason
+        Matcher.abort_flag = False
+
+        ### ref_df FILES ###
+        # If not an API call, run this first
+        if not in_api:
+            Matcher = check_ref_data_exists(Matcher, ref_data_state, in_ref, in_refcol, in_api, in_api_key, query_type=in_api)
+
+        ### MATCH/SEARCH FILES ###
+        # If doing API calls, we need to know the search data before querying for specific addresses/postcodes
+        Matcher = check_match_data_filter(Matcher, data_state, results_data_state, in_file, in_text, in_colnames, in_joincol, in_existing, in_api)
+
+
+        # If an API call, ref_df data is loaded after
+        if in_api:
+            Matcher = check_ref_data_exists(Matcher, ref_data_state, in_ref, in_refcol, in_api, in_api_key, query_type=in_api)
+            
+            #print("Resetting index.")
+            # API-called data will often have duplicate indexes in it - drop them to avoid conflicts down the line
+            #Matcher.ref_df = Matcher.ref_df.reset_index(drop = True)
+
+        print("Shape of ref_df after filtering is: ", Matcher.ref_df.shape)
+        print("Shape of search_df after filtering is: ", Matcher.search_df.shape)
+    
+        Matcher.match_outputs_name = "diagnostics_initial_" + today_rev + ".csv" 
+        Matcher.results_orig_df_name = "results_initial_" + today_rev + ".csv" 
+    
+        #Matcher.match_results_output.to_csv(Matcher.match_outputs_name, index = None)
+        #Matcher.results_on_orig_df.to_csv(Matcher.results_orig_df_name, index = None)
+        
+        return Matcher
+
+# DF preparation functions
+
+# Run batch of matches
+def run_match_batch(InitialMatch, batch_n, total_batches, progress=gr.Progress()):
+    if run_fuzzy_match == True:
+    
+        overall_tic = time.perf_counter()
+        
+        progress(0, desc= "Batch " + str(batch_n+1) + " of " + str(total_batches) + ". Fuzzy match - non-standardised dataset")
+        df_name = "Fuzzy not standardised"
+                                    
+        ''' FUZZY MATCHING '''
+            
+        ''' Run fuzzy match on non-standardised dataset '''
+        
+        FuzzyNotStdMatch = orchestrate_match_run(Matcher = copy.copy(InitialMatch), standardise = False, nnet = False, file_stub= "not_std_", df_name = df_name)
+
+        if FuzzyNotStdMatch.abort_flag == True:
+            message = "Nothing to match! Aborting address check."
+            print(message)
+            return message, InitialMatch
+
+        FuzzyNotStdMatch = combine_two_matches(InitialMatch, FuzzyNotStdMatch, df_name)
+        
+        if (len(FuzzyNotStdMatch.search_df_not_matched) == 0) | (sum(FuzzyNotStdMatch.match_results_output[FuzzyNotStdMatch.match_results_output['full_match']==False]['fuzzy_score'])==0): 
+            overall_toc = time.perf_counter()
+            time_out = f"The fuzzy match script took {overall_toc - overall_tic:0.1f} seconds"
+            FuzzyNotStdMatch.output_summary = FuzzyNotStdMatch.output_summary + " Neural net match not attempted. "# + time_out
+            return FuzzyNotStdMatch.output_summary, FuzzyNotStdMatch
+    
+        ''' Run fuzzy match on standardised dataset '''
+        
+        progress(.25, desc="Batch " + str(batch_n+1) + " of " + str(total_batches) + ". Fuzzy match - standardised dataset")
+        df_name = "Fuzzy standardised"
+        
+        FuzzyStdMatch = orchestrate_match_run(Matcher = copy.copy(FuzzyNotStdMatch), standardise = True, nnet = False, file_stub= "std_", df_name = df_name)
+        FuzzyStdMatch = combine_two_matches(FuzzyNotStdMatch, FuzzyStdMatch, df_name)
+    
+        ''' Continue if reference file in correct format, and neural net model exists. Also if data not too long '''
+        if ((len(FuzzyStdMatch.search_df_not_matched) == 0) | (FuzzyStdMatch.standard_llpg_format == False) |\
+            (os.path.exists(FuzzyStdMatch.model_dir_name + '/saved_model.zip') == False) | (run_nnet_match == False)):
+            overall_toc = time.perf_counter()
+            time_out = f"The fuzzy match script took {overall_toc - overall_tic:0.1f} seconds"
+            FuzzyStdMatch.output_summary = FuzzyStdMatch.output_summary + " Neural net match not attempted. "# + time_out
+            return FuzzyStdMatch.output_summary, FuzzyStdMatch
+
+    if run_nnet_match == True:
+    
+        ''' NEURAL NET '''
+
+        if run_fuzzy_match == False:
+            FuzzyStdMatch = copy.copy(InitialMatch)
+            overall_tic = time.perf_counter()
+    
+        ''' First on non-standardised addresses '''
+        progress(.50, desc="Batch " + str(batch_n+1) + " of " + str(total_batches) + ". Neural net - non-standardised dataset")
+        df_name = "Neural net not standardised"
+        
+        FuzzyNNetNotStdMatch = orchestrate_match_run(Matcher = copy.copy(FuzzyStdMatch), standardise = False, nnet = True, file_stub= "nnet_not_std_", df_name = df_name)
+        FuzzyNNetNotStdMatch = combine_two_matches(FuzzyStdMatch, FuzzyNNetNotStdMatch, df_name)
+    
+        if (len(FuzzyNNetNotStdMatch.search_df_not_matched) == 0):
+            overall_toc = time.perf_counter()
+            time_out = f"The whole match script took {overall_toc - overall_tic:0.1f} seconds"
+            FuzzyNNetNotStdMatch.output_summary = FuzzyNNetNotStdMatch.output_summary# + time_out
+            return FuzzyNNetNotStdMatch.output_summary, FuzzyNNetNotStdMatch
+    
+        ''' Next on standardised addresses '''
+        progress(.75, desc="Batch " + str(batch_n+1) + " of " + str(total_batches) + ". Neural net - standardised dataset")
+        df_name = "Neural net standardised"
+        
+        FuzzyNNetStdMatch = orchestrate_match_run(Matcher = copy.copy(FuzzyNNetNotStdMatch), standardise = True, nnet = True, file_stub= "nnet_std_", df_name = df_name)
+        FuzzyNNetStdMatch = combine_two_matches(FuzzyNNetNotStdMatch, FuzzyNNetStdMatch, df_name)
+ 
+        if run_fuzzy_match == False:
+            overall_toc = time.perf_counter()
+            time_out = f"The neural net match script took {overall_toc - overall_tic:0.1f} seconds"
+            FuzzyNNetStdMatch.output_summary = FuzzyNNetStdMatch.output_summary + " Only Neural net match attempted. "# + time_out
+            return FuzzyNNetStdMatch.output_summary, FuzzyNNetStdMatch
+    
+    overall_toc = time.perf_counter()
+    time_out = f"The whole match script took {overall_toc - overall_tic:0.1f} seconds"
+
+    summary_of_summaries = FuzzyNotStdMatch.output_summary + "\n" + FuzzyStdMatch.output_summary + "\n" + FuzzyNNetStdMatch.output_summary + "\n" + time_out
+
+    return summary_of_summaries, FuzzyNNetStdMatch
+
+# Overarching functions
+def orchestrate_match_run(Matcher, standardise = False, nnet = False, file_stub= "not_std_", df_name = "Fuzzy not standardised"):
+
+        today_rev = datetime.now().strftime("%Y%m%d")
+        
+        #print(Matcher.standardise)
+        Matcher.standardise = standardise
+
+        if Matcher.search_df_not_matched.empty:
+            print("Nothing to match! At start of preparing run.")
+            return Matcher
+    
+        if nnet == False:
+            diag_shortlist,\
+            diag_best_match,\
+            match_results_output,\
+            results_on_orig_df,\
+            summary,\
+            search_address_cols =\
+        full_fuzzy_match(Matcher.search_df_not_matched.copy(),
+                        Matcher.standardise, 
+                        Matcher.search_df_key_field,
+                        Matcher.search_address_cols,
+                        Matcher.search_df_cleaned,
+                        Matcher.search_df_after_stand,
+                        Matcher.search_df_after_full_stand,
+                        Matcher.ref_df_cleaned,
+                        Matcher.ref_df_after_stand,
+                        Matcher.ref_df_after_full_stand,                        
+                        Matcher.fuzzy_match_limit,
+                        Matcher.fuzzy_scorer_used)
+            if match_results_output.empty: 
+                print("Match results empty")
+                Matcher.abort_flag = True
+                return Matcher    
+        
+            else:
+                Matcher.diag_shortlist = diag_shortlist
+                Matcher.diag_best_match = diag_best_match
+                Matcher.match_results_output = match_results_output      
+            
+        else:
+            match_results_output,\
+            results_on_orig_df,\
+            summary,\
+            predict_df_nnet =\
+            full_nn_match(
+                    Matcher.ref_address_cols,
+                    Matcher.search_df_not_matched.copy(),
+                    Matcher.search_address_cols,
+                    Matcher.search_df_key_field,
+                    Matcher.standardise, 
+                    Matcher.exported_model[0],
+                    Matcher.matching_variables,
+                    Matcher.text_columns,
+                    Matcher.weights,
+                    Matcher.fuzzy_method,
+                    Matcher.score_cut_off,
+                    Matcher.match_results_output.copy(), 
+                    Matcher.filter_to_lambeth_pcodes,
+                    Matcher.model_type, 
+                    Matcher.word_to_index, 
+                    Matcher.cat_to_idx, 
+                    Matcher.device,
+                    Matcher.vocab,
+                    Matcher.labels_list,
+                    Matcher.search_df_cleaned,
+                    Matcher.ref_df_after_stand,
+                    Matcher.search_df_after_stand,
+                    Matcher.search_df_after_full_stand)
+            
+            if match_results_output.empty: 
+                print("Match results empty")
+                Matcher.abort_flag = True
+                return Matcher
+            else:
+                Matcher.match_results_output = match_results_output
+                Matcher.predict_df_nnet = predict_df_nnet 
+        
+        # Save to file
+        Matcher.results_on_orig_df = results_on_orig_df
+
+        Matcher.summary = summary
+  
+        Matcher.output_summary = create_match_summary(Matcher.match_results_output, df_name = df_name)       
+        
+        Matcher.match_outputs_name = "diagnostics_" + file_stub + today_rev + ".csv"
+        Matcher.results_orig_df_name = "results_" + file_stub + today_rev + ".csv"
+    
+        Matcher.match_results_output.to_csv(Matcher.match_outputs_name, index = None)
+        Matcher.results_on_orig_df.to_csv(Matcher.results_orig_df_name, index = None)
+    
+        return Matcher 
+
+# Overarching fuzzy match function
+def full_fuzzy_match(search_df:PandasDataFrame,
+                     standardise:bool,
+                     search_df_key_field:str,
+                     search_address_cols:List[str],
+                     search_df_cleaned:PandasDataFrame,
+                     search_df_after_stand:PandasDataFrame,
+                     search_df_after_full_stand:PandasDataFrame,
+                     ref_df_cleaned:PandasDataFrame,
+                     ref_df_after_stand:PandasDataFrame,
+                     ref_df_after_full_stand:PandasDataFrame,                     
+                     fuzzy_match_limit:float,
+                     fuzzy_scorer_used:str,
+                     new_join_col:List[str]=["UPRN"],
+                     fuzzy_search_addr_limit:float = 100,
+                     filter_to_lambeth_pcodes:bool=False):
+
+    '''
+    Compare addresses in a 'search address' dataframe with a 'reference address' dataframe by using fuzzy matching from the rapidfuzz package, blocked by postcode and then street.
+    '''
+
+    # Break if search item has length 0
+    if search_df.empty:
+        out_error = "Nothing to match! Just started fuzzy match."
+        print(out_error)
+        return pd.DataFrame(),pd.DataFrame(),pd.DataFrame(),pd.DataFrame(), out_error,search_address_cols
+
+    # If standardise is true, replace relevant variables with standardised versions
+    if standardise == True: 
+        df_name = "standardised address"
+        search_df_after_stand = search_df_after_full_stand
+        ref_df_after_stand = ref_df_after_full_stand
+    else: 
+        df_name = "non-standardised address"
+    
+    # RUN WITH POSTCODE AS A BLOCKER #
+    # Fuzzy match against reference addresses
+    
+    # Remove rows from ref search series where postcode is not found in the search_df
+    search_df_after_stand_series = search_df_after_stand.copy().set_index('postcode_search')['search_address_stand'].sort_index()
+    ref_df_after_stand_series = ref_df_after_stand.copy().set_index('postcode_search')['ref_address_stand'].sort_index()
+
+    #print(search_df_after_stand_series.index.tolist())
+    #print(ref_df_after_stand_series.index.tolist())
+    
+    ref_df_after_stand_series_checked = ref_df_after_stand_series.copy()[ref_df_after_stand_series.index.isin(search_df_after_stand_series.index.tolist())]
+
+    # pd.DataFrame(ref_df_after_stand_series_checked.to_csv("ref_df_after_stand_series_checked.csv"))
+
+    if len(ref_df_after_stand_series_checked) == 0: 
+        print("Nothing relevant in reference data to match!")
+        return pd.DataFrame(), pd.DataFrame(),  pd.DataFrame(),pd.DataFrame(),"Nothing relevant in reference data to match!",search_address_cols
+
+    # 'matched' is the list for which every single row is searched for in the reference list (the ref_df).
+    
+    print("Starting the fuzzy match")
+    
+    tic = time.perf_counter()
+    results = string_match_by_post_code_multiple(match_address_series = search_df_after_stand_series.copy(),
+                          reference_address_series = ref_df_after_stand_series_checked,
+                          search_limit = fuzzy_search_addr_limit, 
+                          scorer_name = fuzzy_scorer_used)
+
+    toc = time.perf_counter()
+    print(f"Performed the fuzzy match in {toc - tic:0.1f} seconds")
+
+
+    # Create result dfs
+    match_results_output, diag_shortlist, diag_best_match = _create_fuzzy_match_results_output(results, search_df_after_stand, ref_df_cleaned, ref_df_after_stand, fuzzy_match_limit, search_df_cleaned, search_df_key_field, new_join_col, standardise, blocker_col = "Postcode")
+    
+    match_results_output['match_method'] = "Fuzzy match - postcode"
+    
+    search_df_not_matched = filter_not_matched(match_results_output, search_df_after_stand, search_df_key_field)
+
+                        
+    # If nothing left to match, break
+    if (sum(match_results_output['full_match']==False) == 0) | (sum(match_results_output[match_results_output['full_match']==False]['fuzzy_score'])==0):
+        print("Nothing left to match!")
+        
+        summary = create_match_summary(match_results_output, df_name)
+        
+        if type(search_df) != str:
+            results_on_orig_df = join_to_orig_df(match_results_output, search_df_cleaned, search_df_key_field, new_join_col)
+        else: results_on_orig_df = match_results_output
+        
+        return diag_shortlist, diag_best_match, match_results_output, results_on_orig_df, summary, search_address_cols
+    
+
+    # RUN WITH STREET AS A BLOCKER #
+    
+    ### Redo with street as blocker
+    search_df_after_stand_street = search_df_not_matched.copy()
+    search_df_after_stand_street['search_address_stand_w_pcode'] = search_df_after_stand_street['search_address_stand'] + " " + search_df_after_stand_street['postcode_search']
+    ref_df_after_stand['ref_address_stand_w_pcode'] = ref_df_after_stand['ref_address_stand'] + " " + ref_df_after_stand['postcode_search']
+        
+    search_df_after_stand_street['street']= search_df_after_stand_street['full_address_search'].apply(extract_street_name)
+    # Exclude non-postal addresses from street-blocked search
+    search_df_after_stand_street.loc[search_df_after_stand_street['Excluded from search'] == "Excluded - non-postal address", 'street'] = ""
+        
+    ### Create lookup lists
+    search_df_match_series_street = search_df_after_stand_street.copy().set_index('street')['search_address_stand']
+    ref_df_after_stand_series_street = ref_df_after_stand.copy().set_index('Street')['ref_address_stand']
+        
+    # Remove rows where street is not in ref_df df
+    #index_check = ref_df_after_stand_series_street.index.isin(search_df_match_series_street.index)
+    #ref_df_after_stand_series_street_checked = ref_df_after_stand_series_street.copy()[index_check == True]
+
+    ref_df_after_stand_series_street_checked = ref_df_after_stand_series_street.copy()[ref_df_after_stand_series_street.index.isin(search_df_match_series_street.index.tolist())]
+
+    # If nothing left to match, break
+    if (len(ref_df_after_stand_series_street_checked) == 0) | ((len(search_df_match_series_street) == 0)):
+        
+        summary = create_match_summary(match_results_output, df_name)
+        
+        if type(search_df) != str:
+            results_on_orig_df = join_to_orig_df(match_results_output, search_df_after_stand, search_df_key_field, new_join_col)
+        else: results_on_orig_df = match_results_output
+        
+        return diag_shortlist, diag_best_match,\
+        match_results_output, results_on_orig_df, summary, search_address_cols
+    
+    print("Starting the fuzzy match with street as blocker")
+    
+    tic = time.perf_counter()
+    results_st = string_match_by_post_code_multiple(match_address_series = search_df_match_series_street.copy(),
+                          reference_address_series = ref_df_after_stand_series_street_checked.copy(),
+                          search_limit = fuzzy_search_addr_limit, 
+                          scorer_name = fuzzy_scorer_used)
+
+    toc = time.perf_counter()
+
+    print(f"Performed the fuzzy match in {toc - tic:0.1f} seconds")
+    
+    match_results_output_st, diag_shortlist_st, diag_best_match_st = _create_fuzzy_match_results_output(results_st, search_df_after_stand_street, ref_df_cleaned, ref_df_after_stand,\
+    fuzzy_match_limit, search_df_cleaned, search_df_key_field, new_join_col, standardise, blocker_col = "Street")
+    match_results_output_st['match_method'] = "Fuzzy match - street"
+
+    match_results_output_st_out = combine_std_df_remove_dups(match_results_output, match_results_output_st, orig_addr_col = search_df_key_field)
+        
+    match_results_output = match_results_output_st_out
+    
+    summary = create_match_summary(match_results_output, df_name)
+
+    ### Join URPN back onto orig df
+
+    if type(search_df) != str:
+        results_on_orig_df = join_to_orig_df(match_results_output, search_df_cleaned, search_df_key_field, new_join_col)
+    else: results_on_orig_df = match_results_output
+        
+    return diag_shortlist, diag_best_match, match_results_output, results_on_orig_df, summary, search_address_cols
+ 
+# Overarching NN function
+def full_nn_match(ref_address_cols:List[str], 
+                  search_df:PandasDataFrame,
+                  search_address_cols:List[str],
+                  search_df_key_field:str, 
+                  standardise:bool,
+                  exported_model:list,
+                  matching_variables:List[str],
+                  text_columns:List[str],
+                  weights:dict,
+                  fuzzy_method:str,
+                  score_cut_off:float,
+                  match_results:PandasDataFrame,
+                  filter_to_lambeth_pcodes:bool, 
+                  model_type:str,
+                  word_to_index:dict,
+                  cat_to_idx:dict,
+                  device:str,
+                  vocab:List[str],
+                  labels_list:List[str],
+                  search_df_cleaned:PandasDataFrame,
+                  ref_df_after_stand:PandasDataFrame,
+                  search_df_after_stand:PandasDataFrame,
+                  search_df_after_full_stand:PandasDataFrame,
+                  new_join_col:List=["UPRN"]):
+    '''
+    Use a neural network model to partition 'search addresses' into consituent parts in the format of UK Ordnance Survey Land Property Identifier (LPI) addresses. These address components are compared individually against reference addresses in the same format to give an overall match score using the recordlinkage package.
+    '''
+    
+    # Break if search item has length 0
+    if search_df.empty:
+        out_error = "Nothing to match!"
+        print(out_error)
+        return pd.DataFrame(),pd.DataFrame(),pd.DataFrame(),pd.DataFrame(),pd.DataFrame(), out_error, search_address_cols
+
+    # If it is the standardisation step, or you have come from the fuzzy match area
+    if (standardise == True): # | (run_fuzzy_match == True & standardise == False): 
+        df_name = "standardised address"
+
+        search_df_after_stand = search_df_after_full_stand     
+
+    else: 
+        df_name = "non-standardised address"
+
+    print(search_df_after_stand.shape[0])
+    print(ref_df_after_stand.shape[0])
+
+    # Predict on search data to extract LPI address components
+
+    #predict_len = len(search_df_cleaned["full_address"])
+    all_columns = list(search_df_cleaned) # Creates list of all column headers
+    search_df_cleaned[all_columns] = search_df_cleaned[all_columns].astype(str)
+    predict_data = list(search_df_after_stand['search_address_stand'])
+    
+    ### Run predict function
+    print("Starting neural net prediction for " + str(len(predict_data)) + " addresses")
+
+    tic = time.perf_counter()
+    
+    # Determine the number of chunks
+    num_chunks = math.ceil(len(predict_data) / max_predict_len)
+    list_out_all = []
+    predict_df_all = []
+    
+    for i in range(num_chunks):
+        print("Starting to predict batch " + str(i+ 1) + " of " + str(num_chunks) + " batches.")
+        
+        start_idx = i * max_predict_len
+        end_idx = start_idx + max_predict_len
+        
+        # Extract the current chunk of data
+        chunk_data = predict_data[start_idx:end_idx]
+
+        # Replace blank strings with a single space
+        chunk_data = [" " if s in ("") else s for s in chunk_data]
+        
+        if (model_type == "gru") | (model_type == "lstm"):
+            list_out, predict_df = full_predict_torch(model=exported_model, model_type=model_type, 
+                input_text=chunk_data, word_to_index=word_to_index, 
+                cat_to_idx=cat_to_idx, device=device)
+        else:
+            list_out, predict_df = full_predict_func(chunk_data, exported_model, vocab, labels_list)
+            
+        # Append the results
+        list_out_all.extend(list_out)
+        predict_df_all.append(predict_df)
+    
+    # Concatenate all the results dataframes
+    predict_df_all = pd.concat(predict_df_all, ignore_index=True)
+    
+    toc = time.perf_counter()
+    
+    print(f"Performed the NN prediction in {toc - tic:0.1f} seconds")
+    
+    predict_df = post_predict_clean(predict_df=predict_df_all, orig_search_df=search_df_cleaned, 
+        ref_address_cols=ref_address_cols, search_df_key_field=search_df_key_field)
+
+    # Score-based matching between neural net predictions and fuzzy match results
+
+    # Example of recordlinkage package in use: https://towardsdatascience.com/how-to-perform-fuzzy-dataframe-row-matching-with-recordlinkage-b53ca0cb944c
+
+    ## Run with Postcode as blocker column
+
+    blocker_column = ["Postcode"]
+
+    scoresSBM_best_pc, matched_output_SBM_pc = score_based_match(predict_df_search = predict_df.copy(), ref_search = ref_df_after_stand.copy(),
+        orig_search_df = search_df_after_stand, matching_variables = matching_variables,
+                      text_columns = text_columns, blocker_column = blocker_column, weights = weights, fuzzy_method = fuzzy_method, score_cut_off = score_cut_off, search_df_key_field=search_df_key_field, standardise=standardise, new_join_col=new_join_col)
+
+    if matched_output_SBM_pc.empty:
+        error_message = "Match results empty. Exiting neural net match."
+        print(error_message)
+
+        return pd.DataFrame(),pd.DataFrame(), error_message, predict_df
+    
+    else:
+        matched_output_SBM_pc["match_method"] = "Neural net - Postcode"
+       
+        match_results_output_final_pc = combine_std_df_remove_dups(match_results, matched_output_SBM_pc, orig_addr_col = search_df_key_field)       
+        
+    summary_pc = create_match_summary(match_results_output_final_pc, df_name = "NNet blocked by Postcode " + df_name)
+    print(summary_pc)
+    
+    ## Run with Street as blocker column
+
+    blocker_column = ["Street"]
+
+    scoresSBM_best_st, matched_output_SBM_st = score_based_match(predict_df_search = predict_df.copy(), ref_search = ref_df_after_stand.copy(), 
+                    orig_search_df = search_df_after_stand, matching_variables = matching_variables,
+                      text_columns = text_columns, blocker_column = blocker_column, weights = weights, fuzzy_method = fuzzy_method, score_cut_off = score_cut_off, search_df_key_field=search_df_key_field, standardise=standardise, new_join_col=new_join_col)
+    
+    # If no matching pairs are found in the function above then it returns 0 - below we replace these values with the postcode blocker values (which should almost always find at least one pair unless it's a very unusual situation)
+    if (type(matched_output_SBM_st) == int) | matched_output_SBM_st.empty:
+        print("Nothing to match for street block")
+        
+        matched_output_SBM_st = matched_output_SBM_pc
+        matched_output_SBM_st["match_method"] = "Neural net - Postcode" #+ standard_label
+    else: matched_output_SBM_st["match_method"] = "Neural net - Street" #+ standard_label
+ 
+    ### Join together old match df with new (model) match df
+
+    match_results_output_final_st = combine_std_df_remove_dups(match_results_output_final_pc,matched_output_SBM_st, orig_addr_col = search_df_key_field)
+      
+    summary_street = create_match_summary(match_results_output_final_st, df_name = "NNet blocked by Street " + df_name)
+    print(summary_street)
+
+    # I decided in the end not to use PaoStartNumber as a blocker column. I get only a couple more matches in general for a big increase in processing time
+
+    matched_output_SBM_po = matched_output_SBM_st
+    matched_output_SBM_po["match_method"] = "Neural net - Street" #+ standard_label
+    
+    match_results_output_final_po = match_results_output_final_st
+    match_results_output_final_three = match_results_output_final_po
+    
+    summary_three = create_match_summary(match_results_output_final_three, df_name = "fuzzy and nn model street + postcode " + df_name)
+   
+    ### Join URPN back onto orig df
+
+    if type(search_df) != str:
+        results_on_orig_df = join_to_orig_df(match_results_output_final_three, search_df_after_stand, search_df_key_field, new_join_col)
+    else: results_on_orig_df = match_results_output_final_three
+    
+    return match_results_output_final_three, results_on_orig_df, summary_three, predict_df
+
+
+# Combiner/summary functions
+def combine_std_df_remove_dups(df_not_std, df_std, orig_addr_col = "search_orig_address", match_address_series = "full_match", keep_only_duplicated = False):
+
+    if (df_not_std.empty) & (df_std.empty):
+        return df_not_std
+
+    combined_std_not_matches = pd.concat([df_not_std, df_std])#, ignore_index=True)
+
+    if combined_std_not_matches.empty: #| ~(match_address_series in combined_std_not_matches.columns) | ~(orig_addr_col in combined_std_not_matches.columns):
+        combined_std_not_matches[match_address_series] = False
+
+        if "full_address" in combined_std_not_matches.columns:
+            combined_std_not_matches[orig_addr_col] = combined_std_not_matches["full_address"]
+        combined_std_not_matches["fuzzy_score"] = 0
+        return combined_std_not_matches
+
+    combined_std_not_matches = combined_std_not_matches.sort_values([orig_addr_col, match_address_series], ascending=False)
+
+    if keep_only_duplicated == True:
+        combined_std_not_matches = combined_std_not_matches[combined_std_not_matches.duplicated(orig_addr_col)]
+    
+    combined_std_not_matches_no_dups = combined_std_not_matches.drop_duplicates(orig_addr_col).sort_index()
+    
+    return combined_std_not_matches_no_dups
+
+def combine_two_matches(OrigMatchClass, NewMatchClass, df_name):
+
+        today_rev = datetime.now().strftime("%Y%m%d")
+
+        NewMatchClass.match_results_output = combine_std_df_remove_dups(OrigMatchClass.match_results_output, NewMatchClass.match_results_output, orig_addr_col = NewMatchClass.search_df_key_field)
+
+        NewMatchClass.results_on_orig_df = combine_std_df_remove_dups(OrigMatchClass.pre_filter_search_df, NewMatchClass.results_on_orig_df, orig_addr_col = NewMatchClass.search_df_key_field, match_address_series = 'Matched with reference address')
+        
+        
+        # Filter out search results where a match was found
+        NewMatchClass.pre_filter_search_df = NewMatchClass.results_on_orig_df
+
+        found_index = NewMatchClass.results_on_orig_df.loc[NewMatchClass.results_on_orig_df["Matched with reference address"] == True, NewMatchClass.search_df_key_field].astype(int)
+        #print(found_index)[NewMatchClass.search_df_key_field]
+
+        key_field_values = NewMatchClass.search_df_not_matched[NewMatchClass.search_df_key_field].astype(int)  # Assuming list conversion is suitable
+        rows_to_drop = key_field_values[key_field_values.isin(found_index)].tolist()
+        NewMatchClass.search_df_not_matched = NewMatchClass.search_df_not_matched.loc[~NewMatchClass.search_df_not_matched[NewMatchClass.search_df_key_field].isin(rows_to_drop),:]#.drop(rows_to_drop, axis = 0)
+
+        # Filter out rows from NewMatchClass.search_df_cleaned
+
+        filtered_rows_to_keep = NewMatchClass.search_df_cleaned[NewMatchClass.search_df_key_field].astype(int).isin(NewMatchClass.search_df_not_matched[NewMatchClass.search_df_key_field].astype(int)).to_list()
+
+        NewMatchClass.search_df_cleaned = NewMatchClass.search_df_cleaned.loc[filtered_rows_to_keep,:]#.drop(rows_to_drop, axis = 0)
+        NewMatchClass.search_df_after_stand = NewMatchClass.search_df_after_stand.loc[filtered_rows_to_keep,:]#.drop(rows_to_drop)
+        NewMatchClass.search_df_after_full_stand = NewMatchClass.search_df_after_full_stand.loc[filtered_rows_to_keep,:]#.drop(rows_to_drop)
+        
+        ### Create lookup lists
+        NewMatchClass.search_df_after_stand_series = NewMatchClass.search_df_after_stand.copy().set_index('postcode_search')['search_address_stand'].str.lower().str.strip()
+        NewMatchClass.search_df_after_stand_series_full_stand = NewMatchClass.search_df_after_full_stand.copy().set_index('postcode_search')['search_address_stand'].str.lower().str.strip()
+            
+
+        match_results_output_match_score_is_0 = NewMatchClass.match_results_output[NewMatchClass.match_results_output['fuzzy_score']==0.0][["index", "fuzzy_score"]].drop_duplicates(subset='index')
+        match_results_output_match_score_is_0["index"] = match_results_output_match_score_is_0["index"].astype(str)
+        #NewMatchClass.results_on_orig_df["index"] = NewMatchClass.results_on_orig_df["index"].astype(str)
+        NewMatchClass.results_on_orig_df = NewMatchClass.results_on_orig_df.merge(match_results_output_match_score_is_0, on = "index", how = "left")
+    
+        NewMatchClass.results_on_orig_df.loc[NewMatchClass.results_on_orig_df["fuzzy_score"] == 0.0, "Excluded from search"] = "Match score is 0"
+        NewMatchClass.results_on_orig_df = NewMatchClass.results_on_orig_df.drop("fuzzy_score", axis = 1)
+
+        # Drop any duplicates, prioritise any matches
+        NewMatchClass.results_on_orig_df = NewMatchClass.results_on_orig_df.sort_values(by=["index", "Matched with reference address"], ascending=[True,False]).drop_duplicates(subset="index")
+    
+        NewMatchClass.output_summary = create_match_summary(NewMatchClass.match_results_output, df_name = df_name)
+        print(NewMatchClass.output_summary)
+    
+
+        NewMatchClass.search_df_not_matched = filter_not_matched(NewMatchClass.match_results_output, NewMatchClass.search_df, NewMatchClass.search_df_key_field)
+
+        ### Rejoin the excluded matches onto the output file
+        #NewMatchClass.results_on_orig_df = pd.concat([NewMatchClass.results_on_orig_df, NewMatchClass.excluded_df])
+    
+        NewMatchClass.match_outputs_name = "match_results_output_std_" + today_rev + ".csv" # + NewMatchClass.file_name + "_" 
+        NewMatchClass.results_orig_df_name = "results_on_orig_df_std_" + today_rev + ".csv" # + NewMatchClass.file_name + "_"
+
+        # Only keep essential columns
+        essential_results_cols = [NewMatchClass.search_df_key_field, "Excluded from search", "Matched with reference address", "ref_index", "Reference matched address", "Reference file"]
+        essential_results_cols.extend(NewMatchClass.new_join_col) 
+    
+        NewMatchClass.match_results_output.to_csv(NewMatchClass.match_outputs_name, index = None)
+        NewMatchClass.results_on_orig_df[essential_results_cols].to_csv(NewMatchClass.results_orig_df_name, index = None)
+        
+        return NewMatchClass
+
+def create_match_summary(match_results_output:PandasDataFrame, df_name:str):
+    
+    # Check if match_results_output is a dictionary-like object and has the key 'full_match'
+   
+    if not isinstance(match_results_output, dict) or 'full_match' not in match_results_output or (len(match_results_output) == 0):
+        "Nothing in match_results_output"
+        full_match_count = 0
+        match_fail_count = 0
+        records_attempted = 0
+        dataset_length = 0
+        records_not_attempted = 0
+        match_rate = 0
+        match_fail_count_without_excluded = 0
+        match_fail_rate = 0
+        not_attempted_rate = 0
+        
+    ''' Create a summary paragraph '''
+    full_match_count = match_results_output['full_match'][match_results_output['full_match'] == True].count()
+    match_fail_count = match_results_output['full_match'][match_results_output['full_match'] == False].count()
+    records_attempted = int(sum((match_results_output['fuzzy_score']!=0.0) & ~(match_results_output['fuzzy_score'].isna())))
+    dataset_length = len(match_results_output["full_match"])
+    records_not_attempted = int(dataset_length - records_attempted)
+    match_rate = str(round((full_match_count / dataset_length) * 100,1))
+    match_fail_count_without_excluded = match_fail_count - records_not_attempted
+    match_fail_rate = str(round(((match_fail_count_without_excluded) / dataset_length) * 100,1))
+    not_attempted_rate = str(round((records_not_attempted / dataset_length) * 100,1))
+
+    summary = ("For the " + df_name + " dataset (" + str(dataset_length) + " records), the fuzzy matching algorithm successfully matched " + str(full_match_count) +
+               " records (" + match_rate + "%). The algorithm could not attempt to match " + str(records_not_attempted) +
+               " records (" + not_attempted_rate +  "%). There are " + str(match_fail_count_without_excluded) + " records left to potentially match.")
+    
+    return summary

tools/model_predict.py

@@ -0,0 +1,318 @@
+#import tensorflow as tf # Tensorflow use deprecated
+import torch
+import pandas as pd
+import numpy as np
+from typing import Type, Dict, List, Tuple
+from datetime import datetime
+
+PandasDataFrame = Type[pd.DataFrame]
+PandasSeries = Type[pd.Series]
+MatchedResults = Dict[str,Tuple[str,int]]
+array = List[str]
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+# # Neural net functions
+
+
+
+
+
+def vocab_lookup(characters: str, vocab) -> (int, np.ndarray):
+    """
+    Taken from the function from the addressnet package by Jason Rigby
+    
+    Converts a string into a list of vocab indices
+    :param characters: the string to convert
+    :param training: if True, artificial typos will be introduced
+    :return: the string length and an array of vocab indices
+    """
+    result = list()
+    for c in characters.lower():
+        try:
+            result.append(vocab.index(c) + 1)
+        except ValueError:
+            result.append(0)
+    return len(characters), np.array(result, dtype=np.int64)
+
+
+# ## Neural net predictor functions
+
+def text_to_model_input_local(in_text, vocab, model_type = "estimator"):
+    addresses_out = []
+    model_input_out = []
+    encoded_text = []
+    
+    # Calculate longest string length
+    import heapq
+ 
+    # get the index of the largest element in the list
+    index = heapq.nlargest(1, range(len(in_text)), key=lambda x: len(in_text[x]))[0]
+ 
+    # use the index to get the corresponding string
+    longest_string = len(in_text[index])
+ 
+    #print("Longest string is: " + str(longest_string))
+
+    for x in range(0, len(in_text)):
+        
+        out = vocab_lookup(in_text[x], vocab)
+        addresses_out.append(out)
+        
+        #print(out)
+        
+        # Tensorflow model use deprecated
+        # if model_type == "estimator":
+        #     model_input_add= tf.train.Example(features=tf.train.Features(feature={
+        #     'lengths': tf.train.Feature(int64_list=tf.train.Int64List(value=[out[0]])),
+        #     'encoded_text': tf.train.Feature(int64_list=tf.train.Int64List(value=out[1].tolist()))  
+        #     })).SerializeToString()
+
+        #     model_input_out.append(model_input_add)
+        
+        if model_type == "keras":
+            encoded_text.append(out[1])
+            
+    # Tensorflow model use deprecated      
+    # if model_type == "keras":
+    #     # Pad out the strings so they're all the same length. 69 seems to be the value for spaces
+    #     model_input_out = tf.keras.utils.pad_sequences(encoded_text, maxlen=longest_string, padding="post", truncating="post", value=0)#69)
+        
+        
+    return addresses_out, model_input_out
+
+
+def reformat_predictions_local(predict_out):
+
+    predictions_list_reformat = []
+
+    for x in range(0,len(predict_out['pred_output_classes'])):
+
+        new_entry = {'class_ids': predict_out['pred_output_classes'][x], 'probabilities': predict_out['probabilities'][x]}
+        predictions_list_reformat.append(new_entry)
+        
+    return predictions_list_reformat
+
+
+def predict_serve_conv_local(in_text:List[str], labels_list, predictions) -> List[Dict[str, str]]:
+ 
+    class_names = [l.replace("_code", "") for l in labels_list]
+    class_names = [l.replace("_abbreviation", "") for l in class_names]
+    
+    #print(input_text)
+    
+    #print(list(zip(input_text, predictions)))
+    
+    for addr, res in zip(in_text, predictions):
+        
+        #print(zip(input_text, predictions))
+        
+        mappings = dict()
+        
+        
+        #print(addr.upper())
+        #print(res['class_ids'])
+        
+        for char, class_id in zip(addr.upper(), res['class_ids']):
+            #print(char)
+            if class_id == 0:
+                continue
+            cls = class_names[class_id - 1]
+            mappings[cls] = mappings.get(cls, "") + char
+            
+        
+        #print(mappings)
+        yield mappings
+        #return mappings
+
+
+def prep_predict_export(prediction_outputs, in_text):
+    
+    out_list = list(prediction_outputs)
+    
+    df_out = pd.DataFrame(out_list)
+    
+    #print(in_text)
+    #print(df_out)
+    
+    df_out["address"] = in_text
+    
+    return out_list, df_out
+
+
+
+def full_predict_func(list_to_predict, model, vocab, labels_list):
+    
+    if hasattr(model, "summary"): # Indicates this is a keras model rather than an estimator
+        model_type = "keras"
+    else: model_type = "estimator"
+    
+    list_to_predict = [x.upper() for x in list_to_predict]
+    
+    addresses_out, model_input = text_to_model_input_local(list_to_predict, vocab, model_type) 
+
+    if hasattr(model, "summary"):
+        probs = model.predict(model_input, use_multiprocessing=True)
+
+        classes = probs.argmax(axis=-1)
+
+        predictions = {'pred_output_classes':classes, 'probabilities':probs}
+        
+    else:
+        print("Tensorflow use deprecated")
+        #predictions = model.signatures["predict_output"](predictor_inputs=tf.constant(model_input)) # This was for when using the contrib module
+        #predictions = model.signatures["serving_default"](predictor_inputs=tf.constant(model_input))
+    
+    predictions_list_reformat = reformat_predictions_local(predictions)
+    
+
+    #### Final output as list or dataframe
+
+    output = predict_serve_conv_local(list(list_to_predict), labels_list, predictions_list_reformat)
+
+    list_out, predict_df = prep_predict_export(output, list_to_predict)
+
+    # Add organisation as a column if it doesn't already exist    
+    if 'Organisation' not in predict_df.columns:
+        predict_df['Organisation'] = ""
+
+    return list_out, predict_df
+
+# -
+
+def predict_torch(model, model_type, input_text, word_to_index, device):
+    #print(device)
+    
+    # Convert input_text to tensor of character indices
+    indexed_texts = [[word_to_index.get(char, word_to_index['<UNK>']) for char in text] for text in input_text]
+    
+    # Calculate max_len based on indexed_texts
+    max_len = max(len(text) for text in indexed_texts)
+    
+    # Pad sequences and convert to tensor
+    padded_texts = torch.tensor([text + [word_to_index['<pad>']] * (max_len - len(text)) for text in indexed_texts])
+    
+    with torch.no_grad():
+        texts = padded_texts.to(device)
+        
+        if (model_type == "lstm") | (model_type == "gru"):
+            text_lengths = texts.ne(word_to_index['<pad>']).sum(dim=1)
+            predictions = model(texts, text_lengths)
+        
+        if model_type == "transformer":
+            # Call model with texts and pad_idx
+            predictions = model(texts, word_to_index['<pad>'])
+        
+    # Convert predictions to most likely category indices
+    _, predicted_indices = predictions.max(2)
+    return predicted_indices
+
+
+def torch_predictions_to_dicts(input_text, predicted_indices, index_to_category):
+    results = []
+    for i, text in enumerate(input_text):
+        # Treat each character in the input text as a "token"
+        tokens = list(text)  # Convert string to a list of characters
+        
+        # Create a dictionary for the current text
+        curr_dict = {}
+        
+        # Iterate over the predicted categories and the tokens together
+        for category_index, token in zip(predicted_indices[i], tokens):
+            # Convert the category index to its name
+            category_name = index_to_category[category_index.item()]
+            
+            # Append the token to the category in the dictionary (or create the category if it doesn't exist)
+            if category_name in curr_dict:
+                curr_dict[category_name] += token  # No space needed between characters
+            else:
+                curr_dict[category_name] = token
+        
+        results.append(curr_dict)
+    
+    return results
+
+
+def torch_prep_predict_export(prediction_outputs, in_text):
+    
+    #out_list = list(prediction_outputs)
+    
+    df_out = pd.DataFrame(prediction_outputs).drop("IGNORE", axis = 1)
+    
+    #print(in_text)
+    #print(df_out)
+    
+    df_out["address"] = in_text
+    
+    return df_out
+
+
+def full_predict_torch(model,  model_type, input_text, word_to_index, cat_to_idx, device):
+    
+    input_text = [x.upper() for x in input_text]
+    
+    predicted_indices = predict_torch(model, model_type, input_text, word_to_index, device)
+    
+    index_to_category = {v: k for k, v in cat_to_idx.items()}
+
+    results_dict = torch_predictions_to_dicts(input_text, predicted_indices, index_to_category)
+    
+    df_out = torch_prep_predict_export(results_dict, input_text)
+       
+    return results_dict, df_out
+
+
+def post_predict_clean(predict_df, orig_search_df, ref_address_cols, search_df_key_field):
+
+    
+    # Add address to ref_address_cols
+    ref_address_cols_add = ref_address_cols.copy()
+    ref_address_cols_add.extend(['address'])                
+    
+    # Create column if it doesn't exist
+    for x in ref_address_cols:
+
+        predict_df[x] = predict_df.get(x, np.nan)
+    
+    predict_df = predict_df[ref_address_cols_add]
+    
+    #Columns that are in the ref and model, but are not matched in this instance, need to be filled in with blanks
+
+    predict_cols_match = list(predict_df.drop(["address"],axis=1).columns)
+    predict_cols_match_uprn = predict_cols_match.copy()
+    predict_cols_match_uprn.append("UPRN")
+
+    pred_output_missing_cols = list(set(ref_address_cols) - set(predict_cols_match))
+    predict_df[pred_output_missing_cols] = np.nan
+    predict_df = predict_df.fillna("").infer_objects(copy=False)
+
+    #Convert all columns to string
+
+    all_columns = list(predict_df) # Creates list of all column headers
+    predict_df[all_columns] = predict_df[all_columns].astype(str)
+
+    predict_df = predict_df.replace("\.0","",regex=True)
+
+    #When comparing with ref, the postcode existing in the data will be used to compare rather than the postcode predicted by the model. This is to minimise errors in matching
+
+    predict_df = predict_df.rename(columns={"Postcode":"Postcode_predict"})
+
+    #orig_search_df.to_csv("orig_search_df_pre_predict.csv")
+
+    orig_search_df_pc = orig_search_df[[search_df_key_field, "postcode"]].rename(columns={"postcode":"Postcode"}).reset_index(drop=True)
+    predict_df = predict_df.merge(orig_search_df_pc, left_index=True, right_index=True, how = "left")
+
+    #predict_df = pd.concat([predict_df, orig_search_df_pc], axis = 1)
+    
+    #predict_df[search_df_key_field] = orig_search_df[search_df_key_field]
+
+    #predict_df = predict_df.drop("index", axis=1)
+
+    #predict_df['index'] = predict_df.index
+    predict_df[search_df_key_field] = predict_df[search_df_key_field].astype(str)
+
+    #predict_df.to_csv("predict_end_of_clean.csv")
+    
+    return predict_df
+

tools/preparation.py

@@ -0,0 +1,456 @@
+import pandas as pd
+from typing import Type, Dict, List, Tuple
+from datetime import datetime
+#import polars as pl
+import re
+
+PandasDataFrame = Type[pd.DataFrame]
+PandasSeries = Type[pd.Series]
+MatchedResults = Dict[str,Tuple[str,int]]
+array = List[str]
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+
+def prepare_search_address_string(
+    search_str: str
+) -> Tuple[pd.DataFrame, str, List[str], List[str]]:
+    """Extracts address and postcode from search_str into new DataFrame"""
+
+    # Validate input
+    if not isinstance(search_str, str):
+        raise TypeError("search_str must be a string")
+        
+    search_df = pd.DataFrame(data={"full_address":[search_str]})
+
+    #print(search_df)
+    
+    # Extract postcode 
+    postcode_series = extract_postcode(search_df, "full_address").dropna(axis=1)[0]
+
+    # Remove postcode from address
+    address_series = remove_postcode(search_df, "full_address")
+
+    # Construct output DataFrame
+    search_df_out = pd.DataFrame()
+    search_df_out["full_address"] = address_series
+    search_df_out["postcode"] = postcode_series
+
+    # Set key field for joining
+    key_field = "index"
+
+    # Reset index to use as key field
+    search_df_out = search_df_out.reset_index()
+
+    # Define column names to return
+    address_cols = ["full_address"]
+    postcode_col = ["postcode"]
+
+    return search_df_out, key_field, address_cols, postcode_col
+
+# def prepare_search_address(
+#     search_df: pd.DataFrame, 
+#     address_cols: list,
+#     postcode_col: list,
+#     key_col: str
+# ) -> Tuple[pd.DataFrame, str]:
+    
+#     # Validate inputs
+#     if not isinstance(search_df, pd.DataFrame):
+#         raise TypeError("search_df must be a Pandas DataFrame")
+        
+#     if not isinstance(address_cols, list):
+#         raise TypeError("address_cols must be a list")
+        
+#     if not isinstance(postcode_col, list):
+#         raise TypeError("postcode_col must be a list")
+        
+#     if not isinstance(key_col, str):
+#         raise TypeError("key_col must be a string")
+        
+#     # Clean address columns
+#     clean_addresses = _clean_columns(search_df, address_cols)
+    
+#     # Join address columns into one
+#     full_addresses = _join_address(clean_addresses, address_cols)
+    
+#     # Add postcode column 
+#     full_df = _add_postcode_column(full_addresses, postcode_col)
+    
+#     # Remove postcode from main address if there was only one column in the input
+#     if postcode_col == "full_address_postcode":
+#         # Remove postcode from address
+#         address_series = remove_postcode(search_df, "full_address")
+#         search_df["full_address"] == address_series
+    
+#     # Ensure index column
+#     final_df = _ensure_index(full_df, key_col)
+
+#     #print(final_df)
+
+    
+#     return final_df, key_col
+
+def prepare_search_address(
+    search_df: pd.DataFrame, 
+    address_cols: list,
+    postcode_col: list,
+    key_col: str
+) -> Tuple[pd.DataFrame, str]:
+    
+    # Validate inputs
+    if not isinstance(search_df, pd.DataFrame):
+        raise TypeError("search_df must be a Pandas DataFrame")
+        
+    if not isinstance(address_cols, list):
+        raise TypeError("address_cols must be a list")
+        
+    if not isinstance(postcode_col, list):
+        raise TypeError("postcode_col must be a list")
+        
+    if not isinstance(key_col, str):
+        raise TypeError("key_col must be a string")
+        
+    # Clean address columns
+    #search_df_polars = pl.from_dataframe(search_df)
+    clean_addresses = _clean_columns(search_df, address_cols)
+    
+    # Join address columns into one
+    full_addresses = _join_address(clean_addresses, address_cols)
+    
+    # Add postcode column 
+    full_df = _add_postcode_column(full_addresses, postcode_col)
+    
+    # Remove postcode from main address if there was only one column in the input
+    if postcode_col == "full_address_postcode":
+        # Remove postcode from address
+        address_series = remove_postcode(search_df, "full_address")
+        search_df["full_address"] == address_series
+    
+    # Ensure index column
+    final_df = _ensure_index(full_df, key_col)
+
+    #print(final_df)
+
+    
+    return final_df
+
+# Helper functions
+def _clean_columns(df, cols):
+   # Cleaning logic
+   def clean_col(col):
+       return col.astype(str).fillna("").infer_objects(copy=False).str.replace("nan","").str.replace("\s{2,}", " ", regex=True).str.replace(","," ").str.strip()
+
+   df[cols] = df[cols].apply(clean_col)
+    
+   return df
+
+# def _clean_columns(df, cols):
+#     # Cleaning logic
+#     #print(df)
+
+#     #if isinstance(df, pl.DataFrame):
+#     #    print("It's a Polars DataFrame")
+
+#     def clean_col(col):
+#         col = col.str.replace("nan", "")
+#         col = col.apply(lambda x: re.sub(r'\s{2,}', ' ', str(x)), skip_nulls=False, return_dtype=str)  # replace any spaces greater than one with one
+#         return col.str.replace(",", " ").str.strip()  # replace commas with a space
+
+#     for col in cols:
+#         df = df.with_columns(clean_col(df[col]).alias(col))
+
+#     return df
+
+   
+def _join_address(df, cols):
+   # Joining logic
+   full_address = df[cols].apply(lambda row: ' '.join(row.values.astype(str)), axis=1)
+   df["full_address"] = full_address.str.replace("\s{2,}", " ", regex=True).str.strip()
+   
+   return df
+   
+def _add_postcode_column(df, postcodes):
+   # Add postcode column
+   if isinstance(postcodes, list):
+        postcodes = postcodes[0]
+    
+   if postcodes != "full_address_postcode":
+        df = df.rename(columns={postcodes:"postcode"})
+   else:
+        #print(df["full_address_postcode"])
+        #print(extract_postcode(df,"full_address_postcode"))
+        df["full_address_postcode"] = extract_postcode(df,"full_address_postcode")[0] # 
+        df = df.rename(columns={postcodes:"postcode"})
+        #print(df)
+   
+   return df
+   
+def _ensure_index(df, index_col):
+   # Ensure index column exists
+   if ((index_col == "index") & ~("index" in df.columns)):
+        print("Resetting index in _ensure_index function")   
+        df = df.reset_index()
+
+   df[index_col] = df[index_col].astype(str)
+
+   return df
+
+def create_full_address(df):
+
+    df = df.fillna("").infer_objects(copy=False)
+
+    if "Organisation" not in df.columns:
+        df["Organisation"] = ""
+
+    df["full_address"] = df['Organisation'] + " " + df['SaoText'].str.replace(" - ", " REPL ").str.replace("- ", " REPLEFT ").str.replace(" -", " REPLRIGHT ") + " " + df["SaoStartNumber"].astype(str) + df["SaoStartSuffix"] + "-" + df["SaoEndNumber"].astype(str) + df["SaoEndSuffix"] + " " + df["PaoText"].str.replace(" - ", " REPL ").str.replace("- ", " REPLEFT ").str.replace(" -", " REPLRIGHT ") + " " + df["PaoStartNumber"].astype(str) + df["PaoStartSuffix"] + "-" + df["PaoEndNumber"].astype(str) + df["PaoEndSuffix"] + " " + df["Street"] + " " + df["PostTown"] + " " + df["Postcode"]
+
+    #.str.replace(r'(?<=[a-zA-Z])-(?![a-zA-Z])|(?<![a-zA-Z])-(?=[a-zA-Z])', ' ', regex=True)\
+    
+    #.str.replace(".0","", regex=False)\
+    
+    df["full_address"] = df["full_address"]\
+    .str.replace("-999","")\
+    .str.replace(" -"," ")\
+    .str.replace("- "," ")\
+    .str.replace(" REPL "," - ")\
+    .str.replace(" REPLEFT ","- ")\
+    .str.replace(" REPLRIGHT "," -")\
+    .str.replace("\s+"," ", regex=True)\
+    .str.strip()
+    #.str.replace("  "," ")\
+    
+    return df["full_address"]
+
+def prepare_ref_address(ref_df, ref_address_cols, new_join_col = ['UPRN'], standard_cols = True):
+    
+    if ('SaoText' in ref_df.columns) | ("Secondary_Name_LPI" in ref_df.columns): standard_cols = True
+    else: standard_cols = False
+    
+    ref_address_cols_uprn = ref_address_cols.copy()
+
+    ref_address_cols_uprn.extend(new_join_col)
+    ref_address_cols_uprn_w_ref = ref_address_cols_uprn.copy()
+    ref_address_cols_uprn_w_ref.extend(["Reference file"])
+
+    ref_df_cleaned = ref_df.copy()
+      
+    # In on-prem LPI db street has been excluded, so put this back in
+    if ('Street' not in ref_df_cleaned.columns) & ('Address_LPI' in ref_df_cleaned.columns): 
+            ref_df_cleaned['Street'] = ref_df_cleaned['Address_LPI'].str.replace("\\n", " ", regex = True).apply(extract_street_name)#
+        
+    if ('Organisation' not in ref_df_cleaned.columns) & ('SaoText' in ref_df_cleaned.columns):
+        ref_df_cleaned['Organisation'] = ""
+     
+    ref_df_cleaned = ref_df_cleaned[ref_address_cols_uprn_w_ref]
+
+    ref_df_cleaned = ref_df_cleaned.fillna("").infer_objects(copy=False)
+
+    all_columns = list(ref_df_cleaned) # Creates list of all column headers
+    ref_df_cleaned[all_columns] = ref_df_cleaned[all_columns].astype(str).fillna('').infer_objects(copy=False).replace('nan','')
+
+    ref_df_cleaned = ref_df_cleaned.replace("\.0","",regex=True)
+
+    # Create full address
+
+    all_columns = list(ref_df_cleaned) # Creates list of all column headers
+    ref_df_cleaned[all_columns] = ref_df_cleaned[all_columns].astype(str)
+
+    ref_df_cleaned = ref_df_cleaned.replace("nan","")
+    ref_df_cleaned = ref_df_cleaned.replace("\.0","",regex=True)
+      
+    if standard_cols == True:
+        ref_df_cleaned= ref_df_cleaned[ref_address_cols_uprn_w_ref].fillna('').infer_objects(copy=False)
+
+        ref_df_cleaned["fulladdress"] = create_full_address(ref_df_cleaned[ref_address_cols_uprn_w_ref])
+    
+    else: 
+        ref_df_cleaned= ref_df_cleaned[ref_address_cols_uprn_w_ref].fillna('').infer_objects(copy=False)
+        
+        full_address  = ref_df_cleaned[ref_address_cols].apply(lambda row: ' '.join(row.values.astype(str)), axis=1) 
+        ref_df_cleaned["fulladdress"] = full_address
+
+    ref_df_cleaned["fulladdress"] = ref_df_cleaned["fulladdress"]\
+    .str.replace("-999","")\
+    .str.replace(" -"," ")\
+    .str.replace("- "," ")\
+    .str.replace(".0","", regex=False)\
+    .str.replace("\s{2,}", " ", regex=True)\
+    .str.strip()
+    
+    # Create a street column if it doesn't exist by extracting street from the full address
+    
+    if 'Street' not in ref_df_cleaned.columns:        
+        ref_df_cleaned['Street'] = ref_df_cleaned["fulladdress"].apply(extract_street_name)
+
+    # Add index column
+        ref_df_cleaned['ref_index'] = ref_df_cleaned.index
+        
+    return ref_df_cleaned
+
+# def prepare_ref_address(ref_df:pl.DataFrame, ref_address_cols, new_join_col = ['UPRN'], standard_cols = True):
+    
+#     if ('SaoText' in ref_df.columns) | ("Secondary_Name_LPI" in ref_df.columns): 
+#         standard_cols = True
+#     else: 
+#         standard_cols = False
+
+#     ref_address_cols_uprn = list(ref_address_cols) + new_join_col
+#     ref_df_cleaned = ref_df[ref_address_cols_uprn].fill_null("")
+    
+#     # In on-prem LPI db street has been excluded, so put this back in
+#     if ('Street' not in ref_df_cleaned.columns) & ('Address_LPI' in ref_df_cleaned.columns): 
+#         ref_df_cleaned = ref_df_cleaned.with_column(pl.col('Address_LPI').apply(lambda x: extract_street_name(x)).alias('Street'))  
+        
+#     if ('Organisation' not in ref_df_cleaned.columns) & ('SaoText' in ref_df_cleaned.columns): 
+#         ref_df_cleaned = ref_df_cleaned.with_column(pl.lit("").alias('Organisation'))
+     
+#     #ref_df_cleaned['fulladdress'] = 
+        
+#     if standard_cols:
+#         pass
+#         # I can not write the full address code here as it depends on your extract_street_name and create_full_address function implementations. 
+#         # However, you might need to convert string types to object type for full address creation which may require more than just a few lines of codes. 
+#     else:
+#         pass    
+        
+#         # I can not write the full address code here as it depends on your extract_street_name and create_full_address function implementations.
+      
+#     if 'Street' not in ref_df_cleaned.columns:  
+#         ref_df_cleaned = ref_df_cleaned.with_column(pl.col('fulladdress').apply(extract_street_name).alias("Street"))
+    
+#     # Add index column
+#     ref_df_cleaned = ref_df_cleaned.with_column(pl.lit('').alias('ref_index'))
+        
+#     return ref_df_cleaned
+
+
+def extract_postcode(df, col:str) -> PandasSeries:
+    '''
+    Extract a postcode from a string column in a dataframe
+    '''
+    postcode_series = df[col].str.upper().str.extract(pat = \
+    "(\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\\b$)")
+    
+    return postcode_series
+
+
+# Remove addresses with no numbers in at all - too high a risk of badly assigning an address
+def check_no_number_addresses(df, in_address_series) -> PandasSeries:
+    '''
+    Highlight addresses from a pandas df where there are no numbers in the address.
+    '''
+    df["in_address_series_temp"] = df[in_address_series].str.lower()
+
+    no_numbers_series = df["in_address_series_temp"].str.contains("^(?!.*\d+).*$", regex=True)
+
+    df.loc[no_numbers_series == True, 'Excluded from search'] = "Excluded - no numbers in address"
+
+    df = df.drop("in_address_series_temp", axis = 1)
+
+    #print(df[["full_address", "Excluded from search"]])
+
+    return df
+
+
+def remove_postcode(df, col:str) -> PandasSeries:
+    '''
+    Remove a postcode from a string column in a dataframe
+    '''
+    address_series_no_pcode = df[col].str.upper().str.replace(\
+    "\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2}|GIR ?0A{2})\\b$|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$|\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\\b$","", regex=True).str.lower()
+    
+    return address_series_no_pcode
+
+def extract_street_name(address:str) -> str:
+    """
+    Extracts the street name from the given address.
+
+    Args:
+        address (str): The input address string.
+
+    Returns:
+        str: The extracted street name, or an empty string if no match is found.
+
+    Examples:
+        >>> address1 = "1 Ash Park Road SE54 3HB"
+        >>> extract_street_name(address1)
+        'Ash Park Road'
+
+        >>> address2 = "Flat 14 1 Ash Park Road SE54 3HB"
+        >>> extract_street_name(address2)
+        'Ash Park Road'
+
+        >>> address3 = "123 Main Blvd"
+        >>> extract_street_name(address3)
+        'Main Blvd'
+
+        >>> address4 = "456 Maple AvEnUe"
+        >>> extract_street_name(address4)
+        'Maple AvEnUe'
+
+        >>> address5 = "789 Oak Street"
+        >>> extract_street_name(address5)
+        'Oak Street'
+    """
+    
+   
+    street_types = [
+        'Street', 'St', 'Boulevard', 'Blvd', 'Highway', 'Hwy', 'Broadway', 'Freeway',
+        'Causeway', 'Cswy', 'Expressway', 'Way', 'Walk', 'Lane', 'Ln', 'Road', 'Rd',
+        'Avenue', 'Ave', 'Circle', 'Cir', 'Cove', 'Cv', 'Drive', 'Dr', 'Parkway', 'Pkwy',
+        'Park', 'Court', 'Ct', 'Square', 'Sq', 'Loop', 'Place', 'Pl', 'Parade', 'Estate',
+        'Alley', 'Arcade','Avenue', 'Ave','Bay','Bend','Brae','Byway','Close','Corner','Cove',
+        'Crescent', 'Cres','Cul-de-sac','Dell','Drive', 'Dr','Esplanade','Glen','Green','Grove','Heights', 'Hts',
+        'Mews','Parade','Path','Piazza','Promenade','Quay','Ridge','Row','Terrace', 'Ter','Track','Trail','View','Villas',
+        'Marsh', 'Embankment', 'Cut', 'Hill', 'Passage', 'Rise', 'Vale', 'Side'
+    ]
+
+    # Dynamically construct the regex pattern with all possible street types
+    street_types_pattern = '|'.join(rf"{re.escape(street_type)}" for street_type in street_types)
+
+    # The overall regex pattern to capture the street name
+    pattern = rf'(?:\d+\s+|\w+\s+\d+\s+|.*\d+[a-z]+\s+|.*\d+\s+)*(?P<street_name>[\w\s]+(?:{street_types_pattern}))'
+
+    def replace_postcode(address):
+        pattern = r'\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2}|GIR ?0A{2})\b$|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$|\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$'
+        return re.sub(pattern, "", address)
+
+    
+    modified_address = replace_postcode(address.upper())
+    #print(modified_address)
+    #print(address)
+       
+    # Perform a case-insensitive search
+    match = re.search(pattern, modified_address, re.IGNORECASE)
+
+    if match:
+        street_name = match.group('street_name')
+        return street_name.strip()
+    else:
+        return ""
+    
+    
+    # Exclude non-postal addresses
+
+def remove_non_postal(df, in_address_series):
+    '''
+    Highlight non-postal addresses from a polars df where a string series that contain specific substrings
+    indicating non-postal addresses like 'garage', 'parking', 'shed', etc.
+    '''
+    df["in_address_series_temp"] = df[in_address_series].str.lower()
+
+    garage_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bgarage\\b|\\bgarages\\b)", regex=True)
+    parking_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bparking\\b)", regex=True)
+    shed_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bshed\\b|\\bsheds\\b)", regex=True)
+    bike_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bbike\\b|\\bbikes\\b)", regex=True)
+    bicycle_store_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bbicycle store\\b|\\bbicycle store\\b)", regex=True)
+
+    non_postal_series = (garage_address_series | parking_address_series | shed_address_series | bike_address_series | bicycle_store_address_series)
+    
+    df.loc[non_postal_series == True, 'Excluded from search'] = "Excluded - non-postal address"
+
+    df = df.drop("in_address_series_temp", axis = 1)
+
+    return df

tools/pytorch_models.py

@@ -0,0 +1,155 @@
+import torch.nn as nn
+
+class TextClassifier(nn.Module):
+    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers,
+                 dropout, pad_idx):
+        super(TextClassifier, self).__init__()
+        
+        # Embedding layer
+        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_idx)
+        
+        # GRU layers
+        self.rnn = nn.GRU(embedding_dim, 
+                          hidden_dim, 
+                          num_layers=n_layers, 
+                          bidirectional=True, 
+                          dropout=dropout, 
+                          batch_first=True)
+        
+        # Fully connected layer
+        self.fc = nn.Linear(hidden_dim * 2, output_dim)  # Multiply by 2 for bidirection
+        
+        # Dropout layer
+        self.dropout = nn.Dropout(dropout)
+        
+    def forward(self, text, text_lengths):
+        embedded = self.dropout(self.embedding(text))
+    
+        # Pack sequence
+        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths, batch_first=True, enforce_sorted=False)
+        packed_output, _ = self.rnn(packed_embedded)
+    
+        # Unpack sequence
+        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output, batch_first=True)
+    
+        # Pass the entire output tensor to the FC layer for token-level classification
+        return self.fc(output)
+
+class LSTMTextClassifier(nn.Module):
+    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers,
+                 dropout, pad_idx):
+        super(LSTMTextClassifier, self).__init__()
+        
+        # Embedding layer
+        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_idx)
+        
+        # LSTM layers
+        self.rnn = nn.LSTM(embedding_dim, 
+                          hidden_dim, 
+                          num_layers=n_layers, 
+                          bidirectional=True, 
+                          dropout=dropout, 
+                          batch_first=True)
+        
+        # Fully connected layer
+        self.fc = nn.Linear(hidden_dim * 2, output_dim)  # Multiply by 2 for bidirection
+        
+        # Dropout layer
+        self.dropout = nn.Dropout(dropout)
+        
+    def forward(self, text, text_lengths):
+        embedded = self.dropout(self.embedding(text))
+    
+        # Pack sequence
+        packed_embedded = nn.utils.rnn.pack_padded_sequence(embedded, text_lengths, batch_first=True, enforce_sorted=False)
+        
+        # Note: LSTM returns both the output and a tuple of (hidden state, cell state)
+        packed_output, (hidden, cell) = self.rnn(packed_embedded)
+    
+        # Unpack sequence
+        output, output_lengths = nn.utils.rnn.pad_packed_sequence(packed_output, batch_first=True)
+    
+        # Pass the entire output tensor to the FC layer for token-level classification
+        return self.fc(output)
+
+class PositionalEncoding(nn.Module):
+    def __init__(self, d_model, max_len=120):
+        super(PositionalEncoding, self).__init__()
+        self.d_model = d_model
+
+    def forward(self, x):
+        # If pe doesn't exist or its sequence length is different from x's sequence length
+        if not hasattr(self, 'pe') or self.pe.size(0) != x.size(1):
+            max_len = x.size(1)
+            pe = torch.zeros(max_len, self.d_model)
+            position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+            div_term = torch.exp(torch.arange(0, self.d_model, 2).float() * (-math.log(10000.0) / self.d_model))
+            pe[:, 0::2] = torch.sin(position * div_term)
+            pe[:, 1::2] = torch.cos(position * div_term)
+            pe = pe.unsqueeze(0)
+            self.register_buffer('pe', pe.to(x.device))
+
+        return x + self.pe[:, :x.size(1), :]
+
+import torch.nn as nn
+import torch.nn.init as init
+
+def weights_init_kaiming(m):
+    if isinstance(m, nn.Linear):
+        init.kaiming_uniform_(m.weight, nonlinearity='relu')
+        if m.bias is not None:
+            init.zeros_(m.bias)
+    elif isinstance(m, nn.Embedding):
+        init.kaiming_uniform_(m.weight, nonlinearity='relu')
+
+class TransformerClassifier(nn.Module):
+    def __init__(self, vocab_size, embedding_dim, nhead, num_encoder_layers,
+                 num_classes, dropout, pad_idx):
+        super(TransformerClassifier, self).__init__()
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        
+
+        
+        # Embedding layer
+        self.embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx=pad_idx)
+        
+        # Positional encoding
+        self.pos_encoder = PositionalEncoding(embedding_dim)
+        
+        # Transformer with dropout
+        transformer_encoder = nn.TransformerEncoderLayer(d_model=embedding_dim, nhead=nhead, dropout=dropout, activation="gelu")
+        self.transformer = nn.TransformerEncoder(transformer_encoder, num_layers=num_encoder_layers)
+        
+        # Classifier with dropout
+        self.classifier = nn.Sequential(
+            nn.Dropout(dropout),
+            nn.Linear(embedding_dim, num_classes)
+        )
+    
+    def create_attention_mask(self, src, pad_idx):
+        return (src == pad_idx)
+        
+    def forward(self, src, pad_idx):
+        
+        # Check pad_idx
+        if isinstance(pad_idx, torch.Tensor) and torch.numel(pad_idx) > 1:
+            raise ValueError("Expected pad_idx to be a scalar value, but got a tensor with multiple elements.")
+    
+        # Transpose src to have shape (seq_len, batch_size)
+        src = src.transpose(0, 1)
+    
+        # Embedding
+        x = self.embedding(src)
+    
+        # Positional Encoding
+        x = self.pos_encoder(x.to(self.device))
+    
+        # Create attention mask
+        src_key_padding_mask = self.create_attention_mask(src.transpose(0, 1), pad_idx)  # Transpose back to (batch_size, sequence_length)
+    
+        # Transformer
+        x = self.transformer(x, src_key_padding_mask=src_key_padding_mask)
+    
+        #print(model.state_dict())
+        # Classification
+        return self.classifier(x)

tools/recordlinkage_funcs.py

@@ -0,0 +1,384 @@
+import pandas as pd
+from typing import Type, Dict, List, Tuple
+import recordlinkage
+from datetime import datetime
+
+PandasDataFrame = Type[pd.DataFrame]
+PandasSeries = Type[pd.Series]
+MatchedResults = Dict[str,Tuple[str,int]]
+array = List[str]
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+from tools.constants import score_cut_off_nnet_street
+
+# ## Recordlinkage matching functions
+def compute_match(predict_df_search, ref_search, orig_search_df, matching_variables,
+                      text_columns, blocker_column,  weights, fuzzy_method):
+        # Use the merge command to match group1 and group2
+        predict_df_search[matching_variables] = predict_df_search[matching_variables].astype(str)
+        ref_search[matching_variables] = ref_search[matching_variables].astype(str).replace("-999","")
+
+        # SaoText needs to be exactly the same to get a 'full' match. So I moved that to the exact match group
+        exact_columns = list(set(matching_variables) - set(text_columns))
+
+        # Replace all blanks with a space, so they can be included in the fuzzy match searches
+        for column in text_columns:
+            predict_df_search.loc[predict_df_search[column] == '', column] = ' '
+            ref_search.loc[ref_search[column] == '', column] = ' '
+
+        # Score based match functions
+        
+        # Create an index of all pairs
+        indexer = recordlinkage.Index()
+
+        # Block on selected blocker column
+        
+        ## Remove all NAs from predict_df blocker column
+        if blocker_column[0] == "PaoStartNumber":
+            predict_df_search = predict_df_search[~(predict_df_search[blocker_column[0]].isna()) & ~(predict_df_search[blocker_column[0]] == '')& ~(predict_df_search[blocker_column[0]].str.contains(r'^\s*$', na=False))]
+        
+        
+        indexer.block(blocker_column) #matchkey.block(["Postcode", "PaoStartNumber"])
+
+        # Generate candidate pairs
+
+        pairsSBM = indexer.index(predict_df_search,ref_search)
+
+        print('Running with ' + blocker_column[0] + ' as blocker has created', len(pairsSBM), 'pairs.')
+        
+        # If no pairs are found, break
+        if len(pairsSBM) == 0: return pd.DataFrame()
+
+        # Call the compare class from the toolkit 
+        compareSBM = recordlinkage.Compare()
+
+        # Assign variables to matching technique - exact
+        for columns in exact_columns:
+            compareSBM.exact(columns, columns, label = columns, missing_value = 0)
+
+        # Assign variables to matching technique - fuzzy
+        for columns in text_columns:
+            if columns == "Postcode":
+                compareSBM.string(columns, columns, label = columns, missing_value = 0, method = "levenshtein")
+            else:
+                compareSBM.string(columns, columns, label = columns, missing_value = 0, method = fuzzy_method)
+
+        ## Run the match - compare each column within the blocks according to exact or fuzzy matching (defined in cells above)
+
+        scoresSBM = compareSBM.compute(pairs = pairsSBM, x = predict_df_search, x_link = ref_search)
+
+        return scoresSBM
+
+def calc_final_nnet_scores(scoresSBM, weights, matching_variables):
+        #Modify the output scores by the weights set at the start of the code
+        scoresSBM_w = scoresSBM*weights
+
+        ### Determine matched roles that score above a threshold
+
+        # Sum all columns 
+        scoresSBM_r = scoresSBM_w
+
+        scoresSBM_r['score'] = scoresSBM_r[matching_variables].sum(axis = 1)
+        scoresSBM_r['score_max'] = sum(weights.values()) # + 2 for the additional scoring from the weighted variables a couple of cells above
+        scoresSBM_r['score_perc'] = (scoresSBM_r['score'] / scoresSBM_r['score_max'])*100
+
+        scoresSBM_r = scoresSBM_r.reset_index()
+
+        # Rename the index if misnamed
+        scoresSBM_r = scoresSBM_r.rename(columns={"index":"level_1"}, errors = "ignore")
+
+        # Sort all comparisons by score in descending order  
+        scoresSBM_r = scoresSBM_r.sort_values(by=["level_0","score_perc"], ascending = False)
+        
+        # Within each search address, remove anything below the max
+        #scoresSBM_r.to_csv("scoresSBM_r.csv")
+        scoresSBM_g = scoresSBM_r.reset_index()
+        
+        # Get maximum score to join on
+        scoresSBM_g = scoresSBM_g.groupby("level_0").max("score_perc").reset_index()[["level_0", "score_perc"]]
+        scoresSBM_g =scoresSBM_g.rename(columns={"score_perc":"score_perc_max"})
+        scoresSBM_search = scoresSBM_r.merge(scoresSBM_g, on = "level_0", how="left")
+
+        scoresSBM_search['score_perc'] = round(scoresSBM_search['score_perc'],1).astype(float)
+        scoresSBM_search['score_perc_max'] = round(scoresSBM_search['score_perc_max'],1).astype(float)
+
+        return scoresSBM_search
+
+def join_on_pred_ref_details(scoresSBM_search_m, ref_search, predict_df_search):
+    ## Join back search and ref_df address details onto matching df
+    scoresSBM_search_m_j = scoresSBM_search_m.merge(ref_search, left_on="level_1", right_index=True, how = "left", suffixes=("", "_ref"))
+
+    scoresSBM_search_m_j = scoresSBM_search_m_j.merge(predict_df_search, left_on="level_0", right_index=True,how="left", suffixes=("", "_pred"))
+
+    scoresSBM_search_m_j = scoresSBM_search_m_j.reindex(sorted(scoresSBM_search_m_j.columns), axis=1)
+
+    #scoresSBM_search_m_j.to_csv("scoresSBM_search_m_j.csv") 
+
+    return scoresSBM_search_m_j 
+
+def rearrange_columns(scoresSBM_search_m_j, new_join_col, search_df_key_field, blocker_column, standardise):
+
+    start_columns = new_join_col.copy()
+
+    start_columns.extend(["address", "fulladdress", "level_0", "level_1","score","score_max","score_perc","score_perc_max"])
+    
+    other_columns = list(set(scoresSBM_search_m_j.columns) - set(start_columns))
+
+    all_columns_order = start_columns.copy()
+    all_columns_order.extend(sorted(other_columns))
+    
+
+    # Place important columns at start
+
+    scoresSBM_search_m_j = scoresSBM_search_m_j.reindex(all_columns_order, axis=1)
+
+    scoresSBM_search_m_j = scoresSBM_search_m_j.rename(columns={'address':'address_pred',
+    'fulladdress':'address_ref',
+    'level_0':'index_pred',
+    'level_1':'index_ref',
+    'score':'match_score',
+    'score_max':'max_possible_score',
+    'score_perc':'perc_weighted_columns_matched',
+    'score_perc_max':'perc_weighted_columns_matched_max_for_pred_address'})
+
+    scoresSBM_search_m_j = scoresSBM_search_m_j.sort_values("index_pred", ascending = True)
+    
+    # ref_index is just a duplicate of index_ref, needed for outputs
+    scoresSBM_search_m_j["ref_index"] = scoresSBM_search_m_j["index_ref"]
+
+    #search_df_j = orig_search_df[["full_address_search", search_df_key_field]]
+
+    #scoresSBM_out = scoresSBM_search_m_j.merge(search_df_j, left_on = "address_pred", right_on = "full_address_search", how = "left")
+
+    final_cols = new_join_col.copy()
+    final_cols.extend([search_df_key_field, 'full_match_score_based', 'address_pred', 'address_ref',\
+                                                'match_score', 'max_possible_score', 'perc_weighted_columns_matched',\
+                                                'perc_weighted_columns_matched_max_for_pred_address',\
+                                                'Organisation', 'Organisation_ref', 'Organisation_pred',\
+                                                'SaoText', 'SaoText_ref', 'SaoText_pred',\
+                                                'SaoStartNumber', 'SaoStartNumber_ref', 'SaoStartNumber_pred',\
+                                                'SaoStartSuffix', 'SaoStartSuffix_ref', 'SaoStartSuffix_pred',\
+                                                'SaoEndNumber', 'SaoEndNumber_ref', 'SaoEndNumber_pred',\
+                                                'SaoEndSuffix', 'SaoEndSuffix_ref', 'SaoEndSuffix_pred',\
+                                                'PaoStartNumber', 'PaoStartNumber_ref', 'PaoStartNumber_pred',\
+                                                'PaoStartSuffix', 'PaoStartSuffix_ref', 'PaoStartSuffix_pred',\
+                                                'PaoEndNumber', 'PaoEndNumber_ref', 'PaoEndNumber_pred',\
+                                                'PaoEndSuffix', 'PaoEndSuffix_ref', 'PaoEndSuffix_pred',\
+                                                'PaoText', 'PaoText_ref', 'PaoText_pred',\
+                                                'Street', 'Street_ref', 'Street_pred',\
+                                                'PostTown', 'PostTown_ref', 'PostTown_pred',\
+                                                'Postcode', 'Postcode_ref', 'Postcode_pred', 'Postcode_predict',\
+                                                'index_pred', 'index_ref',  'Reference file'
+                                                ])
+    
+    scoresSBM_out = scoresSBM_search_m_j[final_cols]
+
+    #scoresSBM_out.to_csv("scoresSBM_out" + "_" + blocker_column[0] + "_" + str(standardise) + ".csv")
+
+    return scoresSBM_out, start_columns
+
+def create_matched_results_nnet(scoresSBM_best, search_df_key_field, orig_search_df, new_join_col, standardise, ref_search, blocker_column, score_cut_off):
+
+    #scoresSBM_best.to_csv("scores_sbm_best_" + str(standardise) + ".csv")
+
+    ### Make the final 'matched output' file
+    scoresSBM_best_pred_cols = scoresSBM_best.filter(regex='_pred$').iloc[:,1:-1]
+    scoresSBM_best["search_orig_address"] = (scoresSBM_best_pred_cols.agg(' '.join, axis=1)).str.strip().str.replace("\s{2,}", " ", regex=True)
+
+    scoresSBM_best_ref_cols = scoresSBM_best.filter(regex='_ref$').iloc[:,1:-1]
+    scoresSBM_best['reference_mod_address'] = (scoresSBM_best_ref_cols.agg(' '.join, axis=1)).str.strip().str.replace("\s{2,}", " ", regex=True)
+
+    ## Create matched output df
+    matched_output_SBM = orig_search_df[[search_df_key_field, "full_address", "postcode", "property_number", "prop_number",	"flat_number",	"apart_number",	"block_number",	'unit_number', "room_number", "house_court_name"]].replace(r"\bnan\b", "", regex=True).infer_objects(copy=False)
+    matched_output_SBM[search_df_key_field] = matched_output_SBM[search_df_key_field].astype(str)
+
+    ###
+    matched_output_SBM = matched_output_SBM.merge(scoresSBM_best[[search_df_key_field, 'index_ref','address_ref', 
+        'full_match_score_based', 'Reference file']], on = search_df_key_field, how = "left").\
+        rename(columns={"full_address":"search_orig_address"})
+    
+    #ref_search.to_csv("ref_search.csv")
+
+    if 'index' not in ref_search.columns:
+        ref_search['ref_index'] = ref_search.index
+
+    matched_output_SBM = matched_output_SBM.merge(ref_search.drop_duplicates("fulladdress")[["ref_index", "fulladdress", "Postcode", "property_number", "prop_number", "flat_number", "apart_number", "block_number", 'unit_number', "room_number", "house_court_name", "ref_address_stand"]], left_on = "address_ref", right_on = "fulladdress", how = "left", suffixes=('_search', '_reference')).rename(columns={"fulladdress":"reference_orig_address", "ref_address_stand":"reference_list_address"})
+
+    #matched_output_SBM.to_csv("matched_output_SBM_earlier_" + str(standardise) + ".csv")
+
+    # To replace with number check
+   
+
+    matched_output_SBM = matched_output_SBM.rename(columns={"full_match_score_based":"full_match"})
+
+    matched_output_SBM['property_number_match'] = matched_output_SBM['full_match']
+    #
+
+    scores_SBM_best_cols = [search_df_key_field, 'full_match_score_based',  'perc_weighted_columns_matched', 'address_pred']#, "reference_mod_address"]
+    scores_SBM_best_cols.extend(new_join_col)
+
+    matched_output_SBM_b = scoresSBM_best[scores_SBM_best_cols]
+
+    matched_output_SBM = matched_output_SBM.merge(matched_output_SBM_b.drop_duplicates(search_df_key_field), on = search_df_key_field,  how = "left")
+
+    #matched_output_SBM.to_csv("matched_output_SBM_later_" + str(standardise) + ".csv")
+
+    from tools.fuzzy_match import create_diag_shortlist
+    matched_output_SBM = create_diag_shortlist(matched_output_SBM, "search_orig_address", score_cut_off, blocker_column, fuzzy_col='perc_weighted_columns_matched', search_mod_address="address_pred", resolve_tie_breaks=False)
+
+    #matched_output_SBM.to_csv("matched_output_after.csv")
+    
+    #matched_output_SBM["UPRN"] = scoresSBM_best['UPRN']
+
+    matched_output_SBM['standardised_address'] = standardise
+
+    matched_output_SBM = matched_output_SBM.rename(columns={"address_pred":"search_mod_address",
+        #"address_ref":"reference_orig_address",
+        #"full_match_score_based":"fuzzy_score_match",                        
+        'perc_weighted_columns_matched':"fuzzy_score"})
+    
+    matched_output_SBM_cols = [search_df_key_field, 'search_orig_address','reference_orig_address',
+        'full_match',
+        'full_number_match',
+        'flat_number_match',
+        'room_number_match',
+        'block_number_match',
+        'property_number_match',
+        'close_postcode_match',
+        'house_court_name_match',
+        'fuzzy_score_match',
+        "fuzzy_score", 
+        'property_number_search', 'property_number_reference',  
+        'flat_number_search', 'flat_number_reference', 
+        'room_number_search', 'room_number_reference',
+        'block_number_search', 'block_number_reference',
+        "unit_number_search","unit_number_reference",
+        'house_court_name_search', 'house_court_name_reference',
+        "search_mod_address", 'reference_mod_address','Postcode', 'postcode', 'ref_index', 'Reference file']
+
+    #matched_output_SBM_cols = [search_df_key_field, 'search_orig_address', 'reference_orig_address',
+    #'full_match', 'fuzzy_score_match', 'property_number_match', 'full_number_match', 
+    #'fuzzy_score', 'search_mod_address', 'reference_mod_address', 'Reference file']
+    
+    matched_output_SBM_cols.extend(new_join_col)
+    matched_output_SBM_cols.extend(['standardised_address'])
+    matched_output_SBM = matched_output_SBM[matched_output_SBM_cols]
+    
+    matched_output_SBM = matched_output_SBM.sort_values(search_df_key_field, ascending=True)
+
+    #matched_output_SBM.to_csv("matched_output_SBM_out.csv")
+
+    return matched_output_SBM
+
+def score_based_match(predict_df_search, ref_search, orig_search_df, matching_variables, text_columns, blocker_column,  weights, fuzzy_method, score_cut_off, search_df_key_field, standardise, new_join_col, score_cut_off_nnet_street=score_cut_off_nnet_street):
+    
+    scoresSBM = compute_match(predict_df_search, ref_search, orig_search_df, matching_variables, text_columns, blocker_column,  weights, fuzzy_method)   
+    
+    if scoresSBM.empty:
+        # If no pairs are found, break
+        return pd.DataFrame(), pd.DataFrame()
+
+    scoresSBM_search = calc_final_nnet_scores(scoresSBM, weights, matching_variables)
+
+    # Filter potential matched address scores to those with highest scores only
+    scoresSBM_search_m = scoresSBM_search[scoresSBM_search["score_perc"] == scoresSBM_search["score_perc_max"]]
+
+    scoresSBM_search_m_j = join_on_pred_ref_details(scoresSBM_search_m, ref_search, predict_df_search)
+
+    #scoresSBM_search_m_j.to_csv("scoresSBM_search_m_j.csv")
+
+    # When blocking by street, may to have an increased threshold as this is more prone to making mistakes
+    if blocker_column[0] == "Street": scoresSBM_search_m_j['full_match_score_based'] = (scoresSBM_search_m_j['score_perc'] >= score_cut_off_nnet_street)
+
+    else: scoresSBM_search_m_j['full_match_score_based'] = (scoresSBM_search_m_j['score_perc'] >= score_cut_off)
+    
+    ### Reorder some columns
+    scoresSBM_out, start_columns = rearrange_columns(scoresSBM_search_m_j, new_join_col, search_df_key_field, blocker_column, standardise)
+
+    #scoresSBM_out.to_csv("scoresSBM_out.csv")
+
+    matched_output_SBM = create_matched_results_nnet(scoresSBM_out, search_df_key_field, orig_search_df, new_join_col, standardise, ref_search, blocker_column, score_cut_off)
+
+    matched_output_SBM_best = matched_output_SBM.sort_values([search_df_key_field, "full_match"], ascending = [True, False]).drop_duplicates(search_df_key_field)
+
+    #matched_output_SBM.to_csv("matched_output_SBM.csv")
+    #matched_output_SBM_best.to_csv("matched_output_SBM_best.csv")
+
+    scoresSBM_best = scoresSBM_out[scoresSBM_out[search_df_key_field].isin(matched_output_SBM_best[search_df_key_field])]
+
+    return scoresSBM_best, matched_output_SBM_best
+
+def check_matches_against_fuzzy(match_results, scoresSBM, search_df_key_field):
+
+    if not match_results.empty:
+
+        if 'fuzz_full_match' not in match_results.columns:
+            match_results['fuzz_full_match'] = False
+
+        match_results = match_results.add_prefix("fuzz_").rename(columns={"fuzz_"+search_df_key_field:search_df_key_field})
+
+        #Merge fuzzy match full matches onto model data
+
+        scoresSBM_m = scoresSBM.merge(match_results.drop_duplicates(search_df_key_field), on = search_df_key_field, how = "left")
+        
+    else:
+         scoresSBM_m = scoresSBM
+         scoresSBM_m["fuzz_full_match"] = False
+         scoresSBM_m['fuzz_fuzzy_score_match'] = False
+         scoresSBM_m['fuzz_property_number_match'] = False
+         scoresSBM_m['fuzz_fuzzy_score'] = 0
+         scoresSBM_m['fuzz_reference_orig_address'] = ""
+
+    scoresSBM_t = scoresSBM[scoresSBM["full_match_score_based"]==True]
+
+    ### Create a df of matches the model finds that the fuzzy matching work did not
+
+    scoresSBM_m_model_add_matches = scoresSBM_m[(scoresSBM_m["full_match_score_based"] == True) &\
+                                                            (scoresSBM_m["fuzz_full_match"] == False)]
+
+    # Drop some irrelevant columns
+
+    first_cols = ['UPRN', search_df_key_field, 'full_match_score_based', 'fuzz_full_match', 'fuzz_fuzzy_score_match', 'fuzz_property_number_match',\
+                                   'fuzz_fuzzy_score', 'match_score', 'max_possible_score', 'perc_weighted_columns_matched',\
+                                   'perc_weighted_columns_matched_max_for_pred_address', 'address_pred',\
+                                   'address_ref', 'fuzz_reference_orig_address']
+
+    last_cols = [col for col in scoresSBM_m_model_add_matches.columns if col not in first_cols]
+
+    scoresSBM_m_model_add_matches = scoresSBM_m_model_add_matches[first_cols+last_cols].drop(['fuzz_search_mod_address',
+       'fuzz_reference_mod_address', 'fuzz_fulladdress', 'fuzz_UPRN'], axis=1, errors="ignore")
+
+    ### Create a df for matches the fuzzy matching found that the neural net model does not
+
+    if not match_results.empty:
+        scoresSBM_t_model_failed = match_results[(~match_results[search_df_key_field].isin(scoresSBM_t[search_df_key_field])) &\
+                                                        (match_results["fuzz_full_match"] == True)]
+
+        scoresSBM_t_model_failed = scoresSBM_t_model_failed.\
+            merge(scoresSBM.drop_duplicates(search_df_key_field), on = search_df_key_field, how = "left")
+
+        scoresSBM_t_model_failed = scoresSBM_t_model_failed[first_cols+last_cols].drop(['fuzz_search_mod_address',
+        'fuzz_reference_mod_address', 'fuzz_fulladdress', 'fuzz_UPRN'], axis=1, errors="ignore")
+    else:
+          scoresSBM_t_model_failed = pd.DataFrame()
+
+    ## Join back onto original results file and export
+
+    scoresSBM_new_matches_from_model = scoresSBM_m_model_add_matches.drop_duplicates(search_df_key_field)
+
+    if not match_results.empty:
+        match_results_out = match_results.merge(scoresSBM_new_matches_from_model[[search_df_key_field, 'full_match_score_based', 'address_pred',
+        'address_ref']], on = search_df_key_field, how = "left")
+
+        match_results_out.loc[match_results_out['full_match_score_based'].isna(),'full_match_score_based'] = False
+
+        #match_results_out['full_match_score_based'].value_counts()
+
+        match_results_out["full_match_fuzzy_or_score_based"] = (match_results_out["fuzz_full_match"] == True) |\
+        (match_results_out["full_match_score_based"] == True)
+    else: match_results_out = match_results
+
+    return scoresSBM_m_model_add_matches, scoresSBM_t_model_failed, match_results_out
+

tools/standardise.py

@@ -0,0 +1,722 @@
+import pandas as pd
+import numpy as np
+import re
+from typing import Type, Dict, List, Tuple
+from datetime import datetime
+import warnings
+warnings.filterwarnings("ignore", 'This pattern is interpreted as a regular expression')
+
+PandasDataFrame = Type[pd.DataFrame]
+PandasSeries = Type[pd.Series]
+MatchedResults = Dict[str,Tuple[str,int]]
+array = List[str]
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+# # Standardisation functions
+
+def standardise_wrapper_func(search_df_cleaned:PandasDataFrame, ref_df_cleaned:PandasDataFrame,\
+                               standardise = False, filter_to_lambeth_pcodes = True, match_task = "fuzzy"):
+    '''
+    Initial standardisation of search and reference dataframes before passing addresses and postcodes to the main standardisation function
+    '''
+    
+    ## Search df - lower case addresses, replace spaces in postcode
+    search_df_cleaned["full_address_search"] = search_df_cleaned["full_address"].str.lower().str.strip()
+    search_df_cleaned['postcode_search'] = search_df_cleaned['postcode'].str.lower().str.strip().str.replace("\s+", "", regex=True)
+
+    # Filter out records where 'Excluded from search' is not a postal address by making the postcode blank
+    search_df_cleaned.loc[search_df_cleaned['Excluded from search'] == "Excluded - non-postal address", 'postcode_search'] = ""
+
+    # Remove nulls from ref_df postcode
+    ref_df_cleaned = ref_df_cleaned[ref_df_cleaned['Postcode'].notna()]
+    
+    ref_df_cleaned["full_address_search"] = ref_df_cleaned["fulladdress"].str.lower().str.strip()
+    ref_df_cleaned['postcode_search'] = ref_df_cleaned['Postcode'].str.lower().str.strip().str.replace("\s+", "", regex=True)
+    
+    # Block only on first 5 characters of postcode string - Doesn't give more matches and makes everything a bit slower
+    #search_df_cleaned['postcode_search'] = search_df_cleaned['postcode_search'].str[:-1]
+    #ref_df_cleaned['postcode_search'] = ref_df_cleaned['postcode_search'].str[:-1]
+
+
+    ### Use standardise function
+
+    ### Remove 'non-housing' places from the list - not included as want to check all
+    #search_df_after_stand = remove_non_housing(search_df_cleaned, 'full_address_search')
+    search_df_after_stand = standardise_address(search_df_cleaned, "full_address_search", "search_address_stand", standardise = standardise, out_london = True)
+
+    ## Standardise ref_df addresses
+
+    if match_task == "fuzzy":
+        ref_df_after_stand = standardise_address(ref_df_cleaned, "full_address_search", "ref_address_stand", standardise = standardise, out_london = True)
+    else:
+        # For the neural net matching, I didn't find that standardising the reference addresses helped at all, in fact it made things worse. So reference addresses are not standardised at this step.
+        ref_df_after_stand = standardise_address(ref_df_cleaned, "full_address_search", "ref_address_stand", standardise = False, out_london = True)
+
+
+    return search_df_after_stand, ref_df_after_stand#, search_df_after_stand_series, ref_df_after_stand_series
+
+def standardise_address(df:PandasDataFrame, col:str, out_col:str, standardise:bool = True, out_london = True) -> PandasDataFrame:
+    
+    ''' 
+    This function takes a 'full address' column and then standardises so that extraneous
+    information is removed (i.e. postcodes & London, as this algorithm is used for London
+    addresses only), and so that room/flat/property numbers can be accurately extracted. The
+    standardised addresses can then be used for the fuzzy matching functions later in this 
+    notebook.
+    
+    The function does the following:
+    
+    - Removes the post code and 'london' (if not dealing with addresses outside of london)
+      from the address to reduce the text the algorithm has to search.
+      Postcode removal uses regex to extract a UK postcode.
+      
+    - Remove the word 'flat' or 'apartment' from an address that has only one number in it
+    
+    - Add 'flat' to the start of any address that contains 'house' or 'court' (which are generally housing association buildings)
+      This is because in the housing list, these addresses never have the word flat in front of them
+
+    - Replace any addresses that don't have a space between the comma and the next word or double spaces
+    
+    - Replace 'number / number' and 'number-number' with 'number' (the first number in pair)
+    
+    - Add 'flat' to the start of addresses that include ground floor/first floor etc. flat 
+      in the text. Replace with flat a,b,c etc.
+    
+    - Pull out property, flat, and room numbers from the address text
+    
+    - Return the data frame with the new columns included
+    
+    '''
+   
+    df_copy = df.copy(deep=True)
+    
+    # Trim the address to remove leading and tailing spaces
+    df_copy[col] = df_copy[col].str.strip()
+    
+    ''' Remove the post code and 'london' from the address to reduce the text the algorithm has to search
+    Using a regex to extract a UK postcode. I got the regex from the following. Need to replace their \b in the solution with \\b
+    https://stackoverflow.com/questions/51828712/r-regular-expression-for-extracting-uk-postcode-from-an-address-is-not-ordered
+        
+    The following will pick up whole postcodes, postcodes with just the first part, and postcodes with the first
+    part and the first number of the second half
+    '''
+    
+    
+    df_copy['add_no_pcode'] = remove_postcode(df_copy, col)
+    
+    if out_london == False:
+        df_copy['add_no_pcode'] = df_copy['add_no_pcode'].str.replace("london","").str.replace(r",,|, ,","", regex=True)
+    
+    # If the user wants to standardise the address
+    if standardise:
+        
+        df_copy['add_no_pcode'] = df_copy['add_no_pcode'].str.lower()
+
+        # If there are dates at the start of the address, change this
+        df_copy['add_no_pcode'] = replace_mistaken_dates(df_copy, 'add_no_pcode')
+
+        # Replace flat name variations with flat, abbreviations with full name of item (e.g. rd to road)
+        df_copy['add_no_pcode'] = df_copy['add_no_pcode'].str.replace(r"\brd\b","road", regex=True).\
+                                                str.replace(r"\bst\b","street", regex=True).\
+                                                str.replace(r"\bave\b","avenue", regex=True).\
+                                                str.replace("'", "", regex=False).\
+                                                str.replace(r"\bat\b ", " ",regex=True).\
+                                                str.replace("apartment", "flat",regex=False).\
+                                                str.replace("studio flat", "flat",regex=False).\
+                                                str.replace("cluster flat", "flats",regex=False).\
+                                                str.replace(r"\bflr\b", "floor", regex=True).\
+                                                str.replace(r"\bflrs\b", "floors", regex=True).\
+                                                str.replace(r"\blwr\b", "lower", regex=True).\
+                                                str.replace(r"\bgnd\b", "ground", regex=True).\
+                                                str.replace(r"\blgnd\b", "lower ground", regex=True).\
+                                                str.replace(r"\bgrd\b", "ground", regex=True).\
+                                                str.replace(r"\bmais\b", "flat", regex=True).\
+                                                str.replace(r"\bmaisonette\b", "flat", regex=True).\
+                                                str.replace(r"\bpt\b", "penthouse", regex=True).\
+                                                str.replace(r"\bbst\b","basement", regex=True).\
+                                                str.replace(r"\bbsmt\b","basement", regex=True)
+        
+        df_copy["add_no_pcode_house"] = move_flat_house_court(df_copy)
+    
+        # Replace any addresses that don't have a space between the comma and the next word. and double spaces # df_copy['add_no_pcode_house']
+        df_copy['add_no_pcode_house_comma'] = df_copy['add_no_pcode_house'].str.replace(r',(\w)', r', \1', regex=True).str.replace('  ', ' ', regex=False)
+
+        # Replace number / number and number-number with number
+        df_copy['add_no_pcode_house_comma_no'] = df_copy['add_no_pcode_house_comma'].str.replace(r'(\d+)\/(\d+)', r'\1', regex=True\
+                        ).str.replace(r'(\d+)-(\d+)', r'\1', regex=True\
+                        ).str.replace(r'(\d+) - (\d+)', r'\1', regex=True)
+
+        # Add 'flat' to the start of addresses that include ground/first/second etc. floor flat in the text
+        df_copy['floor_replacement'] = replace_floor_flat(df_copy, 'add_no_pcode_house_comma_no')
+        df_copy['flat_added_to_start_addresses_begin_letter'] = add_flat_addresses_start_with_letter(df_copy, 'floor_replacement')
+
+        df_copy[out_col] = merge_series(df_copy['add_no_pcode_house_comma_no'], df_copy['flat_added_to_start_addresses_begin_letter'])
+
+        # Write stuff back to the original df
+        df[out_col] = df_copy[out_col]
+    
+    else:
+        df_copy[out_col] = df_copy['add_no_pcode']
+        df[out_col] = df_copy['add_no_pcode']
+
+    ## POST STANDARDISATION CLEANING AND INFORMATION EXTRACTION  
+    # Remove trailing spaces
+    df[out_col] = df[out_col].str.strip()
+    
+    # Pull out property, flat, and room numbers from the address text
+    df['property_number'] = extract_prop_no(df_copy, out_col)
+
+    # Extract flat, apartment numbers
+    df = extract_flat_and_other_no(df, out_col)
+
+    df['flat_number'] = merge_series(df['flat_number'], df['apart_number'])
+    df['flat_number'] = merge_series(df['flat_number'], df['prop_number'])
+    df['flat_number'] = merge_series(df['flat_number'], df['first_sec_number'])
+    df['flat_number'] = merge_series(df['flat_number'], df['first_letter_flat_number'])
+    df['flat_number'] = merge_series(df['flat_number'], df['first_letter_no_more_numbers'])
+    
+    # Extract room numbers
+    df['room_number'] = extract_room_no(df, out_col)
+
+    # Extract block and unit names
+    df = extract_block_and_unit_name(df, out_col)
+
+    # Extract house or court name
+    df['house_court_name'] = extract_house_or_court_name(df, out_col)
+
+    return df
+
+def move_flat_house_court(df:PandasDataFrame):
+            ''' Remove 'flat' from any address that contains 'house' or 'court'
+            From the df address, remove the word 'flat' from any address that contains the word 'house' or 'court'
+            This is because in the housing list, these addresses never have the word flat in front of them
+            '''
+            
+            # Remove the word flat or apartment from addresses that have only one number in it. 'Flat' will be re-added later to relevant addresses 
+            # that need it (replace_floor_flat)
+            df['flat_removed'] = remove_flat_one_number_address(df, 'add_no_pcode')
+        
+        
+            
+            remove_flat_house = df['flat_removed'].str.lower().str.contains(r"\bhouse\b")#(?=\bhouse\b)(?!.*house road)")
+            remove_flat_court = df['flat_removed'].str.lower().str.contains(r"\bcourt\b")#(?=\bcourt\b)(?!.*court road)")
+            remove_flat_terrace = df['flat_removed'].str.lower().str.contains(r"\bterrace\b")#(?=\bterrace\b)(?!.*terrace road)")
+            remove_flat_house_or_court = (remove_flat_house | remove_flat_court | remove_flat_terrace == 1)
+
+            df['remove_flat_house_or_court'] = remove_flat_house_or_court
+
+            # Assuming 'df' is your DataFrame
+            df = df[~df.index.duplicated(keep='first')]
+
+            df['house_court_replacement'] = "flat " + df.loc[df['remove_flat_house_or_court'] == True, 'flat_removed'].str.replace(r"\bflat\b","", regex=True).str.strip().map(str)
+
+            #df["add_no_pcode_house"] = merge_columns(df, "add_no_pcode_house", 'flat_removed', "house_court_replacement")
+
+            #merge_columns(df, "new_col", col1, 'letter_after_number')
+            df["add_no_pcode_house"] = merge_series(df['flat_removed'], df["house_court_replacement"])
+
+            return df["add_no_pcode_house"]
+
+def extract_street_name(address:str) -> str:
+    """
+    Extracts the street name from the given address.
+
+    Args:
+        address (str): The input address string.
+
+    Returns:
+        str: The extracted street name, or an empty string if no match is found.
+
+    Examples:
+        >>> address1 = "1 Ash Park Road SE54 3HB"
+        >>> extract_street_name(address1)
+        'Ash Park Road'
+
+        >>> address2 = "Flat 14 1 Ash Park Road SE54 3HB"
+        >>> extract_street_name(address2)
+        'Ash Park Road'
+
+        >>> address3 = "123 Main Blvd"
+        >>> extract_street_name(address3)
+        'Main Blvd'
+
+        >>> address4 = "456 Maple AvEnUe"
+        >>> extract_street_name(address4)
+        'Maple AvEnUe'
+
+        >>> address5 = "789 Oak Street"
+        >>> extract_street_name(address5)
+        'Oak Street'
+    """
+    
+  
+    
+    street_types = [
+        'Street', 'St', 'Boulevard', 'Blvd', 'Highway', 'Hwy', 'Broadway', 'Freeway',
+        'Causeway', 'Cswy', 'Expressway', 'Way', 'Walk', 'Lane', 'Ln', 'Road', 'Rd',
+        'Avenue', 'Ave', 'Circle', 'Cir', 'Cove', 'Cv', 'Drive', 'Dr', 'Parkway', 'Pkwy',
+        'Park', 'Court', 'Ct', 'Square', 'Sq', 'Loop', 'Place', 'Pl', 'Parade', 'Estate',
+        'Alley', 'Arcade','Avenue', 'Ave','Bay','Bend','Brae','Byway','Close','Corner','Cove',
+        'Crescent', 'Cres','Cul-de-sac','Dell','Drive', 'Dr','Esplanade','Glen','Green','Grove','Heights', 'Hts',
+        'Mews','Parade','Path','Piazza','Promenade','Quay','Ridge','Row','Terrace', 'Ter','Track','Trail','View','Villas',
+        'Marsh', 'Embankment', 'Cut', 'Hill', 'Passage', 'Rise', 'Vale', 'Side'
+    ]
+
+    # Dynamically construct the regex pattern with all possible street types
+    street_types_pattern = '|'.join(rf"{re.escape(street_type)}" for street_type in street_types)
+
+    # The overall regex pattern to capture the street name
+    pattern = rf'(?:\d+\s+|\w+\s+\d+\s+|.*\d+[a-z]+\s+|.*\d+\s+)*(?P<street_name>[\w\s]+(?:{street_types_pattern}))'
+
+    def replace_postcode(address):
+        pattern = r'\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2}|GIR ?0A{2})\b$|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$|\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$'
+        return re.sub(pattern, "", address)
+
+    
+    modified_address = replace_postcode(address.upper())
+    #print(modified_address)
+    #print(address)
+       
+    # Perform a case-insensitive search
+    match = re.search(pattern, modified_address, re.IGNORECASE)
+
+    if match:
+        street_name = match.group('street_name')
+        return street_name.strip()
+    else:
+        return ""
+
+def remove_flat_one_number_address(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+
+    '''
+    If there is only one number in the address, and there is no letter after the number,
+    remove the word flat from the address
+    '''
+    
+    df['contains_letter_after_number'] = df[col1].str.lower().str.contains(r"\d+(?:[a-z]|[A-Z])(?!.*\d+)", regex = True)
+    df['contains_single_letter_before_number'] = df[col1].str.lower().str.contains(r'\b[A-Za-z]\b[^\d]* \d', regex = True)
+    df['two_numbers_in_address'] =  df[col1].str.lower().str.contains(r"(?:\d+.*?)[^a-zA-Z0-9_].*?\d+", regex = True)
+    df['contains_apartment'] = df[col1].str.lower().str.contains(r"\bapartment\b \w+|\bapartments\b \w+", "", regex = True)
+    df['contains_flat'] = df[col1].str.lower().str.contains(r"\bflat\b \w+|\bflats\b \w+", "", regex = True)
+    df['contains_room'] = df[col1].str.lower().str.contains(r"\broom\b \w+|\brooms\b \w+", "", regex = True)
+      
+    
+    df['selected_rows'] = (df['contains_letter_after_number'] == False) &\
+                          (df['two_numbers_in_address'] == False) &\
+                            (df['contains_single_letter_before_number'] == False) &\
+                             ((df['contains_flat'] == True) |\
+                             (df['contains_apartment'] == True) |\
+                             (df['contains_room'] == True))
+        
+    df['one_number_no_flat'] =  df[df['selected_rows'] == True][col1]
+    df['one_number_no_flat'] =  df['one_number_no_flat'].str.replace(r"(\bapartment\b)|(\bapartments\b)", "", regex=True).str.replace(r"(\bflat\b)|(\bflats\b)", "", regex=True).str.replace(r"(\broom\b)|(\brooms\b)", "", regex=True)
+
+    df["new_col"] = merge_series(df[col1], df["one_number_no_flat"])
+    
+    return df['new_col']
+
+def add_flat_addresses_start_with_letter(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+    '''
+    Add the word flat to addresses that start with a letter.
+    '''
+    
+    df['contains_single_letter_at_start_before_number'] = df[col1].str.lower().str.contains(r'^\b[A-Za-z]\b[^\d]* \d', regex = True)
+
+    df['selected_rows'] = (df['contains_single_letter_at_start_before_number'] == True)
+    df['flat_added_to_string_start'] =  "flat " + df[df['selected_rows'] == True][col1]
+    
+    #merge_columns(df, "new_col", col1, 'flat_added_to_string_start')
+    df["new_col"] = merge_series(df[col1], df['flat_added_to_string_start'])
+    
+    
+    return df['new_col']
+
+def extract_letter_one_number_address(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+    '''
+    This function looks for addresses that have a letter after a number, but ONLY one number
+    in the string, and doesn't already have a flat, apartment, or room number. 
+        
+    It then extracts this letter and returns this.
+    
+    This is for addresses such as '2b sycamore road', changes it to
+    flat b 2 sycamore road so that 'b' is selected as the flat number
+
+    
+    '''
+    
+    df['contains_no_numbers_without_letter'] = df[col1].str.lower().str.contains(r"^(?:(?!\d+ ).)*$")
+    df['contains_letter_after_number'] = df[col1].str.lower().str.contains(r"\d+(?:[a-z]|[A-Z])(?!.*\d+)")      
+    df['contains_apartment'] = df[col1].str.lower().str.contains(r"\bapartment\b \w+|\bapartments\b \w+", "")
+    df['contains_flat'] = df[col1].str.lower().str.contains(r"\bflat\b \w+|\bflats\b \w+", "")
+    df['contains_room'] = df[col1].str.lower().str.contains(r"\broom\b \w+|\brooms\b \w+", "")
+        
+    df['selected_rows'] = (df['contains_no_numbers_without_letter'] == True) &\
+                             (df['contains_letter_after_number'] == True) &\
+                             (df['contains_flat'] == False) &\
+                             (df['contains_apartment'] == False) &\
+                             (df['contains_room'] == False)
+            
+    df['extract_letter'] =  df[(df['selected_rows'] == True)\
+                                  ][col1].str.extract(r"\d+([a-z]|[A-Z])")
+    
+    df['extract_number'] =  df[(df['selected_rows'] == True)\
+                                  ][col1].str.extract(r"(\d+)[a-z]|[A-Z]")
+    
+
+    df['letter_after_number'] = "flat " +\
+                                df[(df['selected_rows'] == True)\
+                                  ]['extract_letter'] +\
+                                " " +\
+                                df[(df['selected_rows'] == True)\
+                                  ]['extract_number'] +\
+                                " " +\
+                                df[(df['selected_rows'])\
+                                  ][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\d+([a-z]|[A-Z])","", regex=True).map(str)
+
+    #merge_columns(df, "new_col", col1, 'letter_after_number')
+    df["new_col"] = merge_series(df[col1], df['letter_after_number'])
+    
+    return df['new_col']
+
+# def extract_letter_one_number_address(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+#     '''
+#     This function extracts a letter after a single number in an address, excluding cases with existing flat, apartment, or room numbers.
+#     It transforms addresses like '2b sycamore road' to 'flat b 2 sycamore road' to designate 'b' as the flat number.
+#     '''
+    
+#     df['selected_rows'] = (df[col1].str.lower().str.contains(r"^(?:(?!\d+ ).)*$") & \
+#                            df[col1].str.lower().str.contains(r"\d+(?:[a-z]|[A-Z])(?!.*\d+)") & \
+#                            ~df[col1].str.lower().str.contains(r"\bflat\b \w+|\bflats\b \w+|\bapartment\b \w+|\bapartments\b \w+|\broom\b \w+|\brooms\b \w+"))
+    
+#     df['extract_letter'] = df.loc[df['selected_rows'], col1].str.extract(r"\d+([a-z]|[A-Z])")
+#     df['extract_number'] = df.loc[df['selected_rows'], col1].str.extract(r"(\d+)[a-z]|[A-Z]")
+    
+#     df['letter_after_number'] = "flat " + df['extract_letter'] + " " + df['extract_number'] + " " + \
+#                                 df.loc[df['selected_rows'], col1].str.replace(r"\bflat\b", "", regex=True).str.replace(r"\d+([a-z]|[A-Z])", "", regex=True).map(str)
+    
+#     df["new_col"] = df[col1].copy()
+#     df.loc[df['selected_rows'], "new_col"] = df['letter_after_number']
+    
+#     return df['new_col']
+
+def replace_floor_flat(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+    ''' In references to basement, ground floor, first floor, second floor, and top floor
+    flats, this function moves the word 'flat' to the front of the address. This is so that the
+    following word (e.g. basement, ground floor) is recognised as the flat number in the 'extract_flat_and_other_no' function
+    '''
+    
+    df['letter_after_number'] = extract_letter_one_number_address(df, col1)
+       
+   
+    df['basement'] = "flat basement" + df[df[col1].str.lower().str.contains(r"basement"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bbasement\b","", regex=True).map(str)
+    
+
+    df['ground_floor'] = "flat a " + df[df[col1].str.lower().str.contains(r"\bground floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bground floor\b","", regex=True).map(str)
+
+    df['first_floor'] = "flat b " + df[df[col1].str.lower().str.contains(r"\bfirst floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bfirst floor\b","", regex=True).map(str)
+
+    df['ground_and_first_floor'] = "flat ab " + df[df[col1].str.lower().str.contains(r"\bground and first floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bground and first floor\b","", regex=True).map(str)
+
+    df['basement_ground_and_first_floor'] = "flat basementab " + df[df[col1].str.lower().str.contains(r"\bbasement ground and first floors\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bbasement and ground and first floors\b","", regex=True).map(str)
+
+    df['basement_ground_and_first_floor2'] = "flat basementab " + df[df[col1].str.lower().str.contains(r"\bbasement ground and first floors\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bbasement ground and first floors\b","", regex=True).map(str)
+
+    df['second_floor'] = "flat c " + df[df[col1].str.lower().str.contains(r"\bsecond floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bsecond floor\b","", regex=True).map(str)
+
+    df['first_and_second_floor'] = "flat bc " + df[df[col1].str.lower().str.contains(r"\bfirst and second floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bfirst and second floor\b","", regex=True).map(str)
+
+    df['first1_floor'] = "flat b " + df[df[col1].str.lower().str.contains(r"\b1st floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\b1st floor\b","", regex=True).map(str)
+
+    df['second2_floor'] = "flat c " + df[df[col1].str.lower().str.contains(r"\b2nd floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\b2nd floor\b","", regex=True).map(str)
+
+    df['ground_first_second_floor'] = "flat abc " + df[df[col1].str.lower().str.contains(r"\bground and first and second floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bground and first and second floor\b","", regex=True).map(str)
+
+    df['third_floor'] = "flat d " + df[df[col1].str.lower().str.contains(r"\bthird floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\bthird floor\b","", regex=True).map(str)
+
+    df['third3_floor'] = "flat d " + df[df[col1].str.lower().str.contains(r"\b3rd floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\b3rd floor\b","", regex=True).map(str)   
+
+    df['top_floor'] = "flat top " + df[df[col1].str.lower().str.contains(r"\btop floor\b"\
+                                )][col1].str.replace(r"\bflat\b","", regex=True).str.replace(r"\btop floor\b","", regex=True).map(str)
+    
+    #merge_columns(df, "new_col", col1, 'letter_after_number')
+    df["new_col"] = merge_series(df[col1], df['letter_after_number'])
+    df["new_col"] = merge_series(df["new_col"], df['basement'])
+    df["new_col"] = merge_series(df["new_col"], df['ground_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['first_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['first1_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['ground_and_first_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['basement_ground_and_first_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['basement_ground_and_first_floor2'])
+    df["new_col"] = merge_series(df["new_col"], df['second_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['second2_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['first_and_second_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['ground_first_second_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['third_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['third3_floor'])
+    df["new_col"] = merge_series(df["new_col"], df['top_floor'])
+       
+    return df['new_col']
+
+# def replace_floor_flat(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+#     '''Moves the word 'flat' to the front of addresses with floor references like basement, ground floor, etc.'''
+    
+#     floor_mapping = {
+#         'basement': 'basement',
+#         'ground floor': 'a',
+#         'first floor': 'b',
+#         'ground and first floor': 'ab',
+#         'basement ground and first floors': 'basementab',
+#         'second floor': 'c',
+#         'first and second floor': 'bc',
+#         '1st floor': 'b',
+#         '2nd floor': 'c',
+#         'ground and first and second floor': 'abc',
+#         'third floor': 'd',
+#         '3rd floor': 'd',
+#         'top floor': 'top'
+#     }
+    
+#     for key, value in floor_mapping.items():
+#         df[key] = f"flat {value} " + df[df[col1].str.lower().str.contains(fr"\b{key}\b")][col1].str.replace(r"\bflat\b", "", regex=True).str.replace(fr"\b{key}\b", "", regex=True).map(str)
+    
+#     df["new_col"] = df[col1].copy()
+    
+#     for key in floor_mapping.keys():
+#         df["new_col"] = merge_series(df["new_col"], df[key])
+    
+#     return df["new_col"]
+
+
+def remove_non_housing(df:PandasDataFrame, col1:PandasSeries) -> PandasDataFrame:
+    '''
+    Remove items from the housing list that are not housing. Includes addresses including
+    the text 'parking', 'garage', 'store', 'visitor bay', 'visitors room', and 'bike rack',
+    'yard', 'workshop'
+    '''
+    df_copy = df.copy()[~df[col1].str.lower().str.contains(\
+    r"parking|garage|\bstore\b|\bstores\b|\bvisitor bay\b|visitors room|\bbike rack\b|\byard\b|\bworkshop\b")]
+                                                                 
+    return df_copy
+
+def extract_prop_no(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+    '''
+    Extract property number from an address. Remove flat/apartment/room numbers, 
+    then extract the last number/number + letter in the string.
+    '''
+    try:
+        prop_no = df[col1].str.replace(r"(^\bapartment\b \w+)|(^\bapartments\b \w+)", "", regex=True\
+                                  ).str.replace(r"(^\bflat\b \w+)|(^\bflats\b \w+)", "", regex=True\
+                                               ).str.replace(r"(^\broom\b \w+)|(^\brooms\b \w+)", "", regex=True\
+                                                            ).str.replace(",", "", regex=True\
+                                                              ).str.extract(r"(\d+\w+|\d+)(?!.*\d+)") #"(\d+\w+|\d+)(?!.*\d+)" 
+    except:
+        prop_no = np.nan
+        
+    return prop_no
+
+def extract_room_no(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+    '''
+    Extract room number from an address. Find rows where the address contains 'room', then extract
+    the next word after 'room' in the string.
+    '''
+    try:
+        room_no = df[df[col1].str.lower().str.contains(r"\broom\b|\brooms\b",regex=True\
+                                )][col1].str.replace("no.","").str.extract(r'room. (\w+)',regex=True\
+                                                        ).rename(columns = {0:"room_number"})
+    except:
+        room_no = np.nan
+        
+    return room_no
+
+def extract_flat_and_other_no(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+    '''
+    Extract flat number from an address. 
+    It looks for letters after a property number IF THERE ARE NO MORE NUMBERS IN THE STRING,
+    the words following the words 'flat' or 'apartment', or
+    the last regex selects all characters in a word containing a digit if there are two numbers in the address
+    # ^\d+([a-z]|[A-Z])
+    '''  
+    
+    #  the regex  essentially matches strings that satisfy any of the following conditions:
+
+    # Start with a number followed by a single letter (either lowercase or uppercase) and not followed by any other number.
+    # Contain the word "flat" or "apartment".
+    # Start with a number, followed by any characters that are not alphanumeric (denoted by [^a-zA-Z0-9_]), and then another number.
+
+    replaced_series = df[df[col1].str.lower().str.replace(r"^\bflats\b","flat", regex=True).\
+                         str.contains(\
+            r"^\d+([a-z]|[A-Z])(?!.*\d+)|^([a-z] |[A-Z] )(?!.*\d+)|\bflat\b|\bapartment\b|(\d+.*?)[^a-zA-Z0-9_].*?\d+")][col1].str.replace("no.","", regex=True)
+    
+    extracted_series = replaced_series.str.extract(r'^\d+([a-z]|[A-Z])(?!.*\d+)')[0]
+
+    extracted_series = extracted_series[~extracted_series.index.duplicated()]
+    df = df[~df.index.duplicated(keep='first')]
+
+    df["prop_number"] = extracted_series
+
+    extracted_series = replaced_series.str.extract(r'(?i)(?:flat|flats) (\w+)')
+    if 1 in  extracted_series.columns:  
+        df["flat_number"] = extracted_series[0].fillna(extracted_series[1]).infer_objects(copy=False)
+    else: 
+        df["flat_number"] = extracted_series[0]
+
+    extracted_series = replaced_series.str.extract(r'(?i)(?:apartment|apartments) (\w+)')
+    if 1 in  extracted_series.columns:  
+        df["apart_number"] = extracted_series[0].fillna(extracted_series[1]).infer_objects(copy=False)
+    else: 
+        df["apart_number"] = extracted_series[0]
+    
+    df["first_sec_number"] = replaced_series.str.extract(r'(\d+.*?)[^a-zA-Z0-9_].*?\d+')
+    df["first_letter_flat_number"] = replaced_series.str.extract(r'\b([A-Za-z])\b[^\d]* \d')
+    df["first_letter_no_more_numbers"] = replaced_series.str.extract(r'^([a-z] |[A-Z] )(?!.*\d+)')
+        
+    return df
+
+def extract_house_or_court_name(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+    '''
+    Extract house or court name. Extended to include estate, buildings, and mansions
+    '''
+    extracted_series = df[col1].str.extract(r"(\w+)\s+(house|court|estate|buildings|mansions)")
+    if 1 in  extracted_series.columns:  
+        df["house_court_name"] = extracted_series[0].fillna(extracted_series[1]).infer_objects(copy=False)
+    else: 
+        df["house_court_name"] = extracted_series[0]
+
+    return df["house_court_name"]
+
+def extract_block_and_unit_name(df:PandasDataFrame, col1:PandasSeries) -> PandasSeries:
+    '''
+    Extract house or court name. Extended to include estate, buildings, and mansions
+    '''
+
+    extracted_series = df[col1].str.extract(r'(?i)(?:block|blocks) (\w+)')
+    if 1 in  extracted_series.columns:  
+        df["block_number"] = extracted_series[0].fillna(extracted_series[1]).infer_objects(copy=False)
+    else: 
+        df["block_number"] = extracted_series[0]
+
+    extracted_series = df[col1].str.extract(r'(?i)(?:unit|units) (\w+)')
+    if 1 in  extracted_series.columns:  
+        df["unit_number"] = extracted_series[0].fillna(extracted_series[1]).infer_objects(copy=False)
+    else: 
+        df["unit_number"] = extracted_series[0]
+
+    return df
+
+def extract_postcode(df:PandasDataFrame, col:str) -> PandasSeries:
+    '''
+    Extract a postcode from a string column in a dataframe
+    '''
+    postcode_series = df[col].str.upper().str.extract(pat = \
+    "(\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\\b$)")
+    
+    return postcode_series
+
+def remove_postcode(df:PandasDataFrame, col:str) -> PandasSeries:
+    '''
+    Remove a postcode from a string column in a dataframe
+    '''
+
+    
+    address_series_no_pcode = df[col].str.upper().str.replace(\
+    "\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2}|GIR ?0A{2})\\b$|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$|\\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\\b$","",\
+                                                              regex=True
+                                                               ).str.lower()
+    
+    return address_series_no_pcode
+
+# Remove addresses with no numbers in at all - too high a risk of badly assigning an address
+def check_no_number_addresses(df:PandasDataFrame, in_address_series:PandasSeries) -> PandasSeries:
+    '''
+    Highlight addresses from a pandas df where there are no numbers in the address.
+    '''
+    df["in_address_series_temp"] = df[in_address_series].str.lower()
+
+    no_numbers_series = df["in_address_series_temp"].str.contains("^(?!.*\d+).*$", regex=True)
+
+    df.loc[no_numbers_series == True, 'Excluded from search'] = "Excluded - no numbers in address"
+
+    df = df.drop("in_address_series_temp", axis = 1)
+
+    print(df[["full_address", "Excluded from search"]])
+
+    return df
+
+# Exclude non-postal addresses
+def remove_non_postal(df, in_address_series):
+    '''
+    Highlight non-postal addresses from a pandas df where a string series that contain specific substrings
+    indicating non-postal addresses like 'garage', 'parking', 'shed', etc.
+    '''
+    df["in_address_series_temp"] = df[in_address_series].str.lower()
+
+    garage_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bgarage\\b|\\bgarages\\b)", regex=True)
+    parking_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bparking\\b)", regex=True)
+    shed_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bshed\\b|\\bsheds\\b)", regex=True)
+    bike_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bbike\\b|\\bbikes\\b)", regex=True)
+    bicycle_store_address_series = df["in_address_series_temp"].str.contains("(?i)(?:\\bbicycle store\\b|\\bbicycle store\\b)", regex=True)
+
+    non_postal_series = (garage_address_series | parking_address_series | shed_address_series | bike_address_series | bicycle_store_address_series)
+    
+    df.loc[non_postal_series == True, 'Excluded from search'] = "Excluded - non-postal address"
+
+    df = df.drop("in_address_series_temp", axis = 1)
+
+    return df
+
+def replace_mistaken_dates(df:PandasDataFrame, col:str) -> PandasSeries:
+    '''
+    Identify addresses that mistakenly have dates in them and replace these dates with number values
+    '''
+    # Regex pattern to identify the date-month format
+    pattern = r'(\d{2})-(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)'
+    
+    # Dictionary to map month abbreviations to numbers
+    month_map = {
+        'jan': '1', 'feb': '2', 'mar': '3', 'apr': '4', 'may': '5', 'jun': '6',
+        'jul': '7', 'aug': '8', 'sep': '9', 'oct': '10', 'nov': '11', 'dec': '12'
+    }
+    
+    # Custom replacement function
+    def replace_month(match):
+        day = match.group(1).lstrip('0')  # Get the day and remove leading zeros
+        month = month_map[match.group(2)]  # Convert month abbreviation to number
+        return f"{day}-{month}"
+    
+    # Apply the regex replacement
+    corrected_addresses = df[col].str.replace(pattern, replace_month, regex = True)
+
+    return corrected_addresses
+
+def merge_series(full_series: pd.Series, partially_filled_series: pd.Series) -> pd.Series:
+    '''
+    Merge two series. The 'full_series' is the series you want to replace values in
+    'partially_filled_series' is the replacer series.
+    '''
+    replacer_series_is_null = partially_filled_series.isnull()
+
+    # Start with full_series values
+    merged_series = full_series.copy()
+
+    # Replace values in merged_series where partially_filled_series is not null
+    merged_series[~replacer_series_is_null] = partially_filled_series.dropna()
+
+    return merged_series
+
+def clean_cols(col:str) -> str:
+    return col.lower().strip().replace(r" ", "_").strip()