Adding code

README.md

@@ -11,3 +11,8 @@ license: mit
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+# Installation
+1. Download the repo
+2. Install jupyter
+3. Run the notebooks in order

notebooks/01_get_data.ipynb

@@ -0,0 +1,191 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "883a8a6a-d0b5-40ea-90a0-5b33d3332360",
+   "metadata": {},
+   "source": [
+    "# Get Data\n",
+    "The data from wikipedia starts in XML, this is a relatively simple way to format that into a single json for our purposes."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a7d66da5-185c-409e-9568-f211ca4b725e",
+   "metadata": {},
+   "source": [
+    "## Initialize Variables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ea8ae64c-f597-4c94-b93d-1b78060d7953",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "import sys"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2f9527f9-4756-478b-99ac-a3c8c26ab63e",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "proj_dir = str(Path.cwd().parent)\n",
+    "proj_dir\n",
+    "\n",
+    "# So we can import later\n",
+    "sys.path.append(proj_dir)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "860da614-743b-4060-9d22-673896414cbd",
+   "metadata": {},
+   "source": [
+    "## Install Libraries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8bec29e3-8434-491f-914c-13f303dc68f3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%pip install -q -r \"$proj_dir\"/requirements.txt"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b928c71f-7e34-47ee-b55e-aa12d5118ba7",
+   "metadata": {},
+   "source": [
+    "## Download Latest Simple Wikipedia"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f1dc5f57-c877-43e3-8131-4f351b99168d",
+   "metadata": {},
+   "source": [
+    "Im getting \"latest\" but its good to see what version it is nonetheless."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe4b357f-88fe-44b5-9fce-354404b1447f",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "!curl -I https://dumps.wikimedia.org/simplewiki/latest/simplewiki-latest-pages-articles-multistream.xml.bz2 --silent | grep \"Last-Modified\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fe62d4a3-b59b-40c4-9a8c-bf0a447a9ec2",
+   "metadata": {},
+   "source": [
+    "Download simple wikipedia"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0f309c12-12de-4460-a03f-bd5b6fcc942c",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "!wget -P \"$proj_dir\"/data/raw https://dumps.wikimedia.org/simplewiki/latest/simplewiki-latest-pages-articles-multistream.xml.bz2"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "46af5df6-5785-400a-986c-54a2c98768ea",
+   "metadata": {},
+   "source": [
+    "## Extract XML into jsonl"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c22dedcd-73b3-4aad-8eb7-1063954967ed",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!wikiextractor -o \"$proj_dir\"/data/raw/output  --json simplewiki-latest-pages-articles-multistream.xml.bz2 "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bb8063c6-1bed-49f0-948a-eeb9a7933b4a",
+   "metadata": {},
+   "source": [
+    "## Consolidate into json\n",
+    "\n",
+    "Some of this is boring, and most people dont care how you format it, just that its correct. Feel free to check out the consolidate file for more details."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0a4ce3aa-9c1e-45e4-8219-a1714f482371",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from src.preprocessing.consolidate import folder_to_json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3e93da6a-e304-450c-a81e-ffecaf0d8a9a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "folder = proj_dir / 'data/raw/output'\n",
+    "file_out = proj_dir / 'data/consolidated/simple_wiki.json'\n",
+    "folder_to_json(folder, file_out)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

notebooks/02_preprocessing.ipynb

@@ -0,0 +1,348 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "b1b28232-b65d-41ce-88de-fd70b93a528d",
+   "metadata": {},
+   "source": [
+    "# Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "abb5186b-ee67-4e1e-882d-3d8d5b4575d4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "from pathlib import Path\n",
+    "import pickle\n",
+    "from tqdm.auto import tqdm\n",
+    "\n",
+    "from haystack.nodes.preprocessor import PreProcessor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c4b82ea2-8b30-4c2e-99f0-9a30f2f1bfb7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/Users/derekthomas/projects/spaces/RAGDemo\n"
+     ]
+    }
+   ],
+   "source": [
+    "proj_dir = Path.cwd().parent\n",
+    "print(proj_dir)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "76119e74-f601-436d-a253-63c5a19d1c83",
+   "metadata": {},
+   "source": [
+    "# Config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "f6f74545-54a7-4f41-9f02-96964e1417f0",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "file_in = proj_dir / 'data/consolidated/simple_wiki.json'\n",
+    "file_out = proj_dir / 'data/processed/simple_wiki_processed.plk'"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6a643cf2-abce-48a9-b4e0-478bcbee28c3",
+   "metadata": {},
+   "source": [
+    "# Preprocessing"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a8f9630e-447e-423e-9f6c-e1dbc654f2dd",
+   "metadata": {},
+   "source": [
+    "Its important to choose good pre-processing options. \n",
+    "\n",
+    "Clean whitespace helps each stage of RAG. It adds noise to the embeddings, and wastes space when we prompt with it.\n",
+    "\n",
+    "I chose to split by word as it would be tedious to tokenize here, and that doesnt scale well. The context length for most embedding models ends up being 512 tokens. This is ~400 words. \n",
+    "\n",
+    "I like to respect the sentence boundary, thats why I gave a ~50 word buffer."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "18807aea-24e4-4d74-bf10-55b24f3cb52c",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "pp = PreProcessor(clean_whitespace = True,\n",
+    "             clean_header_footer = False,\n",
+    "             clean_empty_lines = True,\n",
+    "             remove_substrings = None,\n",
+    "             split_by='word',\n",
+    "             split_length = 350,\n",
+    "             split_overlap = 50,\n",
+    "             split_respect_sentence_boundary = True,\n",
+    "             tokenizer_model_folder = None,\n",
+    "             language = \"en\",\n",
+    "             id_hash_keys = None,\n",
+    "             progress_bar = True,\n",
+    "             add_page_number = False,\n",
+    "             max_chars_check = 10_000)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "dab1658a-79a7-40f2-9a8c-1798e0d124bf",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "with open(file_in, 'r', encoding='utf-8') as f:\n",
+    "    list_of_articles = json.load(f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "4ca6e576-4b7d-4c1a-916f-41d1b82be647",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Preprocessing:   0%|█▏                                                                                                                                                                                                                                                        | 1510/332023 [00:01<04:12, 1308.89docs/s]We found one or more sentences whose word count is higher than the split length.\n",
+      "Preprocessing:  83%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                                         | 276351/332023 [01:06<00:10, 5510.66docs/s]Document 81972e5bc1997b1ed4fb86d17f061a41 is 21206 characters long after preprocessing, where the maximum length should be 10000. Something might be wrong with the splitting, check the document affected to prevent issues at query time. This document will be now hard-split at 10000 chars recursively.\n",
+      "Document 5e63e848e42966ddc747257fb7cf4092 is 11206 characters long after preprocessing, where the maximum length should be 10000. Something might be wrong with the splitting, check the document affected to prevent issues at query time. This document will be now hard-split at 10000 chars recursively.\n",
+      "Preprocessing: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 332023/332023 [01:15<00:00, 4403.36docs/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "documents = pp.process(list_of_articles)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f00dbdb2-906f-4d5a-a3f1-b0d84385d85a",
+   "metadata": {},
+   "source": [
+    "When we break a wikipedia article up, we lose some of the context. The local context is somewhat preserved by the `split_overlap`. Im trying to preserve the global context by adding a prefix that has the article's title.\n",
+    "\n",
+    "You could enhance this with the summary as well. This is mostly to help the retrieval step of RAG. Note that the way Im doing it alters some of `haystack`'s features like the hash and the lengths, but those arent too necessary. \n",
+    "\n",
+    "A more advanced way for many business applications would be to summarize the document and add that as a prefix for sub-documents.\n",
+    "\n",
+    "One last thing to note, is that it would be prudent (in some use-cases) to preserve the original document without the summary to give to the reader (retrieve with the summary but prompt without), but since this is a simple use-case I wont be doing that."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "076e115d-3e88-49d2-bc5d-f725a94e4964",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "dbdb8eadba804b4485cc3e7f11a8b863",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/268980 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Prefix each document's content\n",
+    "for document in tqdm(documents):\n",
+    "    if document.meta['_split_id'] != 0:\n",
+    "        document.content = f'Title: {document.meta[\"title\"]}. ' + document.content"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "72c1849c-1f4d-411f-b74b-6208b1e48217",
+   "metadata": {},
+   "source": [
+    "## Pre-processing Examples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "02c1c6c8-6283-49a8-9d29-c355f1b08540",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Document: {'content': \"April (Apr.) is the fourth month of the year in the Julian and Gregorian calendars, and comes between March and May. It is one of the four months to have 30 days.\\nApril always begins on the same day of the week as July, and additionally, January in leap years. April always ends on the same day of the week as December.\\nThe Month.\\nApril comes between March and May, making it the fourth month of the year. It also comes first in the year out of the four months that have 30 days, as June, September and November are later in the year.\\nApril begins on the same day of the week as July every year and on the same day of the week as January in leap years. April ends on the same day of the week as December every year, as each other's last days are exactly 35 weeks (245 days) apart.\\nIn common years, April starts on the same day of the week as October of the previous year, and in leap years, May of the previous year. In common years, April finishes on the same day of the week as July of the previous year, and in leap years, February and October of the previous year. In common years immediately after other common years, April starts on the same day of the week as January of the previous year, and in leap years and years immediately after that, April finishes on the same day of the week as January of the previous year.\\nIn years immediately before common years, April starts on the same day of the week as September and December of the following year, and in years immediately before leap years, June of the following year. In years immediately before common years, April finishes on the same day of the week as September of the following year, and in years immediately before leap years, March and June of the following year.\\nApril is a spring month in the Northern Hemisphere and an autumn/fall month in the Southern Hemisphere. \", 'content_type': 'text', 'score': None, 'meta': {'id': '1', 'revid': '9086769', 'url': 'https://simple.wikipedia.org/wiki?curid=1', 'title': 'April', '_split_id': 0, '_split_overlap': [{'doc_id': '79a74c1e6444dd0a1acd72840e9dd7c0', 'range': (1529, 1835)}]}, 'id_hash_keys': ['content'], 'embedding': None, 'id': 'a1c2acf337dbc3baa6f7f58403dfb95d'}>"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "documents[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "b34890bf-9dba-459a-9b0d-aa4b5929cbe8",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Document: {'content': 'Title: April. In years immediately before common years, April finishes on the same day of the week as September of the following year, and in years immediately before leap years, March and June of the following year.\\nApril is a spring month in the Northern Hemisphere and an autumn/fall month in the Southern Hemisphere. In each hemisphere, it is the seasonal equivalent of October in the other.\\nIt is unclear as to where April got its name. A common theory is that it comes from the Latin word \"aperire\", meaning \"to open\", referring to flowers opening in spring. Another theory is that the name could come from Aphrodite, the Greek goddess of love. It was originally the second month in the old Roman Calendar, before the start of the new year was put to January 1.\\nQuite a few festivals are held in this month. In many Southeast Asian cultures, new year is celebrated in this month (including Songkran). In Western Christianity, Easter can be celebrated on a Sunday between March 22 and April 25. In Orthodox Christianity, it can fall between April 4 and May 8. At the end of the month, Central and Northern European cultures celebrate Walpurgis Night on April 30, marking the transition from winter into summer.\\nApril in poetry.\\nPoets use \"April\" to mean the end of winter. For example: \"April showers bring May flowers.\"', 'content_type': 'text', 'score': None, 'meta': {'id': '1', 'revid': '9086769', 'url': 'https://simple.wikipedia.org/wiki?curid=1', 'title': 'April', '_split_id': 1, '_split_overlap': [{'doc_id': 'a1c2acf337dbc3baa6f7f58403dfb95d', 'range': (0, 306)}]}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '79a74c1e6444dd0a1acd72840e9dd7c0'}>"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "documents[1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "e6f50c27-a486-47e9-ba60-d567f5e530db",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<Document: {'content': 'Title: Chief Joseph. He knew he could not trust them anymore. He was tired of being considered a savage. He felt it was not fair for people who were born on the same land to be treated differently. He delivered a lot of speeches on this subject, which are still really good examples of eloquence. But he did not feel listened to, and when he died in his reservation in 1904, the doctor said he \"died from sadness\". He was buried in Colville Native American Burial Ground, in Washington State.', 'content_type': 'text', 'score': None, 'meta': {'id': '19310', 'revid': '16695', 'url': 'https://simple.wikipedia.org/wiki?curid=19310', 'title': 'Chief Joseph', '_split_id': 1, '_split_overlap': [{'doc_id': '4bdf9cecd46c3bfac6b225aed940e798', 'range': (0, 275)}]}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '91bc8240c5d067ab24f35c11f8916fc6'}>"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "documents[10102]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "5485cc27-3d3f-4b96-8884-accf5324da2d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of Articles: 332023\n",
+      "Number of processed articles: 237724\n",
+      "Number of processed documents: 268980\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f'Number of Articles: {len(list_of_articles)}')\n",
+    "processed_articles = len([d for d in documents if d.meta['_split_id'] == 0])\n",
+    "print(f'Number of processed articles: {processed_articles}')\n",
+    "print(f'Number of processed documents: {len(documents)}')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "23ce57a8-d14e-426d-abc2-0ce5cdbc881a",
+   "metadata": {},
+   "source": [
+    "# Write to file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "0d044870-7a30-4e09-aad2-42f24a52780d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "with open(file_out, 'wb') as handle:\n",
+    "    pickle.dump(documents, handle, protocol=pickle.HIGHEST_PROTOCOL)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c5833dba-1bf6-48aa-be6f-0d70c71e54aa",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -0,0 +1,4 @@
+wikiextractor==3.0.6
+farm-haystack[inference]==1.20.1
+ipywidgets==8.1.1
+tqdm==4.66.1

src/preprocessing/consolidate.py

@@ -0,0 +1,83 @@
+import json
+from pathlib import Path
+from time import perf_counter
+from typing import Any, Dict
+
+from tqdm.auto import tqdm
+
+
+def folder_to_json(folder_in: Path, json_path: Path):
+    """
+    Process JSON lines from files in a given folder and write processed data to a new JSON file.
+
+    Parameters:
+    folder_in (Path): Path to the input folder containing the JSON files to process.
+    json_path (Path): Path to the output JSON file where the processed data will be written.
+
+    Example:
+    folder_to_json(Path("/path/to/input/folder"), Path("/path/to/output.json"))
+    """
+
+    json_out = []  # Initialize list to hold processed JSON data from all files
+
+    process_start = perf_counter()
+    # Use rglob to get all JSON files and sort them by their full path
+    all_files = sorted(folder_in.rglob('*wiki*'), key=lambda x: str(x))
+
+    # Initialize progress bar with total file count, description, and unit of progress
+    with tqdm(total=len(all_files), desc='Processing', unit='file') as pbar:
+        # Iterate through all files in the input folder in order
+        for file_path in all_files:
+            # Update progress bar postfix to display current file and directory
+            pbar.set_postfix_str(f"File: {file_path.name} | Dir: {file_path.parent}", refresh=True)
+
+            # Open and read the current file
+            with open(file_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    # Load JSON data from each line and process it
+                    article = json.loads(line)
+                    # Add restructured article to the output list
+                    json_out.extend([restructure_articles(article)])
+
+            # Update progress bar after processing each file
+            pbar.update(1)
+    time_taken_to_process = perf_counter() - process_start
+    pbar.write(f"Wiki processed in {round(time_taken_to_process, 2)} seconds!")
+
+    # Notify that the writing process is starting
+    pbar.write("Writing file!")
+    write_start = perf_counter()
+    # Open the output file and write the processed data as JSON
+    with open(json_path, "w", encoding='utf-8') as outfile:
+        json.dump(json_out, outfile)
+    time_taken_to_write = perf_counter() - write_start
+    # Notify that the writing process is complete
+    pbar.write(f"File written in {round(time_taken_to_write, 2)} seconds!")
+
+
+def restructure_articles(article: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Restructures the given article into haystack's format, separating content and meta data.
+
+    Args:
+    - article (Dict[str, Any]): The article to restructure.
+
+    Returns:
+    - Dict[str, Any]: The restructured article.
+    """
+
+    # Extract content and separate meta data
+    article_out = {
+        'content': article['text'],
+        'meta': {k: v for k, v in article.items() if k != 'text'}
+        }
+
+    return article_out
+
+
+if __name__ == '__main__':
+    proj_dir = Path(__file__).parents[2]
+    folder = proj_dir / 'data/raw/output'
+    file_out = proj_dir / 'data/consolidated/simple_wiki.json'
+    folder_to_json(folder, file_out)
+    print('Done!')