Initial revision

.gitignore

@@ -1,2 +1,2 @@
 env
-data
+data

Dockerfile

@@ -2,32 +2,24 @@ FROM python:3.11
 
 # Download ID is set as a space variable 
 # By default it is a download of all Solanum preserved specimen records (c600K)
-ARG GBIF_DOWNLOAD_ID=$GBIF_DOWNLOAD_ID
+ARG IHDELTA_REPO_URL=$IHDELTA_REPO_URL
+ARG START_AFTER_COMMIT=$START_AFTER_COMMIT
 WORKDIR /code
 
 COPY ./requirements.txt /code/requirements.txt
 
 RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
 
-# Download GBIF occurrences and prepare for use with datasette
+# Clone ihdelta repo
 RUN mkdir /data
-ADD https://api.gbif.org/v1/occurrence/download/request/${GBIF_DOWNLOAD_ID}.zip /data/gbif-occs.zip
-RUN ls -lh /data
-RUN unzip /data/gbif-occs.zip -d /data
-RUN ls -lh /data
-COPY ./tab2csv.py /code/tab2csv.py
+RUN mkdir /data/ihdelta
+RUN git clone ${IHDELTA_REPO_URL} /data/ihdelta 
+RUN git-history file --start-after ${START_AFTER_COMMIT} --csv --dialect unix --repo /data/ihdelta --id irn /code/ihdelta.db /data/ihdelta/ih-institutions.csv
 
-
-RUN python tab2csv.py --createcols /data ${GBIF_DOWNLOAD_ID} gbifocc.csv 
-RUN csvs-to-sqlite /data/gbifocc.csv /code/gbifocc.db
-RUN ls -l /code
-RUN sqlite-utils tables /code/gbifocc.db --counts
-RUN sqlite-utils enable-fts /code/gbifocc.db gbifocc collectorNameAndNumber
-RUN chmod 755 /code/gbifocc.db
+RUN sqlite-utils tables /code/ihdelta.db --counts
+RUN chmod 755 /code/ihdelta.db
 
 # Create datasette metadata file
-COPY ./getDownloadMetadata.py /code/getDownloadMetadata.py
 COPY ./metadata.json /code/metadata.json
-RUN python getDownloadMetadata.py /code/metadata.json /code/metadata.json --download_id=$GBIF_DOWNLOAD_ID
 
-CMD ["datasette", "/code/gbifocc.db", "-m", "/code/metadata.json", "--host", "0.0.0.0", "--port", "7860", "--setting", "sql_time_limit_ms", "3500"]
+CMD ["datasette", "/code/ihdelta.db", "-m", "/code/metadata.json", "--host", "0.0.0.0", "--port", "7860", "--setting", "sql_time_limit_ms", "3500"]

README.md

@@ -1,6 +1,6 @@
 ---
-title: GBIF occurrence Datasette
-emoji: 🌱
+title: Index Herbariorum
+emoji: 🏛
 colorFrom: red
 colorTo: purple
 sdk: docker

extractcollectorname.py

@@ -1,36 +0,0 @@
-import argparse
-import pandas as pd
-import bananompy
-from tqdm import tqdm
-tqdm.pandas()
-
-def getFirstFamilyName(s):
-    firstFamilyName = None
-    parsed = bananompy.parse(s)
-    try:
-        firstFamilyName = parsed[0]['parsed'][0]['family']
-    except:
-        pass
-    return firstFamilyName
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("inputfile")
-    parser.add_argument("outputfile")    
-    args = parser.parse_args()
-
-    df = pd.read_csv(args.inputfile, 
-                    encoding='utf8', 
-                    keep_default_na=False, 
-                    na_values=['NONE',''],
-                    on_bad_lines='skip', 
-                    sep=',')
-    # Extract unique recordedBy values
-    df_rb = df[['recordedBy']].drop_duplicates()
-    df_rb['recordedBy_first_familyname'] = df_rb.recordedBy.progress_apply(getFirstFamilyName)
-    # Apply back to main dataframe
-    df = pd.merge(left = df, right=df_rb, left_on='recordedBy', right_on='recordedBy', how='left')
-    # Add column holding collector name and number
-    mask = (df.recordNumber.notnull())
-    df.loc[mask,'collectorNameAndNumber']=df[mask].apply(lambda row: '{} {}'.format(row['recordedBy_first_familyname'],row['recordNumber']),axis=1)
-    df.to_csv(args.outputfile, index=False, sep=',')    

getDownloadMetadata.py

@@ -1,28 +0,0 @@
-import argparse
-from pygbif import occurrences as occ
-import json
-
-
-licenses = {'http://creativecommons.org/licenses/by-nc/4.0/legalcode':'CC BY-NC 4.0'}
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("inputfile")
-    parser.add_argument("--download_id", type=str)
-    parser.add_argument("outputfile")    
-
-    args = parser.parse_args()
-
-    datasette_metadata = None
-    with open(args.inputfile, 'r') as f_in:
-        datasette_metadata = json.load(f_in)
- 
-    gbif_metadata = occ.download_meta(key = args.download_id)
-    license_url = gbif_metadata['license']
-    if license_url in licenses:
-        datasette_metadata['license'] = licenses[license_url]
-        datasette_metadata['license_url'] = license_url
-    datasette_metadata['source_url'] = 'https://doi.org/{}'.format(gbif_metadata['doi'])
-
-    datasette_metadata_json = json.dumps(datasette_metadata, indent=4)
-    with open(args.outputfile, 'w') as f_out:
-        f_out.write(datasette_metadata_json)

metadata.json

@@ -1,27 +1,15 @@
 {
-    "title": "GBIF-mediated specimen occurrences",
-    "description": "This is a datasette instance containing GBIF-mediated specimen occurrences. It can be used to browse specimen records (with options to filter and facet records) and to run SQL queries. It is also configured to run an Open Refine compatible reconciliation service on collector name and number, allowing a user to easily link specimen references (as found in taxonomic literature) to these specimen records.",
-    "source": "Global Biodiversity Information Facility (GBIF)",
+    "title": "Revision controlled Index Herbariorum",
+    "description": "This is a datasette instance containing Index Herbariorum, gathered each week using the git scraping data pattern.",
+    "source": "Index Herbariorum (NYBG)",
     "databases": {
         "gbifocc": {
             "tables": {
                 "gbifocc": {
                     "plugins": {
-                        "datasette-reconcile": {
-                            "id_field": "gbifID",
-                            "name_field": "collectorNameAndNumber",
-                            "type_field": "basisOfRecord",
-                            "type_default": [{
-                              "id": "basisOfRecord",
-                              "name": "PRESERVED_SPECIMEN"
-                            }],
-                            "max_limit": 5,
-                            "service_name": "GBIF specimens reconciliation",
-                            "view_url": "https://gbif.org/occurrence/{{id}}"
-                        },
                         "datasette-cluster-map": {
-                            "latitude_column": "decimalLatitude",
-                            "longitude_column": "decimalLongitude"
+                            "latitude_column": "location.lat",
+                            "longitude_column": "location.lon"
                         }
                     }
                 }

requirements.txt

@@ -1,10 +1,4 @@
 datasette
-datasette-reconcile
+git-history
 datasette-cluster-map
-sqlite-utils
-csvs-to-sqlite
-pandas==1.5.3
-bananompy
-datasette-jellyfish
-tqdm
-pygbif
+sqlite-utils

tab2csv.py

@@ -1,112 +0,0 @@
-import argparse
-import pandas as pd
-import requests
-from pygbif import occurrences as occ
-from tqdm import tqdm
-tqdm.pandas()
-import os.path
-
-def getFirstFamilyName(recordedBy):
-    firstFamilyName = None
-    parsed = bananompy.parse(recordedBy)
-    try:
-        firstFamilyName = parsed[0]['parsed'][0]['family']
-    except:
-        pass
-    return firstFamilyName
-
-def getFirstFamilyNames(recordedBy_l):
-    # post to bionomia
-    bionomia_parse_endpoint_url = "https://api.bionomia.net/parse.json"
-    data = dict()
-    data['names'] = '\r\n'.join(recordedBy_l)
-    r = requests.post(bionomia_parse_endpoint_url, data=data)
-    parsed_results = r.json()
-    results = dict()
-    for parsed_result in parsed_results:
-        try:
-            results[parsed_result['original']] = parsed_result['parsed'][0]['family']
-        except:
-            results[parsed_result['original']] = None
-    return results
-
-def getFirstFamilyNameBulk(df, 
-                            recordedByColName="recordedBy", 
-                            firstFamilyNameColName="recordedBy_first_familyname",
-                            batchsize=500):
-    results = dict()
-    recordedBy_l = []
-    for s in tqdm(df[recordedByColName].values):
-        if len(recordedBy_l) == batchsize:
-            # send it
-            results.update(getFirstFamilyNames(recordedBy_l))
-            # clear for next iteration
-            recordedBy_l = []
-        recordedBy_l.append(s)
-    if len(recordedBy_l) > 0:
-        results.update(getFirstFamilyNames(recordedBy_l))
-    df[firstFamilyNameColName] = df[recordedByColName].map(results)
-    return df
-
-GBIF_DOWNLOAD_DESCRIBE_URL_SIMPLE_CSV = 'https://api.gbif.org/v1/occurrence/download/describe/simpleCsv'
-GBIF_DOWNLOAD_DESCRIBE_URL_DWCA = 'https://api.gbif.org/v1/occurrence/download/describe/dwca'
-
-def getGbifDownloadColumnNames(download_format):
-    column_names = None
-    if download_format == 'SIMPLE_CSV':
-        r = requests.get(GBIF_DOWNLOAD_DESCRIBE_URL_SIMPLE_CSV)
-        columns_metadata = r.json()
-        column_names = [column_metadata['name'] for column_metadata in columns_metadata['fields']]
-    elif download_format == 'DWCA':
-        r = requests.get(GBIF_DOWNLOAD_DESCRIBE_URL_DWCA)
-        columns_metadata = r.json()
-        column_names = [column_metadata['name'] for column_metadata in columns_metadata['verbatim']['fields']]
-    return column_names
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument("data_dir")
-    parser.add_argument("download_id")
-    parser.add_argument("-c","--createcols", action='store_true')
-    parser.add_argument("-l","--limit", type=int)
-    parser.add_argument("outputfilename")    
-    args = parser.parse_args()
-
-    # Determine format of datafile by accessing download metadata from GBIF API
-    gbif_metadata = occ.download_meta(key = args.download_id)
-    download_format = gbif_metadata['request']['format']
-    # The GBIF download format determines:
-    # (1) the columns in the download, SIMPLE_CSV being a much restricted set 
-    # of columns than DWCA
-    # (2) The name of the occurrence data file, SIMPLE_CSV : '[download_id].csv'
-    # DWCA : 'occurrence.txt'
-    inputfilename = None
-    column_names_simple_csv = getGbifDownloadColumnNames('SIMPLE_CSV')
-    column_names = None
-    if download_format == 'SIMPLE_CSV':
-        inputfilename = '{}.csv'.format(args.download_id)
-        column_names = column_names_simple_csv
-    elif download_format == 'DWCA':
-        inputfilename = 'occurrence.txt'
-        column_names_dwca = getGbifDownloadColumnNames('DWCA')
-        column_names = [column_name for column_name in column_names_dwca if column_name in column_names_simple_csv]
-        
-    df = pd.read_csv(os.path.join(args.data_dir,inputfilename), 
-                    encoding='utf8', 
-                    keep_default_na=False, 
-                    on_bad_lines='skip', 
-                    sep='\t',
-                    usecols=column_names,
-                    nrows=args.limit)
-    if args.createcols:
-        # Extract unique recordedBy values
-        df_rb = df[['recordedBy']].drop_duplicates()
-        df_rb = getFirstFamilyNameBulk(df_rb)
-        #df_rb['recordedBy_first_familyname'] = df_rb.recordedBy.progress_apply(getFirstFamilyName)
-        # Apply back to main dataframe
-        df = pd.merge(left = df, right=df_rb, left_on='recordedBy', right_on='recordedBy', how='left')
-        # Add column holding collector name and number
-        mask = (df.recordNumber.notnull())
-        df.loc[mask,'collectorNameAndNumber']=df[mask].apply(lambda row: '{} {}'.format(row['recordedBy_first_familyname'],row['recordNumber']),axis=1)
-    df.to_csv(os.path.join(args.data_dir,args.outputfilename), index=False, sep=',')