<commit_before>#!/usr/bin/env python

import itertools
import multiprocessing
import os
import sys
import time

import click
import matplotlib.pyplot as plt
import numpy as np
import numpy.ma as ma
import rasterio
from rasterio.plot import show

from projections.rasterset import RasterSet
from projections.simpleexpr import SimpleExpr
import projections.r2py.modelr as modelr
import projections.predicts as predicts
import projections.utils as utils

class YearRangeParamType(click.ParamType):
  name = 'year range'

  def convert(self, value, param, ctx):
    try:
      try:
        return [int(value)]
      except ValueError:
        low, high = value.split(':')
        return range(int(low), int(high))
    except ValueError:
      self.fail('%s is not a valid year range' % value, param, ctx)

YEAR_RANGE = YearRangeParamType()

def select_models(model, model_dir):
  """Select the appropriate models for abundance, spieces richness, or
compositional similarity depending on what the user wants to project.

  Assumes models have fixed name and live in the folder passed as a
  parameter.

  """

  if model == 'ab':
    mods = ('ab.rds', )
    out = 'Abundance'
  elif model == 'sr':
    mods = ('sr.rds', )
    out = 'Richness'
  elif model == 'cs-ab':
    mods = ('cs-ab.rds', )
    out = 'CompSimAb'
  elif model == 'cs-sr':
    mods = ('cs-sr.rds', )
    out = 'CompSimSR'
  else:
    raise RuntimeError('Unknown model type %s' % model)
  return out, tuple(map(lambda x: os.path.join(model_dir, x), mods))

def project_year(model, model_dir, scenario, year):
  """Run a projection for a single year.  Can be called in parallel when
projecting a range of years.

  """

  print("projecting %s for %d using %s" % (model, year, scenario))

  what, models = select_models(model, model_dir)
  # Read Sam's abundance model (forested and non-forested)
  mod = modelr.load(models[0])
  predicts.predictify(mod)

  # Import standard PREDICTS rasters
  rasters = predicts.rasterset('luh2', scenario, year)
  rs = RasterSet(rasters)

  if what in ('CompSimAb', 'CompSimSR'):
    expr = '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)'
  else:
    expr = '(exp(%s) / exp(%f))'
  rs[what] = SimpleExpr(what, expr % (mod.output, mod.intercept))

  rs[mod.output] = mod

  if what not in rs:
    print('%s not in rasterset' % what)
    print(', '.join(sorted(rs.keys())))
    sys.exit(1)
<commit_msg>Remove a few bits that refered to multiple models<commit_after>#!/usr/bin/env python

import itertools
import multiprocessing
import os
import sys
import time

import click
import matplotlib.pyplot as plt
import numpy as np
import numpy.ma as ma
import rasterio
from rasterio.plot import show

from projections.rasterset import RasterSet
from projections.simpleexpr import SimpleExpr
import projections.r2py.modelr as modelr
import projections.predicts as predicts
import projections.utils as utils

class YearRangeParamType(click.ParamType):
  name = 'year range'

  def convert(self, value, param, ctx):
    try:
      try:
        return [int(value)]
      except ValueError:
        low, high = value.split(':')
        return range(int(low), int(high))
    except ValueError:
      self.fail('%s is not a valid year range' % value, param, ctx)

YEAR_RANGE = YearRangeParamType()

def select_model(model, model_dir):
  """Select the appropriate models for abundance, spieces richness, or
compositional similarity depending on what the user wants to project.

  Assumes models have fixed name and live in the folder passed as a
  parameter.

  """

  if model == 'ab':
    mod = 'ab.rds'
    out = 'Abundance'
  elif model == 'sr':
    mod = 'sr.rds'
    out = 'Richness'
  elif model == 'cs-ab':
    mod = 'cs-ab.rds'
    out = 'CompSimAb'
  elif model == 'cs-sr':
    mod = 'cs-sr.rds'
    out = 'CompSimSR'
  else:
    raise RuntimeError('Unknown model type %s' % model)
  return out, os.path.join(model_dir, mod)

def project_year(model, model_dir, scenario, year):
  """Run a projection for a single year.  Can be called in parallel when
projecting a range of years.

  """

  print("projecting %s for %d using %s" % (model, year, scenario))

  # Import standard PREDICTS rasters
  rasters = predicts.rasterset('luh2', scenario, year)
  rs = RasterSet(rasters)

  what, model = select_model(model, model_dir)
  # Read Sam's models
  if model:
    mod = modelr.load(model)
    predicts.predictify(mod)
    rs[mod.output] = mod

  if what in ('CompSimAb', 'CompSimSR'):
    expr = '(inv_logit(%s) - 0.01) / (inv_logit(%f) - 0.01)'
  else:
    expr = '(exp(%s) / exp(%f))'
  rs[what] = SimpleExpr(what, expr % (mod.output, mod.intercept))


  if what not in rs:
    print('%s not in rasterset' % what)
    print(', '.join(sorted(rs.keys())))
    sys.exit(1)

{'fraction_non_alphanumeric': 0.12011759764804704, 'fraction_numerical': 0.003989920201595968, 'mean_word_length': 3.5491881566380132, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1406995', 'n_tokens_mistral': 1627, 'n_tokens_neox': 1582, 'n_words': 515}