<commit_before>#!/usr/bin/env python
#-*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals


"""
Module for calculating a variogram. 
This is not interactive
The output will be written in the base folder in CSV format.
"""


__author__ = "<NAME>"
__copyright__ = "Copyright 2017, JEM"
__license__ = "GPL"
__mantainer__ = "Juan"
__email__ ="<EMAIL>"


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.formula.api as smf
import logging
import sys
sys.path.append('..')
sys.path.append('/home/hpc/28/escamill/spystats')
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

from tools import toGeoDataFrame, Variogram


def main(csv_path,minx,maxx,miny,maxy,predicted_size=300):
    """
    The main batch processing
    """
    ### Init configuration
    
    ## Use the ggplot style
    
    
    
    plt.style.use('ggplot')
    
    
    
    logger.info("Reading data")
    
    
    # My mac
    #data = pd.read_csv("/RawDataCSV/plotsClimateData_11092017.csv")
    # My Linux desktop
    
    data = pd.read_csv(csv_path)
    #data = pd.read_csv("/RawDataCSV/idiv_share/plotsClimateData_11092017.csv")
    #new_data = tools.toGeoDataFrame(pandas_dataframe=data,xcoord_name='LON',ycoord_name='LAT')

    new_data = toGeoDataFrame(pandas_dataframe=data,xcoord_name='LON',ycoord_name='LAT')

    logger.info("Performing Reprojection to Alberts")
    ## Reproject to alberts
    new_data =  new_data.to_crs("+proj=aea +lat_1=29.5 +lat_2=45.5 +lat_0=37.5 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs ")
    new_data['logBiomass'] = new_data.apply(lambda x : np.log(x.plotBiomass),axis=1)
    new_data['newLon'] = new_data.apply(lambda c : c.geometry.x, axis=1)
    new_data['newLat'] = new_data.apply(lambda c : c.geometry.y, axis=1)
    #new_data.plot(column='SppN')
    new_data['logBiomass'] = np.log(new_data.plotBiomass)
    #new_data.logBiomass.plot.hist()
    
    logger.info("Fitting Linear Model")
    #linear model
    model = smf.ols(formula='logBiomass ~ SppN',data=new_data)
    results = model.fit()
    param_model = results.params
    results.summary()
    
    new_data['residuals1'] = results.resid
    
    
    logger.info("Cliping data")
    ## Select a section
    
    
    section = new_data[lambda x:  (x.LON > minx) & (x.LON < maxx) & (x.LAT > miny) & (x.LAT < maxy) ]
    #vg = tools.Variogram(new_data,'logBiomass')
    #vg.calculate_empirical(n_bins=10)
    
    logger.info("Calculating Empirical Variogram")
    #vg = tools.Variogram(section,'residuals1')
    vg = Variogram(section,'residuals1')
    
    #vg.calculate_empirical(n_bins=50)
    vgplot = vg.plot(num_iterations=40,n_bins=40,plot_filename='test1.png')
    logger.info("Copying to output file")
    vg.envelope.to_csv("data_envelope.csv")
    logger.info("Finished! Results in: tests1.csv")
    

    
    
    
if __name__ == "__main__":
    __package__ = "spystats"
    csv_path = sys.argv[1]
    minx = float(sys.argv[2])
    maxx = float(sys.argv[3])
    miny = float(sys.argv[4])
    maxy = float(sys.argv[5])    
    
    main(csv_path,minx,maxx,miny,maxy)

<commit_msg>added threshold to variogram fia in HEC£<commit_after>#!/usr/bin/env python
#-*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals


"""
Module for calculating a variogram. 
This is not interactive
The output will be written in the base folder in CSV format.
"""


__author__ = "<NAME>"
__copyright__ = "Copyright 2017, JEM"
__license__ = "GPL"
__mantainer__ = "Juan"
__email__ ="<EMAIL>"


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.formula.api as smf
import logging
import sys
sys.path.append('..')
sys.path.append('/home/hpc/28/escamill/spystats')
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

from tools import toGeoDataFrame, Variogram


def main(csv_path,minx,maxx,miny,maxy,predicted_size=300,distance_threshold=500000):
    """
    The main batch processing
    """
    ### Init configuration
    
    ## Use the ggplot style
    
    
    
    plt.style.use('ggplot')
    
    
    
    logger.info("Reading data")
    
    
    # My mac
    #data = pd.read_csv("/RawDataCSV/plotsClimateData_11092017.csv")
    # My Linux desktop
    
    data = pd.read_csv(csv_path)
    #data = pd.read_csv("/RawDataCSV/idiv_share/plotsClimateData_11092017.csv")
    #new_data = tools.toGeoDataFrame(pandas_dataframe=data,xcoord_name='LON',ycoord_name='LAT')

    new_data = toGeoDataFrame(pandas_dataframe=data,xcoord_name='LON',ycoord_name='LAT')

    logger.info("Performing Reprojection to Alberts")
    ## Reproject to alberts
    new_data =  new_data.to_crs("+proj=aea +lat_1=29.5 +lat_2=45.5 +lat_0=37.5 +lon_0=-96 +x_0=0 +y_0=0 +ellps=GRS80 +datum=NAD83 +units=m +no_defs ")
    new_data['logBiomass'] = new_data.apply(lambda x : np.log(x.plotBiomass),axis=1)
    new_data['newLon'] = new_data.apply(lambda c : c.geometry.x, axis=1)
    new_data['newLat'] = new_data.apply(lambda c : c.geometry.y, axis=1)
    #new_data.plot(column='SppN')
    new_data['logBiomass'] = np.log(new_data.plotBiomass)
    #new_data.logBiomass.plot.hist()
    
    logger.info("Fitting Linear Model")
    #linear model
    model = smf.ols(formula='logBiomass ~ SppN',data=new_data)
    results = model.fit()
    param_model = results.params
    results.summary()
    
    new_data['residuals1'] = results.resid
    
    
    logger.info("Cliping data")
    ## Select a section
    
    
    section = new_data[lambda x:  (x.LON > minx) & (x.LON < maxx) & (x.LAT > miny) & (x.LAT < maxy) ]
    #vg = tools.Variogram(new_data,'logBiomass')
    #vg.calculate_empirical(n_bins=10)
    
    logger.info("Calculating Empirical Variogram")
    #vg = tools.Variogram(section,'residuals1')
    vg = Variogram(section,'residuals1',using_distance_threshold=distance_threshold)
    
    #vg.calculate_empirical(n_bins=50)
    vgplot = vg.plot(num_iterations=40,n_bins=40,plot_filename='test1.png')
    logger.info("Copying to output file")
    vg.envelope.to_csv("data_envelope.csv")
    logger.info("Finished! Results in: tests1.csv")
    

    
    
    
if __name__ == "__main__":
    __package__ = "spystats"
    csv_path = sys.argv[1]
    minx = float(sys.argv[2])
    maxx = float(sys.argv[3])
    miny = float(sys.argv[4])
    maxy = float(sys.argv[5])
    dist_thres = float(sys.argv[6])
    
    main(csv_path,minx,maxx,miny,maxy,distance_threshold=dist_thres)


{'fraction_non_alphanumeric': 0.13478194803221394, 'fraction_numerical': 0.022337030846375932, 'mean_word_length': 3.718279569892473, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7215716', 'n_tokens_mistral': 2591, 'n_tokens_neox': 2471, 'n_words': 428}