<commit_before>
# coding: utf-8

# In[20]:

# setup
import planet4 as p4
import pandas as pd
from pathlib import Path
datapath = Path(p4.__path__[0]) / 'data'

from planet4 import clustering, markings, io
data = {'x':[210,211,212], 'y':[100,102, 104], 'image_x':[10010,10011,10012], 
        'image_y':[20000,20002,20004],
        'angle':[34, 37,48], 'radius_1':[20,21,19], 'radius_2':[50,51,52]}
three_blotches_data = pd.DataFrame(data)
blotch = markings.Blotch(three_blotches_data.iloc[0])

import warnings

with warnings.catch_warnings():
    warnings.filterwarnings("ignore",category=DeprecationWarning)


# In[21]:

# test_calc_fnotch
actual = clustering.calc_fnotch(4, 4)
assert actual == 0.5
actual = clustering.calc_fnotch(4, 0)
assert actual == 1
actual = clustering.calc_fnotch(0, 4)
assert actual == 0


# In[22]:

# test_dbscan_xy

# using only x and y (or image_x,image_y)
coords = ['image_x','image_y']
X = three_blotches_data[coords].values
dbscanner = clustering.DBScanner(X, min_samples=2)

assert dbscanner.reduced_data == [[0, 1, 2]]
assert dbscanner.n_rejected == 0


# In[23]:

coords = ['image_x','image_y', 'angle']
X = three_blotches_data[coords].values
dbscanner = clustering.DBScanner(X, min_samples=2)

assert dbscanner.reduced_data == [[0, 1]]
assert dbscanner.n_rejected == 1


# In[24]:

p4id = markings.example_p4id()


# In[25]:

for eps in range(10,16):
    cm = clustering.ClusteringManager(scope='planet4', eps=eps)
    cm.cluster_image_id(p4id.imgid)
    print(cm.confusion_data)


# # Testing test database

# In[26]:

data = pd.read_csv(datapath / 'test_db.csv')


# In[27]:

data.image_id.value_counts()


# In[28]:

imid1 = 'APF000012w'
imid2 = 'APF000012q'
imid1data = data[data.image_id==imid1]
imid2data = data[data.image_id==imid2]


# In[29]:

outputdir = Path('/Users/klay6683/Dropbox/data/planet4/test_clustering')
cm = clustering.ClusteringManager(fnotched_dir=outputdir)


# In[30]:

cm.cluster_image_id(imid1, data=imid1data)


# In[31]:

from planet4 import plotting
get_ipython().magic('matplotlib nbagg')
import seaborn as sns
sns.set_context('notebook')


# In[32]:

plotting.plot_raw_fans('12w')


# In[ ]:

from planet4 import markings, plotting
get_ipython().magic('matplotlib nbagg')


# In[33]:

id_='6t3'
cm = clustering.ClusteringManager(include_angle=False)
cm.cluster_image_id(id_)


# In[34]:

plotting.plot_image_id_pipeline(id_, include_angle=True, include_distance=True,
                                include_radius=False, eps=10)


# In[37]:

plotting.plot_image_id_pipeline(id_, include_angle=False, include_distance=True,
                                include_radius=True, eps=10, min_distance=15)


# In[36]:

plotting.plot_image_id_pipeline(id_, include_angle=False, include_distance=True,
                                include_radius=False, eps=10, min_distance=15)


# In[47]:

plotting.plot_image_id_pipeline('139', include_angle=True, include_distance=True,
                                include_radius=True, eps=20, min_distance=10)


# In[54]:

plotting.plot_image_id_pipeline('13t', include_angle=True, include_distance=True,
                                include_radius=True, eps=20, min_distance=10)


# In[55]:

plotting.plot_finals('13t')


# In[81]:

plotting.plot_image_id_pipeline('15k', include_angle=True, include_distance=True,
                                include_radius=True, eps=20, min_distance=10)


# In[53]:

plotting.plot_finals('15k')


# In[73]:

db = io.DBManager()
data = db.get_image_id_markings('17a')


# In[80]:

plotting.plot_image_id_pipeline('17a', include_angle=True, include_distance=True,
                                include_radius=False, eps=20, min_distance=20)


# In[60]:

plotting.plot_raw_fans('17a')


# In[67]:

plotting.plot_finals('17a')


# In[ ]:

imgid = markings.ImageID(id_)


# In[ ]:

fans = imgid.get_fans()


# In[ ]:

dbscanner = clustering.DBScanner(fans[['x','y']].values)


# In[ ]:

dbscanner.reduced_data


# In[ ]:

fan = fans.iloc[dbscanner.reduced_data[6]]

imgid.plot_fans(fans=markings.FanContainer.from_df(fan).content)

<commit_msg>cleaning up notebook for better git-ing<commit_after>
# coding: utf-8

# In[ ]:

# setup
import planet4 as p4
import pandas as pd
from pathlib import Path
datapath = Path(p4.__path__[0]) / 'data'

from planet4 import clustering, markings, io
data = {'x':[210,211,212], 'y':[100,102, 104], 'image_x':[10010,10011,10012], 
        'image_y':[20000,20002,20004],
        'angle':[34, 37,48], 'radius_1':[20,21,19], 'radius_2':[50,51,52]}
three_blotches_data = pd.DataFrame(data)
blotch = markings.Blotch(three_blotches_data.iloc[0])

import warnings

with warnings.catch_warnings():
    warnings.filterwarnings("ignore",category=DeprecationWarning)


# In[ ]:

# test_calc_fnotch
actual = clustering.calc_fnotch(4, 4)
assert actual == 0.5
actual = clustering.calc_fnotch(4, 0)
assert actual == 1
actual = clustering.calc_fnotch(0, 4)
assert actual == 0


# In[ ]:

# test_dbscan_xy

# using only x and y (or image_x,image_y)
coords = ['image_x','image_y']
X = three_blotches_data[coords].values
dbscanner = clustering.DBScanner(X, min_samples=2)

assert dbscanner.reduced_data == [[0, 1, 2]]
assert dbscanner.n_rejected == 0


# In[ ]:

coords = ['image_x','image_y', 'angle']
X = three_blotches_data[coords].values
dbscanner = clustering.DBScanner(X, min_samples=2)

assert dbscanner.reduced_data == [[0, 1]]
assert dbscanner.n_rejected == 1


# In[ ]:

p4id = markings.example_p4id()


# In[ ]:

for eps in range(10,16):
    cm = clustering.ClusteringManager(scope='planet4', eps=eps)
    cm.cluster_image_id(p4id.imgid)
    print(cm.confusion_data)


# # Testing test database

# In[ ]:

data = pd.read_csv(datapath / 'test_db.csv')


# In[ ]:

data.image_id.value_counts()


# In[ ]:

imid1 = 'APF000012w'
imid2 = 'APF000012q'
imid1data = data[data.image_id==imid1]
imid2data = data[data.image_id==imid2]


# In[ ]:

outputdir = Path('/Users/klay6683/Dropbox/data/planet4/test_clustering')
cm = clustering.ClusteringManager(fnotched_dir=outputdir)


# In[ ]:

cm.cluster_image_id(imid1, data=imid1data)


# In[ ]:

from planet4 import plotting
get_ipython().magic('matplotlib nbagg')
import seaborn as sns
sns.set_context('notebook')


# In[ ]:

plotting.plot_raw_fans('12w')


# In[ ]:

from planet4 import markings, plotting
get_ipython().magic('matplotlib nbagg')


# In[ ]:

id_='6t3'
cm = clustering.ClusteringManager(include_angle=False)
cm.cluster_image_id(id_)


# In[ ]:

plotting.plot_image_id_pipeline(id_, include_angle=True, include_distance=True,
                                include_radius=False, eps=10)


# In[ ]:

plotting.plot_image_id_pipeline(id_, include_angle=False, include_distance=True,
                                include_radius=True, eps=10, min_distance=15)


# In[ ]:

plotting.plot_image_id_pipeline(id_, include_angle=False, include_distance=True,
                                include_radius=False, eps=10, min_distance=15)


# In[ ]:

plotting.plot_image_id_pipeline('139', include_angle=True, include_distance=True,
                                include_radius=True, eps=20, min_distance=10)


# In[ ]:

plotting.plot_image_id_pipeline('13t', include_angle=True, include_distance=True,
                                include_radius=True, eps=20, min_distance=10)


# In[ ]:

plotting.plot_finals('13t')


# In[ ]:

plotting.plot_image_id_pipeline('15k', include_angle=True, include_distance=True,
                                include_radius=True, eps=20, min_distance=10)


# In[ ]:

plotting.plot_finals('15k')


# In[ ]:

db = io.DBManager()
data = db.get_image_id_markings('17a')


# In[ ]:

plotting.plot_image_id_pipeline('17a', include_angle=True, include_distance=True,
                                include_radius=False, eps=20, min_distance=20)


# In[ ]:

plotting.plot_raw_fans('17a')


# In[ ]:

plotting.plot_finals('17a')


# In[ ]:

imgid = markings.ImageID(id_)


# In[ ]:

fans = imgid.get_fans()


# In[ ]:

dbscanner = clustering.DBScanner(fans[['x','y']].values)


# In[ ]:

dbscanner.reduced_data


# In[ ]:

fan = fans.iloc[dbscanner.reduced_data[6]]

imgid.plot_fans(fans=markings.FanContainer.from_df(fan).content)


{'fraction_non_alphanumeric': 0.14549230393354506, 'fraction_numerical': 0.050085511849499144, 'mean_word_length': 4.264951768488746, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2536380', 'n_tokens_mistral': 3529, 'n_tokens_neox': 3177, 'n_words': 499}