<commit_before>---
layout: post
title:  "Is my life boring?"
date:   2017-03-14 14:21:08 +0010
categories: life
---


When my life is interesting I take a lot of photos. Lets see if I can plot these photos to see if my life is getting more interesting. 


```python
base_dir = "/home/visgean/Dropbox/**/*"
```


```python
import glob
import os

def get_extension(filename):
    filename, file_extension = os.path.splitext(filename)
    return file_extension.lower()


picture_extensions = ['.jpg', '.jpeg', '.png']
pictures = list(filter(
    lambda f: get_extension(f) in picture_extensions,
    glob.iglob(base_dir, recursive=True)
))
```


```python
>>> print('# of pics:', len(pictures))
# of pics: 3136
```




```python
>>> import exifread
>>> 
>>> with open(pictures[40], 'rb') as f:
>>>     tags = exifread.process_file(f)
>>>     
>>> print(tags.keys())

dict_keys(['MakerNote Tag 0x0001', 'Image Orientation', 'EXIF FlashPixVersion', 'Image YResolution', 'Image XResolution', 'GPS GPSLongitudeRef', 'GPS GPSImgDirectionRef', 'EXIF SensingMethod', 'Thumbnail ResolutionUnit', 'GPS GPSImgDirection', 'JPEGThumbnail', 'MakerNote Tag 0x0006', 'GPS GPSAltitudeRef', 'EXIF ColorSpace', 'EXIF ExposureBiasValue', 'EXIF SceneCaptureType', 'EXIF FocalLength', 'Image ResolutionUnit', 'Image Make', 'EXIF SubSecTimeOriginal', 'EXIF LensSpecification', 'EXIF BrightnessValue', 'Image DateTime', 'EXIF ApertureValue', 'MakerNote Tag 0x0005', 'EXIF FNumber', 'EXIF MeteringMode', 'GPS Tag 0x001F', 'GPS GPSLongitude', 'Thumbnail JPEGInterchangeFormat', 'EXIF LensMake', 'Image Software', 'MakerNote Tag 0x0003', 'EXIF ExposureTime', 'EXIF ShutterSpeedValue', 'Thumbnail Compression', 'MakerNote Tag 0x0008', 'Image ExifOffset', 'EXIF WhiteBalance', 'GPS GPSLatitude', 'EXIF ExifVersion', 'EXIF ExifImageWidth', 'EXIF DateTimeOriginal', 'Image Model', 'GPS GPSDestBearingRef', 'MakerNote Tag 0x0014', 'GPS GPSDestBearing', 'GPS GPSAltitude', 'EXIF SubSecTimeDigitized', 'GPS GPSSpeedRef', 'EXIF ComponentsConfiguration', 'EXIF FocalLengthIn35mmFilm', 'EXIF ExposureMode', 'Thumbnail JPEGInterchangeFormatLength', 'EXIF Flash', 'Image YCbCrPositioning', 'EXIF MakerNote', 'Image GPSInfo', 'GPS GPSSpeed', 'MakerNote Tag 0x0004', 'GPS GPSLatitudeRef', 'EXIF SceneType', 'EXIF ExifImageLength', 'EXIF ISOSpeedRatings', 'GPS GPSDate', 'EXIF LensModel', 'EXIF DateTimeDigitized', 'Thumbnail YResolution', 'MakerNote Tag 0x0007', 'GPS GPSTimeStamp', 'Thumbnail XResolution', 'EXIF ExposureProgram'])


```




```python
date = tags['Image DateTime']
```


```python
>>> print(date.values)
2016:10:05 16:16:11
```




```python
>>> from datetime import datetime
>>> datetime.strptime(str(date.values), '%Y:%m:%d %H:%M:%S')
datetime.datetime(2016, 10, 5, 16, 16, 11)

```








```python
def parse_date(exif_info):
    if not exif_info:
        return None    
    date = exif_info.get('Image DateTime')
    if not date:
        return None
    try:
        return datetime.strptime(str(date.values), '%Y:%m:%d %H:%M:%S').date()
    except:
        print(date)
```


```python
def get_exif(filename):
    try:
        with open(filename, 'rb') as f:
            return exifread.process_file(f)
    except:
        return None
```


```python
>>> exif_data = list(map(get_exif, pictures)) # i want to iterate through these multiple times
MemoryError at position: 603312351, length: 3945995808
Possibly corrupted field DateTime in Image IFD
Possibly corrupted field DateTime in Image IFD
Possibly corrupted field DateTime in Image IFD
Possibly corrupted field DateTime in Image IFD
Possibly corrupted field DateTime in Image IFD
```




```python
>>> dates = list(map(parse_date, exif_data))
2013:12:03 19:12:81
```

    



```python
from collections import Counter

photos_per_day = Counter(dates)
```


```python
>>> print('Missing dates', photos_per_day[None])
>>> del photos_per_day[None]
Missing dates 1247

```

```python
>>> print('Missing dates', photos_per_day[None])
Missing dates 0
```

    



```python
from bokeh.plotting import figure, show, output_notebook
output_notebook()
```



    <div class="bk-root">
        <a href="http://bokeh.pydata.org" target="_blank" class="bk-logo bk-logo-small bk-logo-notebook"></a>
        <span id="08df9cb6-309b-45a4-a95e-bc9e2665e25c">Loading BokehJS ...</span>
    </div>





```python
time_graph = figure(
    title="Pics over time", 
    background_fill_color="#E8DDCB",
    y_axis_label='# of pics', 
    x_axis_label='Time',
    x_axis_type="datetime"
)

sorted_vals = photos_per_day.most_common()

time_graph.circle([x[0] for x in sorted_vals], [x[1] for x in sorted_vals])
show(time_graph)
```




    <div class="bk-root">
        <div class="bk-plotdiv" id="410736d5-1a85-4d12-84bb-ed6fd5179284"></div>
    </div>
<script type="text/javascript">
  
  (function(global) {
    function now() {
      return new Date();
    }
  
    var force = false;
  
    if (typeof (window._bokeh_onload_callbacks) === "undefined" || force === true) {
      window._bokeh_onload_callbacks = [];
      window._bokeh_is_loading = undefined;<commit_msg>fix graph<commit_after>---
layout: post
title:  "Is my life boring?"
date:   2017-03-14 14:21:08 +0010
categories: life
---


When my life is interesting I take a lot of photos. Lets see if I can plot these photos to see if my life is getting more interesting. 


```python
base_dir = "/home/visgean/Dropbox/**/*"
```


```python
import glob
import os

def get_extension(filename):
    filename, file_extension = os.path.splitext(filename)
    return file_extension.lower()


picture_extensions = ['.jpg', '.jpeg', '.png']
pictures = list(filter(
    lambda f: get_extension(f) in picture_extensions,
    glob.iglob(base_dir, recursive=True)
))
```


```python
>>> print('# of pics:', len(pictures))
# of pics: 3136
```




```python
>>> import exifread
>>> 
>>> with open(pictures[40], 'rb') as f:
>>>     tags = exifread.process_file(f)
>>>     
>>> print(tags.keys())

dict_keys(['MakerNote Tag 0x0001', 'Image Orientation', 'EXIF FlashPixVersion', 'Image YResolution', 'Image XResolution', 'GPS GPSLongitudeRef', 'GPS GPSImgDirectionRef', 'EXIF SensingMethod', 'Thumbnail ResolutionUnit', 'GPS GPSImgDirection', 'JPEGThumbnail', 'MakerNote Tag 0x0006', 'GPS GPSAltitudeRef', 'EXIF ColorSpace', 'EXIF ExposureBiasValue', 'EXIF SceneCaptureType', 'EXIF FocalLength', 'Image ResolutionUnit', 'Image Make', 'EXIF SubSecTimeOriginal', 'EXIF LensSpecification', 'EXIF BrightnessValue', 'Image DateTime', 'EXIF ApertureValue', 'MakerNote Tag 0x0005', 'EXIF FNumber', 'EXIF MeteringMode', 'GPS Tag 0x001F', 'GPS GPSLongitude', 'Thumbnail JPEGInterchangeFormat', 'EXIF LensMake', 'Image Software', 'MakerNote Tag 0x0003', 'EXIF ExposureTime', 'EXIF ShutterSpeedValue', 'Thumbnail Compression', 'MakerNote Tag 0x0008', 'Image ExifOffset', 'EXIF WhiteBalance', 'GPS GPSLatitude', 'EXIF ExifVersion', 'EXIF ExifImageWidth', 'EXIF DateTimeOriginal', 'Image Model', 'GPS GPSDestBearingRef', 'MakerNote Tag 0x0014', 'GPS GPSDestBearing', 'GPS GPSAltitude', 'EXIF SubSecTimeDigitized', 'GPS GPSSpeedRef', 'EXIF ComponentsConfiguration', 'EXIF FocalLengthIn35mmFilm', 'EXIF ExposureMode', 'Thumbnail JPEGInterchangeFormatLength', 'EXIF Flash', 'Image YCbCrPositioning', 'EXIF MakerNote', 'Image GPSInfo', 'GPS GPSSpeed', 'MakerNote Tag 0x0004', 'GPS GPSLatitudeRef', 'EXIF SceneType', 'EXIF ExifImageLength', 'EXIF ISOSpeedRatings', 'GPS GPSDate', 'EXIF LensModel', 'EXIF DateTimeDigitized', 'Thumbnail YResolution', 'MakerNote Tag 0x0007', 'GPS GPSTimeStamp', 'Thumbnail XResolution', 'EXIF ExposureProgram'])


```




```python
date = tags['Image DateTime']
```


```python
>>> print(date.values)
2016:10:05 16:16:11
```




```python
>>> from datetime import datetime
>>> datetime.strptime(str(date.values), '%Y:%m:%d %H:%M:%S')
datetime.datetime(2016, 10, 5, 16, 16, 11)

```








```python
def parse_date(exif_info):
    if not exif_info:
        return None    
    date = exif_info.get('Image DateTime')
    if not date:
        return None
    try:
        return datetime.strptime(str(date.values), '%Y:%m:%d %H:%M:%S').date()
    except:
        print(date)
```


```python
def get_exif(filename):
    try:
        with open(filename, 'rb') as f:
            return exifread.process_file(f)
    except:
        return None
```


```python
>>> exif_data = list(map(get_exif, pictures)) # i want to iterate through these multiple times
MemoryError at position: 603312351, length: 3945995808
Possibly corrupted field DateTime in Image IFD
Possibly corrupted field DateTime in Image IFD
Possibly corrupted field DateTime in Image IFD
Possibly corrupted field DateTime in Image IFD
Possibly corrupted field DateTime in Image IFD
```




```python
>>> dates = list(map(parse_date, exif_data))
2013:12:03 19:12:81
```

    



```python
from collections import Counter

photos_per_day = Counter(dates)
```


```python
>>> print('Missing dates', photos_per_day[None])
>>> del photos_per_day[None]
Missing dates 1247

```

```python
>>> print('Missing dates', photos_per_day[None])
Missing dates 0
```

    



```python
from bokeh.plotting import figure, show, output_notebook
output_notebook()
```



    <div class="bk-root">
        <a href="http://bokeh.pydata.org" target="_blank" class="bk-logo bk-logo-small bk-logo-notebook"></a>
        <span id="08df9cb6-309b-45a4-a95e-bc9e2665e25c">Loading BokehJS ...</span>
    </div>





```python
time_graph = figure(
    title="Pics over time", 
    background_fill_color="#E8DDCB",
    y_axis_label='# of pics', 
    x_axis_label='Time',
    x_axis_type="datetime"
)

sorted_vals = photos_per_day.most_common()

time_graph.circle([x[0] for x in sorted_vals], [x[1] for x in sorted_vals])
show(time_graph)
```




<div class="bk-root">
    <div class="bk-plotdiv" id="410736d5-1a85-4d12-84bb-ed6fd5179284"></div>
</div>
<script type="text/javascript">
  
  (function(global) {
    function now() {
      return new Date();
    }
  
    var force = false;
  
    if (typeof (window._bokeh_onload_callbacks) === "undefined" || force === true) {
      window._bokeh_onload_callbacks = [];
      window._bokeh_is_loading = undefined;
{'fraction_non_alphanumeric': 0.1483959850423145, 'fraction_numerical': 0.03542609722495572, 'mean_word_length': 4.735327313769751, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6252372', 'n_tokens_mistral': 3892, 'n_tokens_neox': 3405, 'n_words': 908}