<commit_before>import numpy as np
import functools
import math
import cv2

import networkx as nx

from opensfm import io
from opensfm import geo
from opensfm import types


def derivative(func, x):
    eps = 1e-10
    d = (func(x+eps)-func(x))/eps
    d /= np.linalg.norm(d)
    return d


def samples_generator_random_count(count):
    return np.random.rand(count)


def samples_generator_interval(start, length, interval, interval_noise):
    samples = np.linspace(start/length, 1, num=length/interval)
    samples += np.random.normal(0.0,
                                float(interval_noise)/float(length),
                                samples.shape)
    return samples


def generate_samples_and_local_frame(samples, shape):
    points = []
    tangents = []
    for i in samples:
        point = shape(i)
        points += [point]
        ex = derivative(shape, i)
        ez = np.array([ex[1], -ex[0]])
        tangents += [np.array([ez, ex])]
    return np.array(points), np.array(tangents)


def generate_samples_shifted(samples, shape, shift):
    plane_points = []
    for i in samples:
        point = shape(i)
        tangent = derivative(shape, i)
        tangent = np.array([-tangent[1], tangent[0]])
        point += tangent*(shift/2)
        plane_points += [point]
    return np.array(plane_points)


def generate_z_plane(samples, shape, thickness):
    plane_points = []
    for i in samples:
        point = shape(i)
        tangent = derivative(shape, i)
        tangent = np.array([-tangent[1], tangent[0]])
        shift = tangent*((np.random.rand()-0.5)*thickness)
        point += shift
        plane_points += [point]
    plane_points = np.array(plane_points)
    return np.insert(plane_points, 2, values=0, axis=1)


def generate_xy_planes(samples, shape, z_size, y_size):
    xy1 = generate_samples_shifted(samples, shape, y_size)
    xy2 = generate_samples_shifted(samples, shape, -y_size)
    xy1 = np.insert(xy1, 2, values=np.random.rand(
        xy1.shape[0])*z_size, axis=1)
    xy2 = np.insert(xy2, 2, values=np.random.rand(
        xy2.shape[0])*z_size, axis=1)
    return np.concatenate((xy1, xy2), axis=0)


def generate_street(samples, shape, height, width):
    walls = generate_xy_planes(samples, shape, height, width)
    floor = generate_z_plane(samples, shape, width)
    return walls, floor


def generate_cameras(samples, shape, height):
    positions, rotations = generate_samples_and_local_frame(samples, shape)
    positions = np.insert(positions, 2, values=height, axis=1)
    rotations = np.insert(rotations, 2, values=0, axis=2)
    rotations = np.insert(rotations, 1, values=np.array([0, 0, -1]), axis=1)
    return positions, rotations


def line_generator(length, point):
    x = point*length
    return np.transpose(np.array([x, 0]))


def weird_curve(length, point):
    y = math.cos(point+0.1*point**2+2.3*point**4 +
                 0.8*point**7+0.1*point**9)
    return length*np.transpose(np.array([point, y]))


def ellipse_generator(x_size, y_size, point):
    y = np.sin(point*2*np.pi)*y_size/2
    x = np.cos(point*2*np.pi)*x_size/2
    return np.transpose(np.array([x, y]))


def perturb_points(points, sigmas):
    eps = 1e-10
    for point in points:
        sigmas = np.array([max(s, eps) for s in sigmas])
        point += np.random.normal(0.0, sigmas,
                                  point.shape)


def generate_exifs(reconstruction, gps_noise, speed_ms=10):
    """
    Return extracted exif information, as dictionary, usually with fields:

    ================  =====  ===================================
    Field             Type   Description
    ================  =====  ===================================
    width             int    Width of image, in pixels
    height            int    Height of image, in pixels
    focal_prior       float  Focal length (real) / sensor width
    ================  =====  ===================================

    :param image: Image name, with extension (i.e. 123.jpg)
    """
    previous_pose = None
    previous_time = 0
    exifs = {}
    for shot_name in sorted(reconstruction.shots.keys()):
        shot = reconstruction.shots[shot_name]
        exif = {}
        exif['width'] = shot.camera.width
        exif['height'] = shot.camera.height
        exif['focal_prior'] = shot.camera.focal_prior
        exif['camera'] = str(shot.camera.id)

        pose = shot.pose.get_origin()

        if previous_pose is not None:
            previous_time += np.linalg.norm(pose-previous_pose)*speed_ms
            previous_pose = pose
        exif['capture_time'] = previous_time

        perturb_points([pose], [gps_noise, gps_noise, gps_noise])
        lat, lon, alt = geo.lla_from_topocentric(pose[0], pose[1], pose[2],
                                                 0, 0, 0)

        exif['gps'] = {}
        exif['gps']['latitude'] = lat
        exif['gps']['longitude'] = lon
        exif['gps']['altitude'] = alt
        exif['gps']['dop'] = gps_noise
        exifs[shot_name] = exif
    return exifs


def perturb_rotations(rotations, angle_sigma):
    for i in range(len(rotations)):
        rotation = rotations[i]
        rodrigues = cv2.Rodrigues(rotation)[0].ravel()
        angle = np.linalg.norm(rodrigues)
        angle_pertubed = angle + np.random.normal(0.0, angle_sigma)
        rodrigues *= (float(angle_pertubed)/float(angle))<commit_msg>feature: added compass data to synth generator<commit_after>import numpy as np
import functools
import math
import copy
import cv2

import networkx as nx

from opensfm import io
from opensfm import geo
from opensfm import types
from opensfm import reconstruction as rc


def derivative(func, x):
    eps = 1e-10
    d = (func(x+eps)-func(x))/eps
    d /= np.linalg.norm(d)
    return d


def samples_generator_random_count(count):
    return np.random.rand(count)


def samples_generator_interval(start, length, interval, interval_noise):
    samples = np.linspace(start/length, 1, num=length/interval)
    samples += np.random.normal(0.0,
                                float(interval_noise)/float(length),
                                samples.shape)
    return samples


def generate_samples_and_local_frame(samples, shape):
    points = []
    tangents = []
    for i in samples:
        point = shape(i)
        points += [point]
        ex = derivative(shape, i)
        ez = np.array([ex[1], -ex[0]])
        tangents += [np.array([ez, ex])]
    return np.array(points), np.array(tangents)


def generate_samples_shifted(samples, shape, shift):
    plane_points = []
    for i in samples:
        point = shape(i)
        tangent = derivative(shape, i)
        tangent = np.array([-tangent[1], tangent[0]])
        point += tangent*(shift/2)
        plane_points += [point]
    return np.array(plane_points)


def generate_z_plane(samples, shape, thickness):
    plane_points = []
    for i in samples:
        point = shape(i)
        tangent = derivative(shape, i)
        tangent = np.array([-tangent[1], tangent[0]])
        shift = tangent*((np.random.rand()-0.5)*thickness)
        point += shift
        plane_points += [point]
    plane_points = np.array(plane_points)
    return np.insert(plane_points, 2, values=0, axis=1)


def generate_xy_planes(samples, shape, z_size, y_size):
    xy1 = generate_samples_shifted(samples, shape, y_size)
    xy2 = generate_samples_shifted(samples, shape, -y_size)
    xy1 = np.insert(xy1, 2, values=np.random.rand(
        xy1.shape[0])*z_size, axis=1)
    xy2 = np.insert(xy2, 2, values=np.random.rand(
        xy2.shape[0])*z_size, axis=1)
    return np.concatenate((xy1, xy2), axis=0)


def generate_street(samples, shape, height, width):
    walls = generate_xy_planes(samples, shape, height, width)
    floor = generate_z_plane(samples, shape, width)
    return walls, floor


def generate_cameras(samples, shape, height):
    positions, rotations = generate_samples_and_local_frame(samples, shape)
    positions = np.insert(positions, 2, values=height, axis=1)
    rotations = np.insert(rotations, 2, values=0, axis=2)
    rotations = np.insert(rotations, 1, values=np.array([0, 0, -1]), axis=1)
    return positions, rotations


def line_generator(length, point):
    x = point*length
    return np.transpose(np.array([x, 0]))


def weird_curve(length, point):
    y = math.cos(point+0.1*point**2+2.3*point**4 +
                 0.8*point**7+0.1*point**9)
    return length*np.transpose(np.array([point, y]))


def ellipse_generator(x_size, y_size, point):
    y = np.sin(point*2*np.pi)*y_size/2
    x = np.cos(point*2*np.pi)*x_size/2
    return np.transpose(np.array([x, y]))


def perturb_points(points, sigmas):
    eps = 1e-10
    for point in points:
        sigmas = np.array([max(s, eps) for s in sigmas])
        point += np.random.normal(0.0, sigmas,
                                  point.shape)


def generate_exifs(reconstruction, gps_noise, speed_ms=10):
    """
    Return extracted exif information, as dictionary, usually with fields:

    ================  =====  ===================================
    Field             Type   Description
    ================  =====  ===================================
    width             int    Width of image, in pixels
    height            int    Height of image, in pixels
    focal_prior       float  Focal length (real) / sensor width
    ================  =====  ===================================

    :param image: Image name, with extension (i.e. 123.jpg)
    """
    previous_pose = None
    previous_time = 0
    exifs = {}
    reference = {'latitude': 0, 'longitude': 0, 'altitude': 0}
    for shot_name in sorted(reconstruction.shots.keys()):
        shot = reconstruction.shots[shot_name]
        exif = {}
        exif['width'] = shot.camera.width
        exif['height'] = shot.camera.height
        exif['focal_prior'] = shot.camera.focal_prior
        exif['camera'] = str(shot.camera.id)

        pose = shot.pose.get_origin()

        if previous_pose is not None:
            previous_time += np.linalg.norm(pose-previous_pose)*speed_ms
            previous_pose = pose
        exif['capture_time'] = previous_time

        perturb_points([pose], [gps_noise, gps_noise, gps_noise])

        shot_copy = copy.deepcopy(shot)
        shot_copy.pose.set_origin(pose)
        lat, lon, alt, comp = rc.shot_lla_and_compass(shot_copy,
                                                      reference)

        exif['gps'] = {}
        exif['gps']['latitude'] = lat
        exif['gps']['longitude'] = lon
        exif['gps']['altitude'] = alt
        exif['gps']['dop'] = gps_noise
        exif['compass'] = {'angle': comp}
        exifs[shot_name] = exif
    return exifs


def perturb_rotations(rotations, angle_sigma):
    for i in range(len(rotations)):
        rotation = rotations[i]
        rodrigues = cv2.Rodrigues(rotation)[0].ravel()
        angle = np.linalg.norm(rodrigues)
        angle_pertubed = angle + np.random.normal(0.0, angle_sigma)
        rodrigues *= (float(angle_pertubed)/float(angle))
{'fraction_non_alphanumeric': 0.1522133526850508, 'fraction_numerical': 0.014967343976777939, 'mean_word_length': 2.8030355294929286, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '924958', 'n_tokens_mistral': 3702, 'n_tokens_neox': 3514, 'n_words': 781}