<commit_before>#!/usr/bin/env python
# coding: utf-8

# Handles urls like dcode://my_project/some/path.py?l=1&c=1

import sys
import os
from os.path import join, exists, normpath, basename, expanduser
from subprocess import check_call
try:
    from urlparse import urlparse, parse_qs
except:
    from urllib.parse import urlparse, parse_qs
import json
from pprint import pprint
from logging import warning
from time import sleep

HOME = expanduser("~")
CONFIG_FILE = join(HOME, '.dcode.json')
CONFIG_DEFAULTS = {
    'editor': 'atom',
}
DEV = os.environ.get('DCODE_DEV')

# Add paths where editors are likely found
os.environ['PATH'] += os.pathsep + '/usr/local/bin'
print('PATH=' + os.environ['PATH'])

if sys.platform == 'darwin':
    editorCommands = {
        'atom': "open -a atom -n --args '{pathLineColumn}'",
        # vscode doesn't honor arguments from "open -a"
        'vscode': "'/Applications/Visual Studio Code.app/Contents/Resources/app/bin/code' --goto --reuse-window '{pathLineColumn}'",
    }
else:
    editorCommands = {
        'atom': "atom '{pathLineColumn}'",
        'vscode': "code --goto --reuse-window '{pathLineColumn}'",
    }


# Paths known to be repositories
repoCache = None
freshCache = False


def enumerateRepos(home=None):
    ' Walk through HOME and yield git repositories. '
    if not home:
        home = HOME
    # Walk all directories
    for dirpath, childDirs, filenames in os.walk(home, topdown=True):
        # Skip hidden directories
        if basename(dirpath).startswith('.'):
            del childDirs[:]
            continue
        # See whether that's a repository root
        if '.git' in childDirs or '.hg' in childDirs:
            print('GIT ' + basename(dirpath) + ' in ' + dirpath)
            yield dirpath


def collectRepos(home=None, refresh=False):
    global repoCache, freshCache
    if repoCache is None or refresh:
        repoCache = list(enumerateRepos(home))
        freshCache = True
    return repoCache


def findRepoWithPath(path, repoName=None):
    ' Find a repo that contains this path. '
    for root in collectRepos():
        fullPath = root + '/' + path
        if exists(fullPath):
            if repoName and basename(root).lower() != repoName.lower():
                print('W Repo name {0} do not match {1}'.format(repoName, root))
            return root
    return None


def cleanPath(path):
    ' Remove quotes from paths '
    return path.replace('"', '').replace("'", '')


def findRepoFromUrl(url):
    # Find erroneous encoding of hashes (#)
    lastHash = url.rfind('%23')
    if lastHash >= 0:
        url = url[:lastHash] + '#' + url[lastHash+3:]

    # Parse url and parameters
    purl = urlparse(url)
    params = parse_qs(purl.query)
    path = purl.path.strip('/')
    if not path:
        return None

    root = findRepoWithPath(
        path=path,
        repoName=purl.hostname,
    )
    if root is None:
        return None

    lines = params.get('line') or params.get('l')
    cols = params.get('column') or params.get('c')
    location = {
        'root': root,
        'path': path,
        'line': lines[0] if lines else '',
        'column': cols[0] if cols else '',
    }
    return location


def makeEditorCommand(config, location):
    # Create path:line:column notation
    # `line` and `column` are optional
    fullPath = join(location['root'], location['path'])
    withLine = fullPath
    withColumn = fullPath
    if location['line']:
        withLine += ':' + location['line']
        withColumn = withLine
        if location['column']:
            withColumn += ':' + location['column']

    # Find the command template
    tpl = config.get('command')
    if not tpl:
        preset = config.get('editor')
        if not preset:
            raise ValueError('Could not make an editor command')

        tpl = editorCommands[preset]

    variables = dict(
        root=cleanPath(location['root']),
        path=cleanPath(location['path']),
        line=location['line'],
        column=location['column'],<commit_msg>Support overriding the default editor in the url<commit_after>#!/usr/bin/env python
# coding: utf-8

# Handles urls like dcode://my_project/some/path.py?l=1&c=1

import sys
import os
from os.path import join, exists, normpath, basename, expanduser
from subprocess import check_call
try:
    from urlparse import urlparse, parse_qs
except:
    from urllib.parse import urlparse, parse_qs
import json
from pprint import pprint
from logging import warning
from time import sleep

HOME = expanduser("~")
CONFIG_FILE = join(HOME, '.dcode.json')
CONFIG_DEFAULTS = {
    'editor': 'atom',
}
DEV = os.environ.get('DCODE_DEV')

# Add paths where editors are likely found
os.environ['PATH'] += os.pathsep + '/usr/local/bin'
print('PATH=' + os.environ['PATH'])

if sys.platform == 'darwin':
    editorCommands = {
        'atom': "open -a atom -n --args '{pathLineColumn}'",
        # vscode doesn't honor arguments from "open -a"
        'vscode': "'/Applications/Visual Studio Code.app/Contents/Resources/app/bin/code' --goto --reuse-window '{pathLineColumn}'",
    }
else:
    editorCommands = {
        'atom': "atom '{pathLineColumn}'",
        'vscode': "code --goto --reuse-window '{pathLineColumn}'",
    }


# Paths known to be repositories
repoCache = None
freshCache = False


def enumerateRepos(home=None):
    ' Walk through HOME and yield git repositories. '
    if not home:
        home = HOME
    # Walk all directories
    for dirpath, childDirs, filenames in os.walk(home, topdown=True):
        # Skip hidden directories
        if basename(dirpath).startswith('.'):
            del childDirs[:]
            continue
        # See whether that's a repository root
        if '.git' in childDirs or '.hg' in childDirs:
            print('GIT ' + basename(dirpath) + ' in ' + dirpath)
            yield dirpath


def collectRepos(home=None, refresh=False):
    global repoCache, freshCache
    if repoCache is None or refresh:
        repoCache = list(enumerateRepos(home))
        freshCache = True
    return repoCache


def findRepoWithPath(path, repoName=None):
    ' Find a repo that contains this path. '
    for root in collectRepos():
        fullPath = root + '/' + path
        if exists(fullPath):
            if repoName and basename(root).lower() != repoName.lower():
                print('W Repo name {0} do not match {1}'.format(repoName, root))
            return root
    return None


def cleanPath(path):
    ' Remove quotes from paths '
    return path.replace('"', '').replace("'", '')


def findRepoFromUrl(url):
    # Find erroneous encoding of hashes (#)
    lastHash = url.rfind('%23')
    if lastHash >= 0:
        url = url[:lastHash] + '#' + url[lastHash+3:]

    # Parse url and parameters
    purl = urlparse(url)
    params = parse_qs(purl.query)
    path = purl.path.strip('/')
    if not path:
        return None

    root = findRepoWithPath(
        path=path,
        repoName=purl.hostname,
    )
    if root is None:
        return None

    lines = params.get('line') or params.get('l')
    cols = params.get('column') or params.get('c')
    editors = params.get('editor')
    location = {
        'root': root,
        'path': path,
        'line': lines[0] if lines else '',
        'column': cols[0] if cols else '',
        'editor': editors[0] if editors else '',
    }
    return location


def makeEditorCommand(config, location):
    # Create path:line:column notation
    # `line` and `column` are optional
    fullPath = join(location['root'], location['path'])
    withLine = fullPath
    withColumn = fullPath
    if location['line']:
        withLine += ':' + location['line']
        withColumn = withLine
        if location['column']:
            withColumn += ':' + location['column']

    # Find the command template...
    # ...in the url itself
    preset = location.get('editor', '')
    tpl = editorCommands.get(preset)
    if preset and not tpl:
        warning('Unknown editor "%s"' % preset)
    # ...as a custom command
    if not tpl:
        preset = ''
        tpl = config.get('command')
    # ...as a preset
    if not tpl:
        preset = config.get('editor')
        tpl = editorCommands.get(preset)

    if not tpl:
        raise ValueError('Could not make an editor command')

    variables = dict(
        root=cleanPath(location['root']),
        path=cleanPath(location['path']),
        line=location['line'],
        column=location['column'],
{'fraction_non_alphanumeric': 0.12880396319886767, 'fraction_numerical': 0.0027129039867893373, 'mean_word_length': 2.6865217391304346, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6280256', 'n_tokens_mistral': 2611, 'n_tokens_neox': 2560, 'n_words': 749}