<commit_before>import os
import re
import sys
import glob
import json
import time
import shlex
import hashlib
import tarfile
import tempfile
import threading
import subprocess

from itertools import chain
from urllib.request import urlopen

from .base import Base

repo_dirs = set(['.git', '.hg', '.svn'])
flag_pattern = ''.join(r""""
-resource-dir|
(?:
    (?:-(?:internal|externc|c|cxx|objc))*?
    -i(?:
        (?:nclude-(?:p[ct]h))|
        quote|prefix|sysroot|system|framework|dirafter|macros|withprefix|
        withprefixbefore|withsysroot
    )
)
""".split())

std_default = {
    'c': 'c11',
    'cpp': 'c++1z',
    'objc': 'c11',
    'objcpp': 'c++1z',
}

lang_names = {
    'c': 'c',
    'cpp': 'c++',
    'objc': 'objective-c',
    'objcpp': 'objective-c++',
}

pch_cache = os.path.join(tempfile.gettempdir(), 'deoplete-clang2')

if sys.platform == 'darwin':
    darwin_sdk_dl_path = os.path.expanduser('~/Library/Developer/Frameworks')
    darwin_sdk_paths = (
        '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs',
        '/Developer/SDKs',
        darwin_sdk_dl_path,
    )
else:
    darwin_sdk_dl_path = os.path.join(os.getenv('XDG_DATA_HOME', '~/.local/share'), 'SDKs')
    darwin_sdk_paths = (darwin_sdk_dl_path,)

darwin_download = threading.Lock()
darwin_versions = {
    '10.1': ('10.1.5', 1010),
    '10.1.5': ('10.1.5', 1010),
    '10.2': ('10.2.8', 1020),
    '10.2.8': ('10.2.8', 1020),
    '10.3': ('10.3.9', 1030),
    '10.3.0': ('10.3.0', 1030),
    '10.3.9': ('10.3.9', 1030),
    '10.4': ('10.4u', 1040),
    '10.5': ('10.5', 1050),
    '10.6': ('10.6', 1060),
    '10.7': ('10.7', 1070),
    '10.8': ('10.8', 1080),
    '10.9': ('10.9', 1090),
    '10.10': ('10.10', 100000),
    '10.11': ('10.11', 101100),
}

darwin_sdk_url = 'https://github.com/phracker/MacOSX-SDKs/releases/download/MacOSX10.11.sdk/MacOSX%s.sdk.tar.xz'


def dl_progress(nvim, msg):
    nvim.async_call(lambda n, m: n.eval('clang2#status(%r)' % m),
                    nvim, msg)


def download_sdk(nvim, version):
    with darwin_download:
        dest = tempfile.NamedTemporaryFile('wb', delete=False)
        chunk = 16 * 1024
        try:
            r = urlopen(darwin_sdk_url % version)
            last = time.time()
            dl_bytes = 0
            while True:
                buf = r.read(chunk)
                if not buf:
                    break
                dest.write(buf)
                dl_bytes += len(buf)

                if time.time() - last >= 1:
                    dl_progress(nvim,
                                'Downloading MacOSX%s.sdk (%d bytes)'
                                % (version, dl_bytes))
                    last = time.time()

            dl_progress(nvim,
                        'Extracting MacOSX%s.sdk to %s'
                        % (version, darwin_sdk_dl_path))
            dest.close()

            with tarfile.open(dest.name) as fp:
                fp.extractall(darwin_sdk_dl_path)
        except Exception as e:
            dl_progress('Error downloading SDK: %s' % e)
        finally:
            dest.remove()


class Source(Base):
    def __init__(self, nvim):
        super(Source, self).__init__(nvim)
        self.clang_path = ''
        self.min_pattern_length = 0
        self.nvim = nvim
        self.name = 'clang2'
        self.mark = '[clang]'
        self.rank = 500
        self.bad_flags = []
        self.clang_flags = {}
        self.pch_flags = {}
        self.filetypes = ['c', 'cpp', 'objc', 'objcpp']
        self.db_files = {}
        self.last = {}
        self.scope_completions = []
        self.user_flags = {}
        self.sys_includes = {}
        self.darwin_version = 0
        self.last_neomake_flags = set()

    def on_event(self, context, filename=''):
        if context['event'] == 'BufWritePost':
            self.find_db_flags(context)
            self.last = {}
            self.scope_completions = []

        self.clang_path = context['vars'].get(
            'deoplete#sources#clang#executable', 'clang')

        if context['vars'].get('deoplete#sources#clang#autofill_neomake', 1):
            cmd, flags = self.build_flags(context)
            if context['filetype'] == 'cpp':
                cmd += ['-stdlib=libc++']
            neomake_flags = set(cmd + flags)
            if self.last_neomake_flags ^ neomake_flags:
                self.last_neomake_flags = neomake_flags
                self.nvim.async_call(lambda n, m:
                                     n.eval('clang2#set_neomake_cflags(%r)' % m),
                                     self.nvim, cmd + flags)

    def get_complete_position(self, context):
        m = re.search(r'^\s*#(?:include|import)\s+["<]', context['input'])
        if m:
            context['clang2_include'] = m.group(0)
            return m.end()

        pat = r'->|\.'
        objc = context.get('filetype', '') in ('objc', 'objcpp')
        if objc:
            pat += r'|[:@\[]|(?:\S\s+)'
        pat = r'(?:' + pat + ')'
        context['clang2_pattern'] = pat

        m = re.search(r'(' + pat + r')(\w*?)$', context['input'])
        if m is not None:
            if objc and m.group(1) == '@':
                return m.start(1) - 1<commit_msg>Trigger completions after cpp scope resolution operator (::)<commit_after>import os
import re
import sys
import glob
import json
import time
import shlex
import hashlib
import tarfile
import tempfile
import threading
import subprocess

from itertools import chain
from urllib.request import urlopen

from .base import Base

repo_dirs = set(['.git', '.hg', '.svn'])
flag_pattern = ''.join(r""""
-resource-dir|
(?:
    (?:-(?:internal|externc|c|cxx|objc))*?
    -i(?:
        (?:nclude-(?:p[ct]h))|
        quote|prefix|sysroot|system|framework|dirafter|macros|withprefix|
        withprefixbefore|withsysroot
    )
)
""".split())

std_default = {
    'c': 'c11',
    'cpp': 'c++1z',
    'objc': 'c11',
    'objcpp': 'c++1z',
}

lang_names = {
    'c': 'c',
    'cpp': 'c++',
    'objc': 'objective-c',
    'objcpp': 'objective-c++',
}

pch_cache = os.path.join(tempfile.gettempdir(), 'deoplete-clang2')

if sys.platform == 'darwin':
    darwin_sdk_dl_path = os.path.expanduser('~/Library/Developer/Frameworks')
    darwin_sdk_paths = (
        '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs',
        '/Developer/SDKs',
        darwin_sdk_dl_path,
    )
else:
    darwin_sdk_dl_path = os.path.join(os.getenv('XDG_DATA_HOME', '~/.local/share'), 'SDKs')
    darwin_sdk_paths = (darwin_sdk_dl_path,)

darwin_download = threading.Lock()
darwin_versions = {
    '10.1': ('10.1.5', 1010),
    '10.1.5': ('10.1.5', 1010),
    '10.2': ('10.2.8', 1020),
    '10.2.8': ('10.2.8', 1020),
    '10.3': ('10.3.9', 1030),
    '10.3.0': ('10.3.0', 1030),
    '10.3.9': ('10.3.9', 1030),
    '10.4': ('10.4u', 1040),
    '10.5': ('10.5', 1050),
    '10.6': ('10.6', 1060),
    '10.7': ('10.7', 1070),
    '10.8': ('10.8', 1080),
    '10.9': ('10.9', 1090),
    '10.10': ('10.10', 100000),
    '10.11': ('10.11', 101100),
}

darwin_sdk_url = 'https://github.com/phracker/MacOSX-SDKs/releases/download/MacOSX10.11.sdk/MacOSX%s.sdk.tar.xz'


def dl_progress(nvim, msg):
    nvim.async_call(lambda n, m: n.eval('clang2#status(%r)' % m),
                    nvim, msg)


def download_sdk(nvim, version):
    with darwin_download:
        dest = tempfile.NamedTemporaryFile('wb', delete=False)
        chunk = 16 * 1024
        try:
            r = urlopen(darwin_sdk_url % version)
            last = time.time()
            dl_bytes = 0
            while True:
                buf = r.read(chunk)
                if not buf:
                    break
                dest.write(buf)
                dl_bytes += len(buf)

                if time.time() - last >= 1:
                    dl_progress(nvim,
                                'Downloading MacOSX%s.sdk (%d bytes)'
                                % (version, dl_bytes))
                    last = time.time()

            dl_progress(nvim,
                        'Extracting MacOSX%s.sdk to %s'
                        % (version, darwin_sdk_dl_path))
            dest.close()

            with tarfile.open(dest.name) as fp:
                fp.extractall(darwin_sdk_dl_path)
        except Exception as e:
            dl_progress('Error downloading SDK: %s' % e)
        finally:
            dest.remove()


class Source(Base):
    def __init__(self, nvim):
        super(Source, self).__init__(nvim)
        self.clang_path = ''
        self.min_pattern_length = 0
        self.nvim = nvim
        self.name = 'clang2'
        self.mark = '[clang]'
        self.rank = 500
        self.bad_flags = []
        self.clang_flags = {}
        self.pch_flags = {}
        self.filetypes = ['c', 'cpp', 'objc', 'objcpp']
        self.db_files = {}
        self.last = {}
        self.scope_completions = []
        self.user_flags = {}
        self.sys_includes = {}
        self.darwin_version = 0
        self.last_neomake_flags = set()

    def on_event(self, context, filename=''):
        if context['event'] == 'BufWritePost':
            self.find_db_flags(context)
            self.last = {}
            self.scope_completions = []

        self.clang_path = context['vars'].get(
            'deoplete#sources#clang#executable', 'clang')

        if context['vars'].get('deoplete#sources#clang#autofill_neomake', 1):
            cmd, flags = self.build_flags(context)
            if context['filetype'] == 'cpp':
                cmd += ['-stdlib=libc++']
            neomake_flags = set(cmd + flags)
            if self.last_neomake_flags ^ neomake_flags:
                self.last_neomake_flags = neomake_flags
                self.nvim.async_call(lambda n, m:
                                     n.eval('clang2#set_neomake_cflags(%r)' % m),
                                     self.nvim, cmd + flags)

    def get_complete_position(self, context):
        m = re.search(r'^\s*#(?:include|import)\s+["<]', context['input'])
        if m:
            context['clang2_include'] = m.group(0)
            return m.end()

        pat = r'->|\.'
        ft = context.get('filetype', '')
        objc = ft in ('objc', 'objcpp')
        if objc:
            pat += r'|[:@\[]|(?:\S\s+)'
        elif ft == 'cpp':
            pat += r'|::'
        pat = r'(?:' + pat + ')'
        context['clang2_pattern'] = pat

        m = re.search(r'(' + pat + r')(\w*?)$', context['input'])
        if m is not None:
            if objc and m.group(1) == '@':
                return m.start(1) - 1
{'fraction_non_alphanumeric': 0.16838034147721914, 'fraction_numerical': 0.03829827374775965, 'mean_word_length': 1.9581473214285714, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 26, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6714646', 'n_tokens_mistral': 3998, 'n_tokens_neox': 3666, 'n_words': 681}