<commit_before>/*
 * Copyright (C) 2021 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <constant.h>
#include <misc/unicode.hpp>

#include "source.h"

namespace ydsh::lsp {

std::shared_ptr<Source> Source::copyAndUpdate(int v, std::string &&c) const {
  auto newSrc = std::make_shared<Source>();
  newSrc->path = this->path;
  newSrc->content = std::move(c);
  newSrc->srcId = this->srcId;
  newSrc->version = v;
  return newSrc;
}

SourcePtr SourceManager::findById(unsigned int id) const {
  if (id > 0 && --id < this->entries.size()) {
    return this->entries[id];
  }
  return nullptr;
}

SourcePtr SourceManager::find(StringRef path) const {
  auto iter = this->indexMap.find(path);
  if (iter != this->indexMap.end()) {
    return this->entries[iter->second];
  }
  return nullptr;
}

SourcePtr SourceManager::update(StringRef path, int version, std::string &&content) {
  std::string tmp = std::move(content);
  if (tmp.empty() || tmp.back() != '\n') {
    tmp += '\n';
  }

  auto iter = this->indexMap.find(path);
  if (iter != this->indexMap.end()) {
    unsigned int i = iter->second;
    this->entries[i] = this->entries[i]->copyAndUpdate(version, std::move(tmp));
    return this->entries[i];
  } else {
    unsigned int id = this->entries.size() + 1;
    if (id == SYS_LIMIT_MOD_ID) {
      return nullptr;
    }
    unsigned int i = this->entries.size();
    auto src = std::make_shared<Source>(path.data(), static_cast<unsigned short>(id),
                                        std::move(tmp), version);
    auto &ret = this->entries.emplace_back(std::move(src));
    this->indexMap.emplace(ret->getPath(), i);
    return this->entries[i];
  }
}

SourcePtr SourceManager::add(SourcePtr other) {
  if (!other) {
    return nullptr;
  }
  if (auto src = this->find(other->getPath()); src == other) {
    return other;
  }
  return this->update(other->getPath(), other->getVersion(), std::string(other->getContent()));
}

static size_t findLineStartPos(StringRef content, unsigned int count) {
  const char *str = content.data();
  for (unsigned int i = 0; i < count; i++) {
    const char *ptr = strchr(str, '\n');
    if (!ptr) {
      break;
    }
    str = ++ptr;
  }
  return str - content.data();
}

static size_t count(StringRef content, unsigned int offset, unsigned int count) {
  size_t limit = offset;
  for (; limit < content.size() && content[limit] != '\n'; limit++)
    ;
  auto line = content.slice(offset, limit + 1);

  const char *iter = line.begin();
  const char *end = line.end();<commit_msg>[LSP] cleanup line start pos finding<commit_after>/*
 * Copyright (C) 2021 <NAME>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <constant.h>
#include <misc/unicode.hpp>

#include "source.h"

namespace ydsh::lsp {

std::shared_ptr<Source> Source::copyAndUpdate(int v, std::string &&c) const {
  auto newSrc = std::make_shared<Source>();
  newSrc->path = this->path;
  newSrc->content = std::move(c);
  newSrc->srcId = this->srcId;
  newSrc->version = v;
  return newSrc;
}

SourcePtr SourceManager::findById(unsigned int id) const {
  if (id > 0 && --id < this->entries.size()) {
    return this->entries[id];
  }
  return nullptr;
}

SourcePtr SourceManager::find(StringRef path) const {
  auto iter = this->indexMap.find(path);
  if (iter != this->indexMap.end()) {
    return this->entries[iter->second];
  }
  return nullptr;
}

SourcePtr SourceManager::update(StringRef path, int version, std::string &&content) {
  std::string tmp = std::move(content);
  if (tmp.empty() || tmp.back() != '\n') {
    tmp += '\n';
  }

  auto iter = this->indexMap.find(path);
  if (iter != this->indexMap.end()) {
    unsigned int i = iter->second;
    this->entries[i] = this->entries[i]->copyAndUpdate(version, std::move(tmp));
    return this->entries[i];
  } else {
    unsigned int id = this->entries.size() + 1;
    if (id == SYS_LIMIT_MOD_ID) {
      return nullptr;
    }
    unsigned int i = this->entries.size();
    auto src = std::make_shared<Source>(path.data(), static_cast<unsigned short>(id),
                                        std::move(tmp), version);
    auto &ret = this->entries.emplace_back(std::move(src));
    this->indexMap.emplace(ret->getPath(), i);
    return this->entries[i];
  }
}

SourcePtr SourceManager::add(SourcePtr other) {
  if (!other) {
    return nullptr;
  }
  if (auto src = this->find(other->getPath()); src == other) {
    return other;
  }
  return this->update(other->getPath(), other->getVersion(), std::string(other->getContent()));
}

static size_t findLineStartPos(StringRef content, unsigned int count) {
  StringRef::size_type pos = 0;
  for (unsigned int i = 0; i < count; i++) {
    auto ret = content.find('\n', pos);
    if (ret == StringRef::npos) {
      break;
    }
    pos = ++ret;
  }
  return pos;
}

static size_t count(StringRef content, unsigned int offset, unsigned int count) {
  size_t limit = offset;
  for (; limit < content.size() && content[limit] != '\n'; limit++)
    ;
  auto line = content.slice(offset, limit + 1);

  const char *iter = line.begin();
  const char *end = line.end();
{'fraction_non_alphanumeric': 0.15538887983042557, 'fraction_numerical': 0.004076308495026904, 'mean_word_length': 3.5437037037037036, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6057660', 'n_tokens_mistral': 1953, 'n_tokens_neox': 1925, 'n_words': 564}