<commit_before>from steps.zotero import Zotero
from munch import *
from behave import given, when, then, use_step_matcher
import urllib.request
import json
import zotero
import time
import os
from hamcrest import assert_that, equal_to
from steps.utils import assert_equal_diff, expand_scenario_variables

import pathlib
for d in pathlib.Path(__file__).resolve().parents:
  if os.path.exists(os.path.join(d, 'behave.ini')):
    ROOT = d
    break

@step('I set preference {pref} to {value}')
def step_impl(context, pref, value):
  # bit of a cheat...
  if pref.endswith('.postscript'):
    value = expand_scenario_variables(context, value)
  context.zotero.preferences[pref] = context.zotero.preferences.parse(value)

@step(r'I restart Zotero with from "{db}" + "{source}"')
def step_impl(context, source, db):
  source = expand_scenario_variables(context, source)
  context.imported = source

  with open(os.path.join(ROOT, 'test', 'fixtures', source)) as f:
    data = json.load(f)
    items = data['items']
    #references = sum([ 1 + len(item.get('attachments', [])) + len(item.get('notes', [])) for item in items ])
    references = len(items)

  dbs = os.path.join(ROOT, 'test', 'db', db)
  if not os.path.exists(dbs): os.makedirs(dbs)
  zotero = os.path.join(dbs, 'zotero.sqlite')
  if not os.path.exists(zotero):
    urllib.request.urlretrieve(f'https://github.com/retorquere/zotero-better-bibtex/releases/download/test-database/{db}.zotero.sqlite', zotero)
  bbt = os.path.join(dbs, 'better-bibtex.sqlite')
  if not os.path.exists(bbt):
    urllib.request.urlretrieve(f'https://github.com/retorquere/zotero-better-bibtex/releases/download/test-database/{db}.better-bibtex.sqlite', bbt)

  timeout = context.zotero.timeout
  context.zotero.shutdown()
  context.zotero = Zotero(context.config.userdata, db=Munch(zotero=zotero, bbt=bbt))
  context.zotero.timeout = timeout
  assert_that(context.zotero.execute('return await Zotero.BetterBibTeX.TestSupport.librarySize()'), equal_to(references))

  context.zotero.import_file(context, source, items=False)

@step(r'I import {references:d} references from "{source}"')
def step_impl(context, references, source):
  source = expand_scenario_variables(context, source)
  context.imported = source
  assert_that(context.zotero.import_file(context, source, collection), equal_to(references))

@step(r'I import 1 reference from "{source}" into "{collection}"')
def step_impl(context, source, collection):
  source = expand_scenario_variables(context, source)
  context.imported = source
  assert_that(context.zotero.import_file(context, source, collection), equal_to(1))

@step(r'I import 1 reference from "{source}"')
def step_impl(context, source):
  source = expand_scenario_variables(context, source)
  context.imported = source
  assert_that(context.zotero.import_file(context, source), equal_to(1))

@given(u'I import 1 reference with 1 attachment from "{source}"')
def step_impl(context, source):
  source = expand_scenario_variables(context, source)
  context.imported = source
  assert_that(context.zotero.import_file(context, source), equal_to(1))

@step(r'I import {references:d} references with {attachments:d} attachments from "{source}" into a new collection')<commit_msg>bulk test<commit_after>from steps.zotero import Zotero
from munch import *
from behave import given, when, then, use_step_matcher
import urllib.request
import json
import zotero
import time
import os
from hamcrest import assert_that, equal_to
from steps.utils import assert_equal_diff, expand_scenario_variables

import pathlib
for d in pathlib.Path(__file__).resolve().parents:
  if os.path.exists(os.path.join(d, 'behave.ini')):
    ROOT = d
    break

@step('I set preference {pref} to {value}')
def step_impl(context, pref, value):
  # bit of a cheat...
  if pref.endswith('.postscript'):
    value = expand_scenario_variables(context, value)
  context.zotero.preferences[pref] = context.zotero.preferences.parse(value)

@step(r'I restart Zotero with from "{db}" + "{source}"')
def step_impl(context, source, db):
  source = expand_scenario_variables(context, source)
  context.imported = source

  with open(os.path.join(ROOT, 'test', 'fixtures', source)) as f:
    data = json.load(f)
    items = data['items']
    #references = sum([ 1 + len(item.get('attachments', [])) + len(item.get('notes', [])) for item in items ])
    references = len(items)

  dbs = os.path.join(ROOT, 'test', 'db', db)
  if not os.path.exists(dbs): os.makedirs(dbs)
  zotero = os.path.join(dbs, 'zotero.sqlite')
  if not os.path.exists(zotero):
    urllib.request.urlretrieve(f'https://github.com/retorquere/zotero-better-bibtex/releases/download/test-database/{db}.zotero.sqlite', zotero)
  bbt = os.path.join(dbs, 'better-bibtex.sqlite')
  if not os.path.exists(bbt):
    urllib.request.urlretrieve(f'https://github.com/retorquere/zotero-better-bibtex/releases/download/test-database/{db}.better-bibtex.sqlite', bbt)

  timeout = context.zotero.timeout
  context.zotero.shutdown()
  context.zotero = Zotero(context.config.userdata, db=Munch(zotero=zotero, bbt=bbt))
  context.zotero.timeout = timeout
  assert_that(context.zotero.execute('return await Zotero.BetterBibTeX.TestSupport.librarySize()'), equal_to(references))

  context.zotero.import_file(context, source, items=False)

@step(r'I import {references:d} references from "{source}"')
def step_impl(context, references, source):
  source = expand_scenario_variables(context, source)
  context.imported = source
  assert_that(context.zotero.import_file(context, source), equal_to(references))

@step(r'I import 1 reference from "{source}" into "{collection}"')
def step_impl(context, source, collection):
  source = expand_scenario_variables(context, source)
  context.imported = source
  assert_that(context.zotero.import_file(context, source, collection), equal_to(1))

@step(r'I import 1 reference from "{source}"')
def step_impl(context, source):
  source = expand_scenario_variables(context, source)
  context.imported = source
  assert_that(context.zotero.import_file(context, source), equal_to(1))

@given(u'I import 1 reference with 1 attachment from "{source}"')
def step_impl(context, source):
  source = expand_scenario_variables(context, source)
  context.imported = source
  assert_that(context.zotero.import_file(context, source), equal_to(1))

@step(r'I import {references:d} references with {attachments:d} attachments from "{source}" into a new collection')
{'fraction_non_alphanumeric': 0.1356168629882207, 'fraction_numerical': 0.0024798512089274642, 'mean_word_length': 5.714880332986472, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6411249', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2184, 'n_words': 471}