<commit_before>
package au.gov.dva.sopapi.tests.parsers;


import au.gov.dva.dvasopapi.tests.TestUtils
import au.gov.dva.sopapi.sopref.data.Conversions
import au.gov.dva.sopapi.sopref.data.sops.StoredSop
import au.gov.dva.sopapi.sopref.parsing.factories.SoPFactoryLocator
import au.gov.dva.sopapi.sopref.parsing.implementations.GenericClenser
import org.junit.Ignore
import org.junit.runner.RunWith
import org.scalatest.FunSuite
import org.scalatest.junit.JUnitRunner



@RunWith(classOf[JUnitRunner])
class LumbarSpondylosisTests extends FunSuite {

  test("CI debug")
  {
    val bytes = ParserTestUtils.resourceToBytes("sops_rh/F2014L00933.pdf")
    val rawText = Conversions.pdfToPlainText(bytes);
    val genericClenser = new GenericClenser();
    val clensedText = genericClenser.clense(rawText)
    // for some reason below returns null in CI only:
    val sopFactory = SoPFactoryLocator.findFactory("F2014L00933")
    //
    val sop = sopFactory.create("F2014L00933", clensedText)
    assert(sop != null)
  }

  ignore("Parse entire RH LS SoP") {
      val result = ParserTestUtils.executeWholeParsingPipeline("F2014L00933", "sops_rh/F2014L00933.pdf")
      System.out.print(TestUtils.prettyPrint(StoredSop.toJson(result)))
      assert(result != null)
  }

  ignore("Parse entire BoP LS SoP")
  {
    val result = ParserTestUtils.executeWholeParsingPipeline("F2014L00930", "sops_bop/F2014L00930.pdf")
    System.out.println(TestUtils.prettyPrint(StoredSop.toJson(result)))
    assert(result != null)
  }


}<commit_msg>confirm line which fails in CI<commit_after>
package au.gov.dva.sopapi.tests.parsers;


import au.gov.dva.dvasopapi.tests.TestUtils
import au.gov.dva.sopapi.sopref.data.Conversions
import au.gov.dva.sopapi.sopref.data.sops.StoredSop
import au.gov.dva.sopapi.sopref.parsing.factories.SoPFactoryLocator
import au.gov.dva.sopapi.sopref.parsing.implementations.GenericClenser
import org.junit.Ignore
import org.junit.runner.RunWith
import org.scalatest.FunSuite
import org.scalatest.junit.JUnitRunner



@RunWith(classOf[JUnitRunner])
class LumbarSpondylosisTests extends FunSuite {

  test("CI debug")
  {
    val bytes = ParserTestUtils.resourceToBytes("sops_rh/F2014L00933.pdf")
    val rawText = Conversions.pdfToPlainText(bytes);
    val genericClenser = new GenericClenser();
    val clensedText = genericClenser.clense(rawText)
    // for some reason below returns null in CI only:
    val sopFactory = SoPFactoryLocator.findFactory("F2014L00933")
    //
    val sop = sopFactory.create("F2014L00933", clensedText)
    assert(sopFactory != null)
  }

  ignore("Parse entire RH LS SoP") {
      val result = ParserTestUtils.executeWholeParsingPipeline("F2014L00933", "sops_rh/F2014L00933.pdf")
      System.out.print(TestUtils.prettyPrint(StoredSop.toJson(result)))
      assert(result != null)
  }

  ignore("Parse entire BoP LS SoP")
  {
    val result = ParserTestUtils.executeWholeParsingPipeline("F2014L00930", "sops_bop/F2014L00930.pdf")
    System.out.println(TestUtils.prettyPrint(StoredSop.toJson(result)))
    assert(result != null)
  }


}
{'fraction_non_alphanumeric': 0.10974025974025974, 'fraction_numerical': 0.04090909090909091, 'mean_word_length': 4.705555555555556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2756226', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1089, 'n_words': 171}