<commit_before>/* (c) 2009-2010 Regents of the University of California */

package org.cdlib.was.ngIndexer.tests;

import java.io.{File,FileOutputStream};

import net.liftweb.json._;
import net.liftweb.json.Serialization.{read, write};
import net.liftweb.json.{DefaultFormats,Serialization};

import org.apache.solr.common.SolrInputDocument;

import org.archive.io.{ArchiveReaderFactory,ArchiveRecord};

import org.cdlib.was.ngIndexer._;

import org.junit.runner.RunWith;

import org.scalatest.{FeatureSpec,GivenWhenThen,Ignore};
import org.scalatest.junit.JUnitRunner;

import scala.collection.mutable.HashMap;

@RunWith(classOf[JUnitRunner])
class WarcSpec extends FeatureSpec {
  val cl = classOf[WarcSpec].getClassLoader;
  val config = new Config {};
  val indexer = new SolrIndexer;
  val warcName = "IAH-20080430204825-00000-blackbook.warc.gz";
  val arcName = "IAH-20080430204825-00000-blackbook.arc.gz";
  implicit val formats = DefaultFormats;

  feature ("We can read a WARC file.") {
    scenario ("(W)ARC files should return the same data.") {
      val arcData = new HashMap[String, String];
      val warcData = new HashMap[String, String];

      Utility.eachRecord (cl.getResourceAsStream(warcName), warcName) { (rec)=>
        if (rec.isHttpResponse) {
          indexer.parseArchiveRecord(rec).map { res =>
            warcData += (rec.getUrl -> Serialization.write(res));
          }
        }
      }
      Utility.eachRecord (cl.getResourceAsStream(arcName), arcName) { (rec)=>
        indexer.parseArchiveRecord(rec).map { res =>
          arcData += (rec.getUrl -> Serialization.write(res))
        }
      }
      for ((k,v) <- arcData) {
        val j1 = parse(v);
        val j2 = parse(warcData.get(k).getOrElse(""));
        for (field <- List("digest", "url", "date", "title", "content")) {
          assert((j1 \ field) === (j2 \ field));
        }
      }
    }
    
    scenario ("We can round trip JSON.") {
      Utility.eachRecord (cl.getResourceAsStream(arcName), arcName) { (rec)=>
        indexer.parseArchiveRecord(rec).map { res =>
          assert (parse(Serialization.write(res)).extract[ParsedArchiveRecord] == res);
        }
      }
    }
    
    scenario ("can serialize an arc to JSON") {
      val tmpfile = File.createTempFile("ng-indexer", "json");
      val os = new FileOutputStream(tmpfile);
      indexer.parseToJson(cl.getResourceAsStream(arcName), arcName, os);
    }
  }
}<commit_msg>ensure we can extract from json<commit_after>/* (c) 2009-2010 Regents of the University of California */

package org.cdlib.was.ngIndexer.tests;

import java.io.{File,FileInputStream,FileOutputStream,InputStreamReader};

import net.liftweb.json._;
import net.liftweb.json.Serialization.{read, write};
import net.liftweb.json.{DefaultFormats,Serialization};

import org.apache.solr.common.SolrInputDocument;

import org.archive.io.{ArchiveReaderFactory,ArchiveRecord};

import org.cdlib.was.ngIndexer._;

import org.junit.runner.RunWith;

import org.scalatest.{FeatureSpec,GivenWhenThen,Ignore};
import org.scalatest.junit.JUnitRunner;

import scala.collection.mutable.HashMap;

@RunWith(classOf[JUnitRunner])
class WarcSpec extends FeatureSpec {
  val cl = classOf[WarcSpec].getClassLoader;
  val config = new Config {};
  val indexer = new SolrIndexer;
  val warcName = "IAH-20080430204825-00000-blackbook.warc.gz";
  val arcName = "IAH-20080430204825-00000-blackbook.arc.gz";
  implicit val formats = DefaultFormats;

  feature ("We can read a WARC file.") {
    scenario ("(W)ARC files should return the same data.") {
      val arcData = new HashMap[String, String];
      val warcData = new HashMap[String, String];

      Utility.eachRecord (cl.getResourceAsStream(warcName), warcName) { (rec)=>
        if (rec.isHttpResponse) {
          indexer.parseArchiveRecord(rec).map { res =>
            warcData += (rec.getUrl -> Serialization.write(res));
          }
        }
      }
      Utility.eachRecord (cl.getResourceAsStream(arcName), arcName) { (rec)=>
        indexer.parseArchiveRecord(rec).map { res =>
          arcData += (rec.getUrl -> Serialization.write(res))
        }
      }
      for ((k,v) <- arcData) {
        val j1 = parse(v);
        val j2 = parse(warcData.get(k).getOrElse(""));
        for (field <- List("digest", "url", "date", "title", "content")) {
          assert((j1 \ field) === (j2 \ field));
        }
      }
    }
    
    scenario ("We can round trip JSON.") {
      Utility.eachRecord (cl.getResourceAsStream(arcName), arcName) { (rec)=>
        indexer.parseArchiveRecord(rec).map { res =>
          assert (parse(Serialization.write(res)).extract[ParsedArchiveRecord] == res);
        }
      }
    }
    
    scenario ("can serialize an arc to JSON") {
      val tmpfile = File.createTempFile("ng-indexer", "json");
      val os = new FileOutputStream(tmpfile);
      indexer.parseToJson(cl.getResourceAsStream(arcName), arcName, os);

      /* and read it */
      val is = new FileInputStream(tmpfile);
      JsonParser.parse(new InputStreamReader(is, "UTF-8"), true).extract[List[ParsedArchiveRecord]];
    }
  }
}
{'fraction_non_alphanumeric': 0.13735081197417334, 'fraction_numerical': 0.019761299158677362, 'mean_word_length': 3.2458471760797343, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5449547', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1641, 'n_words': 332}