<commit_before>package com.roundeights.shnappy

import com.roundeights.shnappy.component.Parser
import com.roundeights.shnappy.admin.AdminData
import com.roundeights.foldout._
import com.roundeights.scalon._

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent._

/** @see Data */
object Data {

    /** Returns a shared data instance */
    def apply( env: Env, parser: Parser ): Data = new Data(
        env.database,
        parser,
        env.couchDB match {
            case Left( Env.CouchDB(host, port, ssl) )
                => CouchDB(host, port, ssl)
            case Right( conf ) => CouchDB.cloudant(
                conf.apiKey, conf.password, Some(conf.username)
            )
        }
    )
}

/**
 * Data access interface
 */
class Data ( database: String, private val parser: Parser, couch: CouchDB ) {

    // Make sure the database exists
    private val db = couch.db( database )
    db.createNow

    /** Admin data access */
    val admin = new AdminData( db )

    // Design interface
    private val design = Await.result( db.designDir( classOf[Data],
        "siteInfoByHost" -> "/couchdb/siteInfoByHost",
        "pagesBySlug" -> "/couchdb/pagesBySlug",
        "pagesByIndex" -> "/couchdb/pagesByIndex",
        "nav" -> "/couchdb/nav"
    ), Duration(3, "second") )

    /** Shutsdown this connection */
    def close: Unit = couch.close

    /** Returns a new Data instance customized for the given host */
    def forSite( host: String ): Future[Option[SiteData]] = {
        val cleanHost = SiteInfo.host.process( host ).require.value

        design.view("siteInfoByHost").limit(1).key( cleanHost ).exec.map(
            _.headOption.map( doc => SiteInfo(doc) ).map( info => {
                new SiteDataCache( new LiveSiteData(info) )
            } )
        )
    }

    /** Provides request specific data access */
    class LiveSiteData ( override val siteInfo: SiteInfo ) extends SiteData {

        /** {@inheritDoc} */
        override def getPage ( slug: String ): Future[Option[Page]] = {
            design.view("pagesBySlug")
                .key( siteInfo.id.toString, slug )
                .limit(1).exec
                .map( _.headOption.map(doc => Page(doc, parser)) )
        }

        /** {@inheritDoc} */
        override lazy val getIndex: Future[Option[Page]] = {
            design.view("pagesByIndex")
                .startKey( siteInfo.id.toString, nObject() )
                .endKey( siteInfo.id.toString, nNull() )
                .limit(1).desc.exec
                .map( _.headOption.map(doc => Page(doc, parser)) )
        }

        /** {@inheritDoc} */
        override lazy val getNavLinks: Future[Seq[NavLink]] = {
            val index: Future[Option[NavLink]]
                = getIndex.map( _.flatMap( _.navLink ) )

            design.view("nav")
                .startKey( siteInfo.id.toString, nNull() )
                .endKey( siteInfo.id.toString, nObject() )
                .asc.exec
                .map( rows => NavLink.parse(rows, parser) )
                .flatMap( links => index.map({
                    case None => links
                    case Some(indexLink) => links.map( link =>
                        if (link == indexLink) link.withURL("/") else link
                    )
                }))
        }

    }

}
<commit_msg>Switch Data over to being an interface<commit_after>package com.roundeights.shnappy

import com.roundeights.shnappy.component.Parser
import com.roundeights.shnappy.admin.AdminData
import com.roundeights.foldout._
import com.roundeights.scalon._

import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration._
import scala.concurrent._

/** @see Data */
object Data {

    /** Returns a shared data instance */
    def apply( env: Env, parser: Parser ): Data = new LiveData(
        env.database,
        parser,
        env.couchDB match {
            case Left( Env.CouchDB(host, port, ssl) )
                => CouchDB(host, port, ssl)
            case Right( conf ) => CouchDB.cloudant(
                conf.apiKey, conf.password, Some(conf.username)
            )
        }
    )
}

/**
 * Data access interface
 */
trait Data {

    /** Returns the interface for requesting admin data */
    def admin: AdminData

    /** Shutsdown this connection */
    def close: Unit

    /** Returns when the SiteData instance given a host name */
    def forSite( host: String ): Future[Option[SiteData]]
}

/**
 * Data access interface
 */
class LiveData (
    database: String, private val parser: Parser, couch: CouchDB
) extends Data {

    // Make sure the database exists
    private val db = couch.db( database )
    db.createNow

    // Design interface
    private val design = Await.result( db.designDir( classOf[Data],
        "siteInfoByHost" -> "/couchdb/siteInfoByHost",
        "pagesBySlug"    -> "/couchdb/pagesBySlug",
        "pagesByIndex"   -> "/couchdb/pagesByIndex",
        "nav"            -> "/couchdb/nav"
    ), Duration(3, "second") )

    /** {@inheritDoc} */
    override val admin = new AdminData( db )

    /** {@inheritDoc} */
    def close: Unit = couch.close

    /** Fetches down the SiteInfo object for a host name */
    private def getSiteInfo ( host: String ): Future[Option[SiteInfo]] = {
        design.view("siteInfoByHost").limit(1).key(
            SiteInfo.host.process(host).require.value
        ).exec.map(
            _.headOption.map( doc =>SiteInfo(doc) )
        )
    }

    /** {@inheritDoc} */
    override def forSite( host: String ): Future[Option[SiteData]]
        = getSiteInfo( host ).map( _.map( info => new LiveSiteData(info) ) )

    /** Provides request specific data access */
    class LiveSiteData ( override val siteInfo: SiteInfo ) extends SiteData {

        /** {@inheritDoc} */
        override def getPage ( slug: String ): Future[Option[Page]] = {
            design.view("pagesBySlug")
                .key( siteInfo.id.toString, slug )
                .limit(1).exec
                .map( _.headOption.map(doc => Page(doc, parser)) )
        }

        /** {@inheritDoc} */
        override lazy val getIndex: Future[Option[Page]] = {
            design.view("pagesByIndex")
                .startKey( siteInfo.id.toString, nObject() )
                .endKey( siteInfo.id.toString, nNull() )
                .limit(1).desc.exec
                .map( _.headOption.map(doc => Page(doc, parser)) )
        }

        /** {@inheritDoc} */
        override lazy val getNavLinks: Future[Seq[NavLink]] = {
            val index: Future[Option[NavLink]]
                = getIndex.map( _.flatMap( _.navLink ) )

            design.view("nav")
                .startKey( siteInfo.id.toString, nNull() )
                .endKey( siteInfo.id.toString, nObject() )
                .asc.exec
                .map( rows => NavLink.parse(rows, parser) )
                .flatMap( links => index.map({
                    case None => links
                    case Some(indexLink) => links.map( link =>
                        if (link == indexLink) link.withURL("/") else link
                    )
                }))
        }
    }

}


{'fraction_non_alphanumeric': 0.12771852057071617, 'fraction_numerical': 0.0011081867294639148, 'mean_word_length': 1.9266315362788813, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5293062', 'n_tokens_mistral': 2119, 'n_tokens_neox': 2041, 'n_words': 501}