<commit_before>package concordances

import (
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	ReadByConceptID(ids []string) (concordances Concordances, found bool, err error)
	ReadByAuthority(authority string, ids []string) (concordances Concordances, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) CypherDriver {
	return CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	return neoutils.Check(pcw.conn)
}

type neoReadStruct struct {
	UUID          string        `json:"UUID"`
	Types         []string      `json:"TYPES"`
	NeoIdentifier neoIdentifier `json:"IDENTIFIERS"`
	PrefUUID      string        `json:"prefUUID"`
}

type neoIdentifier struct {
	Labels []string `json:"labels"`
	Value  string   `json:"value"`
}

type neoResultStrunct struct {
	Rs []neoReadStruct
}

func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Concordances, found bool, err error) {
	c, f, err := pcw.readByConceptIDNewModel(identifiers)
	if !f {
		c, f, err = pcw.readByConceptIDOldModel(identifiers)
	}
	return c, f, err
}

func (pcw CypherDriver) readByConceptIDNewModel(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
		WHERE p.uuid in {identifiers}
		MATCH (cn)-[:EQUIVALENT_TO]-(cnn:Concept)
		RETURN cn.prefUUID as prefUUID, cnn.uuid AS UUID, labels(cnn) AS TYPES, {labels:collect(cnn.authority), value:cnn.authorityValue} as IDENTIFIERS
		`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	log.Info(results)

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	for _, neoCon := range results {
		log.Info(neoCon)
		// Each record is now two identifiers, one UPP and one other.
		var con = Concordance{}
		var concept = Concept{}
		concept.ID = mapper.IDURL(neoCon.PrefUUID)
		concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con.Concept = concept
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		concordances.Concordance = append(concordances.Concordance, con)

		var con2 = Concordance{}
		var concept2 = Concept{}
		concept2.ID = mapper.IDURL(neoCon.PrefUUID)
		concept2.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con2.Concept = concept
		con2.Identifier = Identifier{Authority: UP_AUTHORITY, IdentifierValue: neoCon.UUID}
		concordances.Concordance = append(concordances.Concordance, con2)
	}

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}

	log.Debugf("Returning %v", concordances)
	return concordances, true, nil
}

func (pcw CypherDriver) readByConceptIDOldModel(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)<-[:IDENTIFIES]-(i:UPPIdentifier)
		WHERE i.value in {identifiers}
		MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
		RETURN p.uuid as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, {labels:labels(ids), value:ids.value} as IDENTIFIERS
		`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}
	return processCypherQueryToConcordances(pcw, query, &results)
}

func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	c, f, err := pcw.readByAuthorityNewModel(authority, identifierValues)
	if !f {
		c, f, err = pcw.readByAuthorityOldModel(authority, identifierValues)
	}
	return c, f, err
}

func (pcw CypherDriver) readByAuthorityNewModel(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	log.Info("readByAuthorityNewModel")
	concordances = Concordances{}
	results := []neoReadStruct{}

	authorityProperty := mapAuthorityToAuthorityProperty(authority)
	log.Info(authorityProperty)
	if authorityProperty == "" {
		return Concordances{}, false, nil
	}

	readByAuthorityQueryStatement := `
		MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
		WHERE p.authorityValue in {identifierValues} AND p.authority = {authority}
		RETURN cn.prefUUID as prefUUID, cn.prefUUID AS UUID, labels(cn) AS TYPES, {labels:collect(p.authority), value:p.authorityValue} as IDENTIFIERS
		`
	if authorityProperty == "UPP" {
		readByAuthorityQueryStatement = `
			MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
			WHERE p.uuid in {identifierValues}
			RETURN cn.prefUUID as prefUUID, cn.prefUUID AS UUID, labels(cn) AS TYPES, {labels:collect(p.authority), value:p.authorityValue} as IDENTIFIERS
		`
	}

	query := &neoism.CypherQuery{
		Statement: readByAuthorityQueryStatement,
		Parameters: neoism.Props{
			"identifierValues": identifierValues,
			"authority":        authorityProperty,
		},
		Result: &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	log.Info(results)

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	for _, neoCon := range results {<commit_msg>Make sure identifiers reports the right information<commit_after>package concordances

import (
	"fmt"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	ReadByConceptID(ids []string) (concordances Concordances, found bool, err error)
	ReadByAuthority(authority string, ids []string) (concordances Concordances, found bool, err error)
	CheckConnectivity() error
}

// CypherDriver struct
type CypherDriver struct {
	conn neoutils.NeoConnection
	env  string
}

//NewCypherDriver instantiate driver
func NewCypherDriver(conn neoutils.NeoConnection, env string) CypherDriver {
	return CypherDriver{conn, env}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() error {
	return neoutils.Check(pcw.conn)
}

type neoReadStruct struct {
	UUID          string        `json:"UUID"`
	Types         []string      `json:"TYPES"`
	NeoIdentifier neoIdentifier `json:"IDENTIFIERS"`
	PrefUUID      string        `json:"prefUUID"`
}

type neoIdentifier struct {
	Labels []string `json:"labels"`
	Value  string   `json:"value"`
}

type neoResultStrunct struct {
	Rs []neoReadStruct
}

func (pcw CypherDriver) ReadByConceptID(identifiers []string) (concordances Concordances, found bool, err error) {
	c, f, err := pcw.readByConceptIDNewModel(identifiers)
	if !f {
		c, f, err = pcw.readByConceptIDOldModel(identifiers)
	}
	return c, f, err
}

func (pcw CypherDriver) readByConceptIDNewModel(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
		WHERE p.uuid in {identifiers}
		MATCH (cn)-[:EQUIVALENT_TO]-(cnn:Concept)
		RETURN cn.prefUUID as prefUUID, cnn.uuid AS UUID, labels(cnn) AS TYPES, {labels:collect(cnn.authority), value:cnn.authorityValue} as IDENTIFIERS
		`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	log.Info(results)

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	for _, neoCon := range results {
		log.Info(neoCon)
		// Each record is now two identifiers, one UPP and one other.
		var con = Concordance{}
		var concept = Concept{}
		concept.ID = mapper.IDURL(neoCon.PrefUUID)
		concept.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con.Concept = concept
		con.Identifier = Identifier{Authority: mapNeoLabelsToAuthorityValue(neoCon.NeoIdentifier.Labels), IdentifierValue: neoCon.NeoIdentifier.Value}
		concordances.Concordance = append(concordances.Concordance, con)

		var con2 = Concordance{}
		var concept2 = Concept{}
		concept2.ID = mapper.IDURL(neoCon.PrefUUID)
		concept2.APIURL = mapper.APIURL(neoCon.PrefUUID, neoCon.Types, pcw.env)
		con2.Concept = concept
		con2.Identifier = Identifier{Authority: UP_AUTHORITY, IdentifierValue: neoCon.UUID}
		concordances.Concordance = append(concordances.Concordance, con2)
	}

	if (len(concordances.Concordance)) == 0 {
		return Concordances{}, false, nil
	}

	log.Debugf("Returning %v", concordances)
	return concordances, true, nil
}

func (pcw CypherDriver) readByConceptIDOldModel(identifiers []string) (concordances Concordances, found bool, err error) {
	results := []neoReadStruct{}
	query := &neoism.CypherQuery{
		Statement: `
		MATCH (p:Concept)<-[:IDENTIFIES]-(i:UPPIdentifier)
		WHERE i.value in {identifiers}
		MATCH (p:Concept)<-[:IDENTIFIES]-(ids:Identifier)
		RETURN p.uuid as prefUUID, p.uuid AS UUID, labels(p) AS TYPES, {labels:labels(ids), value:ids.value} as IDENTIFIERS
		`,
		Parameters: neoism.Props{"identifiers": identifiers},
		Result:     &results,
	}
	return processCypherQueryToConcordances(pcw, query, &results)
}

func (pcw CypherDriver) ReadByAuthority(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	c, f, err := pcw.readByAuthorityNewModel(authority, identifierValues)
	if !f {
		c, f, err = pcw.readByAuthorityOldModel(authority, identifierValues)
	}
	return c, f, err
}

func (pcw CypherDriver) readByAuthorityNewModel(authority string, identifierValues []string) (concordances Concordances, found bool, err error) {
	log.Info("readByAuthorityNewModel")
	concordances = Concordances{}
	results := []neoReadStruct{}

	authorityProperty := mapAuthorityToAuthorityProperty(authority)
	log.Info(authorityProperty)
	if authorityProperty == "" {
		return Concordances{}, false, nil
	}

	readByAuthorityQueryStatement := `
		MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
		WHERE p.authorityValue in {identifierValues} AND p.authority = {authority}
		RETURN cn.prefUUID as prefUUID, cn.prefUUID AS UUID, labels(cn) AS TYPES, {labels:collect(p.authority), value:p.authorityValue} as IDENTIFIERS
		`

	// Special handling for UPP authority.  In for compatibility with the old model but may be removed once identifiers are removed.
	if authorityProperty == "UPP" {
		readByAuthorityQueryStatement = `
			MATCH (p:Concept)-[:EQUIVALENT_TO]-(cn:Concept)
			WHERE p.uuid in {identifierValues}
			RETURN cn.prefUUID as prefUUID, cn.prefUUID AS UUID, labels(cn) AS TYPES, {labels:["UPP"], value:p.uuid} as IDENTIFIERS
		`
	}

	query := &neoism.CypherQuery{
		Statement: readByAuthorityQueryStatement,
		Parameters: neoism.Props{
			"identifierValues": identifierValues,
			"authority":        authorityProperty,
		},
		Result: &results,
	}

	err = pcw.conn.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up Concordances with query %s from neoism: %+v\n", query.Statement, err)
		return Concordances{}, false, fmt.Errorf("Error accessing Concordance datastore for identifier:")
	}

	log.Info(results)

	if (len(results)) == 0 {
		return Concordances{}, false, nil
	}

	concordances = Concordances{
		Concordance: []Concordance{},
	}

	for _, neoCon := range results {
{'fraction_non_alphanumeric': 0.11924920127795527, 'fraction_numerical': 0.001757188498402556, 'mean_word_length': 5.137745098039216, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2448575', 'n_tokens_mistral': 4510, 'n_tokens_neox': 4252, 'n_words': 996}