<commit_before>package se.kb.libris.digi

import groovy.transform.MapConstructor
import groovy.util.logging.Log4j2 as Log
import org.codehaus.jackson.JsonParseException
import org.codehaus.jackson.map.ObjectMapper

import javax.servlet.ServletException
import javax.servlet.http.HttpServlet
import javax.servlet.http.HttpServletRequest
import javax.servlet.http.HttpServletResponse
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.time.Duration
import java.util.concurrent.Semaphore

import static RequestException.badRequest
import static RequestException.internalError
import static Util.JSONLD
import static Util.asList
import static Util.asMap
import static Util.asSet
import static Util.getAtPath
import static Util.isLink
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST
import static javax.servlet.http.HttpServletResponse.SC_CREATED
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR
import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT
import static javax.servlet.http.HttpServletResponse.SC_OK
import static javax.servlet.http.HttpServletResponse.SC_PRECONDITION_FAILED
import static se.kb.libris.digi.DigitalReproductionAPI.Type.ARRAY
import static se.kb.libris.digi.DigitalReproductionAPI.Type.STRING

/**
 Creates a record for a digital reproduction.
 Takes JSON-LD with an Electronic describing the reproduction as input.

 - Validates that minimal required data is present in Electronic (all additional data is kept).
 - Extracts and links work entity from reproduction and original (physical thing)
 - Copies title from original
 - Adds DIGI and DST bibliographies if applicable
 - Adds carrierType rda/OnlineResource if applicable
 - Creates holdings if specified in @reverse.itemOf
 - Record data can be specified in meta


 Example (without required authentication headers):

curl -v -XPOST 'http://localhost:8180/_reproduction' -H 'Content-Type: application/ld+json' --data-binary @- << EOF
{
  "@type": "Electronic",
  "reproductionOf": { "@id": "http://kblocalhost.kb.se:5000/q822pht24j3ljjr#it" },
  "production": [ 
    {
      "@type": "Reproduction",
      "agent": { "@id": "http://kblocalhost.kb.se:5000/jgvxv7m23l9rxd3#it" },
      "place": { "@type": "Place", "label": "Stockholm" },
      "year": "2021"
    } 
  ],
  "meta" : {
    "bibliography": [ {"@id" : "https://libris.kb.se/library/ARB"} ]
  },
  "@reverse" : {
    "itemOf": [
      { "heldBy": { "@id": "https://libris.kb.se/library/S" }},
      { "heldBy": { "@id": "https://libris.kb.se/library/Utb1" }}
    ]
  }
}
EOF

 */

@Log
class DigitalReproductionAPI extends HttpServlet {
    private static final ObjectMapper mapper = new ObjectMapper()

    private static final def FORWARD_HEADERS = [
            'XL-Active-Sigel',
            'Authorization'
    ].collect { it.toLowerCase() }
    
    @Override
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        def forwardHeaders = request.headerNames
                .findAll{ it.toLowerCase() in FORWARD_HEADERS }
                .collectEntries { [(it) : request.getHeader(it as String)] }

        def service = new ReproductionService(xl : new XL(headers : forwardHeaders, apiLocation: getXlAPI(request)))

        try {
            String id = service.createDigitalReproduction(parse(request))
            log.info("Created $id")
            response.setHeader('Location', id)
            response.setStatus(SC_CREATED)
        }
        catch (RequestException e) {
            log.warn("${e.code} ${e.msg}")
            sendError(response, e.code, e.msg)
        }
        catch (Exception e) {
            log.error("Internal error: $e", e)
            sendError(response, SC_INTERNAL_SERVER_ERROR, e.getMessage())
        }
    }

    static Map parse(HttpServletRequest request) {
        if (request.getHeader('Content-Type') != JSONLD) {
            throw badRequest("Header Content-Type must be $JSONLD")
        }

        def electronic = readJson(request)
        
        check(electronic, ['@type'], 'Electronic')
        check(electronic, ['reproductionOf', '@id'], STRING)
        check(electronic, ['production'], ARRAY)
        if (!isLink(getAtPath(electronic, ['production', 0]))) { // minimal valid shape, so just check the first one 
            check(electronic, ['production', 0, '@type'], 'Reproduction')
            check(electronic, ['production', 0, 'year'], STRING)
            if (!isLink(getAtPath(electronic, ['production', 0, 'agent']))) {
                check(electronic, ['production', 0, 'place', '@type'], STRING)
            }
            if (!isLink(getAtPath(electronic, ['production', 0, 'place']))) {
                check(electronic, ['production', 0, 'place', '@type'], 'Place')
                check(electronic, ['production', 0, 'place', 'label'], STRING)
            }
        }
               
        return electronic
    }

    static void check(thing, path, expected) {
        def actual = getAtPath(thing, path)
        def ok = expected instanceof Type
                ? expected.type.isInstance(actual)
                : expected == actual

        if (!ok) {
            throw badRequest("Expected $expected at $path, got: ${actual ?: '<MISSING>'}")
        }
    }
    
    static enum Type {
        ARRAY(List.class),
        STRING(String.class)
        
        Class type
        Type(Class type) { this.type = type }
    }
        
    static Map readJson(HttpServletRequest request) {
        try {
            mapper.readValue(request.getInputStream().getBytes(), Map)
        } catch (JsonParseException e) {
            throw badRequest("Bad JSON: ${e.message}")
        }
    }

    static void sendError(HttpServletResponse response, int code, String msg) {
        response.setStatus(code)
        response.setHeader('Content-Type', 'application/json')
        mapper.writeValue(response.getOutputStream(), ['code': code, 'msg' : msg ?: ''])
    }
    
    static String getXlAPI(HttpServletRequest request) {
        //FIXME
        int port = request.getServerPort()
        if (port == 443) {
            port = 8080
        }
        "http://localhost:${port}/"
    }
}

@Log
class ReproductionService {
    private static final def DIGI = ['@id': 'https://libris.kb.se/library/DIGI']
    private static final def DST = ['@id': 'https://libris.kb.se/library/DST']
    private static final def ONLINE = ['@id': 'https://id.kb.se/term/rda/OnlineResource']
    private static final def FREELY_AVAILABLE = ['@id': 'https://id.kb.se/policy/freely-available']

    XL xl
    
    String createDigitalReproduction(Map electronicThing) {
        String requestedId = electronicThing.reproductionOf['@id'] as String
        Map physicalThing = xl.get(requestedId)
                .map{ it.data['@graph'][1] as Map }
                .orElseThrow{ badRequest("Thing linked in reproductionOf does not exist: $requestedId") }

        if (physicalThing['@type'] == 'Electronic') {
            throw badRequest("Thing linked in reproductionOf cannot be Electronic")
        }

        def holdingsToCreate = getAtPath(electronicThing, ['@reverse', 'itemOf', '*', 'heldBy', '@id'], [])
        def badHeldBy = holdingsToCreate.findAll { xl.get(it).isEmpty() }
        if (badHeldBy) {
            throw badRequest("No such library: $badHeldBy")
        }
        
        electronicThing.reproductionOf['@id'] = physicalThing['@id'] // if link was to sameAs
        
        electronicThing.instanceOf = ['@id' : xl.ensureExtractedWork(physicalThing['@id'] as String)]
        
        if (physicalThing.hasTitle) {
            electronicThing.hasTitle = physicalThing.hasTitle
        }
        
        if (isOnline(electronicThing)) {
            electronicThing.carrierType = asSet(electronicThing.carrierType) << ONLINE
        }

        def record = asMap(electronicThing.remove('meta'))

        record.bibliography = asSet(record.bibliography).with { bibliography ->
            if (isDigitaliseratSvensktTryck(physicalThing, electronicThing)) {
                bibliography << DST<commit_msg>DigitalReproductionAPI: Only copy first Title to extracted Work<commit_after>package se.kb.libris.digi

import groovy.transform.MapConstructor
import groovy.util.logging.Log4j2 as Log
import org.codehaus.jackson.JsonParseException
import org.codehaus.jackson.map.ObjectMapper

import javax.servlet.ServletException
import javax.servlet.http.HttpServlet
import javax.servlet.http.HttpServletRequest
import javax.servlet.http.HttpServletResponse
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.time.Duration
import java.util.concurrent.Semaphore

import static RequestException.badRequest
import static RequestException.internalError
import static Util.JSONLD
import static Util.asList
import static Util.asMap
import static Util.asSet
import static Util.getAtPath
import static Util.isLink
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST
import static javax.servlet.http.HttpServletResponse.SC_CREATED
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR
import static javax.servlet.http.HttpServletResponse.SC_NOT_FOUND
import static javax.servlet.http.HttpServletResponse.SC_NO_CONTENT
import static javax.servlet.http.HttpServletResponse.SC_OK
import static javax.servlet.http.HttpServletResponse.SC_PRECONDITION_FAILED
import static se.kb.libris.digi.DigitalReproductionAPI.Type.ARRAY
import static se.kb.libris.digi.DigitalReproductionAPI.Type.STRING

/**
 Creates a record for a digital reproduction.
 Takes JSON-LD with an Electronic describing the reproduction as input.

 - Validates that minimal required data is present in Electronic (all additional data is kept).
 - Extracts and links work entity from reproduction and original (physical thing)
 - Copies title from original
 - Adds DIGI and DST bibliographies if applicable
 - Adds carrierType rda/OnlineResource if applicable
 - Creates holdings if specified in @reverse.itemOf
 - Record data can be specified in meta


 Example (without required authentication headers):

curl -v -XPOST 'http://localhost:8180/_reproduction' -H 'Content-Type: application/ld+json' --data-binary @- << EOF
{
  "@type": "Electronic",
  "reproductionOf": { "@id": "http://kblocalhost.kb.se:5000/q822pht24j3ljjr#it" },
  "production": [ 
    {
      "@type": "Reproduction",
      "agent": { "@id": "http://kblocalhost.kb.se:5000/jgvxv7m23l9rxd3#it" },
      "place": { "@type": "Place", "label": "Stockholm" },
      "year": "2021"
    } 
  ],
  "meta" : {
    "bibliography": [ {"@id" : "https://libris.kb.se/library/ARB"} ]
  },
  "@reverse" : {
    "itemOf": [
      { "heldBy": { "@id": "https://libris.kb.se/library/S" }},
      { "heldBy": { "@id": "https://libris.kb.se/library/Utb1" }}
    ]
  }
}
EOF

 */

@Log
class DigitalReproductionAPI extends HttpServlet {
    private static final ObjectMapper mapper = new ObjectMapper()

    private static final def FORWARD_HEADERS = [
            'XL-Active-Sigel',
            'Authorization'
    ].collect { it.toLowerCase() }
    
    @Override
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        def forwardHeaders = request.headerNames
                .findAll{ it.toLowerCase() in FORWARD_HEADERS }
                .collectEntries { [(it) : request.getHeader(it as String)] }

        def service = new ReproductionService(xl : new XL(headers : forwardHeaders, apiLocation: getXlAPI(request)))

        try {
            String id = service.createDigitalReproduction(parse(request))
            log.info("Created $id")
            response.setHeader('Location', id)
            response.setStatus(SC_CREATED)
        }
        catch (RequestException e) {
            log.warn("${e.code} ${e.msg}")
            sendError(response, e.code, e.msg)
        }
        catch (Exception e) {
            log.error("Internal error: $e", e)
            sendError(response, SC_INTERNAL_SERVER_ERROR, e.getMessage())
        }
    }

    static Map parse(HttpServletRequest request) {
        if (request.getHeader('Content-Type') != JSONLD) {
            throw badRequest("Header Content-Type must be $JSONLD")
        }

        def electronic = readJson(request)
        
        check(electronic, ['@type'], 'Electronic')
        check(electronic, ['reproductionOf', '@id'], STRING)
        check(electronic, ['production'], ARRAY)
        if (!isLink(getAtPath(electronic, ['production', 0]))) { // minimal valid shape, so just check the first one 
            check(electronic, ['production', 0, '@type'], 'Reproduction')
            check(electronic, ['production', 0, 'year'], STRING)
            if (!isLink(getAtPath(electronic, ['production', 0, 'agent']))) {
                check(electronic, ['production', 0, 'place', '@type'], STRING)
            }
            if (!isLink(getAtPath(electronic, ['production', 0, 'place']))) {
                check(electronic, ['production', 0, 'place', '@type'], 'Place')
                check(electronic, ['production', 0, 'place', 'label'], STRING)
            }
        }
               
        return electronic
    }

    static void check(thing, path, expected) {
        def actual = getAtPath(thing, path)
        def ok = expected instanceof Type
                ? expected.type.isInstance(actual)
                : expected == actual

        if (!ok) {
            throw badRequest("Expected $expected at $path, got: ${actual ?: '<MISSING>'}")
        }
    }
    
    static enum Type {
        ARRAY(List.class),
        STRING(String.class)
        
        Class type
        Type(Class type) { this.type = type }
    }
        
    static Map readJson(HttpServletRequest request) {
        try {
            mapper.readValue(request.getInputStream().getBytes(), Map)
        } catch (JsonParseException e) {
            throw badRequest("Bad JSON: ${e.message}")
        }
    }

    static void sendError(HttpServletResponse response, int code, String msg) {
        response.setStatus(code)
        response.setHeader('Content-Type', 'application/json')
        mapper.writeValue(response.getOutputStream(), ['code': code, 'msg' : msg ?: ''])
    }
    
    static String getXlAPI(HttpServletRequest request) {
        //FIXME
        int port = request.getServerPort()
        if (port == 443) {
            port = 8080
        }
        "http://localhost:${port}/"
    }
}

@Log
class ReproductionService {
    private static final def DIGI = ['@id': 'https://libris.kb.se/library/DIGI']
    private static final def DST = ['@id': 'https://libris.kb.se/library/DST']
    private static final def ONLINE = ['@id': 'https://id.kb.se/term/rda/OnlineResource']
    private static final def FREELY_AVAILABLE = ['@id': 'https://id.kb.se/policy/freely-available']

    XL xl
    
    String createDigitalReproduction(Map electronicThing) {
        String requestedId = electronicThing.reproductionOf['@id'] as String
        Map physicalThing = xl.get(requestedId)
                .map{ it.data['@graph'][1] as Map }
                .orElseThrow{ badRequest("Thing linked in reproductionOf does not exist: $requestedId") }

        if (physicalThing['@type'] == 'Electronic') {
            throw badRequest("Thing linked in reproductionOf cannot be Electronic")
        }

        def holdingsToCreate = getAtPath(electronicThing, ['@reverse', 'itemOf', '*', 'heldBy', '@id'], [])
        def badHeldBy = holdingsToCreate.findAll { xl.get(it).isEmpty() }
        if (badHeldBy) {
            throw badRequest("No such library: $badHeldBy")
        }
        
        electronicThing.reproductionOf['@id'] = physicalThing['@id'] // if link was to sameAs
        
        electronicThing.instanceOf = ['@id' : xl.ensureExtractedWork(physicalThing['@id'] as String)]
        
        if (physicalThing.hasTitle) {
            electronicThing.hasTitle = physicalThing.hasTitle.find { it['@type'] == 'Title' } ?: asList(physicalThing.hasTitle).first()
        }
        
        if (isOnline(electronicThing)) {
            electronicThing.carrierType = asSet(electronicThing.carrierType) << ONLINE
        }

        def record = asMap(electronicThing.remove('meta'))

        record.bibliography = asSet(record.bibliography).with { bibliography ->
            if (isDigitaliseratSvensktTryck(physicalThing, electronicThing)) {
                bibliography << DST
{'fraction_non_alphanumeric': 0.1219731479261568, 'fraction_numerical': 0.005514265164229201, 'mean_word_length': 2.8947245564892623, 'pattern_counts': {'":': 34, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '375936', 'n_tokens_mistral': 4941, 'n_tokens_neox': 4730, 'n_words': 1121}