<commit_before>/*
 * Copyright 2015 Next Century Corporation
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package com.ncc.neon.services

import org.springframework.stereotype.Component
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.web.bind.annotation.RequestParam
//import org.springframework.web.multipart.MultipartFile
import org.apache.commons.fileupload.MultipartStream
import org.apache.commons.fileupload.servlet.ServletFileUpload
import com.sun.jersey.multipart.MultiPart
import com.sun.jersey.multipart.BodyPart

import javax.ws.rs.*
import javax.ws.rs.core.MediaType

import groovy.json.JsonOutput

/**
 * Service for importing a file's contents into a database.
 */

@Component
@Path("/importservice")
class ImportService {

//    @Autowired
//    CommonsMultipartResolver resolver

	@POST
    @Produces(MediaType.APPLICATION_JSON)
    @Path("upload/{host}/{databaseType}")
    public String getFile(@PathParam("host") String host,
                          @PathParam("databaseType") String databaseType,
                          @RequestParam("file") MultiPart data) {

        /* From Jersey. Worked for smaller files, failed for > ~10 mB.
        BodyPart part0 = data.getBodyParts().get(0)
        String s = part0.getEntityAs(String.class)
        //System.out.println(s)
        Map m = [data: s]
        return JsonOutput.toJson(m) */

        ByteArrayInputStream bStream = new ByteArrayInputStream(data.getBytes())
        @SuppressWarnings("deprecated")
        MultipartStream mStream = new MultipartStream(bStream, "---------------------------123206693813157131941135385742".toByteArray(), 1024, new MultipartStream.ProgressNotifier())
        Map m = [data: ServletFileUpload.isMultipartContent(data)]
        return JsonOutput.toJson(m)<commit_msg>Got ImportService.groovy reading input using a stream, so now it shouldn't instantly fail on large files.<commit_after>/*
 * Copyright 2015 Next Century Corporation
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package com.ncc.neon.services

import org.springframework.stereotype.Component
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.web.bind.annotation.RequestParam
import org.apache.commons.fileupload.MultipartStream
import org.apache.commons.fileupload.servlet.ServletFileUpload
import com.sun.jersey.multipart.MultiPart
import com.sun.jersey.multipart.BodyPart
import com.sun.jersey.multipart.FormDataParam

import javax.ws.rs.*
import javax.ws.rs.core.MediaType

import groovy.json.JsonOutput

/**
 * Service for importing a file's contents into a database.
 */

@Component
@Path("/importservice")
class ImportService {

	@POST
    @Consumes(MediaType.MULTIPART_FORM_DATA)
    @Produces(MediaType.APPLICATION_JSON)
    @Path("upload/{host}/{databaseType}")
    public String getFile(@PathParam("host") String host,
                          @PathParam("databaseType") String databaseType,
                          @FormDataParam("file") InputStream dataInputStream) {

        /*
         * From Jersey. Uses a stream, so appears to work for even very large files. I tested by giving it a 363.4 mB file and it worked fine.
         * I obviously wasn't processing the whole thing, but I see no reason why it should fail if I do, given the use of a stream to get data.
         */
        byte[] buff = new byte[24]
        dataInputStream.read(buff)
        byte[] buff2 = new byte[24]
        dataInputStream.read(buff2)
        String s = new String(buff)
        s = s + new String(buff2)
        Map m = [data: s]

        /*
         * This tried to use the Apache fileupload library, but it didn't work - kept saying there was no reader for multipart form data.
        ByteArrayInputStream bStream = new ByteArrayInputStream(data.getBytes())
        @SuppressWarnings("deprecated")
        MultipartStream mStream = new MultipartStream(bStream, "---------------------------123206693813157131941135385742".toByteArray(), 1024, new MultipartStream.ProgressNotifier())
        Map m = [data: ServletFileUpload.isMultipartContent(data)]
        */
        return JsonOutput.toJson(m)
{'fraction_non_alphanumeric': 0.09826589595375723, 'fraction_numerical': 0.019267822736030827, 'mean_word_length': 3.36218487394958, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '830499', 'n_tokens_mistral': 1440, 'n_tokens_neox': 1377, 'n_words': 448}