<commit_before>/*
 * Copyright 2016 SimplifyOps, Inc. (http://simplifyops.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.rundeck.plugin.scm.git

import com.dtolabs.rundeck.core.jobs.JobReference
import com.dtolabs.rundeck.core.plugins.configuration.Validator
import com.dtolabs.rundeck.core.plugins.views.Action
import com.dtolabs.rundeck.core.storage.ResourceMeta
import com.dtolabs.rundeck.plugins.scm.*
import com.jcraft.jsch.Session
import org.eclipse.jgit.api.Git
import org.eclipse.jgit.api.ListBranchCommand
import org.eclipse.jgit.api.PullResult
import org.eclipse.jgit.api.RebaseCommand
import org.eclipse.jgit.api.RebaseResult
import org.eclipse.jgit.api.Status
import org.eclipse.jgit.api.TransportCommand
import org.eclipse.jgit.api.TransportConfigCallback
import org.eclipse.jgit.diff.RawTextComparator
import org.eclipse.jgit.lib.BranchConfig
import org.eclipse.jgit.lib.ConfigConstants
import org.eclipse.jgit.lib.Constants
import org.eclipse.jgit.lib.ObjectId
import org.eclipse.jgit.lib.Ref
import org.eclipse.jgit.lib.Repository
import org.eclipse.jgit.merge.MergeStrategy
import org.eclipse.jgit.revwalk.RevCommit
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.eclipse.jgit.transport.JschConfigSessionFactory
import org.eclipse.jgit.transport.OpenSshConfig
import org.eclipse.jgit.transport.SshTransport
import org.eclipse.jgit.transport.TrackingRefUpdate
import org.eclipse.jgit.transport.Transport
import org.eclipse.jgit.transport.URIish
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider
import org.eclipse.jgit.util.FileUtils
import org.rundeck.plugin.scm.git.config.Common
import org.rundeck.storage.api.StorageException
import org.slf4j.Logger
import org.slf4j.LoggerFactory

import java.nio.file.Files
import java.nio.file.StandardCopyOption
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentMap
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern

/**
 * Common features of the import and export plugins
 */
class BaseGitPlugin {
    public static final String REMOTE_NAME = "origin"
    Git git
    Repository repo
    File workingDir
    String branch
    Map<String, String> input
    Common commonConfig
    JobFileMapper mapper
    RawTextComparator COMP = RawTextComparator.DEFAULT
    Map<String, Map> jobStateMap = Collections.synchronizedMap([:])

    BaseGitPlugin(Common commonConfig) {
        this.input = commonConfig.rawInput
        this.commonConfig = commonConfig
    }

    Map<String, String> getSshConfig() {
        def config = [:]

        if (commonConfig.strictHostKeyChecking in ['yes', 'no']) {
            config['StrictHostKeyChecking'] = commonConfig.strictHostKeyChecking
        }
        config
    }
    
    /**
     * maps output file to an AtomicLong used for synchronization and
     * only serializing monotonically increasing revision for the job
     */
    ConcurrentMap<File, AtomicLong> fileSerializeRevisionCounter = new ConcurrentHashMap<>()

    /**
     * Get an AtomicLong used for synchronization and comparing
     * serialized revision number of the file.
     *
     * @param outfile the target file
     * @return atomic long for file serialization revision number
     */
    private AtomicLong fileCounterFor(File outfile) {
        AtomicLong latch = fileSerializeRevisionCounter.get(outfile)
        if (latch == null) {
            latch = new AtomicLong(-1)
            AtomicLong previous = fileSerializeRevisionCounter.putIfAbsent(outfile, latch)
            if (null != previous) {
                latch = previous
            }
        }
        latch
    }

    /**
     * compareAndSet if the new value is greater than the current value
     * @param atomic atomic long
     * @param update the new value
     * @return
     */
    static boolean greaterAndSet(AtomicLong atomic, long update) {
        while (true) {
            long cur = atomic.get();
            if (update <= cur) {
                return false
            }
            //should set if it hasn't changed
            if (atomic.compareAndSet(cur, update)) {
                return true;
            }
            //otherwise try again
        }
    }

    def serialize(
            final JobExportReference job,
            String format,
            boolean preserveId,
            boolean useSourceId,
            File outfile = null
    )
    {
        if (!outfile) {
            outfile = mapper.fileForJob(job)
        }
        AtomicLong counter = fileCounterFor(outfile)
        logger.debug("Start serialize[${Thread.currentThread().name}]...")

        synchronized (counter) {
            //other threads serializing the same job must wait until we complete
            if (greaterAndSet(counter, job.version)) {
                //only bother writing the file if this rev of Job if it is newer than previously serialized rev

                if (!outfile.parentFile.isDirectory()) {
                    if (!outfile.parentFile.mkdirs()) {
                        throw new ScmPluginException(
                                "Cannot create necessary dirs to serialize file to path: ${outfile.absolutePath}"
                        )
                    }
                }

                File temp = new File(outfile.parentFile, outfile.name + ".tmp${job.version}")
                temp.deleteOnExit()
                Throwable thrown = null
                try {
                    try {
                        temp.withOutputStream { out ->
                            try {
                                def sourceId = (job instanceof JobScmReference) ? job.sourceId  : null
                                job.jobSerializer.serialize(
                                        format,
                                        out,
                                        preserveId,
                                        useSourceId ? (sourceId ?: job.id) : null
                                )
                            } catch (Throwable e) {
                                thrown = e;
                            }
                        }
                    } catch (IOException e) {<commit_msg>Fix: failure to serialize jobs for git<commit_after>/*
 * Copyright 2016 SimplifyOps, Inc. (http://simplifyops.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.rundeck.plugin.scm.git

import com.dtolabs.rundeck.core.jobs.JobReference
import com.dtolabs.rundeck.core.plugins.configuration.Validator
import com.dtolabs.rundeck.core.plugins.views.Action
import com.dtolabs.rundeck.core.storage.ResourceMeta
import com.dtolabs.rundeck.plugins.scm.*
import com.jcraft.jsch.Session
import org.eclipse.jgit.api.Git
import org.eclipse.jgit.api.ListBranchCommand
import org.eclipse.jgit.api.PullResult
import org.eclipse.jgit.api.RebaseCommand
import org.eclipse.jgit.api.RebaseResult
import org.eclipse.jgit.api.Status
import org.eclipse.jgit.api.TransportCommand
import org.eclipse.jgit.api.TransportConfigCallback
import org.eclipse.jgit.diff.RawTextComparator
import org.eclipse.jgit.lib.BranchConfig
import org.eclipse.jgit.lib.ConfigConstants
import org.eclipse.jgit.lib.Constants
import org.eclipse.jgit.lib.ObjectId
import org.eclipse.jgit.lib.Ref
import org.eclipse.jgit.lib.Repository
import org.eclipse.jgit.merge.MergeStrategy
import org.eclipse.jgit.revwalk.RevCommit
import org.eclipse.jgit.storage.file.FileRepositoryBuilder
import org.eclipse.jgit.transport.JschConfigSessionFactory
import org.eclipse.jgit.transport.OpenSshConfig
import org.eclipse.jgit.transport.SshTransport
import org.eclipse.jgit.transport.TrackingRefUpdate
import org.eclipse.jgit.transport.Transport
import org.eclipse.jgit.transport.URIish
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider
import org.eclipse.jgit.util.FileUtils
import org.rundeck.plugin.scm.git.config.Common
import org.rundeck.storage.api.StorageException
import org.slf4j.Logger
import org.slf4j.LoggerFactory

import java.nio.file.Files
import java.nio.file.StandardCopyOption
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.ConcurrentMap
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern

/**
 * Common features of the import and export plugins
 */
class BaseGitPlugin {
    public static final String REMOTE_NAME = "origin"
    Git git
    Repository repo
    File workingDir
    String branch
    Map<String, String> input
    Common commonConfig
    JobFileMapper mapper
    RawTextComparator COMP = RawTextComparator.DEFAULT
    Map<String, Map> jobStateMap = Collections.synchronizedMap([:])

    BaseGitPlugin(Common commonConfig) {
        this.input = commonConfig.rawInput
        this.commonConfig = commonConfig
    }

    Map<String, String> getSshConfig() {
        def config = [:]

        if (commonConfig.strictHostKeyChecking in ['yes', 'no']) {
            config['StrictHostKeyChecking'] = commonConfig.strictHostKeyChecking
        }
        config
    }
    
    /**
     * maps output file to an AtomicLong used for synchronization and
     * only serializing monotonically increasing revision for the job
     */
    ConcurrentMap<File, AtomicLong> fileSerializeRevisionCounter = new ConcurrentHashMap<>()

    /**
     * Get an AtomicLong used for synchronization and comparing
     * serialized revision number of the file.
     *
     * @param outfile the target file
     * @return atomic long for file serialization revision number
     */
    private AtomicLong fileCounterFor(File outfile) {
        AtomicLong latch = fileSerializeRevisionCounter.get(outfile)
        if (latch == null) {
            latch = new AtomicLong(-1)
            AtomicLong previous = fileSerializeRevisionCounter.putIfAbsent(outfile, latch)
            if (null != previous) {
                latch = previous
            }
        }
        latch
    }

    /**
     * compareAndSet if the new value is greater than the current value
     * @param atomic atomic long
     * @param update the new value
     * @return
     */
    static boolean greaterAndSet(AtomicLong atomic, long update) {
        while (true) {
            long cur = atomic.get();
            if (update <= cur) {
                return false
            }
            //should set if it hasn't changed
            if (atomic.compareAndSet(cur, update)) {
                return true;
            }
            //otherwise try again
        }
    }

    def serialize(
            final JobExportReference job,
            String format,
            boolean preserveId,
            boolean useSourceId,
            File outfile = null
    )
    {
        if (!outfile) {
            outfile = mapper.fileForJob(job)
        }
        AtomicLong counter = fileCounterFor(outfile)
        logger.debug("Start serialize[${Thread.currentThread().name}]...")

        synchronized (counter) {
            //other threads serializing the same job must wait until we complete
            if (greaterAndSet(counter, job.version)) {
                //only bother writing the file if this rev of Job if it is newer than previously serialized rev

                if (!outfile.parentFile.isDirectory()) {
                    File parentFile = outfile.parentFile
                    AtomicLong parentCounter = fileCounterFor(parentFile)
                    synchronized (parentCounter) {
                        if (!parentFile.isDirectory()) {
                            if (!parentFile.mkdirs()) {
                                throw new ScmPluginException(
                                    "Cannot create necessary dirs to serialize file to path: ${outfile.absolutePath}"
                                )
                            }
                        }
                    }
                }

                File temp = new File(outfile.parentFile, outfile.name + ".tmp${job.version}")
                temp.deleteOnExit()
                Throwable thrown = null
                try {
                    try {
                        temp.withOutputStream { out ->
                            try {
                                def sourceId = (job instanceof JobScmReference) ? job.sourceId  : null
                                job.jobSerializer.serialize(
                                        format,
                                        out,
                                        preserveId,
                                        useSourceId ? (sourceId ?: job.id) : null
                                )
                            } catch (Throwable e) {
                                thrown = e;
                            }
                        }
                    } catch (IOException e) {
{'fraction_non_alphanumeric': 0.07770562770562771, 'fraction_numerical': 0.0015873015873015873, 'mean_word_length': 2.0981224854716136, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6275875', 'n_tokens_mistral': 3613, 'n_tokens_neox': 3496, 'n_words': 948}