<commit_before>#-------------------------------------------------------------------------------
# Copyright 2017 Cognizant Technology Solutions
# 
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.  You may obtain a copy
# of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations under
# the License.
#-------------------------------------------------------------------------------
'''
Created on Jun 22, 2016

@author: 463188
'''
from time import mktime
from dateutil import parser
import requests
import xml.etree.ElementTree as ET
from requests.auth import HTTPBasicAuth
from com.cognizant.devops.platformagents.core.BaseAgent import BaseAgent

class JenkinsAgent(BaseAgent):
    def process(self):
        self.userid = self.config.get("userid", '')
        self.passwd = self.config.get("passwd", '')
        self.BaseUrl = self.config.get("BaseUrl", '')
        startFrom = self.config.get("StartFrom", '')
        startFrom = parser.parse(startFrom)
        startFrom = mktime(startFrom.timetuple()) + startFrom.microsecond/1000000.0
        self.startFrom = long(startFrom * 1000)
        self.responseTemplate = self.getResponseTemplate()
        self.useAllBuildsApi = self.config.get("useAllBuildsApi", False)
        self.buildsApiName = "builds"
        if self.useAllBuildsApi:
            self.buildsApiName = "allBuilds"
        self.data = []
        self.treeApiParams = self.buildApiParameters('', self.responseTemplate)
        self.validateAndCorrectTrackingFormat()
        jenkinsMasters = self.config.get("jenkinsMasters", None)
        if jenkinsMasters:
            for jenkinsMaster in jenkinsMasters:
                self.currentJenkinsMaster = jenkinsMaster
                self.processFolder(jenkinsMasters[jenkinsMaster])
        else:
            self.currentJenkinsMaster = 'master'
            self.processFolder(self.BaseUrl)
        #self.publishToolsData(self.data)
        self.updateTrackingJson(self.tracking)

    def validateAndCorrectTrackingFormat(self):
        tracking = self.tracking
        correctionRequired = False
        for master in tracking:
            keyType = type(tracking[master])
            if keyType is int:
                correctionRequired = True
            break
        if correctionRequired:
            self.tracking = {'master' : self.tracking}
            self.updateTrackingJson(self.tracking)

    def processFolder(self,url):
        restUrl = url + 'api/json?tree=jobs[name,url,buildable,lastBuild[number]]'
        jenkinsProjects = self.getResponse(restUrl, 'GET', self.userid, self.passwd, None)
        jobs = jenkinsProjects.get('jobs', None);
        if jobs:
            for projects in range(len(jenkinsProjects["jobs"])):
                if "buildable" in jenkinsProjects["jobs"][projects]:
                    jobUrl = jenkinsProjects["jobs"][projects]["url"]
                    if "lastBuild" in jenkinsProjects["jobs"][projects] and jenkinsProjects["jobs"][projects]["lastBuild"] is not None:
                        lastBuild = jenkinsProjects["jobs"][projects]["lastBuild"]["number"]
                        self.getJobDetails(jobUrl, lastBuild)
                else:
                    folderUrl = jenkinsProjects["jobs"][projects]["url"]
                    self.processFolder(folderUrl)
        else:
            restUrl = url + 'api/json?tree=lastBuild[number],url'
            jenkinsProjects = self.getResponse(restUrl, 'GET', self.userid, self.passwd, None)
            jobUrl = jenkinsProjects["url"]
            lastBuild = jenkinsProjects.get("lastBuild", {}).get("number", None)
            if lastBuild is not None:
                self.getJobDetails(jobUrl, lastBuild)


    def getJobDetails(self, url, lastBuild):
        tillJobCount = 0
        buildsIdentified = False
        nextBatch = 0
        injectData = self.getJobLevelConfig(url)
        if injectData['disabled'] == 'true':
            return;
        injectData['master'] = self.currentJenkinsMaster
        if self.tracking.get(self.currentJenkinsMaster, {}).get(url,None):
            trackingNum = self.tracking.get(self.currentJenkinsMaster).get(url)
            tillJobCount = lastBuild - trackingNum
        else:
            while not buildsIdentified:
                restUrl = url+'api/json?tree='+self.buildsApiName+'[number,timestamp,duration]{'+str(nextBatch)+','+str(nextBatch+100)+'},name'
                jobDetails = self.getResponse(restUrl, 'GET', self.userid, self.passwd, None)
                injectData['jobName'] = jobDetails["name"]
                builds = jobDetails[self.buildsApiName]
                for build in builds:
                    if self.startFrom < build["timestamp"]:
                        tillJobCount+=1
                    #if not build["duration"] > 0:
                        #startindex=tillJobCount
                    else:
                        self.updateTrackingDetails(url, lastBuild)
                        buildsIdentified = True<commit_msg>Updating Jenkins Agent to consider jobName during incremental run.<commit_after>#-------------------------------------------------------------------------------
# Copyright 2017 Cognizant Technology Solutions
# 
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License.  You may obtain a copy
# of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations under
# the License.
#-------------------------------------------------------------------------------
'''
Created on Jun 22, 2016

@author: 463188
'''
from time import mktime
from dateutil import parser
import requests
import xml.etree.ElementTree as ET
from requests.auth import HTTPBasicAuth
from com.cognizant.devops.platformagents.core.BaseAgent import BaseAgent

class JenkinsAgent(BaseAgent):
    def process(self):
        self.userid = self.config.get("userid", '')
        self.passwd = self.config.get("passwd", '')
        self.BaseUrl = self.config.get("BaseUrl", '')
        startFrom = self.config.get("StartFrom", '')
        startFrom = parser.parse(startFrom)
        startFrom = mktime(startFrom.timetuple()) + startFrom.microsecond/1000000.0
        self.startFrom = long(startFrom * 1000)
        self.responseTemplate = self.getResponseTemplate()
        self.useAllBuildsApi = self.config.get("useAllBuildsApi", False)
        self.buildsApiName = "builds"
        if self.useAllBuildsApi:
            self.buildsApiName = "allBuilds"
        self.data = []
        self.treeApiParams = self.buildApiParameters('', self.responseTemplate)
        self.validateAndCorrectTrackingFormat()
        jenkinsMasters = self.config.get("jenkinsMasters", None)
        if jenkinsMasters:
            for jenkinsMaster in jenkinsMasters:
                self.currentJenkinsMaster = jenkinsMaster
                self.processFolder(jenkinsMasters[jenkinsMaster])
        else:
            self.currentJenkinsMaster = 'master'
            self.processFolder(self.BaseUrl)
        #self.publishToolsData(self.data)
        self.updateTrackingJson(self.tracking)

    def validateAndCorrectTrackingFormat(self):
        tracking = self.tracking
        correctionRequired = False
        for master in tracking:
            keyType = type(tracking[master])
            if keyType is int:
                correctionRequired = True
            break
        if correctionRequired:
            self.tracking = {'master' : self.tracking}
            self.updateTrackingJson(self.tracking)

    def processFolder(self,url):
        restUrl = url + 'api/json?tree=jobs[name,url,buildable,lastBuild[number]]'
        jenkinsProjects = self.getResponse(restUrl, 'GET', self.userid, self.passwd, None)
        jobs = jenkinsProjects.get('jobs', None);
        if jobs:
            for job in jobs:
                url = job.get('url')
                if job.get('buildable', False):
                    lastBuild = job.get('lastBuild', None)
                    if lastBuild:
                        jobName = job.get('name', None)
                        lastBuildNumber = lastBuild.get('number')
                        self.getJobDetails(url, lastBuildNumber, jobName)
                else:
                    self.processFolder(url)
            #for projects in range(len(jenkinsProjects["jobs"])):
            #   if "buildable" in jenkinsProjects["jobs"][projects]:
            #        jobUrl = jenkinsProjects["jobs"][projects]["url"]
            #        if "lastBuild" in jenkinsProjects["jobs"][projects] and jenkinsProjects["jobs"][projects]["lastBuild"] is not None:
            #            lastBuild = jenkinsProjects["jobs"][projects]["lastBuild"]["number"]
            #            jobName = jenkinsProjects["jobs"][projects].get('name', None)
            #            self.getJobDetails(jobUrl, lastBuild, jobName)
            #    else:
            #        folderUrl = jenkinsProjects["jobs"][projects]["url"]
            #        self.processFolder(folderUrl)
        else:
            restUrl = url + 'api/json?tree=lastBuild[number],url,name'
            jenkinsProjects = self.getResponse(restUrl, 'GET', self.userid, self.passwd, None)
            jobUrl = jenkinsProjects['url']
            jobName = jenkinsProjects.get('name', None)
            lastBuild = jenkinsProjects.get('lastBuild', {}).get('number', None)
            if lastBuild is not None:
                self.getJobDetails(jobUrl, lastBuild, jobName)


    def getJobDetails(self, url, lastBuild, jobName):
        tillJobCount = 0
        buildsIdentified = False
        nextBatch = 0
        injectData = self.getJobLevelConfig(url)
        if injectData['disabled'] == 'true':
            return;
        injectData['master'] = self.currentJenkinsMaster
        injectData['jobName'] = jobName
        if self.tracking.get(self.currentJenkinsMaster, {}).get(url,None):
            trackingNum = self.tracking.get(self.currentJenkinsMaster).get(url)
            tillJobCount = lastBuild - trackingNum
        else:
            while not buildsIdentified:
                restUrl = url+'api/json?tree='+self.buildsApiName+'[number,timestamp,duration]{'+str(nextBatch)+','+str(nextBatch+100)+'},name'
                jobDetails = self.getResponse(restUrl, 'GET', self.userid, self.passwd, None)
                builds = jobDetails[self.buildsApiName]
                for build in builds:
                    if self.startFrom < build["timestamp"]:
                        tillJobCount+=1
                    #if not build["duration"] > 0:
                        #startindex=tillJobCount
                    else:
                        self.updateTrackingDetails(url, lastBuild)
                        buildsIdentified = True
{'fraction_non_alphanumeric': 0.12784389219460973, 'fraction_numerical': 0.006825341267063354, 'mean_word_length': 2.1915666015079585, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '422990', 'n_tokens_mistral': 3143, 'n_tokens_neox': 2985, 'n_words': 676}