<commit_before>#!/usr/bin/env python

import sys
import os
import shutil
import re
import json
import time
import rpm
import subprocess
import stat
import tarfile
import time
import hashlib

import anchore.anchore_utils

analyzer_name = "file_checksums"

try:
    config = anchore.anchore_utils.init_analyzer_cmdline(sys.argv, analyzer_name)
except Exception as err:
    print str(err)
    sys.exit(1)

imgname = config['imgid']
imgid = config['imgid_full']
outputdir = config['dirs']['outputdir']
unpackdir = config['dirs']['unpackdir']

#if not os.path.exists(outputdir):
#    os.makedirs(outputdir)

domd5 = True
outfiles_md5 = {}
outfiles_sha256 = {}

try:
    timer = time.time()
    (tmp, allfiles) = anchore.anchore_utils.get_files_from_path(unpackdir + "/rootfs")
    for name in allfiles.keys():
        name = re.sub("^\.", "", name)
        thefile = '/'.join([unpackdir, "rootfs", name])

        csum = "DIRECTORY_OR_OTHER"
        if os.path.isfile(thefile) and not os.path.islink(thefile):
            if domd5:
                csum = "DIRECTORY_OR_OTHER"
                try:
                    with open(thefile, 'r') as FH:
                        csum = hashlib.md5(FH.read()).hexdigest()
                except:
                    pass
                outfiles_md5[name] = csum

            try:
                with open(thefile, 'r') as FH:
                    csum = hashlib.sha256(FH.read()).hexdigest()
            except:
                pass

            outfiles_sha256[name] = csum

        else:
            outfiles_md5[name] = "DIRECTORY_OR_OTHER"
            outfiles_sha256[name] = "DIRECTORY_OR_OTHER"

except Exception as err:
    import traceback
    traceback.print_exc()
    print "ERROR: " + str(err)
    raise err

if outfiles_md5:
    ofile = os.path.join(outputdir, 'files.md5sums')
    anchore.anchore_utils.write_kvfile_fromdict(ofile, outfiles_md5)

if outfiles_sha256:
    ofile = os.path.join(outputdir, 'files.sha256sums')
    anchore.anchore_utils.write_kvfile_fromdict(ofile, outfiles_sha256)


sys.exit(0)<commit_msg>add support for, but disabled by default, sha1 file checksumming for alpine<commit_after>#!/usr/bin/env python

import sys
import os
import shutil
import re
import json
import time
import rpm
import subprocess
import stat
import tarfile
import time
import hashlib

import anchore.anchore_utils

analyzer_name = "file_checksums"

try:
    config = anchore.anchore_utils.init_analyzer_cmdline(sys.argv, analyzer_name)
except Exception as err:
    print str(err)
    sys.exit(1)

imgname = config['imgid']
imgid = config['imgid_full']
outputdir = config['dirs']['outputdir']
unpackdir = config['dirs']['unpackdir']

#if not os.path.exists(outputdir):
#    os.makedirs(outputdir)

domd5 = True
dosha1 = False

outfiles_sha1 = {}
outfiles_md5 = {}
outfiles_sha256 = {}

try:
    timer = time.time()
    (tmp, allfiles) = anchore.anchore_utils.get_files_from_path(unpackdir + "/rootfs")
    for name in allfiles.keys():
        name = re.sub("^\.", "", name)
        thefile = '/'.join([unpackdir, "rootfs", name])

        csum = "DIRECTORY_OR_OTHER"
        if os.path.isfile(thefile) and not os.path.islink(thefile):
            if domd5:
                try:
                    with open(thefile, 'r') as FH:
                        csum = hashlib.md5(FH.read()).hexdigest()
                except:
                    csum = "DIRECTORY_OR_OTHER"
                outfiles_md5[name] = csum

            if dosha1:
                try:
                    with open(thefile, 'r') as FH:
                        csum = hashlib.sha1(FH.read()).hexdigest()
                except:
                    csum = "DIRECTORY_OR_OTHER"
                outfiles_sha1[name] = csum

            try:
                with open(thefile, 'r') as FH:
                    csum = hashlib.sha256(FH.read()).hexdigest()
            except:
                csum = "DIRECTORY_OR_OTHER"
            outfiles_sha256[name] = csum

        else:
            if domd5:
                outfiles_md5[name] = "DIRECTORY_OR_OTHER"
            if dosha1:
                outfiles_sha1[name] = "DIRECTORY_OR_OTHER"

            outfiles_sha256[name] = "DIRECTORY_OR_OTHER"

except Exception as err:
    import traceback
    traceback.print_exc()
    print "ERROR: " + str(err)
    raise err

if outfiles_sha1:
    ofile = os.path.join(outputdir, 'files.sha1sums')
    anchore.anchore_utils.write_kvfile_fromdict(ofile, outfiles_sha1)

if outfiles_md5:
    ofile = os.path.join(outputdir, 'files.md5sums')
    anchore.anchore_utils.write_kvfile_fromdict(ofile, outfiles_md5)

if outfiles_sha256:
    ofile = os.path.join(outputdir, 'files.sha256sums')
    anchore.anchore_utils.write_kvfile_fromdict(ofile, outfiles_sha256)


sys.exit(0)
{'fraction_non_alphanumeric': 0.10817610062893082, 'fraction_numerical': 0.015932914046121592, 'mean_word_length': 2.4397981254506127, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6739205', 'n_tokens_mistral': 1684, 'n_tokens_neox': 1580, 'n_words': 325}