<commit_before>#     bcf_utils.py: utility classes and functions shared between BCF codes
#     Copyright (C) University of Manchester 2013-14 <NAME>
#
########################################################################
#
# bcf_utils.py
#
#########################################################################

__version__ = "1.4.7"

"""bcf_utils

Utility classes and functions shared between BCF codes.

General utility classes:

  AttributeDictionary
  OrderedDictionary

File system wrappers and utilities:

  PathInfo
  mkdir
  mklink
  chmod
  touch
  format_file_size
  commonprefix
  is_gzipped_file
  rootname
  find_program
  get_user_from_uid
  get_uid_from_user
  get_group_from_gid
  get_gid_from_group
  walk
  strip_ext

Symbolic link handling:

  Symlink
  links

Sample name utilities:

  extract_initials
  extract_prefix
  extract_index_as_string
  extract_index
  pretty_print_names
  name_matches

File manipulations:

  concatenate_fastq_files

Text manipulations:

  split_into_lines

"""

#######################################################################
# Import modules that this module depends on
#######################################################################

import os
import logging
import string
import gzip
import shutil
import copy
import stat
import pwd
import grp
import datetime
import re

#######################################################################
# General utility classes
#######################################################################

class AttributeDictionary:
    """Dictionary-like object with items accessible as attributes

    AttributeDict provides a dictionary-like object where the value
    of items can also be accessed as attributes of the object.

    For example:

    >>> d = AttributeDict()
    >>> d['salutation'] = "hello"
    >>> d.salutation
    ... "hello"

    Attributes can only be assigned by using dictionary item assignment
    notation i.e. d['key'] = value. d.key = value doesn't work.

    If the attribute doesn't match a stored item then an
    AttributeError exception is raised.

    len(d) returns the number of stored items.

    The AttributeDict behaves like a dictionary for iterations, for
    example:

    >>> for attr in d:
    >>>    print "%s = %s" % (attr,d[attr])

    """
    def __init__(self,**args):
        self.__dict = dict(args)

    def __getattr__(self,attr):
        try:
            return self.__dict[attr]
        except KeyError:
            raise AttributeError, "No attribute '%s'" % attr

    def __getitem__(self,key):
        return self.__dict[key]

    def __setitem__(self,key,value):
        self.__dict[key] = value

    def __iter__(self):
        return iter(self.__dict)

    def __len__(self):
        return len(self.__dict)

class OrderedDictionary:
    """Augumented dictionary which keeps keys in order

    OrderedDictionary provides an augmented Python dictionary
    class which keeps the dictionary keys in the order they are
    added to the object.

    Items are added, modified and removed as with a standard
    dictionary e.g.:

    >>> d[key] = value
    >>> value = d[key]
    >>> del(d[key])

    The 'keys()' method returns the OrderedDictionary's keys in
    the correct order.
    """
    def __init__(self):
        self.__keys = []
        self.__dict = {}

    def __getitem__(self,key):
        if key not in self.__keys:
            raise KeyError
        return self.__dict[key]

    def __setitem__(self,key,value):
        if key not in self.__keys:
            self.__keys.append(key)
        self.__dict[key] = value

    def __delitem__(self,key):
        try:
            i = self.__keys.index(key)
            del(self.__keys[i])
            del(self.__dict[key])
        except ValueError:
            raise KeyError

    def __len__(self):
        return len(self.__keys)

    def __contains__(self,key):
        return key in self.__keys

    def __iter__(self):
        return iter(self.__keys)

    def keys(self):
        return copy.copy(self.__keys)

    def insert(self,i,key,value):
        if key not in self.__keys:
            self.__keys.insert(i,key)
            self.__dict[key] = value
        else:
            raise KeyError, "Key '%s' already exists" % key

#######################################################################
# File system wrappers and utilities
#######################################################################

class PathInfo:
    """Collect and report information on a file

    The PathInfo class provides an interface to getting general
    information on a path, which may point to a file, directory, link
    or non-existent location.

    The properties provide information on whether the path is
    readable (i.e. accessible) by the current user, whether it is
    readable by members of the same group, who is the owner and
    what group does it belong to, when was it last modified etc.

    """
    def __init__(self,path,basedir=None):
        """Create a new PathInfo object

        Arguments:
          path: a filesystem path, which can be relative or
            absolute, or point to a non-existent location
          basedir: (optional) if supplied then prepended to
            the supplied path

        """
        self.__basedir = basedir
        if self.__basedir is not None:
            self.__path = os.path.join(self.__basedir,path)
        else:
            self.__path = path
        try:
            self.__st = os.lstat(self.__path)
        except OSError:
            self.__st = None

    @property
    def path(self):
        """Return the filesystem path

        """
        return self.__path

    @property
    def is_readable(self):
        """Return True if the path exists and is readable by the owner

        Paths may be reported as unreadable for various reasons,
        e.g. the target doesn't exist, or doesn't have permission
        for this user to read it, or if part of the path doesn't
        allow the user to read the file.

        """
        if self.__st is None:
            return False
        return bool(self.__st.st_mode & stat.S_IRUSR)

    @property
    def is_group_readable(self):
        """Return True if the path exists and is group-readable

        Paths may be reported as unreadable for various reasons,
        e.g. the target doesn't exist, or doesn't have permission
        for this user to read it, or if part of the path doesn't
        allow the user to read the file.

        """
        if self.__st is None:
            return False
        return bool(self.__st.st_mode & stat.S_IRGRP)

    @property
    def is_group_writable(self):
        """Return True if the path exists and is group-writable

        Paths may be reported as unwritable for various reasons,
        e.g. the target doesn't exist, or doesn't have permission
        for this user to write to it, or if part of the path
        doesn't allow the user to read the file.

        """
        if self.__st is None:
            return False
        return bool(self.__st.st_mode & stat.S_IWGRP)

    @property
    def deepest_accessible_parent(self):
        """Return longest accessible directory that leads to path

        Tries to find the longest parent directory above path
        which is accessible by the current user.

        If it's not possible to find a parent that is accessible
        then raise an exception.

        """
        path = os.path.dirname(os.path.abspath(self.__path))
        while path != os.sep:
            if os.access(path,os.R_OK):
                return path
            path = os.path.dirname(path)
        raise OSError,"Unable to find readable parent for %s" % self.__path

    @property
    def resolve_link_via_parent(self):
        """If path or parent directory is a link then return actual path

        Resolves and returns the 'real' path for a path where either
        it or one of its parent directories is a symbolic link.

        It will resolve multiple levels of symlinks to generate a path
        that is free of links (nb it is possible that the resolved path
        will not be an existing file or directory).

        If there are no links in the directory tree then returns the
        full path of the input.

        """
        path = os.path.abspath(self.__path)
        realpath = []
        while path != os.sep:
            if os.path.islink(path):
                # Construct actual path
                link_path = os.readlink(path)
                if os.path.isabs(link_path):
                    path = link_path
                else:
                    path = os.path.normpath(os.path.join(os.path.dirname(path),
                                                         link_path))
                continue
            # Descend to next level
            realpath.append(os.path.basename(path))
            path = os.path.dirname(path)
        # Descended to root, rebuild path
        realpath = os.sep + os.sep.join(realpath[::-1])
        return realpath

    @property
    def uid(self):
        """Return associated UID (user ID)

        Attempts to return the UID (user ID) number associated with
        the path.

        If the UID can't be found then returns None.

        """
        if self.__st is None:
            return None
        return self.__st.st_uid

    @property
    def user(self):
        """Return associated user name

        Attempts to return the user name associated with the path.
        If the name can't be found then tries to return the UID
        instead.

        If neither pieces of information can be found then returns
        None.

        """
        if self.__st is None:
            return None
        user = get_user_from_uid(self.uid)
        if user is not None:
            return user
        else:
            return self.uid

    @property
    def gid(self):
        """Return associated GID (group ID)

        Attempts to return the GID (group ID) number associated with
        the path.

        If the GID can't be found then returns None.

        """
        if self.__st is None:
            return None
        return self.__st.st_gid

    @property
    def group(self):
        """Return associated group name

        Attempts to return the group name associated with the path.
        If the name can't be found then tries to return the GID
        instead.

        If neither pieces of information can be found then returns
        None.

        """
        if self.__st is None:
            return None
        group = get_group_from_gid(self.gid)
        if group is not None:
            return group
        else:
            return self.gid

    @property
    def exists(self):
        """Return True if the path refers to an existing location

        Note that this is a wrapper to os.path.lexists so it reports
        the existence of symbolic links rather than their targets.

        """
        return os.path.lexists(self.__path)

    @property
    def is_link(self):
        """Return True if path refers to a symbolic link

        """
        return os.path.islink(self.__path)

    @property
    def is_file(self):
        """Return True if path refers to a file

        """
        if not self.is_link:
            return os.path.isfile(self.__path)
        else:
            return False

    @property
    def is_dir(self):
        """Return True if path refers to a directory

        """
        if not self.is_link:
            return os.path.isdir(self.__path)
        else:
            return False

    @property
    def is_executable(self):
        """Return True if path refers to an executable file

        """
        if self.__st is None:
            return False
        if self.is_link:
            return PathInfo(Symlink(self.__path).resolve_target()).is_executable
        return bool(self.__st.st_mode & stat.S_IXUSR) and self.is_file

    @property
    def mtime(self):
        """Return last modification timestamp for path

        """
        if self.__st is None:
            return None
        return self.__st.st_mtime

    @property
    def datetime(self):
        """Return last modification time as datetime object

        """
        if self.mtime is None:
            return None
        return datetime.datetime.fromtimestamp(self.mtime)

    def relpath(self,dirn):
        """Return part of path relative to a directory

        Wrapper for os.path.relpath(...).
        
        """
        return os.path.relpath(self.__path,dirn)

    def chown(self,user=None,group=None):
        """Change associated owner and group

        'user' and 'group' must be supplied as UID/GID
        numbers (or None to leave the current values
        unchanged).

        *** Note that chown will fail attempting to
        change the owner if the current process is not
        owned by root ***

        This is actually a wrapper to the os.lchmod
        function, so it doesn't follow symbolic links.

        """
        if user is None and group is None:
            # Nothing to do
            return
        if user is None:
            user = -1
        if group is None:
            group = -1
        # Convert to ints
        user = int(user)
        group = int(group)
        # Do chown - note this will fail if the user
        # performing the operation is not root
        os.lchown(self.__path,user,group)
        # Update the stat information
        try:
            self.__st = os.lstat(self.__path)
        except OSError:
            self.__st = None

    def __repr__(self):
        """Implements the built-in __repr__ function
        """
        return str(self.__path)

def mkdir(dirn,mode=None):
    """Make a directory

    Arguments:
      dirn: the path of the directory to be created
      mode: (optional) a mode specifier to be applied to the
        new directory once it has been created e.g. 0775 or 0664

    """
    logging.debug("Making %s" % dirn)
    if not os.path.isdir(dirn):
        os.mkdir(dirn)
        if mode is not None: chmod(dirn,mode)

def mklink(target,link_name,relative=False):
    """Make a symbolic link

    Arguments:
      target: the file or directory to link to
      link_name: name of the link
      relative: if True then make a relative link (if possible);
        otherwise link to the target as given (default)"""
    logging.debug("Linking to %s from %s" % (target,link_name))
    target_path = target
    if relative:
        # Try to construct relative link to target
        target_abs_path = os.path.abspath(target)
        link_abs_path = os.path.abspath(link_name)
        common_prefix = commonprefix(target_abs_path,link_abs_path)
        if common_prefix:
            # Use relpath to generate the relative path from the link
            # to the target
            target_path = os.path.relpath(target_abs_path,os.path.dirname(link_abs_path))
    os.symlink(target_path,link_name)

def chmod(target,mode):
    """Change mode of file or directory

    This a wrapper for the os.chmod function, with the
    addition that it doesn't follow symbolic links.

    For symbolic links it attempts to use the os.lchmod
    function instead, as this operates on the link
    itself and not the link target. If os.lchmod is not
    available then links are ignored.

    Arguments:
      target: file or directory to apply new mode to
      mode: a valid mode specifier e.g. 0775 or 0664

    """
    logging.debug("Changing mode of %s to %s" % (target,mode))
    try:
        if os.path.islink(target):
            # Try to use lchmod to operate on the link
            try:
                os.lchmod(target,mode)
            except AttributeError,ex:
                # lchmod is not available on all systems
                # If not then just ignore
                logging.debug("os.lchmod not available? Exception: %s" % ex)
        else:
            # Use os.chmod for everything else
            os.chmod(target,mode)
    except OSError, ex:
        logging.warning("Failed to change permissions on %s to %s: %s" % (target,mode,ex))

def touch(filename):
    """Create new empty file, or update modification time if already exists

    Arguments:
      filename: name of the file to create (can include leading path)

    """
    if not os.path.exists(filename):
        open(filename,'wb+').close()
    os.utime(filename,None)

def format_file_size(fsize,units=None):
    """Format a file size from bytes to human-readable form

    Takes a file size in bytes and returns a human-readable
    string, e.g. 4.0K, 186M, 1.5G.

    Alternatively specify the required units via the 'units'
    arguments.

    Arguments:
      fsize: size in bytes
      units: (optional) specify output in kb ('K'), Mb ('M'),
             Gb ('G') or Tb ('T')

    Returns:
      Human-readable version of file size.

    """
    # Return size in human readable form
    if units is not None:
        units = units.upper()
    fsize = float(fsize)/1024
    unit_list = 'KMGT'
    for unit in unit_list:
        if units is None:
            if fsize > 1024:
                fsize = fsize/1024
            else:
                break
        else:
            if units != unit:
                fsize = fsize/1024
            else:
                break
    return "%.1f%s" % (fsize,unit)
            
def commonprefix(path1,path2):
    """Determine common prefix path for path1 and path2

    Use this in preference to os.path.commonprefix as the version
    in os.path compares the two paths in a character-wise fashion
    and so can give counter-intuitive matches; this version compares
    path components which seems more sensible.

    For example: for two paths /mnt/dir1/file and /mnt/dir2/file,
    os.path.commonprefix will return /mnt/dir, whereas this function
    will return /mnt.

    Arguments:
      path1: first path in comparison
      path2: second path in comparison

    Returns:
      Leading part of path which is common to both input paths.
    """
    path1_components = str(path1).split(os.sep)
    path2_components = str(path2).split(os.sep)
    common_components = []
    ncomponents = min(len(path1_components),len(path2_components))
    for i in range(ncomponents):
        if path1_components[i] == path2_components[i]:
            common_components.append(path1_components[i])
        else:
            break
    commonprefix = "%s" % os.sep.join(common_components)
    return commonprefix

def is_gzipped_file(filename):
    """Check if a file has a .gz extension

    Arguments:
      filename: name of the file to be tested (can include leading path)

    Returns:
      True if filename has trailing .gz extension, False if not.

    """
    return os.path.splitext(filename)[1] == '.gz'

def rootname(name):
    """Remove all extensions from name

    Arguments:
      name: name of a file

    Returns:
      Leading part of name up to first dot, i.e. name without any
      trailing extensions.

    """
    try:
        i = name.index('.')
        return name[0:i]
    except ValueError:
        # No dot
        return name

def find_program(name):
    """Find a program on the PATH

    Search the current PATH for the specified program name and return
    the full path, or None if not found.

    """
    if os.path.isabs(name) and PathInfo(name).is_executable:
        return name
    for path in os.environ['PATH'].split(os.pathsep):
        name_path = os.path.abspath(os.path.join(path,name))
        if PathInfo(name_path).is_executable:
            return name_path
    return None

def get_user_from_uid(uid):
    """Return user name from UID

    Looks up user name matching the supplied UID;
    returns None if no matching name can be found.

    """
    try:
        return pwd.getpwuid(int(uid)).pw_name
    except (KeyError,ValueError,OverflowError):
        return None

def get_uid_from_user(user):
    """Return UID from user name

    Looks up UID matching the supplied user name;
    returns None if no matching name can be found.

    NB returned UID will be an integer.

    """
    try:
        return pwd.getpwnam(str(user)).pw_uid
    except KeyError:
        return None

def get_group_from_gid(gid):<commit_msg>bcftbx/utils.py 1.4.8: new function 'get_current_user'.<commit_after>#     bcf_utils.py: utility classes and functions shared between BCF codes
#     Copyright (C) University of Manchester 2013-14 <NAME>
#
########################################################################
#
# bcf_utils.py
#
#########################################################################

__version__ = "1.4.8"

"""bcf_utils

Utility classes and functions shared between BCF codes.

General utility classes:

  AttributeDictionary
  OrderedDictionary

File system wrappers and utilities:

  PathInfo
  mkdir
  mklink
  chmod
  touch
  format_file_size
  commonprefix
  is_gzipped_file
  rootname
  find_program
  get_current_user
  get_user_from_uid
  get_uid_from_user
  get_group_from_gid
  get_gid_from_group
  walk
  strip_ext

Symbolic link handling:

  Symlink
  links

Sample name utilities:

  extract_initials
  extract_prefix
  extract_index_as_string
  extract_index
  pretty_print_names
  name_matches

File manipulations:

  concatenate_fastq_files

Text manipulations:

  split_into_lines

"""

#######################################################################
# Import modules that this module depends on
#######################################################################

import os
import logging
import string
import gzip
import shutil
import copy
import stat
import pwd
import grp
import datetime
import re

#######################################################################
# General utility classes
#######################################################################

class AttributeDictionary:
    """Dictionary-like object with items accessible as attributes

    AttributeDict provides a dictionary-like object where the value
    of items can also be accessed as attributes of the object.

    For example:

    >>> d = AttributeDict()
    >>> d['salutation'] = "hello"
    >>> d.salutation
    ... "hello"

    Attributes can only be assigned by using dictionary item assignment
    notation i.e. d['key'] = value. d.key = value doesn't work.

    If the attribute doesn't match a stored item then an
    AttributeError exception is raised.

    len(d) returns the number of stored items.

    The AttributeDict behaves like a dictionary for iterations, for
    example:

    >>> for attr in d:
    >>>    print "%s = %s" % (attr,d[attr])

    """
    def __init__(self,**args):
        self.__dict = dict(args)

    def __getattr__(self,attr):
        try:
            return self.__dict[attr]
        except KeyError:
            raise AttributeError, "No attribute '%s'" % attr

    def __getitem__(self,key):
        return self.__dict[key]

    def __setitem__(self,key,value):
        self.__dict[key] = value

    def __iter__(self):
        return iter(self.__dict)

    def __len__(self):
        return len(self.__dict)

class OrderedDictionary:
    """Augumented dictionary which keeps keys in order

    OrderedDictionary provides an augmented Python dictionary
    class which keeps the dictionary keys in the order they are
    added to the object.

    Items are added, modified and removed as with a standard
    dictionary e.g.:

    >>> d[key] = value
    >>> value = d[key]
    >>> del(d[key])

    The 'keys()' method returns the OrderedDictionary's keys in
    the correct order.
    """
    def __init__(self):
        self.__keys = []
        self.__dict = {}

    def __getitem__(self,key):
        if key not in self.__keys:
            raise KeyError
        return self.__dict[key]

    def __setitem__(self,key,value):
        if key not in self.__keys:
            self.__keys.append(key)
        self.__dict[key] = value

    def __delitem__(self,key):
        try:
            i = self.__keys.index(key)
            del(self.__keys[i])
            del(self.__dict[key])
        except ValueError:
            raise KeyError

    def __len__(self):
        return len(self.__keys)

    def __contains__(self,key):
        return key in self.__keys

    def __iter__(self):
        return iter(self.__keys)

    def keys(self):
        return copy.copy(self.__keys)

    def insert(self,i,key,value):
        if key not in self.__keys:
            self.__keys.insert(i,key)
            self.__dict[key] = value
        else:
            raise KeyError, "Key '%s' already exists" % key

#######################################################################
# File system wrappers and utilities
#######################################################################

class PathInfo:
    """Collect and report information on a file

    The PathInfo class provides an interface to getting general
    information on a path, which may point to a file, directory, link
    or non-existent location.

    The properties provide information on whether the path is
    readable (i.e. accessible) by the current user, whether it is
    readable by members of the same group, who is the owner and
    what group does it belong to, when was it last modified etc.

    """
    def __init__(self,path,basedir=None):
        """Create a new PathInfo object

        Arguments:
          path: a filesystem path, which can be relative or
            absolute, or point to a non-existent location
          basedir: (optional) if supplied then prepended to
            the supplied path

        """
        self.__basedir = basedir
        if self.__basedir is not None:
            self.__path = os.path.join(self.__basedir,path)
        else:
            self.__path = path
        try:
            self.__st = os.lstat(self.__path)
        except OSError:
            self.__st = None

    @property
    def path(self):
        """Return the filesystem path

        """
        return self.__path

    @property
    def is_readable(self):
        """Return True if the path exists and is readable by the owner

        Paths may be reported as unreadable for various reasons,
        e.g. the target doesn't exist, or doesn't have permission
        for this user to read it, or if part of the path doesn't
        allow the user to read the file.

        """
        if self.__st is None:
            return False
        return bool(self.__st.st_mode & stat.S_IRUSR)

    @property
    def is_group_readable(self):
        """Return True if the path exists and is group-readable

        Paths may be reported as unreadable for various reasons,
        e.g. the target doesn't exist, or doesn't have permission
        for this user to read it, or if part of the path doesn't
        allow the user to read the file.

        """
        if self.__st is None:
            return False
        return bool(self.__st.st_mode & stat.S_IRGRP)

    @property
    def is_group_writable(self):
        """Return True if the path exists and is group-writable

        Paths may be reported as unwritable for various reasons,
        e.g. the target doesn't exist, or doesn't have permission
        for this user to write to it, or if part of the path
        doesn't allow the user to read the file.

        """
        if self.__st is None:
            return False
        return bool(self.__st.st_mode & stat.S_IWGRP)

    @property
    def deepest_accessible_parent(self):
        """Return longest accessible directory that leads to path

        Tries to find the longest parent directory above path
        which is accessible by the current user.

        If it's not possible to find a parent that is accessible
        then raise an exception.

        """
        path = os.path.dirname(os.path.abspath(self.__path))
        while path != os.sep:
            if os.access(path,os.R_OK):
                return path
            path = os.path.dirname(path)
        raise OSError,"Unable to find readable parent for %s" % self.__path

    @property
    def resolve_link_via_parent(self):
        """If path or parent directory is a link then return actual path

        Resolves and returns the 'real' path for a path where either
        it or one of its parent directories is a symbolic link.

        It will resolve multiple levels of symlinks to generate a path
        that is free of links (nb it is possible that the resolved path
        will not be an existing file or directory).

        If there are no links in the directory tree then returns the
        full path of the input.

        """
        path = os.path.abspath(self.__path)
        realpath = []
        while path != os.sep:
            if os.path.islink(path):
                # Construct actual path
                link_path = os.readlink(path)
                if os.path.isabs(link_path):
                    path = link_path
                else:
                    path = os.path.normpath(os.path.join(os.path.dirname(path),
                                                         link_path))
                continue
            # Descend to next level
            realpath.append(os.path.basename(path))
            path = os.path.dirname(path)
        # Descended to root, rebuild path
        realpath = os.sep + os.sep.join(realpath[::-1])
        return realpath

    @property
    def uid(self):
        """Return associated UID (user ID)

        Attempts to return the UID (user ID) number associated with
        the path.

        If the UID can't be found then returns None.

        """
        if self.__st is None:
            return None
        return self.__st.st_uid

    @property
    def user(self):
        """Return associated user name

        Attempts to return the user name associated with the path.
        If the name can't be found then tries to return the UID
        instead.

        If neither pieces of information can be found then returns
        None.

        """
        if self.__st is None:
            return None
        user = get_user_from_uid(self.uid)
        if user is not None:
            return user
        else:
            return self.uid

    @property
    def gid(self):
        """Return associated GID (group ID)

        Attempts to return the GID (group ID) number associated with
        the path.

        If the GID can't be found then returns None.

        """
        if self.__st is None:
            return None
        return self.__st.st_gid

    @property
    def group(self):
        """Return associated group name

        Attempts to return the group name associated with the path.
        If the name can't be found then tries to return the GID
        instead.

        If neither pieces of information can be found then returns
        None.

        """
        if self.__st is None:
            return None
        group = get_group_from_gid(self.gid)
        if group is not None:
            return group
        else:
            return self.gid

    @property
    def exists(self):
        """Return True if the path refers to an existing location

        Note that this is a wrapper to os.path.lexists so it reports
        the existence of symbolic links rather than their targets.

        """
        return os.path.lexists(self.__path)

    @property
    def is_link(self):
        """Return True if path refers to a symbolic link

        """
        return os.path.islink(self.__path)

    @property
    def is_file(self):
        """Return True if path refers to a file

        """
        if not self.is_link:
            return os.path.isfile(self.__path)
        else:
            return False

    @property
    def is_dir(self):
        """Return True if path refers to a directory

        """
        if not self.is_link:
            return os.path.isdir(self.__path)
        else:
            return False

    @property
    def is_executable(self):
        """Return True if path refers to an executable file

        """
        if self.__st is None:
            return False
        if self.is_link:
            return PathInfo(Symlink(self.__path).resolve_target()).is_executable
        return bool(self.__st.st_mode & stat.S_IXUSR) and self.is_file

    @property
    def mtime(self):
        """Return last modification timestamp for path

        """
        if self.__st is None:
            return None
        return self.__st.st_mtime

    @property
    def datetime(self):
        """Return last modification time as datetime object

        """
        if self.mtime is None:
            return None
        return datetime.datetime.fromtimestamp(self.mtime)

    def relpath(self,dirn):
        """Return part of path relative to a directory

        Wrapper for os.path.relpath(...).
        
        """
        return os.path.relpath(self.__path,dirn)

    def chown(self,user=None,group=None):
        """Change associated owner and group

        'user' and 'group' must be supplied as UID/GID
        numbers (or None to leave the current values
        unchanged).

        *** Note that chown will fail attempting to
        change the owner if the current process is not
        owned by root ***

        This is actually a wrapper to the os.lchmod
        function, so it doesn't follow symbolic links.

        """
        if user is None and group is None:
            # Nothing to do
            return
        if user is None:
            user = -1
        if group is None:
            group = -1
        # Convert to ints
        user = int(user)
        group = int(group)
        # Do chown - note this will fail if the user
        # performing the operation is not root
        os.lchown(self.__path,user,group)
        # Update the stat information
        try:
            self.__st = os.lstat(self.__path)
        except OSError:
            self.__st = None

    def __repr__(self):
        """Implements the built-in __repr__ function
        """
        return str(self.__path)

def mkdir(dirn,mode=None):
    """Make a directory

    Arguments:
      dirn: the path of the directory to be created
      mode: (optional) a mode specifier to be applied to the
        new directory once it has been created e.g. 0775 or 0664

    """
    logging.debug("Making %s" % dirn)
    if not os.path.isdir(dirn):
        os.mkdir(dirn)
        if mode is not None: chmod(dirn,mode)

def mklink(target,link_name,relative=False):
    """Make a symbolic link

    Arguments:
      target: the file or directory to link to
      link_name: name of the link
      relative: if True then make a relative link (if possible);
        otherwise link to the target as given (default)"""
    logging.debug("Linking to %s from %s" % (target,link_name))
    target_path = target
    if relative:
        # Try to construct relative link to target
        target_abs_path = os.path.abspath(target)
        link_abs_path = os.path.abspath(link_name)
        common_prefix = commonprefix(target_abs_path,link_abs_path)
        if common_prefix:
            # Use relpath to generate the relative path from the link
            # to the target
            target_path = os.path.relpath(target_abs_path,os.path.dirname(link_abs_path))
    os.symlink(target_path,link_name)

def chmod(target,mode):
    """Change mode of file or directory

    This a wrapper for the os.chmod function, with the
    addition that it doesn't follow symbolic links.

    For symbolic links it attempts to use the os.lchmod
    function instead, as this operates on the link
    itself and not the link target. If os.lchmod is not
    available then links are ignored.

    Arguments:
      target: file or directory to apply new mode to
      mode: a valid mode specifier e.g. 0775 or 0664

    """
    logging.debug("Changing mode of %s to %s" % (target,mode))
    try:
        if os.path.islink(target):
            # Try to use lchmod to operate on the link
            try:
                os.lchmod(target,mode)
            except AttributeError,ex:
                # lchmod is not available on all systems
                # If not then just ignore
                logging.debug("os.lchmod not available? Exception: %s" % ex)
        else:
            # Use os.chmod for everything else
            os.chmod(target,mode)
    except OSError, ex:
        logging.warning("Failed to change permissions on %s to %s: %s" % (target,mode,ex))

def touch(filename):
    """Create new empty file, or update modification time if already exists

    Arguments:
      filename: name of the file to create (can include leading path)

    """
    if not os.path.exists(filename):
        open(filename,'wb+').close()
    os.utime(filename,None)

def format_file_size(fsize,units=None):
    """Format a file size from bytes to human-readable form

    Takes a file size in bytes and returns a human-readable
    string, e.g. 4.0K, 186M, 1.5G.

    Alternatively specify the required units via the 'units'
    arguments.

    Arguments:
      fsize: size in bytes
      units: (optional) specify output in kb ('K'), Mb ('M'),
             Gb ('G') or Tb ('T')

    Returns:
      Human-readable version of file size.

    """
    # Return size in human readable form
    if units is not None:
        units = units.upper()
    fsize = float(fsize)/1024
    unit_list = 'KMGT'
    for unit in unit_list:
        if units is None:
            if fsize > 1024:
                fsize = fsize/1024
            else:
                break
        else:
            if units != unit:
                fsize = fsize/1024
            else:
                break
    return "%.1f%s" % (fsize,unit)
            
def commonprefix(path1,path2):
    """Determine common prefix path for path1 and path2

    Use this in preference to os.path.commonprefix as the version
    in os.path compares the two paths in a character-wise fashion
    and so can give counter-intuitive matches; this version compares
    path components which seems more sensible.

    For example: for two paths /mnt/dir1/file and /mnt/dir2/file,
    os.path.commonprefix will return /mnt/dir, whereas this function
    will return /mnt.

    Arguments:
      path1: first path in comparison
      path2: second path in comparison

    Returns:
      Leading part of path which is common to both input paths.
    """
    path1_components = str(path1).split(os.sep)
    path2_components = str(path2).split(os.sep)
    common_components = []
    ncomponents = min(len(path1_components),len(path2_components))
    for i in range(ncomponents):
        if path1_components[i] == path2_components[i]:
            common_components.append(path1_components[i])
        else:
            break
    commonprefix = "%s" % os.sep.join(common_components)
    return commonprefix

def is_gzipped_file(filename):
    """Check if a file has a .gz extension

    Arguments:
      filename: name of the file to be tested (can include leading path)

    Returns:
      True if filename has trailing .gz extension, False if not.

    """
    return os.path.splitext(filename)[1] == '.gz'

def rootname(name):
    """Remove all extensions from name

    Arguments:
      name: name of a file

    Returns:
      Leading part of name up to first dot, i.e. name without any
      trailing extensions.

    """
    try:
        i = name.index('.')
        return name[0:i]
    except ValueError:
        # No dot
        return name

def find_program(name):
    """Find a program on the PATH

    Search the current PATH for the specified program name and return
    the full path, or None if not found.

    """
    if os.path.isabs(name) and PathInfo(name).is_executable:
        return name
    for path in os.environ['PATH'].split(os.pathsep):
        name_path = os.path.abspath(os.path.join(path,name))
        if PathInfo(name_path).is_executable:
            return name_path
    return None

def get_current_user():
    """Return name of the current user

    Looks up user name for the current user; returns
    None if no matching name can be found.

    """
    try:
        return pwd.getpwuid(os.getuid()).pw_name
    except (KeyError,ValueError,OverflowError):
        return None

def get_user_from_uid(uid):
    """Return user name from UID

    Looks up user name matching the supplied UID;
    returns None if no matching name can be found.

    """
    try:
        return pwd.getpwuid(int(uid)).pw_name
    except (KeyError,ValueError,OverflowError):
        return None

def get_uid_from_user(user):
    """Return UID from user name

    Looks up UID matching the supplied user name;
    returns None if no matching name can be found.

    NB returned UID will be an integer.

    """
    try:
        return pwd.getpwnam(str(user)).pw_uid
    except KeyError:
        return None

def get_group_from_gid(gid):
{'fraction_non_alphanumeric': 0.10017389600529036, 'fraction_numerical': 0.003551397291141101, 'mean_word_length': 2.190092976013751, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2919345', 'n_tokens_mistral': 11442, 'n_tokens_neox': 10855, 'n_words': 4138}