<commit_before>#!/usr/bin/python
# 
# Copyright 2014 University of Southern California
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#    http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Raw network client for HTTP(S) communication with ERMREST service.
"""

import os
import subprocess
import json
import base64
import urlparse
from httplib import HTTPConnection, HTTPSConnection, HTTPException, OK, CREATED, ACCEPTED, NO_CONTENT, CONFLICT, FORBIDDEN, INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, BadStatusLine, GATEWAY_TIMEOUT, METHOD_NOT_ALLOWED
import sys
import traceback
import time
import shutil
import smtplib
import urllib
import re
from email.mime.text import MIMEText
from bioformats import BioformatsClient
from socket import gaierror, EAI_AGAIN
from lxml.etree import XMLSyntaxError
mail_footer = 'Do not reply to this message.  This is an automated message generated by the system, which does not receive email messages.'

class ErmrestHTTPException(Exception):
    def __init__(self, value, status):
        super(ErmrestHTTPException, self).__init__(value)
        self.value = value
        self.status = status
        
    def __str__(self):
        message = "%s." % self.value
        return message

class ErmrestException(Exception):
    def __init__(self, value, cause=None):
        super(ErmrestException, self).__init__(value)
        self.value = value
        self.cause = cause
        
    def __str__(self):
        message = "%s." % self.value
        if self.cause:
            message += " Caused by: %s." % self.cause
        return message

class MalformedURL(ErmrestException):
    """MalformedURL indicates a malformed URL.
    """
    def __init__(self, cause=None):
        super(MalformedURL, self).__init__("URL was malformed", cause)

class UnresolvedAddress(ErmrestException):
    """UnresolvedAddress indicates a failure to resolve the network address of
    the Ermrest service.
    
    This error is raised when a low-level socket.gaierror is caught.
    """
    def __init__(self, cause=None):
        super(UnresolvedAddress, self).__init__("Could not resolve address of host", cause)

class NetworkError(ErmrestException):
    """NetworkError wraps a socket.error exception.
    
    This error is raised when a low-level socket.error is caught.
    """
    def __init__(self, cause=None):
        super(NetworkError, self).__init__("Network I/O failure", cause)

class ProtocolError(ErmrestException):
    """ProtocolError indicates a protocol-level failure.
    
    In other words, you may have tried to add a tag for which no tagdef exists.
    """
    def __init__(self, message='Network protocol failure', errorno=-1, response=None, cause=None):
        super(ProtocolError, self).__init__("Ermrest protocol failure", cause)
        self._errorno = errorno
        self._response = response
        
    def __str__(self):
        message = "%s." % self.value
        if self._errorno >= 0:
            message += " HTTP ERROR %d: %s" % (self._errorno, self._response)
        return message
    
class NotFoundError(ErmrestException):
    """Raised for HTTP NOT_FOUND (i.e., ERROR 404) responses."""
    pass


class ErmrestClient (object):
    """Network client for ERMREST.
    """
    ## Derived from the ermrest iobox service client

    def __init__(self, **kwargs):
        self.metadata = kwargs.get("metadata")
        self.baseuri = kwargs.get("baseuri")
        o = urlparse.urlparse(self.baseuri)
        self.scheme = o[0]
        host_port = o[1].split(":")
        self.host = host_port[0]
        self.path = o.path
        self.port = None
        if len(host_port) > 1:
            self.port = host_port[1]
        self.use_goauth = kwargs.get("use_goauth")
        self.mail_server = kwargs.get("mail_server")
        self.mail_sender = kwargs.get("mail_sender")
        self.mail_receiver = kwargs.get("mail_receiver")
        self.logger = kwargs.get("logger")
        self.thumbnails = kwargs.get("thumbnails")
        self.dzi = kwargs.get("dzi")
        self.czi2dzi = kwargs.get("czi2dzi")
        self.viewer = kwargs.get("viewer")
        self.czi = kwargs.get("czi")
        self.czirules = kwargs.get("czirules")
        self.showinf = kwargs.get("showinf")
        self.data_scratch = kwargs.get("data_scratch")
        self.timeout = kwargs.get("timeout") * 60
        self.limit = kwargs.get("limit")
        self.hatrac = kwargs.get("hatrac")
        self.namespace = kwargs.get("namespace")
        self.cookie = kwargs.get("cookie")
        self.chunk_size = kwargs.get("chunk_size")
        self.header = None
        self.webconn = None
        self.logger.debug('Client initialized.')

    def send_request(self, method, url, body='', headers={}, reconnect=False):
        if self.header:
            headers.update(self.header)
        self.webconn.request(method, url, body, headers)
        try:
            resp = self.webconn.getresponse()
        except BadStatusLine:
            if not reconnect:
                self.close()
                self.connect(True)
                #self.sendMail('WARNING Tiles: HTTP BadStatusLine exception', 'The HTTPSConnection has been restarted\n')
                self.webconn.request(method, url, body, headers)
                resp = self.webconn.getresponse()
            else:
                raise
        if not reconnect and resp.status in [FORBIDDEN, INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT, METHOD_NOT_ALLOWED, CONFLICT]:
            self.close()
            self.connect(True)
            #self.sendMail('WARNING Tiles: HTTP exception: %d' % resp.status, 'The HTTPSConnection has been restarted\n')
            self.webconn.request(method, url, body, headers)
            resp = self.webconn.getresponse()
        if resp.status not in [OK, CREATED, ACCEPTED, NO_CONTENT]:
            raise ErmrestHTTPException("Error response (%i) received: %s" % (resp.status, resp.read()), resp.status)
        return resp

    def connect(self, reconnect=False):
        if self.scheme == 'https':
            self.webconn = HTTPSConnection(host=self.host, port=self.port)
        elif self.scheme == 'http':
            self.webconn = HTTPConnection(host=self.host, port=self.port)
        else:
            raise ValueError('Scheme %s is not supported.' % self.scheme)

        """
        if self.use_goauth:
            auth = base64.encodestring('%s:%s' % (self.username, self.password)).replace('\n', '')
            headers = dict(Authorization='Basic %s' % auth)
            resp = self.send_request('GET', '/service/nexus/goauth/token?grant_type=client_credentials', '', headers, reconnect)
            goauth = json.loads(resp.read())
            self.access_token = goauth['access_token']
            self.header = dict(Authorization='Globus-Goauthtoken %s' % self.access_token)
        else:
            #headers = {}
            #headers["Content-Type"] = "application/x-www-form-urlencoded"
            #resp = self.send_request("POST", "/ermrest/authn/session", "username=%s&password=%s" % (self.username, self.password), headers, reconnect)
            #self.header = dict(Cookie=resp.getheader("set-cookie"))
        """
        self.header = {'Cookie': self.cookie}
        
    def close(self):
        """Closes the connection to the Ermrest service.
        
        The underlying python documentation is not very helpful but it would
        appear that the HTTP[S]Connection.close() could raise a socket.error.
        Thus, this method potentially raises a 'NetworkError'.
        """
        assert self.webconn
        try:
            self.webconn.close()
        except socket.error as e:
            raise NetworkError(e)
        finally:
            self.webconn = None

    def writeThumbnailFile(self, slide_id, scan_id):
        scanDir='%s/%s' % (self.dzi, scan_id)
        channels = []
        for channel in os.listdir(scanDir):
            if os.path.isdir('%s%s%s' % (scanDir, os.sep, channel)):
               channels.append( channel)
        outdir = '%s/%s' % (self.thumbnails, slide_id)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        shutil.copyfile('%s/%s/%s/0/0_0.jpg' % (self.dzi, scan_id, channels[0]), '%s/%s.jpg' % (outdir, scan_id))
        thumbnail = '/thumbnails/%s/%s.jpg' % (slide_id, scan_id)
        urls = []
        for channel in channels:
            urls.append('url=/data/%s/%s/ImageProperties.xml' % (scan_id, channel))
        return (thumbnail, '&'.join(urls))
            
        
    def sendMail(self, subject, text):
        if self.mail_server and self.mail_sender and self.mail_receiver:
            retry = 0
            ready = False
            while not ready:
                try:
                    msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
                    msg['Subject'] = subject
                    msg['From'] = self.mail_sender
                    msg['To'] = self.mail_receiver
                    s = smtplib.SMTP(self.mail_server)
                    s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
                    s.quit()
                    self.logger.debug('Sent email notification')
                    ready = True
                except gaierror as e:
                    if e.errno == EAI_AGAIN:
                        time.sleep(100)
                        retry = retry + 1
                        ready = retry > 10
                    else:
                        ready = True
                    if ready:
                        et, ev, tb = sys.exc_info()
                        self.logger.error('got exception "%s"' % str(ev))
                        self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                except:
                    et, ev, tb = sys.exc_info()
                    self.logger.error('got exception "%s"' % str(ev))
                    self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                    ready = True

    def start(self):
        self.connect()
        ready = False
        while ready == False:
            try:
                self.processScans()
                ready = True
                #time.sleep(self.timeout)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
                raise
        
    def processScans(self):
        url = '%s/entity/Scan/!Filename::null::&DZI::null::&czi2dzi::null::@sort(%s::desc::)?limit=%d' % (self.path,urllib.quote('File Date', safe=''),self.limit)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        scans = json.loads(resp.read())
        scanids = []
        for scan in scans:
            if self.hasCziFile(scan['Slide ID'], scan['ID']) == True:
                scanids.append((scan['Slide ID'], scan['ID']))
            else:
                self.reportFailure(scan['Slide ID'], scan['ID'], 'missing file')
                
        self.logger.debug('Processing %d scans.' % (len(scanids))) 
        for slideId,scanId in scanids:
            f = self.getCziFile(slideId, scanId)
            self.logger.debug('Converting czi to dzi tiles for slide "%s", scan "%s"' % (slideId, scanId))
            
            try:
                currentDirectory=os.getcwd()
                os.chdir(self.dzi)
                args = [self.czi2dzi, f, scanId]
                p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                stdoutdata, stderrdata = p.communicate()
                returncode = p.returncode
                os.chdir(currentDirectory)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: czi2dzi ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                os.chdir(currentDirectory)
                returncode = 1
            
            if returncode != 0:
                self.logger.error('Can not convert czi to dzi for file "%s", slide "%s", scan "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (f, slideId, scanId, stdoutdata, stderrdata)) 
                self.sendMail('FAILURE Tiles', 'Can not convert czi to dzi for file "%s", slide "%s", scan "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (f, slideId, scanId, stdoutdata, stderrdata))
                os.remove(f)
                #os.rename('%s', '%s.err' % (f, f))
                """
                Update the Scan table with the failure result.
                """
                self.reportFailure(slideId, scanId, 'czi2dzi error')
                continue
            try:
                thumbnail,urls = self.writeThumbnailFile(slideId, scanId)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                os.remove(f)
                """
                Update the Scan table with the failure result.
                """
                self.reportFailure(slideId, scanId, 'DZI failure')
                continue
                
            self.logger.debug('Extracting metadata for slide "%s", scan "%s"' % (slideId, scanId)) 
            bioformatsClient = BioformatsClient(showinf=self.showinf, \
                                                czirules=self.czirules, \
                                                czifile=f, \
                                                logger=self.logger)
            columns = ["Thumbnail","DZI","czi2dzi"]
            try:
                metadata = bioformatsClient.getMetadata()
                returncode = 0
            except XMLSyntaxError:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                metadata = {}
                returncode = 1
                    
            if returncode == 0:
                self.logger.debug('Metadata: "%s"' % str(metadata)) 
                os.remove('temp.xml')
                columns.extend(self.metadata)
            
            os.remove(f)
            columns = ','.join([urllib.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Scan/ID;%s' % (self.path, columns)
            body = []
            obj = {'ID': scanId,
                   'Thumbnail': thumbnail,
                   'DZI': '/%s?%s' % (self.viewer, urls),
                   "czi2dzi": 'success'
                   }
            for col in self.metadata:
                if col in metadata and metadata[col] != None:
                    obj[col] = metadata[col]
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED created the tiles directory for the slide id "%s" and scan id "%s".' % (slideId, scanId)) 
            #self.sendMail('SUCCEEDED Tiles', 'The tiles directory for the slide id "%s" and scan id "%s" was created.\n' % (slideId, scanId))
        self.logger.debug('Ended Scans Processing.') 
        
    def reportFailure(self, slideId, scanId, error_message):
        """
            Update the Scan table with the czi2dzi failure result.
        """
        try:
            columns = ["Thumbnail","czi2dzi"]
            columns = ','.join([urllib.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Scan/ID;%s' % (self.path, columns)
            body = []
            obj = {'ID': scanId,
                   'Thumbnail': '/thumbnails/generic/generic_genetic.png',
                   "czi2dzi": '%s' % error_message
                   }
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED updated the Scan table for the slide id "%s" and scan id "%s" with the czi2dzi result.' % (slideId, scanId)) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            
        
    def hasCziFile(self, slideId=None, scanId=None):
        """
            Check if the file exists in hatrac
        """
        ret = False
        try:
            url = '%s/%s/%s/%s.czi' % (self.hatrac, self.namespace, urllib.quote(slideId, safe=''), urllib.quote(scanId, safe=''))
            headers = {'Accept': '*'}
            resp = self.send_request('HEAD', url, '', headers, False)
            resp.read()
            ret = True
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('HEAD exception "%s" for the slide id "%s" and scan id "%s"' % (str(ev), slideId, scanId))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: Missing file', '%s\n' % 'HEAD exception "%s" for the slide id "%s" and scan id "%s"' % (str(ev), slideId, scanId))
            pass
        return ret
                
    def getCziFile(self, slideId, scanId):
        cziFile = '%s/%s/%s.czi' % (self.data_scratch, self.namespace, scanId)
        outdir = '%s/%s' % (self.data_scratch, self.namespace)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        url = '%s/%s/%s/%s.czi;versions' % (self.hatrac, self.namespace, urllib.quote(slideId, safe=''), urllib.quote(scanId, safe=''))
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        srcFile = '%s%s'  % (self.czi, json.loads(resp.read())[0])
        shutil.copyfile(srcFile, cziFile)
        return cziFile

<commit_msg>Update also the "HTTP URL" column of the "Scan" table.<commit_after>#!/usr/bin/python
# 
# Copyright 2014 University of Southern California
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#    http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Raw network client for HTTP(S) communication with ERMREST service.
"""

import os
import subprocess
import json
import base64
import urlparse
from httplib import HTTPConnection, HTTPSConnection, HTTPException, OK, CREATED, ACCEPTED, NO_CONTENT, CONFLICT, FORBIDDEN, INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, BadStatusLine, GATEWAY_TIMEOUT, METHOD_NOT_ALLOWED
import sys
import traceback
import time
import shutil
import smtplib
import urllib
import re
from email.mime.text import MIMEText
from bioformats import BioformatsClient
from socket import gaierror, EAI_AGAIN
from lxml.etree import XMLSyntaxError
mail_footer = 'Do not reply to this message.  This is an automated message generated by the system, which does not receive email messages.'

class ErmrestHTTPException(Exception):
    def __init__(self, value, status):
        super(ErmrestHTTPException, self).__init__(value)
        self.value = value
        self.status = status
        
    def __str__(self):
        message = "%s." % self.value
        return message

class ErmrestException(Exception):
    def __init__(self, value, cause=None):
        super(ErmrestException, self).__init__(value)
        self.value = value
        self.cause = cause
        
    def __str__(self):
        message = "%s." % self.value
        if self.cause:
            message += " Caused by: %s." % self.cause
        return message

class MalformedURL(ErmrestException):
    """MalformedURL indicates a malformed URL.
    """
    def __init__(self, cause=None):
        super(MalformedURL, self).__init__("URL was malformed", cause)

class UnresolvedAddress(ErmrestException):
    """UnresolvedAddress indicates a failure to resolve the network address of
    the Ermrest service.
    
    This error is raised when a low-level socket.gaierror is caught.
    """
    def __init__(self, cause=None):
        super(UnresolvedAddress, self).__init__("Could not resolve address of host", cause)

class NetworkError(ErmrestException):
    """NetworkError wraps a socket.error exception.
    
    This error is raised when a low-level socket.error is caught.
    """
    def __init__(self, cause=None):
        super(NetworkError, self).__init__("Network I/O failure", cause)

class ProtocolError(ErmrestException):
    """ProtocolError indicates a protocol-level failure.
    
    In other words, you may have tried to add a tag for which no tagdef exists.
    """
    def __init__(self, message='Network protocol failure', errorno=-1, response=None, cause=None):
        super(ProtocolError, self).__init__("Ermrest protocol failure", cause)
        self._errorno = errorno
        self._response = response
        
    def __str__(self):
        message = "%s." % self.value
        if self._errorno >= 0:
            message += " HTTP ERROR %d: %s" % (self._errorno, self._response)
        return message
    
class NotFoundError(ErmrestException):
    """Raised for HTTP NOT_FOUND (i.e., ERROR 404) responses."""
    pass


class ErmrestClient (object):
    """Network client for ERMREST.
    """
    ## Derived from the ermrest iobox service client

    def __init__(self, **kwargs):
        self.metadata = kwargs.get("metadata")
        self.baseuri = kwargs.get("baseuri")
        o = urlparse.urlparse(self.baseuri)
        self.scheme = o[0]
        host_port = o[1].split(":")
        self.host = host_port[0]
        self.path = o.path
        self.port = None
        if len(host_port) > 1:
            self.port = host_port[1]
        self.use_goauth = kwargs.get("use_goauth")
        self.mail_server = kwargs.get("mail_server")
        self.mail_sender = kwargs.get("mail_sender")
        self.mail_receiver = kwargs.get("mail_receiver")
        self.logger = kwargs.get("logger")
        self.thumbnails = kwargs.get("thumbnails")
        self.dzi = kwargs.get("dzi")
        self.czi2dzi = kwargs.get("czi2dzi")
        self.viewer = kwargs.get("viewer")
        self.czi = kwargs.get("czi")
        self.czirules = kwargs.get("czirules")
        self.showinf = kwargs.get("showinf")
        self.data_scratch = kwargs.get("data_scratch")
        self.timeout = kwargs.get("timeout") * 60
        self.limit = kwargs.get("limit")
        self.hatrac = kwargs.get("hatrac")
        self.namespace = kwargs.get("namespace")
        self.cookie = kwargs.get("cookie")
        self.chunk_size = kwargs.get("chunk_size")
        self.header = None
        self.webconn = None
        self.logger.debug('Client initialized.')

    def send_request(self, method, url, body='', headers={}, reconnect=False):
        if self.header:
            headers.update(self.header)
        self.webconn.request(method, url, body, headers)
        try:
            resp = self.webconn.getresponse()
        except BadStatusLine:
            if not reconnect:
                self.close()
                self.connect(True)
                #self.sendMail('WARNING Tiles: HTTP BadStatusLine exception', 'The HTTPSConnection has been restarted\n')
                self.webconn.request(method, url, body, headers)
                resp = self.webconn.getresponse()
            else:
                raise
        if not reconnect and resp.status in [FORBIDDEN, INTERNAL_SERVER_ERROR, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT, METHOD_NOT_ALLOWED, CONFLICT]:
            self.close()
            self.connect(True)
            #self.sendMail('WARNING Tiles: HTTP exception: %d' % resp.status, 'The HTTPSConnection has been restarted\n')
            self.webconn.request(method, url, body, headers)
            resp = self.webconn.getresponse()
        if resp.status not in [OK, CREATED, ACCEPTED, NO_CONTENT]:
            raise ErmrestHTTPException("Error response (%i) received: %s" % (resp.status, resp.read()), resp.status)
        return resp

    def connect(self, reconnect=False):
        if self.scheme == 'https':
            self.webconn = HTTPSConnection(host=self.host, port=self.port)
        elif self.scheme == 'http':
            self.webconn = HTTPConnection(host=self.host, port=self.port)
        else:
            raise ValueError('Scheme %s is not supported.' % self.scheme)

        """
        if self.use_goauth:
            auth = base64.encodestring('%s:%s' % (self.username, self.password)).replace('\n', '')
            headers = dict(Authorization='Basic %s' % auth)
            resp = self.send_request('GET', '/service/nexus/goauth/token?grant_type=client_credentials', '', headers, reconnect)
            goauth = json.loads(resp.read())
            self.access_token = goauth['access_token']
            self.header = dict(Authorization='Globus-Goauthtoken %s' % self.access_token)
        else:
            #headers = {}
            #headers["Content-Type"] = "application/x-www-form-urlencoded"
            #resp = self.send_request("POST", "/ermrest/authn/session", "username=%s&password=%s" % (self.username, self.password), headers, reconnect)
            #self.header = dict(Cookie=resp.getheader("set-cookie"))
        """
        self.header = {'Cookie': self.cookie}
        
    def close(self):
        """Closes the connection to the Ermrest service.
        
        The underlying python documentation is not very helpful but it would
        appear that the HTTP[S]Connection.close() could raise a socket.error.
        Thus, this method potentially raises a 'NetworkError'.
        """
        assert self.webconn
        try:
            self.webconn.close()
        except socket.error as e:
            raise NetworkError(e)
        finally:
            self.webconn = None

    def writeThumbnailFile(self, slide_id, scan_id):
        scanDir='%s/%s' % (self.dzi, scan_id)
        channels = []
        for channel in os.listdir(scanDir):
            if os.path.isdir('%s%s%s' % (scanDir, os.sep, channel)):
               channels.append( channel)
        outdir = '%s/%s' % (self.thumbnails, slide_id)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        shutil.copyfile('%s/%s/%s/0/0_0.jpg' % (self.dzi, scan_id, channels[0]), '%s/%s.jpg' % (outdir, scan_id))
        thumbnail = '/thumbnails/%s/%s.jpg' % (slide_id, scan_id)
        urls = []
        for channel in channels:
            urls.append('url=/data/%s/%s/ImageProperties.xml' % (scan_id, channel))
        return (thumbnail, '&'.join(urls))
            
        
    def sendMail(self, subject, text):
        if self.mail_server and self.mail_sender and self.mail_receiver:
            retry = 0
            ready = False
            while not ready:
                try:
                    msg = MIMEText('%s\n\n%s' % (text, mail_footer), 'plain')
                    msg['Subject'] = subject
                    msg['From'] = self.mail_sender
                    msg['To'] = self.mail_receiver
                    s = smtplib.SMTP(self.mail_server)
                    s.sendmail(self.mail_sender, self.mail_receiver.split(','), msg.as_string())
                    s.quit()
                    self.logger.debug('Sent email notification')
                    ready = True
                except gaierror as e:
                    if e.errno == EAI_AGAIN:
                        time.sleep(100)
                        retry = retry + 1
                        ready = retry > 10
                    else:
                        ready = True
                    if ready:
                        et, ev, tb = sys.exc_info()
                        self.logger.error('got exception "%s"' % str(ev))
                        self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                except:
                    et, ev, tb = sys.exc_info()
                    self.logger.error('got exception "%s"' % str(ev))
                    self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                    ready = True

    def start(self):
        self.connect()
        ready = False
        while ready == False:
            try:
                self.processScans()
                ready = True
                #time.sleep(self.timeout)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: unexpected exception', '%s\nThe process might have been stopped\n' % str(traceback.format_exception(et, ev, tb)))
                raise
        
    def processScans(self):
        url = '%s/entity/Scan/!Filename::null::&DZI::null::&czi2dzi::null::@sort(%s::desc::)?limit=%d' % (self.path,urllib.quote('File Date', safe=''),self.limit)
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        scans = json.loads(resp.read())
        scanids = []
        for scan in scans:
            if self.hasCziFile(scan['Slide ID'], scan['ID']) == True:
                scanids.append((scan['Slide ID'], scan['ID']))
            else:
                self.reportFailure(scan['Slide ID'], scan['ID'], 'missing file')
                
        self.logger.debug('Processing %d scans.' % (len(scanids))) 
        for slideId,scanId in scanids:
            f = self.getCziFile(slideId, scanId)
            self.logger.debug('Converting czi to dzi tiles for slide "%s", scan "%s"' % (slideId, scanId))
            
            try:
                currentDirectory=os.getcwd()
                os.chdir(self.dzi)
                args = [self.czi2dzi, f, scanId]
                p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                stdoutdata, stderrdata = p.communicate()
                returncode = p.returncode
                os.chdir(currentDirectory)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: czi2dzi ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                os.chdir(currentDirectory)
                returncode = 1
            
            if returncode != 0:
                self.logger.error('Can not convert czi to dzi for file "%s", slide "%s", scan "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (f, slideId, scanId, stdoutdata, stderrdata)) 
                self.sendMail('FAILURE Tiles', 'Can not convert czi to dzi for file "%s", slide "%s", scan "%s".\nstdoutdata: %s\nstderrdata: %s\n' % (f, slideId, scanId, stdoutdata, stderrdata))
                os.remove(f)
                #os.rename('%s', '%s.err' % (f, f))
                """
                Update the Scan table with the failure result.
                """
                self.reportFailure(slideId, scanId, 'czi2dzi error')
                continue
            try:
                thumbnail,urls = self.writeThumbnailFile(slideId, scanId)
            except:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: write thumbnail ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                os.remove(f)
                """
                Update the Scan table with the failure result.
                """
                self.reportFailure(slideId, scanId, 'DZI failure')
                continue
                
            self.logger.debug('Extracting metadata for slide "%s", scan "%s"' % (slideId, scanId)) 
            bioformatsClient = BioformatsClient(showinf=self.showinf, \
                                                czirules=self.czirules, \
                                                czifile=f, \
                                                logger=self.logger)
            columns = ["Thumbnail","DZI","czi2dzi", "HTTP URL"]
            try:
                metadata = bioformatsClient.getMetadata()
                returncode = 0
            except XMLSyntaxError:
                et, ev, tb = sys.exc_info()
                self.logger.error('got unexpected exception "%s"' % str(ev))
                self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
                self.sendMail('FAILURE Tiles: XMLSyntaxError', '%s\n' % str(traceback.format_exception(et, ev, tb)))
                metadata = {}
                returncode = 1
                    
            if returncode == 0:
                self.logger.debug('Metadata: "%s"' % str(metadata)) 
                os.remove('temp.xml')
                columns.extend(self.metadata)
            
            os.remove(f)
            columns = ','.join([urllib.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Scan/ID;%s' % (self.path, columns)
            body = []
            obj = {'ID': scanId,
                   'Thumbnail': thumbnail,
                   'DZI': '/%s?%s' % (self.viewer, urls),
                   'HTTP URL': '%s/%s/%s/%s.czi' % (self.hatrac, self.namespace, slideId, scanId),
                   "czi2dzi": 'success'
                   }
            for col in self.metadata:
                if col in metadata and metadata[col] != None:
                    obj[col] = metadata[col]
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED created the tiles directory for the slide id "%s" and scan id "%s".' % (slideId, scanId)) 
            #self.sendMail('SUCCEEDED Tiles', 'The tiles directory for the slide id "%s" and scan id "%s" was created.\n' % (slideId, scanId))
        self.logger.debug('Ended Scans Processing.') 
        
    def reportFailure(self, slideId, scanId, error_message):
        """
            Update the Scan table with the czi2dzi failure result.
        """
        try:
            columns = ["Thumbnail","czi2dzi"]
            columns = ','.join([urllib.quote(col, safe='') for col in columns])
            url = '%s/attributegroup/Scan/ID;%s' % (self.path, columns)
            body = []
            obj = {'ID': scanId,
                   'Thumbnail': '/thumbnails/generic/generic_genetic.png',
                   "czi2dzi": '%s' % error_message
                   }
            body.append(obj)
            headers = {'Content-Type': 'application/json'}
            resp = self.send_request('PUT', url, json.dumps(body), headers, False)
            resp.read()
            self.logger.debug('SUCCEEDED updated the Scan table for the slide id "%s" and scan id "%s" with the czi2dzi result.' % (slideId, scanId)) 
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('got unexpected exception "%s"' % str(ev))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: reportFailure ERROR', '%s\n' % str(traceback.format_exception(et, ev, tb)))
            
        
    def hasCziFile(self, slideId=None, scanId=None):
        """
            Check if the file exists in hatrac
        """
        ret = False
        try:
            url = '%s/%s/%s/%s.czi' % (self.hatrac, self.namespace, urllib.quote(slideId, safe=''), urllib.quote(scanId, safe=''))
            headers = {'Accept': '*'}
            resp = self.send_request('HEAD', url, '', headers, False)
            resp.read()
            ret = True
        except:
            et, ev, tb = sys.exc_info()
            self.logger.error('HEAD exception "%s" for the slide id "%s" and scan id "%s"' % (str(ev), slideId, scanId))
            self.logger.error('%s' % str(traceback.format_exception(et, ev, tb)))
            self.sendMail('FAILURE Tiles: Missing file', '%s\n' % 'HEAD exception "%s" for the slide id "%s" and scan id "%s"' % (str(ev), slideId, scanId))
            pass
        return ret
                
    def getCziFile(self, slideId, scanId):
        cziFile = '%s/%s/%s.czi' % (self.data_scratch, self.namespace, scanId)
        outdir = '%s/%s' % (self.data_scratch, self.namespace)
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        url = '%s/%s/%s/%s.czi;versions' % (self.hatrac, self.namespace, urllib.quote(slideId, safe=''), urllib.quote(scanId, safe=''))
        headers = {'Content-Type': 'application/json', 'Accept': 'application/json'}
        resp = self.send_request('GET', url, '', headers, False)
        srcFile = '%s%s'  % (self.czi, json.loads(resp.read())[0])
        shutil.copyfile(srcFile, cziFile)
        return cziFile


{'fraction_non_alphanumeric': 0.12602190343976555, 'fraction_numerical': 0.002725075839374775, 'mean_word_length': 2.036137995629098, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7228315', 'n_tokens_mistral': 11464, 'n_tokens_neox': 11199, 'n_words': 2784}