<commit_before>/*
 * Copyright 2014-2015 OpenEstate.org.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.openestate.is24.restapi.utils;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import javax.xml.bind.JAXBException;
import oauth.signpost.exception.OAuthException;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOExceptionWithCause;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.openestate.is24.restapi.AbstractClient;
import org.openestate.is24.restapi.ImportExport;
import org.openestate.is24.restapi.xml.common.Attachment;
import org.openestate.is24.restapi.xml.common.Attachments;
import org.openestate.is24.restapi.xml.common.Link;
import org.openestate.is24.restapi.xml.common.Message;
import org.openestate.is24.restapi.xml.common.MessageCode;
import org.openestate.is24.restapi.xml.common.Messages;
import org.openestate.is24.restapi.xml.common.PDFDocument;
import org.openestate.is24.restapi.xml.common.Picture;
import org.openestate.is24.restapi.xml.common.PublishChannel;
import org.openestate.is24.restapi.xml.common.PublishChannels;
import org.openestate.is24.restapi.xml.common.PublishObject;
import org.openestate.is24.restapi.xml.common.PublishObjects;
import org.openestate.is24.restapi.xml.common.RealEstateState;
import org.openestate.is24.restapi.xml.common.RealtorContactDetails;
import org.openestate.is24.restapi.xml.common.StreamingVideo;
import org.openestate.is24.restapi.xml.offerlistelement.OfferRealEstateForList;
import org.openestate.is24.restapi.xml.offerlistelement.RealEstateList;
import org.openestate.is24.restapi.xml.realestates.RealEstate;
import org.openestate.is24.restapi.xml.realestates.RealEstates;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Handler for bulk exports.
 * <p>
 * The {@link ExportHandler} is part of the high level API for bulk exports of real
 * estate data.
 * <p>
 * A previously created {@link ExportPool} can be processed with the
 * {@link ExportHandler#export(org.openestate.is24.restapi.AbstractClient, org.openestate.is24.restapi.utils.ExportPool, boolean, boolean)}
 * method. The {@link ExportHandler} will launch the required low level
 * operations of the {@link ImportExport}-API for each pooled object.
 *
 * @since 0.2
 * @author <NAME> <<EMAIL>>
 */
public class ExportHandler
{
  private final static Logger LOGGER = LoggerFactory.getLogger( ExportHandler.class );
  private final List<ExportMessage> messages = new ArrayList<ExportMessage>();
  private final List<String> savedContactIds = new ArrayList<String>();
  private final Map<String,Long> duplicatedContactIds = new HashMap<String, Long>();
  private AbstractClient client = null;
  private ExportPool pool = null;
  private long progress = 0;
  private long totalProgress = 0;
  private boolean useNewEnergySourceEnev2014Values = true;

  /**
   * Creates a new {@link ExportHandler}.
   */
  public ExportHandler()
  {
  }

  /**
   * Calback method to track progress during the export process.
   * <p>
   * This method may be overridden by inheriting classes in order to track the
   * progress of the export process.
   *
   * @param value
   * value of additional progress
   */
  protected final void addProgress( long value )
  {
    this.setProgress( this.progress + Math.abs( value ) );
  }

  /**
   * Callback method to check, if a contact should be updated in the export process.
   *
   * @param contact
   * contact person to update
   *
   * @param poolContactId
   * id of the contact person within export pool
   *
   * @return
   * true, if the contact person can be ignored in the export process
   */
  protected boolean canIgnoreContact( RealtorContactDetails contact, String poolContactId )
  {
    return false;
  }

  /**
   * Callback method to check, if a real estate should be updated in the export process.
   *
   * @param object
   * real estate to update
   *
   * @param poolObjectId
   * id of the real estate within export pool
   *
   * @return
   * true, if the real estate can be ignored in the export process
   */
  protected boolean canIgnoreObject( RealEstate object, String poolObjectId )
  {
    return false;
  }

  /**
   * Archivates a real estate object at the Webservice.
   *
   * @param externalObjectId
   * external real estate ID
   *
   * @throws IOException
   * if the operation failed
   */
  protected final void doArchiveObject( String externalObjectId ) throws IOException
  {
    // Immobilie ermitteln
    final RealEstate is24Object;
    try
    {
      is24Object = ImportExport.RealEstateService.getByExternalId(
        this.client, externalObjectId );
      if (is24Object==null)
      {
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING,
          "Property '" + externalObjectId + "' is not available anymore at the Webservice!" );
        return;
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (RequestFailedException ex)
    {
      LOGGER.error( "Can't get property '" + externalObjectId + "' from the Webservice!" );
      logMessagesAsError( ex.responseMessages );
      LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      this.putObjectMessage(
        externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING, ex.responseMessages );
      return;
    }
    this.doArchiveObject( is24Object );
  }

  /**
   * Archivates a real estate object at the Webservice.
   *
   * @param is24ObjectId
   * real estate ID by IS24
   *
   * @param externalObjectId
   * external real estate ID
   *
   * @throws IOException
   * if the operation failed
   */
  protected final void doArchiveObject( long is24ObjectId, String externalObjectId ) throws IOException
  {
    // Immobilie ermitteln
    final RealEstate is24Object;
    try
    {
      is24Object = ImportExport.RealEstateService.getByIs24Id(
        this.client, is24ObjectId );
      if (is24Object==null)
      {
        if (!StringUtils.isBlank( externalObjectId ))
        {
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING,
            "Property '" + externalObjectId + "' is not available anymore at the Webservice!" );
        }
        else
        {
          this.putGeneralMessage(
            ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING,
            "Property (" + is24ObjectId + ") is not available anymore at the Webservice!" );
        }
        return;
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (RequestFailedException ex)
    {
      if (!StringUtils.isBlank( externalObjectId ))
      {
        LOGGER.error( "Can't get property '" + externalObjectId + "' (" + is24ObjectId + ") from the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING, ex.responseMessages );
      }
      else
      {
        LOGGER.error( "Can't get property (" + is24ObjectId + ") from the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putGeneralMessage(
          ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING, ex.responseMessages );
      }
      return;
    }
    this.doArchiveObject( is24Object );
  }

  /**
   * Archivates a real estate object at the Webservice.
   *
   * @param is24Object
   * real estate to archivate
   *
   * @throws IOException
   * if the operation failed
   */
  protected void doArchiveObject( RealEstate is24Object ) throws IOException
  {
    try
    {
      final Long is24ObjectId = is24Object.getId();
      final String externalObjectId = StringUtils.trimToNull( is24Object.getExternalId() );

      // aktuelle Veröffentlichungen zur Immobilie ermitteln
      PublishObjects is24Publishings = null;
      try
      {
        is24Publishings = ImportExport.PublishService.get(
          this.client, is24ObjectId, 0 );
      }
      catch (RequestFailedException ex)
      {
        if (!StringUtils.isBlank( externalObjectId ))
        {
          LOGGER.error( "Can't get publishings of property '" + externalObjectId + "' (" + is24ObjectId + ") from the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_PUBLISHINGS_NOT_FOUND, ex.responseMessages );
        }
        else
        {
          LOGGER.error( "Can't get publishings of property (" + is24ObjectId + ") from the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putGeneralMessage(
            ExportMessage.Code.OBJECT_PUBLISHINGS_NOT_FOUND, ex.responseMessages );
        }
      }

      // keine Veröffentlichungen gefunden
      if (is24Publishings==null)
      {
        if (!StringUtils.isBlank( externalObjectId ))
        {
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_PUBLISHINGS_NOT_FOUND,
            "Can't get publishings of property '" + externalObjectId + "' (" + is24ObjectId + ") from the Webservice!" );
        }
        else
        {
          this.putGeneralMessage(
            ExportMessage.Code.OBJECT_PUBLISHINGS_NOT_FOUND,
            "Can't get publishings of property (" + is24ObjectId + ") from the Webservice!" );
        }
      }

      // ggf. Veröffentlichungen der Immobilie entfernen
      else
      {
        for (PublishObject publishing : is24Publishings.getPublishObject())
        {
          Long is24ChannelId = publishing.getPublishChannel().getId();
          String publishId = StringUtils.trimToNull( publishing.getId() );
          try
          {
            ImportExport.PublishService.delete(
              this.client, publishId );
          }
          catch (RequestFailedException ex)
          {
            if (!StringUtils.isBlank( externalObjectId ))
            {
              LOGGER.error( "Can't unpublish property '" + externalObjectId + "' (" + is24ObjectId + ") "
                + "in channel '" + publishing.getPublishChannel().getTitle() + "' (" + is24ChannelId +  ") at the Webservice!" );
              logMessagesAsError( ex.responseMessages );
              LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
              this.putObjectMessage(
                externalObjectId, ExportMessage.Code.OBJECT_NOT_UNPUBLISHED, ex.responseMessages );
            }
            else
            {
              LOGGER.error( "Can't unpublish property (" + is24ObjectId + ") "
                + "in channel '" + publishing.getPublishChannel().getTitle() + "' (" + is24ChannelId +  ") at the Webservice!" );
              logMessagesAsError( ex.responseMessages );
              LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
              this.putGeneralMessage(
                ExportMessage.Code.OBJECT_NOT_UNPUBLISHED, ex.responseMessages );
            }
          }
        }
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Downloads an {@link URL} into a {@link File}.
   *
   * @param url
   * URL to download
   *
   * @return
   * downloaded file
   *
   * @throws IOException
   * if the operation failed
   */
  protected File doDownloadFile( URL url ) throws IOException
  {
    if (url==null) return null;
    LOGGER.info( "downloading " + url );
    InputStream input = null;
    OutputStream output = null;
    try
    {
      input = url.openStream();
      File tempFile = File.createTempFile( "is24-export-attachment-", ".bin" );
      tempFile.deleteOnExit();
      output = new FileOutputStream( tempFile );
      IOUtils.copy( input, output );
      output.flush();
      return tempFile;
    }
    finally
    {
      IOUtils.closeQuietly( output );
      IOUtils.closeQuietly( input );
    }
  }

  /**
   * Returns internal and external ID's of real estates from the Webservice,
   * that were not changed during the current export process.
   *
   * @return
   * mapping of internal and external ID's of untouched real estates
   *
   * @throws IOException
   * if the operation failed
   */
  protected Map<Long,String> doListUntouchedObjects() throws IOException
  {
    try
    {
      final Map<Long,String> ids = new TreeMap<Long,String>();

      // Immobilien im Bestand ermitteln
      int page = 1;
      while (true)
      {
        RealEstates is24Objects;
        try
        {
          is24Objects = ImportExport.RealEstateService.getAll(
            this.client, null, null, 0, page, false );
          if (is24Objects==null)
          {
            this.putGeneralMessage(
              ExportMessage.Code.OBJECTS_NOT_FOUND,
              "Can't get available properties from the Webservice!" );
            return new HashMap<Long,String>();
          }
        }
        catch (RequestFailedException ex)
        {
          LOGGER.error( "Can't get available properties from the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putGeneralMessage(
            ExportMessage.Code.OBJECTS_NOT_FOUND, ex.responseMessages );
          return new HashMap<Long,String>();
        }

        Integer totalPages = is24Objects.getPaging().getNumberOfPages();
        if (totalPages==null) totalPages = 0;

        RealEstateList is24ObjectList = is24Objects.getRealEstateList();
        if (is24ObjectList!=null && !is24ObjectList.getRealEstateElement().isEmpty())
        {
          for (OfferRealEstateForList is24Object : is24ObjectList.getRealEstateElement())
          {
            // Immobilien nur löschen,
            // wenn diese als "aktiv" bei IS24 markiert sind.
            //
            // TODO: Es ist hier nicht ganz klar, aus welchem XML-Element
            // der Immobilien-Status zu ermitteln ist.
            boolean isActive = RealEstateState.ACTIVE.equals( is24Object.getState() )
              || RealEstateState.ACTIVE.equals( is24Object.getRealEstateState() );
            if (!isActive) continue;

            // Immobilien nur zurückliefern,
            // wenn diese beim Exportvorgang nicht verändert wurden.
            long is24ObjectId = is24Object.getId();
            String externalObjectId = is24Object.getExternalId();
            if (!ids.containsKey( is24ObjectId ) && !this.pool.hasObjectForExport( externalObjectId ))
            {
              ids.put( is24ObjectId, externalObjectId );
            }
          }
        }
        if (page>=totalPages) break;
        page++;
      }

      return ids;
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Returns publish channels for the authorized agency.
   *
   * @return
   * publish channels
   *
   * @throws IOException
   * if the operation failed
   */
  protected PublishChannels doLoadPublishChannels() throws IOException
  {
    try
    {
      return ImportExport.PublishChannelService.get( this.client );
    }
    catch (RequestFailedException ex)
    {
      LOGGER.error( "Can't get publish channels from the Webservice!" );
      logMessagesAsError( ex.responseMessages );
      LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      this.putGeneralMessage(
        ExportMessage.Code.PUBLISH_CHANNELS_NOT_FOUND, ex.responseMessages );
      return null;
    }
    catch (JAXBException ex)
    {
      //LOGGER.error( "Can't read / write XML while communicating with the Webservice!" );
      //LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      //LOGGER.error( "Can't authorize at the Webservice!" );
      //LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      //LOGGER.error( "Can't communicate with the Webservice!" );
      //LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Removes a real estate object from the Webservice.
   *
   * @param externalObjectId
   * external real estate ID
   *
   * @throws IOException
   * if the operation failed
   */
  protected void doRemoveObject( String externalObjectId ) throws IOException
  {
    try
    {
      // Löschung durchführen
      try
      {
        ImportExport.RealEstateService.deleteByExternalId(
          this.client, externalObjectId );
      }
      catch (RequestFailedException ex)
      {
        LOGGER.error( "Can't delete property '" + externalObjectId + "' at the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_NOT_REMOVED, ex.responseMessages );
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Removes a real estate object from the Webservice.
   *
   * @param is24ObjectId
   * real estate ID by IS24
   *
   * @param externalObjectId
   * external real estate ID
   *
   * @throws IOException
   * if the operation failed
   */
  protected void doRemoveObject( long is24ObjectId, String externalObjectId ) throws IOException
  {
    try
    {
      // Löschung durchführen
      try
      {
        ImportExport.RealEstateService.deleteByIs24Id(
          this.client, is24ObjectId );
      }
      catch (RequestFailedException ex)
      {
        if (!StringUtils.isBlank( externalObjectId ))
        {
          LOGGER.error( "Can't delete property '" + externalObjectId + "' (" + is24ObjectId + ") at the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_NOT_REMOVED, ex.responseMessages );
        }
        else
        {
          LOGGER.error( "Can't delete property (" + is24ObjectId + ") at the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putGeneralMessage(
            ExportMessage.Code.OBJECT_NOT_REMOVED, ex.responseMessages );
        }
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Saves a contact person to the Webservice.
   *
   * @param contact
   * contact to save
   *
   * @param poolContactId
   * contact ID within export pool
   *
   * @return
   * ID of the processed contact person in the Webservice or null, if the object
   * was not updated
   *
   * @throws IOException
   * if the operation failed
   */
  protected Long doUpdateContact( RealtorContactDetails contact, String poolContactId ) throws IOException
  {
    final String externalContactId = contact.getExternalId();
    try
    {
      // prüfen, ob ein Ansprechpartner mit der externen ID bereits im Webservice existiert
      final RealtorContactDetails oldIs24Contact;
      try
      {
        oldIs24Contact = ImportExport.ContactAddressService.getByExternalId(
          this.client, externalContactId );
      }
      catch (RequestFailedException ex)
      {
        LOGGER.error( "Can't get contact person '" + externalContactId + "' from the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putContactMessage(
          externalContactId, ExportMessage.Code.CONTACT_NOT_FOUND, ex.responseMessages );
        return null;
      }

      Long is24ContactId;

      // neuen Ansprechpartner erstellen
      if (oldIs24Contact==null)
      {
        try
        {
          is24ContactId = ImportExport.ContactAddressService.post(
            this.client, contact );
        }
        catch (RequestFailedException ex)
        {
          Resource resource = Resource.getMessageResource( ex.responseMessages );
          if (resource!=null && "duplicated contactDetails".equalsIgnoreCase( resource.type ) && resource.id>0)
          {
            LOGGER.info( "contact '" + externalContactId + "' is already "
              + "available at the Webservice with ID " + resource.id + "." );
            this.duplicatedContactIds.put( externalContactId, resource.id );
            is24ContactId = resource.id;
          }
          else
          {
            LOGGER.error( "Can't add contact person '" + externalContactId + "' to the Webservice!" );
            logMessagesAsError( ex.responseMessages );
            LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
            this.putContactMessage(
              externalContactId, ExportMessage.Code.CONTACT_NOT_SAVED, ex.responseMessages );
            return null;
          }
        }
      }

      // bestehenden Ansprechpartner aktualisieren
      else
      {
        is24ContactId = oldIs24Contact.getId();
        contact.setId( is24ContactId );

        try
        {
          ImportExport.ContactAddressService.putByIs24Id(
            this.client, contact, is24ContactId );
        }
        catch (RequestFailedException ex)
        {
          Resource resource = Resource.getMessageResource( ex.responseMessages );
          if (resource!=null && "duplicated contactDetails".equalsIgnoreCase( resource.type ) && resource.id>0)
          {
            LOGGER.info( "contact '" + externalContactId + "' is already "
              + "available at the Webservice with ID " + resource.id + "." );
            this.duplicatedContactIds.put( externalContactId, resource.id );
            return resource.id;
          }
          else
          {
            LOGGER.error( "Can't update contact person '" + externalContactId + "' at the Webservice!" );
            logMessagesAsError( ex.responseMessages );
            LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
            this.putContactMessage(
              externalContactId, ExportMessage.Code.CONTACT_NOT_SAVED, ex.responseMessages );
            return null;
          }
        }
      }

      // ID des Ansprechpartners als erfolgreich exportiert vormerken
      this.savedContactIds.add( externalContactId );

      // ID des verarbeiteten Ansprechpartners bei IS24 zurückliefern
      return is24ContactId;
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
    finally
    {
      // Fortschritt protokollieren
      this.addProgress(
        this.pool.getContactSize( poolContactId, true ) );
    }
  }

  /**
   * Saves a real estate to the Webservice.
   *
   * @param object
   * real estate to save
   *
   * @param is24PublishChannels
   * channels, where the real estate should be published
   *
   * @param poolObjectId
   * real estate ID within export pool
   *
   * @return
   * ID of the processed real estate in the Webservice or null, if the object
   * was not updated
   *
   * @throws IOException
   * if the operation failed
   */
  protected Long doUpdateObject( RealEstate object, PublishChannels is24PublishChannels, String poolObjectId ) throws IOException
  {
    final String externalObjectId = object.getExternalId();
    final org.openestate.is24.restapi.xml.common.ObjectFactory commonFactory =
      new org.openestate.is24.restapi.xml.common.ObjectFactory();
    final org.openestate.is24.restapi.xml.realestates.ObjectFactory realEstatesFactory =
      new org.openestate.is24.restapi.xml.realestates.ObjectFactory();
    final org.openestate.is24.restapi.xml.attachmentsorder.ObjectFactory attachmentsorderFactory =
      new org.openestate.is24.restapi.xml.attachmentsorder.ObjectFactory();

    object.setRealEstateState( RealEstateState.ACTIVE );

    // Ansprechpartner zuweisen, wenn nicht bereits explizit eine intene
    // Ansprechpartner-ID hinterlegt wurde
    Long is24ContactId = (object.getContact()!=null)?
      object.getContact().getId(): null;
    if (is24ContactId==null || is24ContactId.longValue()<1)
    {
      String externalContactId = (object.getContact()!=null)?
        StringUtils.trimToNull( object.getContact().getExternalId() ): null;

      // Duplikat des Ansprechpartners verwenden
      if (externalContactId!=null && this.duplicatedContactIds.containsKey( externalContactId ))
      {
        is24ContactId = this.duplicatedContactIds.get( externalContactId );
        if (object.getContact()==null)
          object.setContact( realEstatesFactory.createRealEstateContact() );
        object.getContact().setId( is24ContactId );
        object.getContact().setExternalId( null );
      }

      // sicherstellen, dass der Ansprechpartner vorher erfolgreich während des Transports exportiert wurde
      else if (externalContactId!=null && !this.savedContactIds.contains( externalContactId ))
      {
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_WITHOUT_CONTACT,
          "The contact '" + externalContactId + "' was not saved during the export process." );
        object.setContact( null );
      }
    }

    try
    {
      // prüfen, ob eine Immobilie mit der externen ID bereits im Webservice existiert
      RealEstate oldIs24Object;
      try
      {
        oldIs24Object = ImportExport.RealEstateService.getByExternalId(
          this.client, externalObjectId );
      }
      catch (RequestFailedException ex)
      {
        LOGGER.error( "Can't get property '" + externalObjectId + "' from the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND, ex.responseMessages );

        // Fortschritt protokollieren
        this.addProgress(
          this.pool.getObjectSize( poolObjectId, true ) );

        return null;
      }

      //Messages responseMessages;
      final Long is24ObjectId;

      // Immobilie im Portal löschen,
      // wenn diese bereits im Portal existiert
      // und einer anderen Rubrik zugewiesen ist
      if (oldIs24Object!=null && !oldIs24Object.getClass().getName().equals( object.getClass().getName() ))
      {
        //LOGGER.debug( "<NAME>" );
        //LOGGER.debug( "> für Immobilie #" + oldIs24Object.getId() );
        //LOGGER.debug( "> alte Rubrik " + oldIs24Object.getClass().getName() );
        //LOGGER.debug( "> neue Rubrik " + object.getClass().getName() );
        try
        {
          ImportExport.RealEstateService.deleteByIs24Id(
            this.client, oldIs24Object.getId() );
          oldIs24Object = null;
        }
        catch (RequestFailedException ex)
        {
          LOGGER.error( "Can't delete property '" + externalObjectId + "' (" + oldIs24Object.getId() + ") at the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_NOT_SAVED, ex.responseMessages );

          // Fortschritt protokollieren
          this.addProgress(
            this.pool.getObjectSize( poolObjectId, true ) );

          return null;
        }
      }

      // neue Immobilie erstellen
      if (oldIs24Object==null)
      {
        try
        {
          is24ObjectId = ImportExport.RealEstateService.post(
            this.client, object, this.isUseNewEnergySourceEnev2014Values() );
        }
        catch (RequestFailedException ex)
        {
          LOGGER.error( "Can't add property '" + externalObjectId + "' to the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_NOT_SAVED, ex.responseMessages );

          // Fortschritt protokollieren
          this.addProgress(
            this.pool.getObjectSize( poolObjectId, true ) );

          return null;
        }
      }

      // bestehende Immobilie aktualisieren
      else
      {
        is24ObjectId = oldIs24Object.getId();
        object.setId( is24ObjectId );

        try
        {
          ImportExport.RealEstateService.putByIs24Id(
            this.client, object, is24ObjectId, this.isUseNewEnergySourceEnev2014Values() );
        }
        catch (RequestFailedException ex)
        {
          LOGGER.error( "Can't update property '" + externalObjectId + "' (" + is24ObjectId + ") at the Webservice!" );<commit_msg>added some lines for debugging<commit_after>/*
 * Copyright 2014-2015 OpenEstate.org.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.openestate.is24.restapi.utils;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import javax.xml.bind.JAXBException;
import oauth.signpost.exception.OAuthException;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOExceptionWithCause;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.openestate.is24.restapi.AbstractClient;
import org.openestate.is24.restapi.ImportExport;
import org.openestate.is24.restapi.xml.common.Attachment;
import org.openestate.is24.restapi.xml.common.Attachments;
import org.openestate.is24.restapi.xml.common.Link;
import org.openestate.is24.restapi.xml.common.Message;
import org.openestate.is24.restapi.xml.common.MessageCode;
import org.openestate.is24.restapi.xml.common.Messages;
import org.openestate.is24.restapi.xml.common.PDFDocument;
import org.openestate.is24.restapi.xml.common.Picture;
import org.openestate.is24.restapi.xml.common.PublishChannel;
import org.openestate.is24.restapi.xml.common.PublishChannels;
import org.openestate.is24.restapi.xml.common.PublishObject;
import org.openestate.is24.restapi.xml.common.PublishObjects;
import org.openestate.is24.restapi.xml.common.RealEstateState;
import org.openestate.is24.restapi.xml.common.RealtorContactDetails;
import org.openestate.is24.restapi.xml.common.StreamingVideo;
import org.openestate.is24.restapi.xml.offerlistelement.OfferRealEstateForList;
import org.openestate.is24.restapi.xml.offerlistelement.RealEstateList;
import org.openestate.is24.restapi.xml.realestates.RealEstate;
import org.openestate.is24.restapi.xml.realestates.RealEstates;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Handler for bulk exports.
 * <p>
 * The {@link ExportHandler} is part of the high level API for bulk exports of real
 * estate data.
 * <p>
 * A previously created {@link ExportPool} can be processed with the
 * {@link ExportHandler#export(org.openestate.is24.restapi.AbstractClient, org.openestate.is24.restapi.utils.ExportPool, boolean, boolean)}
 * method. The {@link ExportHandler} will launch the required low level
 * operations of the {@link ImportExport}-API for each pooled object.
 *
 * @since 0.2
 * @author <NAME> <<EMAIL>>
 */
public class ExportHandler
{
  private final static Logger LOGGER = LoggerFactory.getLogger( ExportHandler.class );
  private final List<ExportMessage> messages = new ArrayList<ExportMessage>();
  private final List<String> savedContactIds = new ArrayList<String>();
  private final Map<String,Long> duplicatedContactIds = new HashMap<String, Long>();
  private AbstractClient client = null;
  private ExportPool pool = null;
  private long progress = 0;
  private long totalProgress = 0;
  private boolean useNewEnergySourceEnev2014Values = true;

  /**
   * Creates a new {@link ExportHandler}.
   */
  public ExportHandler()
  {
  }

  /**
   * Calback method to track progress during the export process.
   * <p>
   * This method may be overridden by inheriting classes in order to track the
   * progress of the export process.
   *
   * @param value
   * value of additional progress
   */
  protected final void addProgress( long value )
  {
    this.setProgress( this.progress + Math.abs( value ) );
  }

  /**
   * Callback method to check, if a contact should be updated in the export process.
   *
   * @param contact
   * contact person to update
   *
   * @param poolContactId
   * id of the contact person within export pool
   *
   * @return
   * true, if the contact person can be ignored in the export process
   */
  protected boolean canIgnoreContact( RealtorContactDetails contact, String poolContactId )
  {
    return false;
  }

  /**
   * Callback method to check, if a real estate should be updated in the export process.
   *
   * @param object
   * real estate to update
   *
   * @param poolObjectId
   * id of the real estate within export pool
   *
   * @return
   * true, if the real estate can be ignored in the export process
   */
  protected boolean canIgnoreObject( RealEstate object, String poolObjectId )
  {
    return false;
  }

  /**
   * Archivates a real estate object at the Webservice.
   *
   * @param externalObjectId
   * external real estate ID
   *
   * @throws IOException
   * if the operation failed
   */
  protected final void doArchiveObject( String externalObjectId ) throws IOException
  {
    // Immobilie ermitteln
    final RealEstate is24Object;
    try
    {
      is24Object = ImportExport.RealEstateService.getByExternalId(
        this.client, externalObjectId );
      if (is24Object==null)
      {
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING,
          "Property '" + externalObjectId + "' is not available anymore at the Webservice!" );
        return;
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (RequestFailedException ex)
    {
      LOGGER.error( "Can't get property '" + externalObjectId + "' from the Webservice!" );
      logMessagesAsError( ex.responseMessages );
      LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      this.putObjectMessage(
        externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING, ex.responseMessages );
      return;
    }
    this.doArchiveObject( is24Object );
  }

  /**
   * Archivates a real estate object at the Webservice.
   *
   * @param is24ObjectId
   * real estate ID by IS24
   *
   * @param externalObjectId
   * external real estate ID
   *
   * @throws IOException
   * if the operation failed
   */
  protected final void doArchiveObject( long is24ObjectId, String externalObjectId ) throws IOException
  {
    // Immobilie ermitteln
    final RealEstate is24Object;
    try
    {
      is24Object = ImportExport.RealEstateService.getByIs24Id(
        this.client, is24ObjectId );
      if (is24Object==null)
      {
        if (!StringUtils.isBlank( externalObjectId ))
        {
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING,
            "Property '" + externalObjectId + "' is not available anymore at the Webservice!" );
        }
        else
        {
          this.putGeneralMessage(
            ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING,
            "Property (" + is24ObjectId + ") is not available anymore at the Webservice!" );
        }
        return;
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (RequestFailedException ex)
    {
      if (!StringUtils.isBlank( externalObjectId ))
      {
        LOGGER.error( "Can't get property '" + externalObjectId + "' (" + is24ObjectId + ") from the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING, ex.responseMessages );
      }
      else
      {
        LOGGER.error( "Can't get property (" + is24ObjectId + ") from the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putGeneralMessage(
          ExportMessage.Code.OBJECT_NOT_FOUND_FOR_ARCHIVING, ex.responseMessages );
      }
      return;
    }
    this.doArchiveObject( is24Object );
  }

  /**
   * Archivates a real estate object at the Webservice.
   *
   * @param is24Object
   * real estate to archivate
   *
   * @throws IOException
   * if the operation failed
   */
  protected void doArchiveObject( RealEstate is24Object ) throws IOException
  {
    try
    {
      final Long is24ObjectId = is24Object.getId();
      final String externalObjectId = StringUtils.trimToNull( is24Object.getExternalId() );

      // aktuelle Veröffentlichungen zur Immobilie ermitteln
      PublishObjects is24Publishings = null;
      try
      {
        is24Publishings = ImportExport.PublishService.get(
          this.client, is24ObjectId, 0 );
      }
      catch (RequestFailedException ex)
      {
        if (!StringUtils.isBlank( externalObjectId ))
        {
          LOGGER.error( "Can't get publishings of property '" + externalObjectId + "' (" + is24ObjectId + ") from the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_PUBLISHINGS_NOT_FOUND, ex.responseMessages );
        }
        else
        {
          LOGGER.error( "Can't get publishings of property (" + is24ObjectId + ") from the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putGeneralMessage(
            ExportMessage.Code.OBJECT_PUBLISHINGS_NOT_FOUND, ex.responseMessages );
        }
      }

      // keine Veröffentlichungen gefunden
      if (is24Publishings==null)
      {
        if (!StringUtils.isBlank( externalObjectId ))
        {
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_PUBLISHINGS_NOT_FOUND,
            "Can't get publishings of property '" + externalObjectId + "' (" + is24ObjectId + ") from the Webservice!" );
        }
        else
        {
          this.putGeneralMessage(
            ExportMessage.Code.OBJECT_PUBLISHINGS_NOT_FOUND,
            "Can't get publishings of property (" + is24ObjectId + ") from the Webservice!" );
        }
      }

      // ggf. Veröffentlichungen der Immobilie entfernen
      else
      {
        for (PublishObject publishing : is24Publishings.getPublishObject())
        {
          Long is24ChannelId = publishing.getPublishChannel().getId();
          String publishId = StringUtils.trimToNull( publishing.getId() );
          try
          {
            ImportExport.PublishService.delete(
              this.client, publishId );
          }
          catch (RequestFailedException ex)
          {
            if (!StringUtils.isBlank( externalObjectId ))
            {
              LOGGER.error( "Can't unpublish property '" + externalObjectId + "' (" + is24ObjectId + ") "
                + "in channel '" + publishing.getPublishChannel().getTitle() + "' (" + is24ChannelId +  ") at the Webservice!" );
              logMessagesAsError( ex.responseMessages );
              LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
              this.putObjectMessage(
                externalObjectId, ExportMessage.Code.OBJECT_NOT_UNPUBLISHED, ex.responseMessages );
            }
            else
            {
              LOGGER.error( "Can't unpublish property (" + is24ObjectId + ") "
                + "in channel '" + publishing.getPublishChannel().getTitle() + "' (" + is24ChannelId +  ") at the Webservice!" );
              logMessagesAsError( ex.responseMessages );
              LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
              this.putGeneralMessage(
                ExportMessage.Code.OBJECT_NOT_UNPUBLISHED, ex.responseMessages );
            }
          }
        }
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Downloads an {@link URL} into a {@link File}.
   *
   * @param url
   * URL to download
   *
   * @return
   * downloaded file
   *
   * @throws IOException
   * if the operation failed
   */
  protected File doDownloadFile( URL url ) throws IOException
  {
    if (url==null) return null;
    LOGGER.info( "downloading " + url );
    InputStream input = null;
    OutputStream output = null;
    try
    {
      input = url.openStream();
      File tempFile = File.createTempFile( "is24-export-attachment-", ".bin" );
      tempFile.deleteOnExit();
      output = new FileOutputStream( tempFile );
      IOUtils.copy( input, output );
      output.flush();
      return tempFile;
    }
    finally
    {
      IOUtils.closeQuietly( output );
      IOUtils.closeQuietly( input );
    }
  }

  /**
   * Returns internal and external ID's of real estates from the Webservice,
   * that were not changed during the current export process.
   *
   * @return
   * mapping of internal and external ID's of untouched real estates
   *
   * @throws IOException
   * if the operation failed
   */
  protected Map<Long,String> doListUntouchedObjects() throws IOException
  {
    try
    {
      final Map<Long,String> ids = new TreeMap<Long,String>();

      // Immobilien im Bestand ermitteln
      int page = 1;
      while (true)
      {
        RealEstates is24Objects;
        try
        {
          is24Objects = ImportExport.RealEstateService.getAll(
            this.client, null, null, 0, page, false );
          if (is24Objects==null)
          {
            this.putGeneralMessage(
              ExportMessage.Code.OBJECTS_NOT_FOUND,
              "Can't get available properties from the Webservice!" );
            return new HashMap<Long,String>();
          }
        }
        catch (RequestFailedException ex)
        {
          LOGGER.error( "Can't get available properties from the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putGeneralMessage(
            ExportMessage.Code.OBJECTS_NOT_FOUND, ex.responseMessages );
          return new HashMap<Long,String>();
        }

        Integer totalPages = is24Objects.getPaging().getNumberOfPages();
        if (totalPages==null) totalPages = 0;

        RealEstateList is24ObjectList = is24Objects.getRealEstateList();
        if (is24ObjectList!=null && !is24ObjectList.getRealEstateElement().isEmpty())
        {
          for (OfferRealEstateForList is24Object : is24ObjectList.getRealEstateElement())
          {
            // Immobilien nur löschen,
            // wenn diese als "aktiv" bei IS24 markiert sind.
            //
            // TODO: Es ist hier nicht ganz klar, aus welchem XML-Element
            // der Immobilien-Status zu ermitteln ist.
            boolean isActive = RealEstateState.ACTIVE.equals( is24Object.getState() )
              || RealEstateState.ACTIVE.equals( is24Object.getRealEstateState() );
            if (!isActive) continue;

            // Immobilien nur zurückliefern,
            // wenn diese beim Exportvorgang nicht verändert wurden.
            long is24ObjectId = is24Object.getId();
            String externalObjectId = is24Object.getExternalId();
            if (!ids.containsKey( is24ObjectId ) && !this.pool.hasObjectForExport( externalObjectId ))
            {
              ids.put( is24ObjectId, externalObjectId );
            }
          }
        }
        if (page>=totalPages) break;
        page++;
      }

      return ids;
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Returns publish channels for the authorized agency.
   *
   * @return
   * publish channels
   *
   * @throws IOException
   * if the operation failed
   */
  protected PublishChannels doLoadPublishChannels() throws IOException
  {
    try
    {
      return ImportExport.PublishChannelService.get( this.client );
    }
    catch (RequestFailedException ex)
    {
      LOGGER.error( "Can't get publish channels from the Webservice!" );
      logMessagesAsError( ex.responseMessages );
      LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      this.putGeneralMessage(
        ExportMessage.Code.PUBLISH_CHANNELS_NOT_FOUND, ex.responseMessages );
      return null;
    }
    catch (JAXBException ex)
    {
      //LOGGER.error( "Can't read / write XML while communicating with the Webservice!" );
      //LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      //LOGGER.error( "Can't authorize at the Webservice!" );
      //LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      //LOGGER.error( "Can't communicate with the Webservice!" );
      //LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Removes a real estate object from the Webservice.
   *
   * @param externalObjectId
   * external real estate ID
   *
   * @throws IOException
   * if the operation failed
   */
  protected void doRemoveObject( String externalObjectId ) throws IOException
  {
    try
    {
      // Löschung durchführen
      try
      {
        ImportExport.RealEstateService.deleteByExternalId(
          this.client, externalObjectId );
      }
      catch (RequestFailedException ex)
      {
        LOGGER.error( "Can't delete property '" + externalObjectId + "' at the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_NOT_REMOVED, ex.responseMessages );
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Removes a real estate object from the Webservice.
   *
   * @param is24ObjectId
   * real estate ID by IS24
   *
   * @param externalObjectId
   * external real estate ID
   *
   * @throws IOException
   * if the operation failed
   */
  protected void doRemoveObject( long is24ObjectId, String externalObjectId ) throws IOException
  {
    try
    {
      // Löschung durchführen
      try
      {
        ImportExport.RealEstateService.deleteByIs24Id(
          this.client, is24ObjectId );
      }
      catch (RequestFailedException ex)
      {
        if (!StringUtils.isBlank( externalObjectId ))
        {
          LOGGER.error( "Can't delete property '" + externalObjectId + "' (" + is24ObjectId + ") at the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_NOT_REMOVED, ex.responseMessages );
        }
        else
        {
          LOGGER.error( "Can't delete property (" + is24ObjectId + ") at the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putGeneralMessage(
            ExportMessage.Code.OBJECT_NOT_REMOVED, ex.responseMessages );
        }
      }
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
  }

  /**
   * Saves a contact person to the Webservice.
   *
   * @param contact
   * contact to save
   *
   * @param poolContactId
   * contact ID within export pool
   *
   * @return
   * ID of the processed contact person in the Webservice or null, if the object
   * was not updated
   *
   * @throws IOException
   * if the operation failed
   */
  protected Long doUpdateContact( RealtorContactDetails contact, String poolContactId ) throws IOException
  {
    final String externalContactId = contact.getExternalId();
    try
    {
      // prüfen, ob ein Ansprechpartner mit der externen ID bereits im Webservice existiert
      final RealtorContactDetails oldIs24Contact;
      try
      {
        oldIs24Contact = ImportExport.ContactAddressService.getByExternalId(
          this.client, externalContactId );
      }
      catch (RequestFailedException ex)
      {
        LOGGER.error( "Can't get contact person '" + externalContactId + "' from the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putContactMessage(
          externalContactId, ExportMessage.Code.CONTACT_NOT_FOUND, ex.responseMessages );
        return null;
      }

      Long is24ContactId;

      // neuen Ansprechpartner erstellen
      if (oldIs24Contact==null)
      {
        try
        {
          is24ContactId = ImportExport.ContactAddressService.post(
            this.client, contact );
        }
        catch (RequestFailedException ex)
        {
          Resource resource = Resource.getMessageResource( ex.responseMessages );
          if (resource!=null && "duplicated contactDetails".equalsIgnoreCase( resource.type ) && resource.id>0)
          {
            LOGGER.info( "contact '" + externalContactId + "' is already "
              + "available at the Webservice with ID " + resource.id + "." );
            this.duplicatedContactIds.put( externalContactId, resource.id );
            is24ContactId = resource.id;
          }
          else
          {
            LOGGER.error( "Can't add contact person '" + externalContactId + "' to the Webservice!" );
            logMessagesAsError( ex.responseMessages );
            LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
            this.putContactMessage(
              externalContactId, ExportMessage.Code.CONTACT_NOT_SAVED, ex.responseMessages );
            return null;
          }
        }
      }

      // bestehenden Ansprechpartner aktualisieren
      else
      {
        is24ContactId = oldIs24Contact.getId();
        contact.setId( is24ContactId );

        try
        {
          ImportExport.ContactAddressService.putByIs24Id(
            this.client, contact, is24ContactId );
        }
        catch (RequestFailedException ex)
        {
          Resource resource = Resource.getMessageResource( ex.responseMessages );
          if (resource!=null && "duplicated contactDetails".equalsIgnoreCase( resource.type ) && resource.id>0)
          {
            LOGGER.info( "contact '" + externalContactId + "' is already "
              + "available at the Webservice with ID " + resource.id + "." );
            this.duplicatedContactIds.put( externalContactId, resource.id );
            return resource.id;
          }
          else
          {
            LOGGER.error( "Can't update contact person '" + externalContactId + "' at the Webservice!" );
            logMessagesAsError( ex.responseMessages );
            LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
            this.putContactMessage(
              externalContactId, ExportMessage.Code.CONTACT_NOT_SAVED, ex.responseMessages );
            return null;
          }
        }
      }

      // ID des Ansprechpartners als erfolgreich exportiert vormerken
      this.savedContactIds.add( externalContactId );

      // ID des verarbeiteten Ansprechpartners bei IS24 zurückliefern
      return is24ContactId;
    }
    catch (JAXBException ex)
    {
      throw new IOExceptionWithCause(
        "Can't read / write XML while communicating with the Webservice!", ex );
    }
    catch (OAuthException ex)
    {
      throw new IOExceptionWithCause(
        "Authorization failed!", ex );
    }
    catch (IOException ex)
    {
      throw new IOExceptionWithCause(
        "Communication failed!", ex );
    }
    finally
    {
      // Fortschritt protokollieren
      this.addProgress(
        this.pool.getContactSize( poolContactId, true ) );
    }
  }

  /**
   * Saves a real estate to the Webservice.
   *
   * @param object
   * real estate to save
   *
   * @param is24PublishChannels
   * channels, where the real estate should be published
   *
   * @param poolObjectId
   * real estate ID within export pool
   *
   * @return
   * ID of the processed real estate in the Webservice or null, if the object
   * was not updated
   *
   * @throws IOException
   * if the operation failed
   */
  protected Long doUpdateObject( RealEstate object, PublishChannels is24PublishChannels, String poolObjectId ) throws IOException
  {
    final String externalObjectId = object.getExternalId();
    final org.openestate.is24.restapi.xml.common.ObjectFactory commonFactory =
      new org.openestate.is24.restapi.xml.common.ObjectFactory();
    final org.openestate.is24.restapi.xml.realestates.ObjectFactory realEstatesFactory =
      new org.openestate.is24.restapi.xml.realestates.ObjectFactory();
    final org.openestate.is24.restapi.xml.attachmentsorder.ObjectFactory attachmentsorderFactory =
      new org.openestate.is24.restapi.xml.attachmentsorder.ObjectFactory();

    object.setRealEstateState( RealEstateState.ACTIVE );

    // Ansprechpartner zuweisen, wenn nicht bereits explizit eine intene
    // Ansprechpartner-ID hinterlegt wurde
    Long is24ContactId = (object.getContact()!=null)?
      object.getContact().getId(): null;
    if (is24ContactId==null || is24ContactId.longValue()<1)
    {
      String externalContactId = (object.getContact()!=null)?
        StringUtils.trimToNull( object.getContact().getExternalId() ): null;

      // Duplikat des Ansprechpartners verwenden
      if (externalContactId!=null && this.duplicatedContactIds.containsKey( externalContactId ))
      {
        is24ContactId = this.duplicatedContactIds.get( externalContactId );
        if (object.getContact()==null)
          object.setContact( realEstatesFactory.createRealEstateContact() );
        object.getContact().setId( is24ContactId );
        object.getContact().setExternalId( null );
      }

      // sicherstellen, dass der Ansprechpartner vorher erfolgreich während des Transports exportiert wurde
      else if (externalContactId!=null && !this.savedContactIds.contains( externalContactId ))
      {
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_WITHOUT_CONTACT,
          "The contact '" + externalContactId + "' was not saved during the export process." );
        object.setContact( null );
      }
    }

    try
    {
      // prüfen, ob eine Immobilie mit der externen ID bereits im Webservice existiert
      RealEstate oldIs24Object;
      try
      {
        oldIs24Object = ImportExport.RealEstateService.getByExternalId(
          this.client, externalObjectId );
      }
      catch (RequestFailedException ex)
      {
        LOGGER.error( "Can't get property '" + externalObjectId + "' from the Webservice!" );
        logMessagesAsError( ex.responseMessages );
        LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
        this.putObjectMessage(
          externalObjectId, ExportMessage.Code.OBJECT_NOT_FOUND, ex.responseMessages );

        // Fortschritt protokollieren
        this.addProgress(
          this.pool.getObjectSize( poolObjectId, true ) );

        return null;
      }

      //Messages responseMessages;
      final Long is24ObjectId;

      // Immobilie im Portal löschen,
      // wenn diese bereits im Portal existiert
      // und einer anderen Rubrik zugewiesen ist
      if (oldIs24Object!=null && !oldIs24Object.getClass().getName().equals( object.getClass().getName() ))
      {
        //LOGGER.debug( "<NAME>" );
        //LOGGER.debug( "> für Immobilie #" + oldIs24Object.getId() );
        //LOGGER.debug( "> alte Rubrik " + oldIs24Object.getClass().getName() );
        //LOGGER.debug( "> neue Rubrik " + object.getClass().getName() );
        try
        {
          ImportExport.RealEstateService.deleteByIs24Id(
            this.client, oldIs24Object.getId() );
          oldIs24Object = null;
        }
        catch (RequestFailedException ex)
        {
          LOGGER.error( "Can't delete property '" + externalObjectId + "' (" + oldIs24Object.getId() + ") at the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_NOT_SAVED, ex.responseMessages );

          // Fortschritt protokollieren
          this.addProgress(
            this.pool.getObjectSize( poolObjectId, true ) );

          return null;
        }
      }

      // neue Immobilie erstellen
      if (oldIs24Object==null)
      {
        try
        {
          is24ObjectId = ImportExport.RealEstateService.post(
            this.client, object, this.isUseNewEnergySourceEnev2014Values() );
          //LOGGER.debug( "created object with IS24-ID #" + is24ObjectId );
        }
        catch (RequestFailedException ex)
        {
          LOGGER.error( "Can't add property '" + externalObjectId + "' to the Webservice!" );
          logMessagesAsError( ex.responseMessages );
          LOGGER.error( "> " + ex.getLocalizedMessage(), ex );
          this.putObjectMessage(
            externalObjectId, ExportMessage.Code.OBJECT_NOT_SAVED, ex.responseMessages );

          // Fortschritt protokollieren
          this.addProgress(
            this.pool.getObjectSize( poolObjectId, true ) );

          return null;
        }
      }

      // bestehende Immobilie aktualisieren
      else
      {
        is24ObjectId = oldIs24Object.getId();
        object.setId( is24ObjectId );

        try
        {
          ImportExport.RealEstateService.putByIs24Id(
            this.client, object, is24ObjectId, this.isUseNewEnergySourceEnev2014Values() );
          //LOGGER.debug( "updated object with IS24-ID #" + is24ObjectId );
        }
        catch (RequestFailedException ex)
        {
          LOGGER.error( "Can't update property '" + externalObjectId + "' (" + is24ObjectId + ") at the Webservice!" );
{'fraction_non_alphanumeric': 0.09621734979733008, 'fraction_numerical': 0.00948401333395934, 'mean_word_length': 2.505678389202831, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 52}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6651815', 'n_tokens_mistral': 18526, 'n_tokens_neox': 18042, 'n_words': 4322}