<commit_before>/*
 * Copyright 2012 SFB 632.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package annis.sqlgen;

import annis.exceptions.AnnisQLSemanticsException;
import annis.service.objects.FrequencyTable;
import annis.model.QueryNode;
import annis.ql.parser.QueryData;
import annis.service.objects.FrequencyTableEntry;
import annis.service.objects.FrequencyTableEntryType;
import static annis.sqlgen.AbstractSqlGenerator.TABSTOP;
import static annis.sqlgen.TableAccessStrategy.NODE_ANNOTATION_TABLE;
import static annis.sqlgen.TableAccessStrategy.NODE_TABLE;
import annis.sqlgen.extensions.FrequencyTableQueryData;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.springframework.dao.DataAccessException;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class FrequencySqlGenerator extends AbstractSqlGenerator
  implements WhereClauseSqlGenerator<QueryData>,
  SelectClauseSqlGenerator<QueryData>,
  GroupByClauseSqlGenerator<QueryData>, FromClauseSqlGenerator<QueryData>,
  SqlGeneratorAndExtractor<QueryData, FrequencyTable>
{

  private SolutionSqlGenerator solutionSqlGenerator;
  
  @Override
  public FrequencyTable extractData(ResultSet rs) throws SQLException, DataAccessException
  {
    FrequencyTable result = new FrequencyTable();

    ResultSetMetaData meta = rs.getMetaData();

    while (rs.next())
    {
      Validate.isTrue(meta.getColumnCount() > 1,
        "frequency table extractor needs at least 2 columns");

      Validate.isTrue(
        "count".equalsIgnoreCase(meta.getColumnName(meta.getColumnCount())),
        "last column name must be \"count\"");

      long count = rs.getLong("count");
      String[] tupel = new String[meta.getColumnCount() - 1];

      for (int i = 1; i <= tupel.length; i++)
      {
        String colVal = rs.getString(i);
        tupel[i - 1] = colVal;
      } // end for each column (except last "count" column) 

      result.addEntry(new FrequencyTable.Entry(tupel, count));

    } // end for complete result

    return result;
  }

  @Override
  public String groupByAttributes(QueryData queryData,
    List<QueryNode> alternative)
  {
    FrequencyTableQueryData ext;
    List<FrequencyTableQueryData> freqQueryData = queryData.getExtensions(
      FrequencyTableQueryData.class);
    Validate.notNull(freqQueryData);
    Validate.notEmpty(freqQueryData);
    ext = freqQueryData.get(0);

    StringBuilder sb = new StringBuilder();
    for (int i = 1; i <= ext.size(); i++)
    {
      sb.append("value").append(i);
      if (i < ext.size())
      {
        sb.append(", ");
      }
    }

    return sb.toString();
  }

  @Override
  public String fromClause(QueryData queryData, List<QueryNode> alternative,
    String indent)
  {
    FrequencyTableQueryData ext;
    List<FrequencyTableQueryData> freqQueryData = queryData.getExtensions(
      FrequencyTableQueryData.class);
    Validate.notNull(freqQueryData);
    Validate.notEmpty(freqQueryData);
    ext = freqQueryData.get(0);

    StringBuilder sb = new StringBuilder();

    sb.append(indent).append("(\n");
    sb.append(indent);

    sb.append(getSolutionSqlGenerator().toSql(queryData, indent + TABSTOP));
    sb.append(indent).append(") AS solutions,\n");

    int i = 1;
    Iterator<FrequencyTableEntry> itEntry = ext.iterator();
    while (itEntry.hasNext())
    {
      FrequencyTableEntry e = itEntry.next();

      sb.append(indent).append(TABSTOP);

      String factsSql = SelectedFactsFromClauseGenerator.selectedFactsSQL(
        queryData.getCorpusList(), indent);
      
      sb.append(factsSql);
      sb.append(" AS v").append(i);

      if (itEntry.hasNext())
      {
        sb.append(",");
      }

      sb.append("\n");

      i++;
    }
    return sb.toString();
  }

  @Override
  public String selectClause(QueryData queryData, List<QueryNode> alternative,
    String indent)
  {
    TableAccessStrategy tas = tables(null);

    FrequencyTableQueryData ext;
    List<FrequencyTableQueryData> freqQueryData = queryData.getExtensions(
      FrequencyTableQueryData.class);
    Validate.notNull(freqQueryData);
    Validate.notEmpty(freqQueryData);
    ext = freqQueryData.get(0);

    StringBuilder sb = new StringBuilder();
    int i = 1;
    for (FrequencyTableEntry e : ext)
    {
      if (e.getType() == FrequencyTableEntryType.annotation)
      {
        sb
          .append("(splitanno(")
          .append("v").append(i).append(".")
          .append(tas.columnName(NODE_ANNOTATION_TABLE, "qannotext"))
          .append("))[3]");
      }
      else
      {
        sb.append("v").append(i).append(".").append(
          tas.columnName(NODE_TABLE,
            "span"));
      }
      sb.append(" AS value").append(i).append(", ");
      i++;
    }

    sb.append("count(*) AS \"count\"");
<commit_msg>don't call "splitanno" in select clause, instead to the splitting in the extractResult function<commit_after>/*
 * Copyright 2012 SFB 632.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package annis.sqlgen;

import annis.exceptions.AnnisQLSemanticsException;
import annis.service.objects.FrequencyTable;
import annis.model.QueryNode;
import annis.ql.parser.QueryData;
import annis.service.objects.FrequencyTableEntry;
import annis.service.objects.FrequencyTableEntryType;
import static annis.sqlgen.AbstractSqlGenerator.TABSTOP;
import static annis.sqlgen.TableAccessStrategy.NODE_ANNOTATION_TABLE;
import static annis.sqlgen.TableAccessStrategy.NODE_TABLE;
import annis.sqlgen.extensions.FrequencyTableQueryData;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Validate;
import org.springframework.dao.DataAccessException;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class FrequencySqlGenerator extends AbstractSqlGenerator
  implements WhereClauseSqlGenerator<QueryData>,
  SelectClauseSqlGenerator<QueryData>,
  GroupByClauseSqlGenerator<QueryData>, FromClauseSqlGenerator<QueryData>,
  SqlGeneratorAndExtractor<QueryData, FrequencyTable>
{

  private SolutionSqlGenerator solutionSqlGenerator;
  
  @Override
  public FrequencyTable extractData(ResultSet rs) throws SQLException, DataAccessException
  {
    FrequencyTable result = new FrequencyTable();

    ResultSetMetaData meta = rs.getMetaData();

    while (rs.next())
    {
      Validate.isTrue(meta.getColumnCount() > 1,
        "frequency table extractor needs at least 2 columns");

      Validate.isTrue(
        "count".equalsIgnoreCase(meta.getColumnName(meta.getColumnCount())),
        "last column name must be \"count\"");

      long count = rs.getLong("count");
      String[] tupel = new String[meta.getColumnCount() - 1];

      for (int i = 1; i <= tupel.length; i++)
      {
        String colVal = rs.getString(i);
        
        String[] splitted = colVal.split(":", 3);
        if(splitted.length > 0)
        {
          colVal = splitted[splitted.length-1];
        }

        tupel[i - 1] = colVal;
      } // end for each column (except last "count" column) 

      result.addEntry(new FrequencyTable.Entry(tupel, count));

    } // end for complete result

    return result;
  }

  @Override
  public String groupByAttributes(QueryData queryData,
    List<QueryNode> alternative)
  {
    FrequencyTableQueryData ext;
    List<FrequencyTableQueryData> freqQueryData = queryData.getExtensions(
      FrequencyTableQueryData.class);
    Validate.notNull(freqQueryData);
    Validate.notEmpty(freqQueryData);
    ext = freqQueryData.get(0);

    StringBuilder sb = new StringBuilder();
    for (int i = 1; i <= ext.size(); i++)
    {
      sb.append("value").append(i);
      if (i < ext.size())
      {
        sb.append(", ");
      }
    }

    return sb.toString();
  }

  @Override
  public String fromClause(QueryData queryData, List<QueryNode> alternative,
    String indent)
  {
    FrequencyTableQueryData ext;
    List<FrequencyTableQueryData> freqQueryData = queryData.getExtensions(
      FrequencyTableQueryData.class);
    Validate.notNull(freqQueryData);
    Validate.notEmpty(freqQueryData);
    ext = freqQueryData.get(0);

    StringBuilder sb = new StringBuilder();

    sb.append(indent).append("(\n");
    sb.append(indent);

    sb.append(getSolutionSqlGenerator().toSql(queryData, indent + TABSTOP));
    sb.append(indent).append(") AS solutions,\n");

    int i = 1;
    Iterator<FrequencyTableEntry> itEntry = ext.iterator();
    while (itEntry.hasNext())
    {
      FrequencyTableEntry e = itEntry.next();

      sb.append(indent).append(TABSTOP);

      String factsSql = SelectedFactsFromClauseGenerator.selectedFactsSQL(
        queryData.getCorpusList(), indent);
      
      sb.append(factsSql);
      sb.append(" AS v").append(i);

      if (itEntry.hasNext())
      {
        sb.append(",");
      }

      sb.append("\n");

      i++;
    }
    return sb.toString();
  }

  @Override
  public String selectClause(QueryData queryData, List<QueryNode> alternative,
    String indent)
  {
    TableAccessStrategy tas = tables(null);

    FrequencyTableQueryData ext;
    List<FrequencyTableQueryData> freqQueryData = queryData.getExtensions(
      FrequencyTableQueryData.class);
    Validate.notNull(freqQueryData);
    Validate.notEmpty(freqQueryData);
    ext = freqQueryData.get(0);

    StringBuilder sb = new StringBuilder();
    int i = 1;
    for (FrequencyTableEntry e : ext)
    {
      if (e.getType() == FrequencyTableEntryType.annotation)
      {
        sb
          .append("v").append(i).append(".")
          .append(tas.columnName(NODE_ANNOTATION_TABLE, "qannotext"));
      }
      else
      {
        sb.append("('annis:tok:' || ").append("v").append(i).append(".").append(
          tas.columnName(NODE_TABLE,
            "span)"));
      }
      sb.append(" AS value").append(i).append(", ");
      i++;
    }

    sb.append("count(*) AS \"count\"");

{'fraction_non_alphanumeric': 0.11602257924446374, 'fraction_numerical': 0.004515848892748589, 'mean_word_length': 3.484423676012461, 'pattern_counts': {'":': 1, '<': 39, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2769657', 'n_tokens_mistral': 3449, 'n_tokens_neox': 3355, 'n_words': 764}