<commit_before>//  SelectIdeologicalTweets.scala
//
//  Copyright (C) 2012 <NAME>, The University of Texas at Austin
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
///////////////////////////////////////////////////////////////////////////////

package opennlp.textgrounder.preprocess

import collection.mutable

import java.io._

import org.apache.commons.logging

import com.nicta.scoobi.Scoobi._

import opennlp.textgrounder.{util => tgutil}
import tgutil.argparser._
import tgutil.corpusutil._
import tgutil.hadoop._
import tgutil.ioutil._
import tgutil.collectionutil._
import tgutil.osutil._
import tgutil.printutil._

class SelectIdeologicalTweetsParams(val ap: ArgParser) extends
    ScoobiProcessFilesParams(ap) {
  var political_twitter_accounts = ap.option[String](
    "political-twitter-accounts", "pta",
    help="""File containing list of politicians and associated twitter
    accounts, for identifying liberal and conservative tweeters.""")
  var min_accounts = ap.option[Int]("min-accounts", default = 2,
    help="""Minimum number of political accounts mentioned by Twitter users
    in order for users to be considered.  Default %default.""")
  var min_conservative = ap.option[Double]("min-conservative", "mc",
    default = 0.75,
    help="""Minimum ideology score to consider a user as an "ideological
    conservative".  On the ideology scale, greater values indicate more
    conservative.  Currently, the scale runs from 0 to 1; hence, this value
    should be greater than 0.5.  Default %default.""")
  var max_liberal = ap.option[Double]("max-liberal", "ml",
    help="""Maximum ideology score to consider a user as an "ideological
    liberal".  On the ideology scale, greater values indicate more
    conservative.  Currently, the scale runs from 0 to 1; hence, this value
    should be less than 0.5.  If unspecified, computed as the mirror image of
    the value of '--min-conservative' (e.g. 0.25 if
    --min-conservative=0.75).""")
  var corpus_name = ap.option[String]("corpus-name",
    help="""Name of output corpus; for identification purposes.
    Default to name taken from input directory.""")
}

object SelectIdeologicalTweets extends
    ScoobiProcessFilesApp[SelectIdeologicalTweetsParams] {
  abstract class SelectIdeologicalTweetsShared(opts: SelectIdeologicalTweetsParams)
    extends ScoobiProcessFilesShared {
    val progname = "SelectIdeologicalTweets"
  }

  /**
   * Description of a "politico" -- a politician along their party and
   * known twitter accounts.
   */
  case class Politico(last: String, first: String, title: String,
      party: String, where: String, accounts: Set[String]) {
    def full_name = first + " " + last
  }
  implicit val politico_wire =
    mkCaseWireFormat(Politico.apply _, Politico.unapply _)

  /**
   * Description of a user and the accounts mentioned, both political and
   * nonpolitical, along with ideology.
   */
  case class IdeologicalUser(user: String, ideology: Double,
    num_mentions: Int, mentions: Seq[(String, Int)],
    num_dem_mentions: Int, dem_mentions: Seq[(Politico, Int)],
    num_rep_mentions: Int, rep_mentions: Seq[(Politico, Int)]) {
    def encode_politico_count_map(seq: Seq[(Politico, Int)]) =
      encode_word_count_map(
        seq.map { case (politico, count) =>
          (politico.full_name.replace(" ", "."), count) })
    def to_row =
      Seq(user, "%.3f" format ideology,
        num_mentions.toString, encode_word_count_map(mentions),
        num_dem_mentions.toString, encode_politico_count_map(dem_mentions),
        num_rep_mentions.toString, encode_politico_count_map(rep_mentions)
      ) mkString "\t"
  }
  implicit val ideological_user_wire =
    mkCaseWireFormat(IdeologicalUser.apply _, IdeologicalUser.unapply _)

  object IdeologicalUser {

    def row_fields =
      Seq("user", "ideology", "num-mentions", "mentions",
        "num-dem-mentions", "dem-mentions",
        "num-rep-mentions", "rep-mentions")

    /**
     * For a given user, determine if the user is an "ideological user"
     * and if so, return an object describing the user.
     */
    def get_ideological_user(line: String, schema: Schema,
        accounts: Map[String, Politico]) = {
      error_wrap(line, None: Option[IdeologicalUser]) { line => {
        val fields = line.split("\t", -1)
        // get list of (user mentions, times) pairs
        val mentions =
          decode_word_count_map(schema.get_field(fields, "user-mentions"))
        val num_mentions = mentions.map(_._2).sum
        val user = schema.get_field(fields, "user")
        // errprint("For user %s, mentions: %s", user, mentions.toList)
        // find mentions of a politician
        val demrep_mentions =
          for {(mention, times) <- mentions
               account = accounts.getOrElse(mention.toLowerCase, null)
               if account != null && {
                 //errprint("saw account %s, party %s", account, account.party);
                 "DR".contains(account.party)}}
            yield (account, times)
        //errprint("demrep_mentions: %s", demrep_mentions.toList)
        val (dem_mentions, rep_mentions) = demrep_mentions.partition {
            case (account, times) => account.party == "D" }
        val num_dem_mentions = dem_mentions.map(_._2).sum
        val num_rep_mentions = rep_mentions.map(_._2).sum
        val num_demrep_mentions = num_dem_mentions + num_rep_mentions
        val ideology = num_rep_mentions.toFloat/num_demrep_mentions
        if (num_demrep_mentions > 0) {
          val ideo_user =
            IdeologicalUser(user, ideology,
              num_mentions, mentions,
              num_dem_mentions, dem_mentions,
              num_rep_mentions, rep_mentions)
          Some(ideo_user)
        } else None
      }}
    }
  }

  /**
   * Description of a potential politico -- someone mentioned by an
   * ideological user.
   *
   * @param lcuser Twitter account of user, lowercased
   * @param spellings Map of actual (non-lowercased) spellings of account
   *   by usage
   * @param num_mentions Total number of mentions
   *   a noticeably "liberal" ideology
   * @param num_lib_mentions Number of times mentioned by people with
   *   a noticeably "liberal" ideology
   * @param num_conserv_mentions Number of times mentioned by people with
   *   a noticeably "conservative" ideology
   * @param num_ideo_mentions Sum of mentions weighted by ideology of
   *   person doing the mentioning, so that we can compute a weighted
   *   average to determine their ideology.
   */
  case class PotentialPolitico(lcuser: String, spellings: Map[String, Int],
    num_mentions: Int, num_lib_mentions: Int, num_conserv_mentions: Int,
    num_ideo_mentions: Double) {
    def to_row =
      Seq(lcuser, encode_word_count_map(spellings.toSeq), num_mentions,
        num_lib_mentions, num_conserv_mentions,
        num_ideo_mentions/num_mentions) mkString "\t"
  }

  object PotentialPolitico {

    def row_fields =
      Seq("lcuser", "spellings", "num-mentions", "num-lib-mentions",
        "num-conserv-mentions", "ideology")
    /**
     * For a given ideological user, generate the "potential politicos": other
     * people mentioned, along with their ideological scores.
     */
    def get_potential_politicos(user: IdeologicalUser,
        opts: SelectIdeologicalTweetsParams) = {
      for {(mention, times) <- user.mentions
           lcmention = mention.toLowerCase } yield {
        val is_lib = user.ideology <= opts.max_liberal
        val is_conserv = user.ideology >= opts.min_conservative
        PotentialPolitico(
          lcmention, Map(mention->times), times,
          if (is_lib) times else 0,
          if (is_conserv) times else 0,
          times * user.ideology
        )
      }
    }

    /**
     * Merge two PotentialPolitico objects, which must refer to the same user.
     * Add up the mentions and combine the set of spellings.
     */
    def merge_potential_politicos(u1: PotentialPolitico, u2: PotentialPolitico) = {
      assert(u1.lcuser == u2.lcuser)
      PotentialPolitico(u1.lcuser, combine_maps(u1.spellings, u2.spellings),
        u1.num_mentions + u2.num_mentions,
        u1.num_lib_mentions + u2.num_lib_mentions,
        u1.num_conserv_mentions + u2.num_conserv_mentions,
        u1.num_ideo_mentions + u2.num_ideo_mentions)
    }
  }

  implicit val potential_politico =
    mkCaseWireFormat(PotentialPolitico.apply _, PotentialPolitico.unapply _)

  class SelectIdeologicalTweets(opts: SelectIdeologicalTweetsParams)<commit_msg>Also propagate text field in SelectIdeologicalTweets<commit_after>//  SelectIdeologicalTweets.scala
//
//  Copyright (C) 2012 <NAME>, The University of Texas at Austin
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
///////////////////////////////////////////////////////////////////////////////

package opennlp.textgrounder.preprocess

import collection.mutable

import java.io._

import org.apache.commons.logging

import com.nicta.scoobi.Scoobi._

import opennlp.textgrounder.{util => tgutil}
import tgutil.argparser._
import tgutil.corpusutil._
import tgutil.hadoop._
import tgutil.ioutil._
import tgutil.collectionutil._
import tgutil.osutil._
import tgutil.printutil._

class SelectIdeologicalTweetsParams(val ap: ArgParser) extends
    ScoobiProcessFilesParams(ap) {
  var political_twitter_accounts = ap.option[String](
    "political-twitter-accounts", "pta",
    help="""File containing list of politicians and associated twitter
    accounts, for identifying liberal and conservative tweeters.""")
  var min_accounts = ap.option[Int]("min-accounts", default = 2,
    help="""Minimum number of political accounts mentioned by Twitter users
    in order for users to be considered.  Default %default.""")
  var min_conservative = ap.option[Double]("min-conservative", "mc",
    default = 0.75,
    help="""Minimum ideology score to consider a user as an "ideological
    conservative".  On the ideology scale, greater values indicate more
    conservative.  Currently, the scale runs from 0 to 1; hence, this value
    should be greater than 0.5.  Default %default.""")
  var max_liberal = ap.option[Double]("max-liberal", "ml",
    help="""Maximum ideology score to consider a user as an "ideological
    liberal".  On the ideology scale, greater values indicate more
    conservative.  Currently, the scale runs from 0 to 1; hence, this value
    should be less than 0.5.  If unspecified, computed as the mirror image of
    the value of '--min-conservative' (e.g. 0.25 if
    --min-conservative=0.75).""")
  var corpus_name = ap.option[String]("corpus-name",
    help="""Name of output corpus; for identification purposes.
    Default to name taken from input directory.""")
}

object SelectIdeologicalTweets extends
    ScoobiProcessFilesApp[SelectIdeologicalTweetsParams] {
  abstract class SelectIdeologicalTweetsShared(opts: SelectIdeologicalTweetsParams)
    extends ScoobiProcessFilesShared {
    val progname = "SelectIdeologicalTweets"
  }

  /**
   * Description of a "politico" -- a politician along their party and
   * known twitter accounts.
   */
  case class Politico(last: String, first: String, title: String,
      party: String, where: String, accounts: Set[String]) {
    def full_name = first + " " + last
  }
  implicit val politico_wire =
    mkCaseWireFormat(Politico.apply _, Politico.unapply _)

  /**
   * Description of a user and the accounts mentioned, both political and
   * nonpolitical, along with ideology.
   */
  case class IdeologicalUser(user: String, ideology: Double,
    num_mentions: Int, mentions: Seq[(String, Int)],
    num_dem_mentions: Int, dem_mentions: Seq[(Politico, Int)],
    num_rep_mentions: Int, rep_mentions: Seq[(Politico, Int)],
    text: String) {
    def encode_politico_count_map(seq: Seq[(Politico, Int)]) =
      encode_word_count_map(
        seq.map { case (politico, count) =>
          (politico.full_name.replace(" ", "."), count) })
    def to_row =
      Seq(user, "%.3f" format ideology,
        num_mentions.toString, encode_word_count_map(mentions),
        num_dem_mentions.toString, encode_politico_count_map(dem_mentions),
        num_rep_mentions.toString, encode_politico_count_map(rep_mentions),
        text
      ) mkString "\t"
  }
  implicit val ideological_user_wire =
    mkCaseWireFormat(IdeologicalUser.apply _, IdeologicalUser.unapply _)

  object IdeologicalUser {

    def row_fields =
      Seq("user", "ideology", "num-mentions", "mentions",
        "num-dem-mentions", "dem-mentions",
        "num-rep-mentions", "rep-mentions",
        "text")

    /**
     * For a given user, determine if the user is an "ideological user"
     * and if so, return an object describing the user.
     */
    def get_ideological_user(line: String, schema: Schema,
        accounts: Map[String, Politico]) = {
      error_wrap(line, None: Option[IdeologicalUser]) { line => {
        val fields = line.split("\t", -1)
        // get list of (user mentions, times) pairs
        val mentions =
          decode_word_count_map(schema.get_field(fields, "user-mentions"))
        val num_mentions = mentions.map(_._2).sum
        val text = schema.get_field(fields, "text")
        val user = schema.get_field(fields, "user")
        // errprint("For user %s, mentions: %s", user, mentions.toList)
        // find mentions of a politician
        val demrep_mentions =
          for {(mention, times) <- mentions
               account = accounts.getOrElse(mention.toLowerCase, null)
               if account != null && {
                 //errprint("saw account %s, party %s", account, account.party);
                 "DR".contains(account.party)}}
            yield (account, times)
        //errprint("demrep_mentions: %s", demrep_mentions.toList)
        val (dem_mentions, rep_mentions) = demrep_mentions.partition {
            case (account, times) => account.party == "D" }
        val num_dem_mentions = dem_mentions.map(_._2).sum
        val num_rep_mentions = rep_mentions.map(_._2).sum
        val num_demrep_mentions = num_dem_mentions + num_rep_mentions
        val ideology = num_rep_mentions.toFloat/num_demrep_mentions
        if (num_demrep_mentions > 0) {
          val ideo_user =
            IdeologicalUser(user, ideology,
              num_mentions, mentions,
              num_dem_mentions, dem_mentions,
              num_rep_mentions, rep_mentions,
              text)
          Some(ideo_user)
        } else None
      }}
    }
  }

  /**
   * Description of a potential politico -- someone mentioned by an
   * ideological user.
   *
   * @param lcuser Twitter account of user, lowercased
   * @param spellings Map of actual (non-lowercased) spellings of account
   *   by usage
   * @param num_mentions Total number of mentions
   *   a noticeably "liberal" ideology
   * @param num_lib_mentions Number of times mentioned by people with
   *   a noticeably "liberal" ideology
   * @param num_conserv_mentions Number of times mentioned by people with
   *   a noticeably "conservative" ideology
   * @param num_ideo_mentions Sum of mentions weighted by ideology of
   *   person doing the mentioning, so that we can compute a weighted
   *   average to determine their ideology.
   * @param all_text Text of all users mentioning the politico.
   */
  case class PotentialPolitico(lcuser: String, spellings: Map[String, Int],
    num_mentions: Int, num_lib_mentions: Int, num_conserv_mentions: Int,
    num_ideo_mentions: Double, all_text: Seq[String]) {
    def to_row =
      Seq(lcuser, encode_word_count_map(spellings.toSeq), num_mentions,
        num_lib_mentions, num_conserv_mentions,
        num_ideo_mentions/num_mentions,
        all_text mkString " !! ") mkString "\t"
  }

  object PotentialPolitico {

    def row_fields =
      Seq("lcuser", "spellings", "num-mentions", "num-lib-mentions",
        "num-conserv-mentions", "ideology", "all-text")
    /**
     * For a given ideological user, generate the "potential politicos": other
     * people mentioned, along with their ideological scores.
     */
    def get_potential_politicos(user: IdeologicalUser,
        opts: SelectIdeologicalTweetsParams) = {
      for {(mention, times) <- user.mentions
           lcmention = mention.toLowerCase } yield {
        val is_lib = user.ideology <= opts.max_liberal
        val is_conserv = user.ideology >= opts.min_conservative
        PotentialPolitico(
          lcmention, Map(mention->times), times,
          if (is_lib) times else 0,
          if (is_conserv) times else 0,
          times * user.ideology,
          Seq(user.text)
        )
      }
    }

    /**
     * Merge two PotentialPolitico objects, which must refer to the same user.
     * Add up the mentions and combine the set of spellings.
     */
    def merge_potential_politicos(u1: PotentialPolitico, u2: PotentialPolitico) = {
      assert(u1.lcuser == u2.lcuser)
      PotentialPolitico(u1.lcuser, combine_maps(u1.spellings, u2.spellings),
        u1.num_mentions + u2.num_mentions,
        u1.num_lib_mentions + u2.num_lib_mentions,
        u1.num_conserv_mentions + u2.num_conserv_mentions,
        u1.num_ideo_mentions + u2.num_ideo_mentions,
        u1.all_text ++ u2.all_text)
    }
  }

  implicit val potential_politico =
    mkCaseWireFormat(PotentialPolitico.apply _, PotentialPolitico.unapply _)

  class SelectIdeologicalTweets(opts: SelectIdeologicalTweetsParams)
{'fraction_non_alphanumeric': 0.10328562134027326, 'fraction_numerical': 0.005421817393190197, 'mean_word_length': 3.052955394418809, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2407379', 'n_tokens_mistral': 5736, 'n_tokens_neox': 5468, 'n_words': 1562}