<commit_before># -*- coding: utf-8 -*-

'''
Created on Nov 12, 2012

Script that gets news items from newskeeper.ro in JSON format.
The data is then converter to XML for further processing.

note: get dict2xml from https://github.com/quandyfactory/dict2xml 

@author cornel

'''

import urllib, urllib2, os
import simplejson
import sys, time
import dict2xml
import re,codecs
from bs4 import BeautifulSoup

NO_CACHE = False

SOURCES = {
      'ZF' : 'ziarul financiar',
      'RL' : 'romania libera',
      'JURNALUL' : 'jurnalul national',
      'WS' : 'wall-street.ro',
      'GANDUL' : 'gandul',
      'ADEVARUL': 'adevarul'
    }

def extract_article_image(source, html):
  """
  Extracts image link from raw html
  """

  image = ""
  try:
    soup = BeautifulSoup(html, "lxml")
  except HTMLParseError:
    print "\terror: can't parse raw HTML"
    return image

  try:
    if source in ['RL', 'GANDUL', 'JURNALUL']:
      elem = soup.find('meta', {'property': 'og:image'})
      if elem:
        image = elem['content']
      else:
        # RL
        elem = soup.find('div', {'class': 'main_pic'})
        if elem:
          elem2 = elem.find('div', id='item_0').find('img')['src']
          if elem2:
            image = "http://www.romanialibera.ro" + elem2
        else: # GANDUL
          elem = soup.find('div', {'class': 'image'})
          if elem:
            image = elem.find('a', id='jsMainImage').find('img')['src']

    elif source in ['EVZ', 'CAPITAL']:
      image = soup.find('div', id='gallery').find('img')['src']
      domain = source.lower() + '.ro'
      if not domain in image:
        image = 'http://www.' + domain +'/' + image

    elif source == 'ADEVARUL':
      # we'll have to change this soon, as they seem to work on a new version of the website
      elem = soup.find('div', {'class': 'bb-wg-media'})
      if elem:
        image = elem.find('div', {'class':'mg'}).find('img')['src']
        # get a smaller image if pssible
        image = "http://www.adevarul.ro" + re.sub("/4\.jpg", "/2.jpg", image)

    elif source == 'ZF':
      elem = soup.find('div', id='content').find('div', id='mainVideo').find('div', {'class':'image'})
      if elem:
        image = re.sub("=\d{3}$", "=100", elem.find('img')['src'])
    else:
      image = ''
  except AttributeError, e: pass
  except TypeError, e: pass

  return image

#----------------------------------------------------------------
# main
#

WORK_DIR = 'python/src/ro/vivi/news_parser/newskeeper'
if len(sys.argv) > 1:
  WORK_DIR = sys.argv[1]

try:
  f = urllib2.urlopen('http://newskeeper.ro/hpexport/list')
  #f = file('/tmp/lista.json')
except IOError, e:
  print "Error: ", e
  sys.exit(1)

items = simplejson.load(f)

all_xml = ''

skipped = 0

for i in items:

  # we don't care about libertatea, right?!
  # we already get hotnews articles
  if i['newspaper'] in ['LIBERTATEA', 'HOTNEWS', 'MEDIAFAX'] :
    continue

  #if i['newspaper'] not in ['EVZ', 'CAPITAL'] :
  #  continue

  cache_file = "%s/nk_%s.json" % (WORK_DIR, i['md5'])
  if os.path.exists(cache_file):
    print 'skipping: ', i['nkUrl']
    skipped += 1
    if skipped >= 10:
      break
    else:
      continue

  try:
    nk_full_uri = 'http://newskeeper.ro%s' % i['nkUrl']
    print "  + ", nk_full_uri
    f = urllib2.urlopen(nk_full_uri)
  except urllib2.URLError, e:
    print " - skipping %s" % i['nkUrl']
    continue
  try:
    item = simplejson.load(f)
  except simplejson.decoder.JSONDecodeError:
    print " - ERR: can't get json"
    continue

  if item.has_key('optionalArticle') \
      and item['optionalArticle'] \
      and item['optionalArticle'].has_key('title'):
    news_title = item['optionalArticle']['title']
  else:
    continue

  if item['optionalArticle'].has_key('content'):
    content = item['optionalArticle']['content']
  else:
    continue

  if SOURCES.has_key(i['newspaper']):
    news_source = SOURCES[ i['newspaper'] ]
  else:
    news_source = i['newspaper'].lower()

  category = item['optionalArticle']['category'].lower()

  # Only look at articles that are in categories potentially related to Politics
  cat_ok_re = "politic|parlament|alegeri|justi"
  if re.search(cat_ok_re, category):
    print '     we got politics as a category [' + \
          category.encode('UTF-8') + ']'

    # extract image
    image_url = extract_article_image(i['newspaper'], item['rawHtml'])
<commit_msg>Get the criminal articles out of the parser<commit_after># -*- coding: utf-8 -*-

'''
Created on Nov 12, 2012

Script that gets news items from newskeeper.ro in JSON format.
The data is then converter to XML for further processing.

note: get dict2xml from https://github.com/quandyfactory/dict2xml 

@author cornel

'''

import urllib, urllib2, os
import simplejson
import sys, time
import dict2xml
import re,codecs
from bs4 import BeautifulSoup

NO_CACHE = False

SOURCES = {
      'ZF' : 'ziarul financiar',
      'RL' : 'romania libera',
      'JURNALUL' : 'jurnalul national',
      'WS' : 'wall-street.ro',
      'GANDUL' : 'gandul',
      'ADEVARUL': 'adevarul'
    }

def extract_article_image(source, html):
  """
  Extracts image link from raw html
  """

  image = ""
  try:
    soup = BeautifulSoup(html, "lxml")
  except HTMLParseError:
    print "\terror: can't parse raw HTML"
    return image

  try:
    if source in ['RL', 'GANDUL', 'JURNALUL']:
      elem = soup.find('meta', {'property': 'og:image'})
      if elem:
        image = elem['content']
      else:
        # RL
        elem = soup.find('div', {'class': 'main_pic'})
        if elem:
          elem2 = elem.find('div', id='item_0').find('img')['src']
          if elem2:
            image = "http://www.romanialibera.ro" + elem2
        else: # GANDUL
          elem = soup.find('div', {'class': 'image'})
          if elem:
            image = elem.find('a', id='jsMainImage').find('img')['src']

    elif source in ['EVZ', 'CAPITAL']:
      image = soup.find('div', id='gallery').find('img')['src']
      domain = source.lower() + '.ro'
      if not domain in image:
        image = 'http://www.' + domain +'/' + image

    elif source == 'ADEVARUL':
      # we'll have to change this soon, as they seem to work on a new version of the website
      elem = soup.find('div', {'class': 'bb-wg-media'})
      if elem:
        image = elem.find('div', {'class':'mg'}).find('img')['src']
        # get a smaller image if pssible
        image = "http://www.adevarul.ro" + re.sub("/4\.jpg", "/2.jpg", image)

    elif source == 'ZF':
      elem = soup.find('div', id='content').find('div', id='mainVideo').find('div', {'class':'image'})
      if elem:
        image = re.sub("=\d{3}$", "=100", elem.find('img')['src'])
    else:
      image = ''
  except AttributeError, e: pass
  except TypeError, e: pass

  return image

#----------------------------------------------------------------
# main
#

WORK_DIR = 'python/src/ro/vivi/news_parser/newskeeper'
if len(sys.argv) > 1:
  WORK_DIR = sys.argv[1]

try:
  f = urllib2.urlopen('http://newskeeper.ro/hpexport/list')
  #f = file('/tmp/lista.json')
except IOError, e:
  print "Error: ", e
  sys.exit(1)

items = simplejson.load(f)

all_xml = ''

skipped = 0

for i in items:

  # we don't care about libertatea, right?!
  # we already get hotnews articles
  if i['newspaper'] in ['LIBERTATEA', 'HOTNEWS', 'MEDIAFAX'] :
    continue

  #if i['newspaper'] not in ['EVZ', 'CAPITAL'] :
  #  continue

  cache_file = "%s/nk_%s.json" % (WORK_DIR, i['md5'])
  if os.path.exists(cache_file):
    print 'skipping: ', i['nkUrl']
    skipped += 1
    if skipped >= 10:
      break
    else:
      continue

  try:
    nk_full_uri = 'http://newskeeper.ro%s' % i['nkUrl']
    print "  + ", nk_full_uri
    f = urllib2.urlopen(nk_full_uri)
  except urllib2.URLError, e:
    print " - skipping %s" % i['nkUrl']
    continue
  try:
    item = simplejson.load(f)
  except simplejson.decoder.JSONDecodeError:
    print " - ERR: can't get json"
    continue

  if item.has_key('optionalArticle') \
      and item['optionalArticle'] \
      and item['optionalArticle'].has_key('title'):
    news_title = item['optionalArticle']['title']
  else:
    continue

  if item['optionalArticle'].has_key('content'):
    content = item['optionalArticle']['content']
  else:
    continue

  if SOURCES.has_key(i['newspaper']):
    news_source = SOURCES[ i['newspaper'] ]
  else:
    news_source = i['newspaper'].lower()

  category = item['optionalArticle']['category'].lower()

  # Only look at articles that are in categories potentially related to Politics
  cat_ok_re = "politic|parlament|alegeri"
  if re.search(cat_ok_re, category):
    print '     we got politics as a category [' + \
          category.encode('UTF-8') + ']'

    # extract image
    image_url = extract_article_image(i['newspaper'], item['rawHtml'])

{'fraction_non_alphanumeric': 0.1756235827664399, 'fraction_numerical': 0.007709750566893424, 'mean_word_length': 2.782590051457976, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2010314', 'n_tokens_mistral': 3133, 'n_tokens_neox': 2988, 'n_words': 761}