<commit_before># coding=utf-8

import http.client
import re
from urllib.parse import quote

import module_states_file as xmlparser

class DDGSearch:
  def __init__(self, terms):
    self.terms = terms
    self.curRT = -1
    (res, page) = getPage(terms)
    if res == http.client.OK:
      self.ddgres = xmlparser.parse_string(page)
    else:
      self.ddgres = None

  @property
  def type(self):
    if self.ddgres.hasNode("Type"):
      return self.ddgres.getFirstNode("Type").getContent()
    else:
      return ""

  @property
  def definition(self):
    if self.ddgres.hasNode("Definition"):
      return self.ddgres.getFirstNode("Definition").getContent()
    else:
      return "Sorry, no definition found for %s" % self.terms

  @property
  def nextRes(self):
    if self.type == "D":
      if len(self.ddgres.getFirstNode("RelatedTopics").getNodes("RelatedTopic")) > self.curRT + 1:
        self.curRT += 1
        node = self.ddgres.getFirstNode("RelatedTopics").getNodes("RelatedTopic")[self.curRT]
        return node.getFirstNode("Text").getContent()
    elif self.ddgres.hasNode("Answer"):
      if self.curRT < 0:
        self.curRT = 0
        return striphtml(self.ddgres.getFirstNode("Answer").getContent())
    elif self.ddgres.hasNode("Abstract") and len (self.ddgres.getNode("Abstract").getContent()) > 0:
      if self.curRT < 0:
        self.curRT = 0
        return self.ddgres.getNode("Abstract").getContent() + " <" + self.ddgres.getNode("AbstractURL").getContent() + ">"
      elif len(self.ddgres.getFirstNode("RelatedTopics").getNodes("RelatedTopic")) > self.curRT:
        node = self.ddgres.getFirstNode("RelatedTopics").getNodes("RelatedTopic")[self.curRT]
        self.curRT += 1
        return node.getFirstNode("Text").getContent()
    return "No more results"


def striphtml(data):
    p = re.compile(r'<.*?>')
    return p.sub('', data)

def getPage(terms):
  conn = http.client.HTTPConnection("api.duckduckgo.com")
  try:
    conn.request("GET", "/?q=%s&format=xml" % quote(terms))
  except socket.gaierror:
    print ("impossible de récupérer la page %s."%(p))
    return (http.client.INTERNAL_SERVER_ERROR, None)

  res = conn.getresponse()
  data = res.read()

  conn.close()
  return (res.status, data)
<commit_msg>DDG module: implement last features from DDG API not implemented<commit_after># coding=utf-8

import http.client
import re
from urllib.parse import quote

import module_states_file as xmlparser

class DDGSearch:
  def __init__(self, terms):
    self.terms = terms
    self.curRT = -1
    (res, page) = getPage(terms)
    if res == http.client.OK or res == http.client.SEE_OTHER:
      self.ddgres = xmlparser.parse_string(page)
    else:
      self.ddgres = None

  @property
  def type(self):
    if self.ddgres and self.ddgres.hasNode("Type"):
      return self.ddgres.getFirstNode("Type").getContent()
    else:
      return ""

  @property
  def definition(self):
    if self.ddgres.hasNode("Definition"):
      return self.ddgres.getFirstNode("Definition").getContent()
    else:
      return "Sorry, no definition found for %s" % self.terms

  @property
  def nextRes(self):
    if (self.type == "D" or self.type == "C") and len(self.ddgres.getFirstNode("RelatedTopics").getNodes("RelatedTopic")) > self.curRT + 1:
      self.curRT += 1
      node = self.ddgres.getFirstNode("RelatedTopics").getNodes("RelatedTopic")[self.curRT]
      return node.getFirstNode("Text").getContent()
    elif self.ddgres.hasNode("Redirect") and self.ddgres.getFirstNode("Redirect").getContent() != "":
      return self.ddgres.getFirstNode("Redirect").getContent()
    elif self.ddgres.hasNode("Results") and self.ddgres.getFirstNode("Results").hasNode("Result") and self.curRT < 0:
      self.curRT = 0
      node = self.ddgres.getFirstNode("Results").getFirstNode("Result")
      return node.getFirstNode("Text").getContent() + ": " + node.getFirstNode("FirstURL").getContent() 
    elif self.ddgres.hasNode("Answer") and self.curRT < 0:
      self.curRT = 0
      return striphtml(self.ddgres.getFirstNode("Answer").getContent())
    elif self.ddgres.hasNode("Abstract") and len (self.ddgres.getNode("Abstract").getContent()) > 0:
      if self.curRT < 0:
        self.curRT = 0
        return self.ddgres.getNode("Abstract").getContent() + " <" + self.ddgres.getNode("AbstractURL").getContent() + ">"
      elif len(self.ddgres.getFirstNode("RelatedTopics").getNodes("RelatedTopic")) > self.curRT:
        node = self.ddgres.getFirstNode("RelatedTopics").getNodes("RelatedTopic")[self.curRT]
        self.curRT += 1
        return node.getFirstNode("Text").getContent()
    return "No more results"


def striphtml(data):
  p = re.compile(r'<.*?>')
  return p.sub('', data).replace("&#x28;", "/(").replace("&#x29;", ")/").replace("&#x22;", "\"")

def getPage(terms):
  conn = http.client.HTTPConnection("api.duckduckgo.com")
  try:
    conn.request("GET", "/?q=%s&format=xml" % quote(terms))
  except socket.gaierror:
    print ("impossible de récupérer la page %s."%(p))
    return (http.client.INTERNAL_SERVER_ERROR, None)

  res = conn.getresponse()
  data = res.read()

  conn.close()
  return (res.status, data)

{'fraction_non_alphanumeric': 0.15712074303405574, 'fraction_numerical': 0.005417956656346749, 'mean_word_length': 3.6442048517520216, 'pattern_counts': {'":': 3, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7187076', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1688, 'n_words': 331}