<commit_before>mysql:
  image: mysql
  environment:
    MYSQL_ROOT_PASSWORD: <PASSWORD>
  ports:
    - "3336:3306"
  volumes:
    - /home/ubuntu/trail-selections/dev-env:/dev-env

zookeeper:
  image: jplock/zookeeper
  ports:
    - "2181:2181"

kafka:
  image: wurstmeister/kafka:0.8.1.1-1
  ports:
    - "9092:9092"
  links:
    - zookeeper:zk
  environment:
    KAFKA_ADVERTISED_HOST_NAME: 172.31.31.203
    KAFKA_MESSAGE_MAX_BYTES: 2000000
    KAFKA_REPLICA_FETCH_MAX_BYTES: 3000000
  volumes:
    - /var/run/docker.sock:/var/run/docker.sock

dwstream:
  build: ../memex-datawake-stream
  links:
    - mysql:mysql
    - kafka:kafka
    - zookeeper:zk
  environment:
      DW_DB: memex_sotera
      DW_DB_USER: root
      DW_DB_PASSWORD: <PASSWORD>
      DW_DB_HOST: 172.31.31.203
      DW_DB_PORT: 3336
  volumes:
    - /home/ubuntu/trail-selections/memex-datawake-stream:/memex-datawake-stream
    - /home/ubuntu/trail-selections/dev-env:/dev-env


dwsearch:
  build: ../trail-specific-search
  working_dir: /trail-specific-search
  links:
    - mysql:mysql
    - kafka:kafka
    - zookeeper:zk
  environment:
    DW_DB_USER: root
    DW_DB_PASSWORD: <PASSWORD>
    DW_DB_JDBC: jdbc:mysql://172.31.31.203:3336/memex_sotera
    DW_CRAWLER_TOPIC: memex.crawled_trail-entities-search
    DW_BING_KEY: AeLgwrJ49hIsdsivwmnGs6VOHq3cNs3vkKdnkYlwffQ
    DW_GOOGLE_PUBLIC_KEY: <KEY>
    DW_GOOGLE_CUSTOM_SEARCH_CX: 017987995273492241149:qjaqo2emn3w
    DW_KAFKA_BROKERS: 172.31.31.203:9092
    DW_USE_DISTRIBUTED: true
    DW_INCOMING_URLS: memex.incoming_urls
    DW_KAFKA_CRAWLER_BROKERS: k01.istresearch.com:9092,k02.istresearch.com:9092,k03.istresearch.com:9092
    DW_ZK_QUORUM: 172.31.31.203:2181
    DW_SEARCH_API: http://172.31.44.172:8787/search/
  volumes:
    - /home/ubuntu/trail-selections/trail-specific-search:/tss
  command: scala -classpath "build/libs/Search-Crawler-1.0-all.jar" com.soteradefense.datawake.trails.topology.search.SearchTopology

dwurlcontents:
  build: ../trail-specific-search
  working_dir: /trail-specific-search
  links:
    - mysql:mysql
    - kafka:kafka
    - zookeeper:zk
  environment:
    DW_DB_JDBC: jdbc:mysql://172.31.31.203:3336/memex_sotera
    DW_CRAWLER_OUT: crawler-out
    DW_USE_DISTRIBUTED: true
    DW_KAFKA_BROKERS: 172.31.31.203:9092
    DW_LOCAL_ZK_QUORUM: 172.31.31.203:2181
    # DW_ZK_QUORUM: 172.31.31.203:2181
    DW_ZK_QUORUM: 172.16.58.3:2181
    DW_DB_USER: root
    DW_DB_PASSWORD: <PASSWORD>
    DW_CRAWLER_TOPIC: memex.crawled_trail-entities-search
  command: scala -classpath "build/libs/Search-Crawler-1.0-all.jar" com.soteradefense.datawake.trails.topology.save.UrlContentsTopology

dwurlrank:
  build: ../trail-specific-search
  working_dir: /trail-specific-search
  links:
    - mysql:mysql
    - kafka:kafka
    - zookeeper:zk
  environment:
      DW_DB_USER: root
      DW_DB_PASSWORD: <PASSWORD>
      DW_DB_JDBC: jdbc:mysql://172.31.31.203:3336/memex_sotera
      DW_KAFKA_BROKERS: 172.31.31.203:9092
      DW_ZK_QUORUM: 172.16.58.3:2181
  command: scala -classpath "build/libs/Search-Crawler-1.0-all.jar" com.soteradefense.datawake.trails.topology.rank.UrlRankTopology

datawake:
  build: ../server
  links:
    - mysql:mysql
    - kafka:kafka
    - dwsearch
    - dwurlcontents
    - dwstream
  ports:
    - "80:80"
  environment:
    DW_DB: memex_sotera<commit_msg>working rank<commit_after>mysql:
  image: mysql
  environment:
    MYSQL_ROOT_PASSWORD: root
  ports:
    - "3336:3306"
  volumes:
    - /home/ubuntu/trail-selections/dev-env:/dev-env

zookeeper:
  image: jplock/zookeeper
  ports:
    - "2181:2181"

kafka:
  image: wurstmeister/kafka:0.8.1.1-1
  ports:
    - "9092:9092"
  links:
    - zookeeper:zk
  environment:
    KAFKA_ADVERTISED_HOST_NAME: 172.31.31.203
    KAFKA_MESSAGE_MAX_BYTES: 2000000
    KAFKA_REPLICA_FETCH_MAX_BYTES: 3000000
  volumes:
    - /var/run/docker.sock:/var/run/docker.sock

dwstream:
  build: ../memex-datawake-stream
  links:
    - mysql:mysql
    - kafka:kafka
    - zookeeper:zk
  environment:
      DW_DB: memex_sotera
      DW_DB_USER: root
      DW_DB_PASSWORD: <PASSWORD>
      DW_DB_HOST: 172.31.31.203
      DW_DB_PORT: 3336
  volumes:
    - /home/ubuntu/trail-selections/memex-datawake-stream:/memex-datawake-stream
    - /home/ubuntu/trail-selections/dev-env:/dev-env


dwsearch:
  build: ../trail-specific-search
  working_dir: /trail-specific-search
  links:
    - mysql:mysql
    - kafka:kafka
    - zookeeper:zk
  environment:
    DW_DB_USER: root
    DW_DB_PASSWORD: <PASSWORD>
    DW_DB_JDBC: jdbc:mysql://172.31.31.203:3336/memex_sotera
    DW_CRAWLER_TOPIC: memex.crawled_trail-entities-search
    DW_BING_KEY: <KEY>
    DW_GOOGLE_PUBLIC_KEY: <KEY>
    DW_GOOGLE_CUSTOM_SEARCH_CX: 017987995273492241149:qjaqo2emn3w
    DW_KAFKA_BROKERS: 172.31.31.203:9092
    DW_USE_DISTRIBUTED: true
    DW_INCOMING_URLS: memex.incoming_urls
    DW_KAFKA_CRAWLER_BROKERS: k01.istresearch.com:9092,k02.istresearch.com:9092,k03.istresearch.com:9092
    DW_ZK_QUORUM: 172.31.31.203:2181
    DW_SEARCH_API: http://172.31.44.172:8787/search/
  volumes:
    - /home/ubuntu/trail-selections/trail-specific-search:/tss
  command: scala -classpath "build/libs/Search-Crawler-1.0-all.jar" com.soteradefense.datawake.trails.topology.search.SearchTopology

dwurlcontents:
  build: ../trail-specific-search
  working_dir: /trail-specific-search
  links:
    - mysql:mysql
    - kafka:kafka
    - zookeeper:zk
  environment:
    DW_DB_JDBC: jdbc:mysql://172.31.31.203:3336/memex_sotera
    DW_CRAWLER_OUT: crawler-out
    DW_USE_DISTRIBUTED: true
    DW_KAFKA_BROKERS: 172.31.31.203:9092
    DW_LOCAL_ZK_QUORUM: 172.31.31.203:2181
    # DW_ZK_QUORUM: 172.31.31.203:2181
    DW_ZK_QUORUM: 172.16.58.3:2181
    DW_DB_USER: root
    DW_DB_PASSWORD: <PASSWORD>
    DW_CRAWLER_TOPIC: memex.crawled_trail-entities-search
  command: scala -classpath "build/libs/Search-Crawler-1.0-all.jar" com.soteradefense.datawake.trails.topology.save.UrlContentsTopology

dwurlrank:
  build: ../trail-specific-search
  working_dir: /trail-specific-search
  links:
    - mysql:mysql
    - kafka:kafka
    - zookeeper:zk
  environment:
      DW_DB_USER: root
      DW_DB_PASSWORD: <PASSWORD>
      DW_DB_JDBC: jdbc:mysql://172.31.31.203:3336/memex_sotera
      DW_KAFKA_BROKERS: 172.31.31.203:9092
    #  DW_ZK_QUORUM: 172.16.58.3:2181
      DW_ZK_QUORUM: 172.31.31.203:2181
  command: scala -classpath "build/libs/Search-Crawler-1.0-all.jar" com.soteradefense.datawake.trails.topology.rank.UrlRankTopology

datawake:
  build: ../server
  links:
    - mysql:mysql
    - kafka:kafka
    - dwsearch
    - dwurlcontents
    - dwstream
  ports:
    - "80:80"
  environment:
    DW_DB: memex_sotera
{'fraction_non_alphanumeric': 0.11435413860200569, 'fraction_numerical': 0.08726238587037868, 'mean_word_length': 3.291586384071933, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 31, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1495220', 'n_tokens_mistral': 3365, 'n_tokens_neox': 2879, 'n_words': 340}