<commit_before>const Influx = require('influx');

async function createInfluxClient() {

  const influxClicksDb = new Influx.InfluxDB({
    host: 'influx',
    database: 'clicks', // doh, should have been something more generic
    schema: [
      {
        measurement: 'click',
        fields: {
          url: Influx.FieldType.STRING,
          cost: Influx.FieldType.FLOAT,
          unitCost: Influx.FieldType.FLOAT,
          count: Influx.FieldType.INTEGER,
          name: Influx.FieldType.STRING,
        },
        tags: [
          'userAgent',
          'vendor',
          'province',
          'brand',
          'calibre',
        ]
      },
      {
        measurement: 'view',
        fields: { v: Influx.FieldType.BOOLEAN },
        tags: [
          'userAgent',
          'brand',
          'calibre',
        ]
      },
      {
        measurement: 'item',
        fields: {
          name: Influx.FieldType.STRING,
          price: Influx.FieldType.FLOAT,
          unitCost: Influx.FieldType.FLOAT,
        },
        tags: [
          'vendor',
          'province',
          'brand',
          'calibre',
        ]
      },
      {
        measurement: 'scrape',
        fields: {
          results: Influx.FieldType.INTEGER,
          time: Influx.FieldType.INTEGER,
        },
        tags: [
          'vendor',
          'type',
        ]
      },
      {
        measurement: 'scrape-failure',
        fields: {
          time: Influx.FieldType.INTEGER,
          error: Influx.FieldType.STRING
        },
        tags: [
          'vendor',
          'type',
        ]
      }
    ]
  })

  const names = await influxClicksDb.getDatabaseNames()
  if (!names.includes('clicks')) {
    await influxClicksDb.createDatabase('clicks');
  }

  return influxClicksDb;
}

module.exports = {
  logClick(url, userAgent, item) {
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'clicks',
            tags: {
              userAgent,
              vendor: item.vendor,
              province: item.province,
              brand: item.brand,
              calibre: item.calibre
            },
            fields: {
              url,
              price: item.price || 0,
              unitCost: item.unitCost || 0,
              count: item.count || 0,
              name: item.name,
            }
          }
        ])
      )
  },
  logView(userAgent, brand, calibre) {
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'view',
            tags: {
              userAgent,
              brand,
              calibre
            },
            fields: { v: true }
          }
        ])
      )
  },
  logItem(item) {
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'item',
            tags: {
              vendor: item.vendor,
              province: item.province,
              brand: item.brand,
              calibre: item.calibre
            },
            fields: {
              price: item.price || 0,
              unitCost: item.unitCost || null,
              name: item.name,
            }
          }
        ])
      )
  },
  logScrapeResult(type, vendor, results, time) {
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'scrape',
            tags: {
              vendor,
              type
            },
            fields: {
              results,
              time
            }
          }
        ])
      )
  },
  logScrapeFail(type, vendor, time, error) {
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'scrape-failure',
            tags: {
              vendor,
              type
            },
            fields: {
              time,
              error
            }
          }
        ])
      )
  },
}<commit_msg>created DONT_USE_INFLUX flag to skip writting to inlfux (#122)<commit_after>const Influx = require('influx');
const dontUseInflux = process.env.DONT_USE_INFLUX === 'true';
async function createInfluxClient() {

  const influxClicksDb = new Influx.InfluxDB({
    host: 'influx',
    database: 'clicks', // doh, should have been something more generic
    schema: [
      {
        measurement: 'click',
        fields: {
          url: Influx.FieldType.STRING,
          cost: Influx.FieldType.FLOAT,
          unitCost: Influx.FieldType.FLOAT,
          count: Influx.FieldType.INTEGER,
          name: Influx.FieldType.STRING,
        },
        tags: [
          'userAgent',
          'vendor',
          'province',
          'brand',
          'calibre',
        ]
      },
      {
        measurement: 'view',
        fields: { v: Influx.FieldType.BOOLEAN },
        tags: [
          'userAgent',
          'brand',
          'calibre',
        ]
      },
      {
        measurement: 'item',
        fields: {
          name: Influx.FieldType.STRING,
          price: Influx.FieldType.FLOAT,
          unitCost: Influx.FieldType.FLOAT,
        },
        tags: [
          'vendor',
          'province',
          'brand',
          'calibre',
        ]
      },
      {
        measurement: 'scrape',
        fields: {
          results: Influx.FieldType.INTEGER,
          time: Influx.FieldType.INTEGER,
        },
        tags: [
          'vendor',
          'type',
        ]
      },
      {
        measurement: 'scrape-failure',
        fields: {
          time: Influx.FieldType.INTEGER,
          error: Influx.FieldType.STRING
        },
        tags: [
          'vendor',
          'type',
        ]
      }
    ]
  })

  const names = await influxClicksDb.getDatabaseNames()
  if (!names.includes('clicks')) {
    await influxClicksDb.createDatabase('clicks');
  }

  return influxClicksDb;
}

module.exports = {
  logClick(url, userAgent, item) {
    if (dontUseInflux) {
        return Promise.resolve();
    }
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'clicks',
            tags: {
              userAgent,
              vendor: item.vendor,
              province: item.province,
              brand: item.brand,
              calibre: item.calibre
            },
            fields: {
              url,
              price: item.price || 0,
              unitCost: item.unitCost || 0,
              count: item.count || 0,
              name: item.name,
            }
          }
        ])
      )
  },
  logView(userAgent, brand, calibre) {
    if (dontUseInflux) {
        return Promise.resolve();
    }
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'view',
            tags: {
              userAgent,
              brand,
              calibre
            },
            fields: { v: true }
          }
        ])
      )
  },
  logItem(item) {
    if (dontUseInflux) {
        return Promise.resolve();
    }
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'item',
            tags: {
              vendor: item.vendor,
              province: item.province,
              brand: item.brand,
              calibre: item.calibre
            },
            fields: {
              price: item.price || 0,
              unitCost: item.unitCost || null,
              name: item.name,
            }
          }
        ])
      )
  },
  logScrapeResult(type, vendor, results, time) {
    if (dontUseInflux) {
        return Promise.resolve();
    }
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'scrape',
            tags: {
              vendor,
              type
            },
            fields: {
              results,
              time
            }
          }
        ])
      )
  },
  logScrapeFail(type, vendor, time, error) {
    if (dontUseInflux) {
        return Promise.resolve();
    }
    return createInfluxClient()
      .then(influxClicksDb =>
        influxClicksDb.writePoints([
          {
            measurement: 'scrape-failure',
            tags: {
              vendor,
              type
            },
            fields: {
              time,
              error
            }
          }
        ])
      )
  },
}
{'fraction_non_alphanumeric': 0.10981389434747428, 'fraction_numerical': 0.0012715293029707548, 'mean_word_length': 1.2099616858237547, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5171110', 'n_tokens_mistral': 2558, 'n_tokens_neox': 2503, 'n_words': 432}