<commit_before>const { isJson5, json_parse, handleJson, joinPath, safeArray } = require('./utils')
const constants = require('../client/constants/variable.js')
const _ = require("underscore")
const URL = require('url')
const utils = require('./power-string.js').utils;
const HTTP_METHOD = constants.HTTP_METHOD;
const axios = require('axios');
const qs = require('qs');

const isNode = typeof global == 'object' && global.global === global;
const ContentTypeMap = {
  'application/json': 'json',
  'application/xml': 'xml',
  'other': 'text',
  'application/html': 'html'
}

async function httpRequestByNode(options) {
  function handleRes(response){
    return {
      res: {
        header: response.headers,
        status: response.status,
        body: response.data
      }
    }
  }

  function handleData(){
    let contentTypeItem;
    if(!options) return;
    if(typeof options.headers === 'object' && options.headers ){
      Object.keys(options.headers).forEach(key => {
        if (/content-type/i.test(key)) {
          contentTypeItem = options.headers[key].split(";")[0].trim().toLowerCase();        
        }
      })
      if(contentTypeItem === 'application/x-www-form-urlencoded' && typeof options.data === 'object' && options.data){
        options.data = qs.stringify(options.data);    
      }
    }
  }
  
  try{
    handleData(options);
    let response=await axios({
      method: options.method,
      url: options.url,
      headers: options.headers,
      timeout: 5000,
      data: options.data
    })
    return handleRes(response)
  }catch(response){
    return handleRes(response.response)
  }
}

function handleContentType(headers) {
  if (!headers || typeof headers !== 'object') return ContentTypeMap.other;
  let contentTypeItem = 'other';
  try {
    Object.keys(headers).forEach(key => {
      if (/content-type/i.test(key)) {
        contentTypeItem = headers[key].split(";")[0].trim().toLowerCase();
      }
    })
    return ContentTypeMap[contentTypeItem] ? ContentTypeMap[contentTypeItem] : ContentTypeMap.other;
  } catch (err) {
    return ContentTypeMap.other
  }
<commit_msg>fix: content-type is empty bug<commit_after>const { isJson5, json_parse, handleJson, joinPath, safeArray } = require('./utils')
const constants = require('../client/constants/variable.js')
const _ = require("underscore")
const URL = require('url')
const utils = require('./power-string.js').utils;
const HTTP_METHOD = constants.HTTP_METHOD;
const axios = require('axios');
const qs = require('qs');

const isNode = typeof global == 'object' && global.global === global;
const ContentTypeMap = {
  'application/json': 'json',
  'application/xml': 'xml',
  'other': 'text',
  'application/html': 'html'
}

async function httpRequestByNode(options) {
  function handleRes(response){
      return {
        res: {
          header: response.headers,
          status: response.status,
          body: response.data
        }
      }
  }

  function handleData(){
    let contentTypeItem;
    if(!options) return;
    if(typeof options.headers === 'object' && options.headers ){
      Object.keys(options.headers).forEach(key => {
        if (/content-type/i.test(key)) {
          if(options.headers[key]){
            contentTypeItem = options.headers[key].split(";")[0].trim().toLowerCase();        
          }          
        }
        if(!options.headers[key]) delete options.headers[key];
      })

      if(contentTypeItem === 'application/x-www-form-urlencoded' && typeof options.data === 'object' && options.data){
        options.data = qs.stringify(options.data);    
      }
    }
  }
  
  try{
    handleData(options);
    let response=await axios({
      method: options.method,
      url: options.url,
      headers: options.headers,
      timeout: 5000,
      data: options.data
    })
    return handleRes(response)
  }catch(err){
    if(err.response === undefined){
      handleRes({
        headers: {},
        status: null,
        data: err.message
      })
    }
    return handleRes(err.response)
  }
}

function handleContentType(headers) {
  if (!headers || typeof headers !== 'object') return ContentTypeMap.other;
  let contentTypeItem = 'other';
  try {
    Object.keys(headers).forEach(key => {
      if (/content-type/i.test(key)) {
        contentTypeItem = headers[key].split(";")[0].trim().toLowerCase();
      }
    })
    return ContentTypeMap[contentTypeItem] ? ContentTypeMap[contentTypeItem] : ContentTypeMap.other;
  } catch (err) {
    return ContentTypeMap.other
  }

{'fraction_non_alphanumeric': 0.15073935113661444, 'fraction_numerical': 0.003089825645552858, 'mean_word_length': 3.0141718334809564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4512409', 'n_tokens_mistral': 1356, 'n_tokens_neox': 1352, 'n_words': 268}