<commit_before><script src="https://unpkg.com/deeplearn@0.5.1/dist/deeplearn.js"> </script> 

<script>
const EPSILON = dl.scalar(1e-7);
const HIDDEN_UNITS = 10;
const one = dl.scalar(1);

const input = [[0, 0], [0, 1], [1, 0], [1, 1]];
const target = [[0], [1], [1], [0]];

const inputTensor: dl.Tensor2D = dl.tensor2d(input, [4, 2]);

const targetTensor: dl.Tensor2D = dl.tensor2d(target, [4, 1]);

const fullyConnectedWeights1 =
    dl.variable(initializeWeights([2, HIDDEN_UNITS], 2) );

const fullyConnectedBias1 = dl.variable(dl.scalar(0));

const fullyConnectedWeights2 = dl.variable(
    initializeWeights([HIDDEN_UNITS, 1], HIDDEN_UNITS));

const fullyConnectedBias2 = dl.variable(dl.scalar(0));

const optimizer = new dl.SGDOptimizer(0.1);

const getRandomIntegerInRange = (min, max) => {
  return Math.floor(Math.random() * (max - min + 1)) + min;
};

function initializeWeights(shape, prevLayerSize) {
  return dl.randomNormal(shape).mul(dl.scalar(Math.sqrt(2.0 / prevLayerSize)));
}

function calculateCost(y, output){
  return dl.add(
               y.mul(output.add(EPSILON).log()),
               one.sub(y).mul(one.sub(output).add(EPSILON).log())).sum().neg();
}

function model(xs){
  const hiddenLayer = dl.tidy(() => {
    return xs.matMul(fullyConnectedWeights1).add(fullyConnectedBias1).relu() 
  });

  return hiddenLayer.matMul(fullyConnectedWeights2)
             .add(fullyConnectedBias2)
             .sigmoid();
}

// Train the model.
export async function train(iterations: number) {
  const returnCost = true;
  let cost;

  for (let i = 0; i < iterations; i++) {
    cost = optimizer.minimize(() => {
      return calculateCost(targetTensor, model(inputTensor));
    }, returnCost);

    if (i % 10 === 0) {
      console.log(`loss[${i}]: ${cost.getValues()}`);
    }

    await dl.nextFrame();
  }

  return cost.dataSync();
}

export const learnXOR = async () => {
  const iterations = getRandomIntegerInRange(800, 1000);
  const timeStart = performance.now();
  const result = [];

  const loss = await train(iterations);<commit_msg>Update neural03.html<commit_after><script src="https://unpkg.com/deeplearn@0.5.1/dist/deeplearn.js"> </script> 

<script>
const EPSILON = dl.scalar(1e-7);
const HIDDEN_UNITS = 10;
const one = dl.scalar(1);

const input = [[0, 0], [0, 1], [1, 0], [1, 1]];
const target = [[0], [1], [1], [0]];

const inputTensor: dl.Tensor2D = dl.tensor2d(input, [4, 2]);

const targetTensor: dl.Tensor2D = dl.tensor2d(target, [4, 1]);

const fullyConnectedWeights1 =
    dl.variable(initializeWeights([2, HIDDEN_UNITS], 2) );

const fullyConnectedBias1 = dl.variable(dl.scalar(0));

const fullyConnectedWeights2 = dl.variable(
    initializeWeights([HIDDEN_UNITS, 1], HIDDEN_UNITS));

const fullyConnectedBias2 = dl.variable(dl.scalar(0));

const optimizer = new dl.SGDOptimizer(0.1);

const getRandomIntegerInRange = (min, max) => {
  return Math.floor(Math.random() * (max - min + 1)) + min;
};

function initializeWeights(shape, prevLayerSize) {
  return dl.randomNormal(shape).mul(dl.scalar(Math.sqrt(2.0 / prevLayerSize)));
}

function calculateCost(y, output){
  return dl.add(
               y.mul(output.add(EPSILON).log()),
               one.sub(y).mul(one.sub(output).add(EPSILON).log())).sum().neg();
}

function model(xs){
  const hiddenLayer = dl.tidy(() => {
    return xs.matMul(fullyConnectedWeights1).add(fullyConnectedBias1).relu() 
  });

  return hiddenLayer.matMul(fullyConnectedWeights2)
             .add(fullyConnectedBias2)
             .sigmoid();
}

// Train the model.
async function train(iterations: number) {
  const returnCost = true;
  let cost;

  for (let i = 0; i < iterations; i++) {
    cost = optimizer.minimize(() => {
      return calculateCost(targetTensor, model(inputTensor));
    }, returnCost);

    if (i % 10 === 0) {
      console.log(`loss[${i}]: ${cost.getValues()}`);
    }

    await dl.nextFrame();
  }

  return cost.dataSync();
}

const learnXOR = async () => {
  const iterations = getRandomIntegerInRange(800, 1000);
  const timeStart = performance.now();
  const result = [];

  const loss = await train(iterations);
{'fraction_non_alphanumeric': 0.16993939393939395, 'fraction_numerical': 0.02812121212121212, 'mean_word_length': 3.941317365269461, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3710721', 'n_tokens_mistral': 1532, 'n_tokens_neox': 1454, 'n_words': 296}