<commit_before>/// <reference path="../typings/main.d.ts" />
/// <reference path="../node_modules/typescript/lib/typescript.d.ts" />

import {readFileSync} from "fs";
import * as ts from "typescript";
import d = require("./Domain");
import o = require("./JsonOutput");

export module SonarTypeScript {
    export class Analyzer {
        private scanner = ts.createScanner(ts.ScriptTarget.ES5, /*skipTrivia*/ false);

        constructor(private outputter: o.Output.IFileMetricOutput) {
        }

        public analyzeFile(sourceFile: ts.SourceFile) {
            var fileMetrics = new d.Domain.FileMetric();
            fileMetrics.FileName = sourceFile.fileName;

            this.initializeState(sourceFile.getFullText());
            this.scanFile(sourceFile, fileMetrics);

            this.outputter.writeMetric(fileMetrics);
        }

        private scanFile(sourceFile: ts.SourceFile, metrics: d.Domain.FileMetric) {
            var token = this.scanner.scan();
            while (token != ts.SyntaxKind.EndOfFileToken) {
                this.updateMetricFor(<ts.SyntaxKind>token, metrics);
                console.log(this.scanner.getTokenText());
                token = this.scanner.scan();
            }
        }

        private initializeState(text: string): void {
            this.scanner.setText(text);
            this.scanner.setOnError((message: ts.DiagnosticMessage, length: number) => {
                console.error(message);
            });
            this.scanner.setScriptTarget(ts.ScriptTarget.ES5);
            this.scanner.setLanguageVariant(ts.LanguageVariant.Standard);
        }

        private prevTokenKind: ts.SyntaxKind;

        private updateMetricFor(kind: ts.SyntaxKind, fileMetrics: d.Domain.FileMetric) {
            console.log("kind: " + ts.SyntaxKind[kind]);
            switch (kind) {
                case ts.SyntaxKind.ModuleDeclaration:
                    fileMetrics.NumberOfModules++;
                    break;
                case ts.SyntaxKind.InterfaceDeclaration:
                    fileMetrics.NumberOfInterfaces++;
                    break;
                case ts.SyntaxKind.ClassDeclaration:
                    fileMetrics.NumberOfClasses++;
                    break;
                case ts.SyntaxKind.MethodDeclaration:
                case ts.SyntaxKind.FunctionDeclaration:
                    fileMetrics.NumberOfMethods++;
                    break;
                case ts.SyntaxKind.NewLineTrivia:
                    fileMetrics.NumberOfLines++;
                    fileMetrics.LinesOfCode++;
             
                    if (this.prevTokenKind != null && this.prevTokenKind == ts.SyntaxKind.NewLineTrivia) {
                        fileMetrics.LinesOfCode--;
                    }
                    
                    break;
                default:
                    break;
            }

            this.prevTokenKind = kind;
        }
    }
}<commit_msg>Detect lines with only whitespace<commit_after>/// <reference path="../typings/main.d.ts" />
/// <reference path="../node_modules/typescript/lib/typescript.d.ts" />

import {readFileSync} from "fs";
import * as ts from "typescript";
import d = require("./Domain");
import o = require("./JsonOutput");

export module SonarTypeScript {
    export class Analyzer {
        private scanner = ts.createScanner(ts.ScriptTarget.ES5, /*skipTrivia*/ false);

        constructor(private outputter: o.Output.IFileMetricOutput) {
        }

        public analyzeFile(sourceFile: ts.SourceFile) {
            var fileMetrics = new d.Domain.FileMetric();
            fileMetrics.FileName = sourceFile.fileName;

            this.initializeState(sourceFile.getFullText());
            this.scanFile(sourceFile, fileMetrics);

            this.outputter.writeMetric(fileMetrics);
        }

        private scanFile(sourceFile: ts.SourceFile, metrics: d.Domain.FileMetric) {
            var token = this.scanner.scan();
            while (token != ts.SyntaxKind.EndOfFileToken) {
                this.updateMetricFor(<ts.SyntaxKind>token, metrics);
                console.log(this.scanner.getTokenText());
                token = this.scanner.scan();
            }
        }

        private initializeState(text: string): void {
            this.scanner.setText(text);
            this.scanner.setOnError((message: ts.DiagnosticMessage, length: number) => {
                console.error(message);
            });
            this.scanner.setScriptTarget(ts.ScriptTarget.ES5);
            this.scanner.setLanguageVariant(ts.LanguageVariant.Standard);
        }

        private prevTokenKind: ts.SyntaxKind;
        private onlyWhitespaceSinceLastNewLine: boolean = true;

        private updateMetricFor(kind: ts.SyntaxKind, fileMetrics: d.Domain.FileMetric) {
            console.log("kind: " + ts.SyntaxKind[kind]);
            switch (kind) {
                case ts.SyntaxKind.ModuleDeclaration:
                    fileMetrics.NumberOfModules++;
                    break;
                case ts.SyntaxKind.InterfaceDeclaration:
                    fileMetrics.NumberOfInterfaces++;
                    break;
                case ts.SyntaxKind.ClassDeclaration:
                    fileMetrics.NumberOfClasses++;
                    break;
                case ts.SyntaxKind.MethodDeclaration:
                case ts.SyntaxKind.FunctionDeclaration:
                    fileMetrics.NumberOfMethods++;
                    break;
                case ts.SyntaxKind.NewLineTrivia:
                    fileMetrics.NumberOfLines++;
                    fileMetrics.LinesOfCode++;
             
                    if (this.onlyWhitespaceSinceLastNewLine) {
                        fileMetrics.LinesOfCode--;
                    }
                    
                    // Reset whitespace test
                    this.onlyWhitespaceSinceLastNewLine = true;
                    
                    break;
                default:
                    break;
            }

            this.prevTokenKind = kind;
            
            if(kind != ts.SyntaxKind.WhitespaceTrivia && kind != ts.SyntaxKind.NewLineTrivia) {
                this.onlyWhitespaceSinceLastNewLine = false;
            }
        }
    }
}
{'fraction_non_alphanumeric': 0.09974424552429667, 'fraction_numerical': 0.0006393861892583121, 'mean_word_length': 1.6693686006825939, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1496107', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1518, 'n_words': 256}