<commit_before>// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert' as convert;
import 'dart:io';

import 'package:logging/logging.dart';
import 'package:pana/pana.dart' hide Pubspec;
import 'package:path/path.dart' as p;
import 'package:uuid/uuid.dart';

import 'package:pub_dartdoc_data/pub_dartdoc_data.dart';

import '../job/backend.dart';
import '../job/job.dart';
import '../scorecard/backend.dart';
import '../scorecard/models.dart';
import '../shared/configuration.dart';
import '../shared/tool_env.dart';
import '../shared/urls.dart';
import '../shared/versions.dart' as versions;

import 'backend.dart';
import 'customization.dart';
import 'models.dart';

final Logger _logger = Logger('pub.dartdoc.runner');
final Uuid _uuid = Uuid();

const statusFilePath = 'status.json';
const _archiveFilePath = 'package.tar.gz';
const _buildLogFilePath = 'log.txt';
const _packageTimeout = Duration(minutes: 10);
const _pubDataFileName = 'pub-data.json';
const _sdkTimeout = Duration(minutes: 20);
final Duration _twoYears = const Duration(days: 2 * 365);

const _excludedLibraries = <String>[
  'dart:async',
  'dart:collection',
  'dart:convert',
  'dart:core',
  'dart:developer',
  'dart:io',
  'dart:isolate',
  'dart:math',
  'dart:typed_data',
  'dart:ui',
];

final _pkgPubDartdocDir =
    Platform.script.resolve('../../pkg/pub_dartdoc').toFilePath();

class DartdocJobProcessor extends JobProcessor {
  DartdocJobProcessor({Duration lockDuration})
      : super(service: JobService.dartdoc, lockDuration: lockDuration);

  /// Uses the tool environment's SDK (the one that is used for analysis too) to
  /// generate dartdoc documentation and extracted data file for SDK API indexing.
  /// Only the extracted data file will be used and uploaded.<commit_msg>Few more dart:* packages on the dartdoc blacklist (#3161)<commit_after>// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert' as convert;
import 'dart:io';

import 'package:logging/logging.dart';
import 'package:pana/pana.dart' hide Pubspec;
import 'package:path/path.dart' as p;
import 'package:uuid/uuid.dart';

import 'package:pub_dartdoc_data/pub_dartdoc_data.dart';

import '../job/backend.dart';
import '../job/job.dart';
import '../scorecard/backend.dart';
import '../scorecard/models.dart';
import '../shared/configuration.dart';
import '../shared/tool_env.dart';
import '../shared/urls.dart';
import '../shared/versions.dart' as versions;

import 'backend.dart';
import 'customization.dart';
import 'models.dart';

final Logger _logger = Logger('pub.dartdoc.runner');
final Uuid _uuid = Uuid();

const statusFilePath = 'status.json';
const _archiveFilePath = 'package.tar.gz';
const _buildLogFilePath = 'log.txt';
const _packageTimeout = Duration(minutes: 10);
const _pubDataFileName = 'pub-data.json';
const _sdkTimeout = Duration(minutes: 20);
final Duration _twoYears = const Duration(days: 2 * 365);

const _excludedLibraries = <String>[
  'dart:async',
  'dart:collection',
  'dart:convert',
  'dart:core',
  'dart:developer',
  'dart:ffi',
  'dart:html',
  'dart:io',
  'dart:isolate',
  'dart:js',
  'dart:js_util',
  'dart:math',
  'dart:typed_data',
  'dart:ui',
];

final _pkgPubDartdocDir =
    Platform.script.resolve('../../pkg/pub_dartdoc').toFilePath();

class DartdocJobProcessor extends JobProcessor {
  DartdocJobProcessor({Duration lockDuration})
      : super(service: JobService.dartdoc, lockDuration: lockDuration);

  /// Uses the tool environment's SDK (the one that is used for analysis too) to
  /// generate dartdoc documentation and extracted data file for SDK API indexing.
  /// Only the extracted data file will be used and uploaded.
{'fraction_non_alphanumeric': 0.1414116485686081, 'fraction_numerical': 0.006910167818361303, 'mean_word_length': 4.748936170212766, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '651664', 'n_tokens_mistral': 1365, 'n_tokens_neox': 1277, 'n_words': 369}