<commit_before>﻿/* ****************************************************************************
 *
 * Copyright (c) Microsoft Corporation. 
 *
 * This source code is subject to terms and conditions of the Apache License, Version 2.0. A 
 * copy of the license can be found in the License.html file at the root of this distribution. If 
 * you cannot locate the Apache License, Version 2.0, please send an email to 
 * <EMAIL>. By using this source code in any fashion, you are agreeing to be bound 
 * by the terms of the Apache License, Version 2.0.
 *
 * You must not remove this notice, or any other, from this software.
 *
 * ***************************************************************************/

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.Remoting.Messaging;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.VisualStudioTools.Project;
using Newtonsoft.Json.Linq;

namespace Microsoft.NodejsTools.Npm.SPI {
    internal class NpmGetCatalogCommand : NpmCommand, IPackageCatalog {
        private IDictionary<string, IPackage> _byName = new Dictionary<string, IPackage>(); 
        private readonly bool _forceDownload;
        private string _cachePath;

        public NpmGetCatalogCommand(
            string fullPathToRootPackageDirectory,
            string cachePath,
            bool forceDownload,
            string pathToNpm = null,
            bool useFallbackIfNpmNotFound = true
        )
            : base(
                fullPathToRootPackageDirectory,
                pathToNpm,
                useFallbackIfNpmNotFound
        ) {
            _cachePath = cachePath;
            Arguments = "search";
            _forceDownload = forceDownload;
            LastRefreshed = DateTime.MinValue;
       }

        internal List<IPackage> ParseResultsFromReader(TextReader reader) {
            var builder = new NodeModuleBuilder();
            var results = new List<IPackage>();
            using (var jsonReader = new Newtonsoft.Json.JsonTextReader(reader)) {
                var token = JObject.ReadFrom(jsonReader);
                foreach (var module in token.Values()) {
                    try {
                        builder.Name = (string)module["name"];
                        if (string.IsNullOrEmpty(builder.Name)) {
                            continue;
                        }

                        builder.AppendToDescription((string)module["description"] ?? string.Empty);

                        var time = module["time"];
                        if (time != null) {
                            builder.AppendToDate((string)time["modified"]);
                        }

                        var versions = module["versions"];
                        if (versions != null) {
                            var latestVersion = versions
                                .OfType<JProperty>()
                                .Where(v => (string)v == "latest")
                                .Select(v => v.Name)
                                .FirstOrDefault();

                            if (!string.IsNullOrEmpty(latestVersion)) {
                                builder.Version = SemverVersion.Parse(latestVersion);
                            }
                        }

                        var keywords = module["keywords"];
                        if (keywords != null) {
                            foreach (var keyword in keywords.Select(v => (string)v)) {
                                builder.AddKeyword(keyword);
                            }
                        }

                        AddAuthor(builder, module["author"]);

                        AddHomepage(builder, module["homepage"]);

                        results.Add(builder.Build());
                    } catch (InvalidOperationException) {
                        // Occurs if a JValue appears where we expect JProperty
                    } catch (ArgumentException) {
                        OnOutputLogged(string.Format("Parsing error - Cannot display package {0}", builder.Name));
                    } finally {
                        builder.Reset();
                    }
                }
            }

            return results;<commit_msg>Don't skip packages with parsing errors. Will allow people to still browse/download those packages, and also help resolve issues like #1404<commit_after>﻿/* ****************************************************************************
 *
 * Copyright (c) Microsoft Corporation. 
 *
 * This source code is subject to terms and conditions of the Apache License, Version 2.0. A 
 * copy of the license can be found in the License.html file at the root of this distribution. If 
 * you cannot locate the Apache License, Version 2.0, please send an email to 
 * <EMAIL>. By using this source code in any fashion, you are agreeing to be bound 
 * by the terms of the Apache License, Version 2.0.
 *
 * You must not remove this notice, or any other, from this software.
 *
 * ***************************************************************************/

using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.Remoting.Messaging;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.VisualStudioTools.Project;
using Newtonsoft.Json.Linq;

namespace Microsoft.NodejsTools.Npm.SPI {
    internal class NpmGetCatalogCommand : NpmCommand, IPackageCatalog {
        private IDictionary<string, IPackage> _byName = new Dictionary<string, IPackage>(); 
        private readonly bool _forceDownload;
        private string _cachePath;

        public NpmGetCatalogCommand(
            string fullPathToRootPackageDirectory,
            string cachePath,
            bool forceDownload,
            string pathToNpm = null,
            bool useFallbackIfNpmNotFound = true
        )
            : base(
                fullPathToRootPackageDirectory,
                pathToNpm,
                useFallbackIfNpmNotFound
        ) {
            _cachePath = cachePath;
            Arguments = "search";
            _forceDownload = forceDownload;
            LastRefreshed = DateTime.MinValue;
       }

        internal List<IPackage> ParseResultsFromReader(TextReader reader) {
            var builder = new NodeModuleBuilder();
            var results = new List<IPackage>();
            using (var jsonReader = new Newtonsoft.Json.JsonTextReader(reader)) {
                var token = JObject.ReadFrom(jsonReader);
                foreach (var module in token.Values()) {
                    try {
                        builder.Name = (string)module["name"];
                        if (string.IsNullOrEmpty(builder.Name)) {
                            continue;
                        }

                        builder.AppendToDescription((string)module["description"] ?? string.Empty);

                        var time = module["time"];
                        if (time != null) {
                            builder.AppendToDate((string)time["modified"]);
                        }

                        var versions = module["versions"];
                        if (versions != null) {
                            var latestVersion = versions
                                .OfType<JProperty>()
                                .Where(v => (string)v == "latest")
                                .Select(v => v.Name)
                                .FirstOrDefault();

                            if (!string.IsNullOrEmpty(latestVersion)) {
                                builder.Version = SemverVersion.Parse(latestVersion);
                            }
                        }

                        var keywords = module["keywords"];
                        if (keywords != null) {
                            foreach (var keyword in keywords.Select(v => (string)v)) {
                                builder.AddKeyword(keyword);
                            }
                        }

                        AddAuthor(builder, module["author"]);

                        AddHomepage(builder, module["homepage"]);

                        results.Add(builder.Build());
                    } catch (InvalidOperationException) {
                        // Occurs if a JValue appears where we expect JProperty
                    } catch (ArgumentException) {
                        OnOutputLogged(string.Format("Parsing error (package {0})", builder.Name));
                        if (!string.IsNullOrEmpty(builder.Name)) {
                            results.Add(builder.Build());
                        }
                    } finally {
                        builder.Reset();
                    }
                }
            }

            return results;
{'fraction_non_alphanumeric': 0.11183035714285715, 'fraction_numerical': 0.0020089285714285712, 'mean_word_length': 1.5370894677236693, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7050424', 'n_tokens_mistral': 2032, 'n_tokens_neox': 1897, 'n_words': 545}