<commit_before>package migrations

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"os"
	"path"

	"github.com/OpenBazaar/jsonpb"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/golang/protobuf/proto"
	ipfscore "github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
)

const migration012_ListingVersionForMarkupListings = 4

type Migration012 struct{}

type Migration012_ListingData struct {
	Hash         string   `json:"hash"`
	Slug         string   `json:"slug"`
	Title        string   `json:"title"`
	Categories   []string `json:"categories"`
	NSFW         bool     `json:"nsfw"`
	ContractType string   `json:"contractType"`
	Description  string   `json:"description"`
	Thumbnail    struct {
		Tiny   string `json:"tiny"`
		Small  string `json:"small"`
		Medium string `json:"medium"`
	} `json:"thumbnail"`
	Price struct {
		CurrencyCode string  `json:"currencyCode"`
		Amount       uint64  `json:"amount"`
		Modifier     float32 `json:"modifier"`
	} `json:"price"`
	ShipsTo            []string `json:"shipsTo"`
	FreeShipping       []string `json:"freeShipping"`
	Language           string   `json:"language"`
	AverageRating      float32  `json:"averageRating"`
	RatingCount        uint32   `json:"ratingCount"`
	ModeratorIDs       []string `json:"moderators"`
	AcceptedCurrencies []string `json:"acceptedCurrencies"`
	CoinType           string   `json:"coinType"`
}

func Migration012_listingHasNewFeaturesAndOldVersion(sl *pb.SignedListing) bool {
	metadata := sl.Listing.Metadata
	if metadata == nil {
		return false
	}
	return metadata.Version == 3 &&
		metadata.PriceModifier != 0
}

func Migration012_GetIdentityKey(repoPath, databasePassword string, testnetEnabled bool) ([]byte, error) {
	db, err := OpenDB(repoPath, databasePassword, testnetEnabled)
	if err != nil {
		return nil, err
	}
	defer db.Close()

	var identityKey []byte
	err = db.
		QueryRow("select value from config where key=?", "identityKey").
		Scan(&identityKey)
	if err != nil {
		return nil, err
	}
	return identityKey, nil
}

func (Migration012) Up(repoPath, databasePassword string, testnetEnabled bool) error {
	listingsIndexFilePath := path.Join(repoPath, "root", "listings.json")

	// Find all crypto listings
	listingsIndexJSONBytes, err := ioutil.ReadFile(listingsIndexFilePath)
	if err != nil {
		return err
	}

	listingsIndex := []*Migration012_ListingData{}
	err = json.Unmarshal(listingsIndexJSONBytes, &listingsIndex)
	if err != nil {
		return err
	}

	cryptoListings := []*Migration012_ListingData{}
	for _, listingAbstract := range listingsIndex {
		if listingAbstract.ContractType == "CRYPTOCURRENCY" {
			cryptoListings = append(cryptoListings, listingAbstract)
		}
	}

	// Finish early If no crypto listings
	if len(cryptoListings) == 0 {
		return writeRepoVer(repoPath, 13)
	}

	// Check each crypto listing for markup
	markupListings := []*pb.SignedListing{}
	for _, listingAbstract := range cryptoListings {
		listingJSONBytes, err := ioutil.ReadFile(migration012_listingFilePath(repoPath, listingAbstract.Slug))
		if err != nil {
			return err
		}
		sl := new(pb.SignedListing)
		err = jsonpb.UnmarshalString(string(listingJSONBytes), sl)
		if err != nil {
			return err
		}

		if Migration012_listingHasNewFeaturesAndOldVersion(sl) {
			markupListings = append(markupListings, sl)
		}
	}

	// Finish early If no crypto listings with new features are found
	if len(markupListings) == 0 {
		return writeRepoVer(repoPath, 13)
	}

	// Setup signing capabilities
	identityKey, err := Migration012_GetIdentityKey(repoPath, databasePassword, testnetEnabled)
	if err != nil {
		return err
	}

	identity, err := ipfs.IdentityFromKey(identityKey)
	if err != nil {
		return err
	}

	// IPFS node setup
	r, err := fsrepo.Open(repoPath)
	if err != nil {
		return err
	}
	cctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	cfg, err := r.Config()
	if err != nil {
		return err
	}

	cfg.Identity = identity

	ncfg := &ipfscore.BuildCfg{
		Repo:   r,
		Online: true,
		ExtraOpts: map[string]bool{
			"mplex": true,
		},
		DNSResolver: nil,
		Routing:     nil,
	}

	nd, err := ipfscore.NewNode(cctx, ncfg)
	if err != nil {
		return err
	}
	defer nd.Close()<commit_msg>TWEAK: Make ipfs node in Migration012 be offline.<commit_after>package migrations

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"os"
	"path"

	"github.com/OpenBazaar/jsonpb"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"github.com/golang/protobuf/proto"
	ipfscore "github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
)

const migration012_ListingVersionForMarkupListings = 4

type Migration012 struct{}

type Migration012_ListingData struct {
	Hash         string   `json:"hash"`
	Slug         string   `json:"slug"`
	Title        string   `json:"title"`
	Categories   []string `json:"categories"`
	NSFW         bool     `json:"nsfw"`
	ContractType string   `json:"contractType"`
	Description  string   `json:"description"`
	Thumbnail    struct {
		Tiny   string `json:"tiny"`
		Small  string `json:"small"`
		Medium string `json:"medium"`
	} `json:"thumbnail"`
	Price struct {
		CurrencyCode string  `json:"currencyCode"`
		Amount       uint64  `json:"amount"`
		Modifier     float32 `json:"modifier"`
	} `json:"price"`
	ShipsTo            []string `json:"shipsTo"`
	FreeShipping       []string `json:"freeShipping"`
	Language           string   `json:"language"`
	AverageRating      float32  `json:"averageRating"`
	RatingCount        uint32   `json:"ratingCount"`
	ModeratorIDs       []string `json:"moderators"`
	AcceptedCurrencies []string `json:"acceptedCurrencies"`
	CoinType           string   `json:"coinType"`
}

func Migration012_listingHasNewFeaturesAndOldVersion(sl *pb.SignedListing) bool {
	metadata := sl.Listing.Metadata
	if metadata == nil {
		return false
	}
	return metadata.Version == 3 &&
		metadata.PriceModifier != 0
}

func Migration012_GetIdentityKey(repoPath, databasePassword string, testnetEnabled bool) ([]byte, error) {
	db, err := OpenDB(repoPath, databasePassword, testnetEnabled)
	if err != nil {
		return nil, err
	}
	defer db.Close()

	var identityKey []byte
	err = db.
		QueryRow("select value from config where key=?", "identityKey").
		Scan(&identityKey)
	if err != nil {
		return nil, err
	}
	return identityKey, nil
}

func (Migration012) Up(repoPath, databasePassword string, testnetEnabled bool) error {
	listingsIndexFilePath := path.Join(repoPath, "root", "listings.json")

	// Find all crypto listings
	listingsIndexJSONBytes, err := ioutil.ReadFile(listingsIndexFilePath)
	if err != nil {
		return err
	}

	listingsIndex := []*Migration012_ListingData{}
	err = json.Unmarshal(listingsIndexJSONBytes, &listingsIndex)
	if err != nil {
		return err
	}

	cryptoListings := []*Migration012_ListingData{}
	for _, listingAbstract := range listingsIndex {
		if listingAbstract.ContractType == "CRYPTOCURRENCY" {
			cryptoListings = append(cryptoListings, listingAbstract)
		}
	}

	// Finish early If no crypto listings
	if len(cryptoListings) == 0 {
		return writeRepoVer(repoPath, 13)
	}

	// Check each crypto listing for markup
	markupListings := []*pb.SignedListing{}
	for _, listingAbstract := range cryptoListings {
		listingJSONBytes, err := ioutil.ReadFile(migration012_listingFilePath(repoPath, listingAbstract.Slug))
		if err != nil {
			return err
		}
		sl := new(pb.SignedListing)
		err = jsonpb.UnmarshalString(string(listingJSONBytes), sl)
		if err != nil {
			return err
		}

		if Migration012_listingHasNewFeaturesAndOldVersion(sl) {
			markupListings = append(markupListings, sl)
		}
	}

	// Finish early If no crypto listings with new features are found
	if len(markupListings) == 0 {
		return writeRepoVer(repoPath, 13)
	}

	// Setup signing capabilities
	identityKey, err := Migration012_GetIdentityKey(repoPath, databasePassword, testnetEnabled)
	if err != nil {
		return err
	}

	identity, err := ipfs.IdentityFromKey(identityKey)
	if err != nil {
		return err
	}

	// IPFS node setup
	r, err := fsrepo.Open(repoPath)
	if err != nil {
		return err
	}
	cctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	cfg, err := r.Config()
	if err != nil {
		return err
	}

	cfg.Identity = identity

	ncfg := &ipfscore.BuildCfg{
		Repo:   r,
		Online: false,
		ExtraOpts: map[string]bool{
			"mplex": true,
		},
		DNSResolver: nil,
		Routing:     nil,
	}

	nd, err := ipfscore.NewNode(cctx, ncfg)
	if err != nil {
		return err
	}
	defer nd.Close()
{'fraction_non_alphanumeric': 0.11841950990737483, 'fraction_numerical': 0.012076445069761988, 'mean_word_length': 4.044352454169131, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2112902', 'n_tokens_mistral': 3081, 'n_tokens_neox': 2840, 'n_words': 676}