<commit_before>/*
 * Copyright (c) 2010, Regents of the University of California
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 *  * Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *  * Neither the name of the University of California, Berkeley
 * nor the names of its contributors may be used to endorse or promote
 * products derived from this software without specific prior written
 * permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Author: <NAME> (<EMAIL>), 2013.
 */

package radlab.rain.workload.rubis;


import java.util.Arrays;
import java.util.Calendar;
import java.util.concurrent.atomic.AtomicInteger;
//import java.util.concurrent.Semaphore;
import java.util.Date;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.HttpStatus;
import radlab.rain.util.HttpTransport;
import radlab.rain.workload.rubis.model.RubisCategory;
import radlab.rain.workload.rubis.model.RubisComment;
import radlab.rain.workload.rubis.model.RubisItem;
import radlab.rain.workload.rubis.model.RubisRegion;
import radlab.rain.workload.rubis.model.RubisUser;
import radlab.rain.workload.rubis.util.DiscreteDistribution;


/**
 * Collection of RUBiS utilities.
 *
 * @author <NAME> (<EMAIL>)
 */
public final class RubisUtility
{
	public static final int ANONYMOUS_USER_ID = -1;
	public static final int INVALID_CATEGORY_ID = -1;
	public static final int INVALID_REGION_ID = -1;
	public static final int INVALID_ITEM_ID = -1;
	public static final int INVALID_OPERATION_ID = -1;
	public static final int MILLISECS_PER_DAY = 1000*60*60*24;


	private static final char[] ALNUM_CHARS = { '0', '1', '2', '3', '4', '5',
												'6', '7', '8', '9', 'A', 'B',
												'C', 'D', 'E', 'F', 'G', 'H',
												'I', 'J', 'K', 'L', 'M', 'N',
												'O', 'P', 'Q', 'R', 'S', 'T',
												'U', 'V', 'W', 'X', 'Y', 'Z',
												'a', 'b', 'c', 'd', 'e', 'f',
												'g', 'h', 'i', 'j', 'k', 'l',
												'm', 'n', 'o', 'p', 'q', 'r',
												's', 't', 'u', 'v', 'w', 'x',
												'y', 'z'}; ///< The set of alphanumeric characters
	private static final String[] COMMENTS = {"This is a very bad comment. Stay away from this seller!!",
											  "This is a comment below average. I don't recommend this user!!",
											  "This is a neutral comment. It is neither a good or a bad seller!!",
											  "This is a comment above average. You can trust this seller even if it is not the best deal!!",
											  "This is an excellent comment. You can make really great deals with this seller!!"}; ///< Descriptions associated to comment ratings
	private static final int[] COMMENT_RATINGS = {-5, // Bad
												  -3, // Below average
												   0, // Neutral
												   3, // Average
												   5  /* Excellent */ }; ///< Possible comment ratings
	private static final int MIN_USER_ID = 1; ///< Mininum value for user IDs
	private static final int MIN_ITEM_ID = 1; ///< Mininum value for item IDs
	private static final int MIN_REGION_ID = 1; ///< Mininum value for region IDs
	private static final int MIN_CATEGORY_ID = 1; ///< Mininum value for category IDs
	private static AtomicInteger _userId;
	private static AtomicInteger _itemId;
//	private static Semaphore _userLock = new Semaphore(1, true);
//	private static Semaphore _itemLock = new Semaphore(1, true);


	private Random _rng = null;
	private RubisConfiguration _conf = null;
	private DiscreteDistribution _catDistr = null; ///< Probability distribution for generating random categories
	private Pattern _pageRegex = Pattern.compile("^.*?[&?]page=(\\d+).*?(?:[&?]page=(\\d+).*?)?$");


	private static int nextUserId()
	{
		return _userId.incrementAndGet();
	}

	private static int lastUserId()
	{
		return _userId.get();
	}

	private static int nextItemId()
	{
		return _itemId.incrementAndGet();
	}

	private static int lastItemId()
	{
		return _itemId.get();
	}

//	private static void lockUsers() throws InterruptedException
//	{
//		_userLock.acquire();
//	}

//	private static void unlockUsers()
//	{
//		_userLock.release();
//	}

//	private static void lockItems() throws InterruptedException
//	{
//		_itemLock.acquire();
//	}

//	private static void unlockItems()
//	{
//		_itemLock.release();
//	}

	private static synchronized void initUserId(int numPreloadUsers)
	{
		_userId = new AtomicInteger(MIN_USER_ID+numPreloadUsers-1);
	}

	private static synchronized void initItemId(int numPreloadItems)
	{
		_itemId = new AtomicInteger(MIN_ITEM_ID+numPreloadItems-1);
	}


	public RubisUtility()
	{
	}

	public RubisUtility(Random rng, RubisConfiguration conf)
	{
		this._rng = rng;
		this._conf = conf;

		initUserId(this._conf.getNumOfPreloadedUsers());
		initItemId(this._conf.getTotalActiveItems()+this._conf.getNumOfOldItems());

		int nc = this._conf.getCategories().size();
		if (nc > 0)
		{
			double[] catProbs = new double[nc];
			for (int i = 0; i < nc; ++i)
			{
				catProbs[i] = this._conf.getNumOfItemsPerCategory(i) / this._conf.getTotalActiveItems();
			}

			this._catDistr = new DiscreteDistribution(catProbs);
		}
	}

	public void setRandomGenerator(Random rng)
	{
		this._rng = rng;
	}

	public Random getRandomGenerator()
	{
		return this._rng;
	}

	public void setConfiguration(RubisConfiguration conf)
	{
		this._conf = conf;

		initUserId(this._conf.getNumOfPreloadedUsers());
	}
<commit_msg>[rubis] (bug-fix:major) Replace bad integer division with real division in the computation of category probabilities.<commit_after>/*
 * Copyright (c) 2010, Regents of the University of California
 * All rights reserved.
 * 
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 * 
 *  * Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *  * Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *  * Neither the name of the University of California, Berkeley
 * nor the names of its contributors may be used to endorse or promote
 * products derived from this software without specific prior written
 * permission.
 * 
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Author: <NAME> (<EMAIL>), 2013.
 */

package radlab.rain.workload.rubis;


import java.util.Arrays;
import java.util.Calendar;
import java.util.concurrent.atomic.AtomicInteger;
//import java.util.concurrent.Semaphore;
import java.util.Date;
import java.util.Random;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.http.HttpStatus;
import radlab.rain.util.HttpTransport;
import radlab.rain.workload.rubis.model.RubisCategory;
import radlab.rain.workload.rubis.model.RubisComment;
import radlab.rain.workload.rubis.model.RubisItem;
import radlab.rain.workload.rubis.model.RubisRegion;
import radlab.rain.workload.rubis.model.RubisUser;
import radlab.rain.workload.rubis.util.DiscreteDistribution;


/**
 * Collection of RUBiS utilities.
 *
 * @author <NAME> (<EMAIL>)
 */
public final class RubisUtility
{
	public static final int ANONYMOUS_USER_ID = -1;
	public static final int INVALID_CATEGORY_ID = -1;
	public static final int INVALID_REGION_ID = -1;
	public static final int INVALID_ITEM_ID = -1;
	public static final int INVALID_OPERATION_ID = -1;
	public static final int MILLISECS_PER_DAY = 1000*60*60*24;


	private static final char[] ALNUM_CHARS = { '0', '1', '2', '3', '4', '5',
												'6', '7', '8', '9', 'A', 'B',
												'C', 'D', 'E', 'F', 'G', 'H',
												'I', 'J', 'K', 'L', 'M', 'N',
												'O', 'P', 'Q', 'R', 'S', 'T',
												'U', 'V', 'W', 'X', 'Y', 'Z',
												'a', 'b', 'c', 'd', 'e', 'f',
												'g', 'h', 'i', 'j', 'k', 'l',
												'm', 'n', 'o', 'p', 'q', 'r',
												's', 't', 'u', 'v', 'w', 'x',
												'y', 'z'}; ///< The set of alphanumeric characters
	private static final String[] COMMENTS = {"This is a very bad comment. Stay away from this seller!!",
											  "This is a comment below average. I don't recommend this user!!",
											  "This is a neutral comment. It is neither a good or a bad seller!!",
											  "This is a comment above average. You can trust this seller even if it is not the best deal!!",
											  "This is an excellent comment. You can make really great deals with this seller!!"}; ///< Descriptions associated to comment ratings
	private static final int[] COMMENT_RATINGS = {-5, // Bad
												  -3, // Below average
												   0, // Neutral
												   3, // Average
												   5  /* Excellent */ }; ///< Possible comment ratings
	private static final int MIN_USER_ID = 1; ///< Mininum value for user IDs
	private static final int MIN_ITEM_ID = 1; ///< Mininum value for item IDs
	private static final int MIN_REGION_ID = 1; ///< Mininum value for region IDs
	private static final int MIN_CATEGORY_ID = 1; ///< Mininum value for category IDs
	private static AtomicInteger _userId;
	private static AtomicInteger _itemId;
//	private static Semaphore _userLock = new Semaphore(1, true);
//	private static Semaphore _itemLock = new Semaphore(1, true);


	private Random _rng = null;
	private RubisConfiguration _conf = null;
	private DiscreteDistribution _catDistr = null; ///< Probability distribution for generating random categories
	private Pattern _pageRegex = Pattern.compile("^.*?[&?]page=(\\d+).*?(?:[&?]page=(\\d+).*?)?$");


	private static int nextUserId()
	{
		return _userId.incrementAndGet();
	}

	private static int lastUserId()
	{
		return _userId.get();
	}

	private static int nextItemId()
	{
		return _itemId.incrementAndGet();
	}

	private static int lastItemId()
	{
		return _itemId.get();
	}

//	private static void lockUsers() throws InterruptedException
//	{
//		_userLock.acquire();
//	}

//	private static void unlockUsers()
//	{
//		_userLock.release();
//	}

//	private static void lockItems() throws InterruptedException
//	{
//		_itemLock.acquire();
//	}

//	private static void unlockItems()
//	{
//		_itemLock.release();
//	}

	private static synchronized void initUserId(int numPreloadUsers)
	{
		_userId = new AtomicInteger(MIN_USER_ID+numPreloadUsers-1);
	}

	private static synchronized void initItemId(int numPreloadItems)
	{
		_itemId = new AtomicInteger(MIN_ITEM_ID+numPreloadItems-1);
	}


	public RubisUtility()
	{
	}

	public RubisUtility(Random rng, RubisConfiguration conf)
	{
		this._rng = rng;
		this._conf = conf;

		initUserId(this._conf.getNumOfPreloadedUsers());
		initItemId(this._conf.getTotalActiveItems()+this._conf.getNumOfOldItems());

		int nc = this._conf.getCategories().size();
		if (nc > 0)
		{
			double[] catProbs = new double[nc];
			for (int i = 0; i < nc; ++i)
			{
				catProbs[i] = this._conf.getNumOfItemsPerCategory(i) / ((double) this._conf.getTotalActiveItems());
			}

			this._catDistr = new DiscreteDistribution(catProbs);
		}
	}

	public void setRandomGenerator(Random rng)
	{
		this._rng = rng;
	}

	public Random getRandomGenerator()
	{
		return this._rng;
	}

	public void setConfiguration(RubisConfiguration conf)
	{
		this._conf = conf;

		initUserId(this._conf.getNumOfPreloadedUsers());
	}

{'fraction_non_alphanumeric': 0.11935459549121616, 'fraction_numerical': 0.007238181406921511, 'mean_word_length': 3.5674931129476586, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6432613', 'n_tokens_mistral': 4767, 'n_tokens_neox': 4077, 'n_words': 1397}