<commit_before>package com.mishiranu.dashchan.content.net;

import android.net.Uri;
import android.util.Base64;
import chan.content.ChanLocator;
import chan.http.HttpException;
import chan.http.HttpHolder;
import chan.http.HttpRequest;
import com.mishiranu.dashchan.content.Preferences;
import com.mishiranu.dashchan.content.async.HttpHolderTask;
import com.mishiranu.dashchan.util.ConcurrentUtils;
import com.mishiranu.dashchan.util.Log;
import java.util.HashMap;
import java.util.Locale;
import org.json.JSONArray;
import org.json.JSONObject;

public class UserAgentProvider {
	private static final UserAgentProvider INSTANCE = new UserAgentProvider();

	public static UserAgentProvider getInstance() {
		return INSTANCE;
	}

	private UserAgentProvider() {
		String reference = Preferences.getUserAgentReference();
		if (reference == null) {
			reference = "chromium:74.0.3729.169";
		}
		if (reference.startsWith("chromium:")) {
			String version = reference.substring(9);
			userAgent = formatChromiumUserAgent(version);
		}
		loadChromiumUserAgentReference();
	}

	private String userAgent = "Mozilla/5.0";

	private String formatChromiumUserAgent(String version) {
		return String.format(Locale.US, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
				"(KHTML, like Gecko) Chrome/%s Safari/537.36", version);
	}

	private void loadChromiumUserAgentReference() {
		new HttpHolderTask<Void, Void, String>() {
			@Override
			protected String doInBackground(HttpHolder holder, Void... params) {
				JSONObject object;
				try {
					Uri uri = ChanLocator.getDefault().buildQueryWithHost("www.googleapis.com",
							"storage/v1/b/chromium-browser-snapshots/o",
							"prefix", "Linux_x64/LAST_CHANGE", "fields", "items(metadata)");
					object = new HttpRequest(uri, holder).read().getJsonObject();
				} catch (HttpException e) {
					return null;
				}
				JSONArray array = object != null ? object.optJSONArray("items") : null;
				object = array != null && array.length() > 0 ? array.optJSONObject(0) : null;
				object = object != null ? object.optJSONObject("metadata") : null;
				String commit = object != null ? object.optString("cr-git-commit") : null;

				if (commit != null) {
					String response;
					try {
						Uri uri = ChanLocator.getDefault().buildQueryWithHost("chromium.googlesource.com",
								"chromium/src/+/" + commit + "/chrome/VERSION", "format", "TEXT");
						response = new HttpRequest(uri, holder).read().getString();
					} catch (HttpException e) {
						return null;
					}
					if (response != null) {
						String versionData;
						try {
							versionData = new String(Base64.decode(response, Base64.DEFAULT));
						} catch (Exception e) {
							Log.persistent().stack(e);
							return null;
						}
						HashMap<String, Integer> map = new HashMap<>();
						for (String line : versionData.split("\n")) {
							int index = line.indexOf('=');
							if (index >= 0) {
								String key = line.substring(0, index);
								String valueString = line.substring(index + 1);
								try {
									int value = Integer.parseInt(valueString);
									map.put(key.toLowerCase(Locale.US), value);
								} catch (NumberFormatException e) {
									// Ignore
								}
							}
						}
						Integer major = map.get("major");
						Integer minor = map.get("minor");
						Integer build = map.get("build");
						Integer patch = map.get("patch");
						if (major != null && minor != null) {
							return major + "." + minor + "." + (build != null ? build : 0) + "." +
									(patch != null ? patch : 0);
						}
					}
				}
				return null;
			}

			@Override
			protected void onPostExecute(String version) {
				if (version != null) {
					Preferences.setUserAgentReference("chromium:" + version);
					userAgent = formatChromiumUserAgent(version);
				}
			}
		}.executeOnExecutor(ConcurrentUtils.SEPARATE_EXECUTOR);
	}

	public String getUserAgent() {
		return userAgent;
	}
}<commit_msg>Use mobile User-Agent<commit_after>package com.mishiranu.dashchan.content.net;

import android.net.Uri;
import android.os.Build;
import android.util.Base64;
import chan.content.ChanLocator;
import chan.http.HttpException;
import chan.http.HttpHolder;
import chan.http.HttpRequest;
import com.mishiranu.dashchan.content.Preferences;
import com.mishiranu.dashchan.content.async.HttpHolderTask;
import com.mishiranu.dashchan.util.ConcurrentUtils;
import com.mishiranu.dashchan.util.Log;
import java.util.HashMap;
import java.util.Locale;
import org.json.JSONArray;
import org.json.JSONObject;

public class UserAgentProvider {
	private static final UserAgentProvider INSTANCE = new UserAgentProvider();

	public static UserAgentProvider getInstance() {
		return INSTANCE;
	}

	private static final String PREFIX = "chromium.v1:";
	private static final String DEFAULT_REFERENCE = PREFIX + "74.0.3729.169.537.36";

	private String userAgent = "Mozilla/5.0";

	private UserAgentProvider() {
		String userAgent = formatUserAgentForReference(Preferences.getUserAgentReference());
		if (userAgent == null) {
			userAgent = formatUserAgentForReference(DEFAULT_REFERENCE);
		}
		if (userAgent != null) {
			this.userAgent = userAgent;
		}
		loadChromiumUserAgentReference();
	}

	private String formatUserAgentForReference(String reference) {
		if (reference != null && reference.startsWith(PREFIX)) {
			String version = reference.substring(PREFIX.length());
			return formatChromiumUserAgent(version);
		}
		return null;
	}

	private String formatChromiumUserAgent(String version) {
		try {
			String[] numbers = version.split("\\.");
			String chromiumVersion = numbers[0] + "." + numbers[1] + "." + numbers[2] + "." + numbers[3];
			String webKitVersion = numbers[4] + "." + numbers[5];
			return String.format(Locale.US, "Mozilla/5.0 (Linux; Android %s; wv) AppleWebKit/%s " +
							"(KHTML, like Gecko) Version/4.0 Chrome/%s Mobile Safari/%s", Build.VERSION.RELEASE,
					webKitVersion, chromiumVersion, webKitVersion);
		} catch (Exception e) {
			return null;
		}
	}

	private void loadChromiumUserAgentReference() {
		new HttpHolderTask<Void, Void, String>() {
			@Override
			protected String doInBackground(HttpHolder holder, Void... params) {
				JSONObject object;
				try {
					Uri uri = ChanLocator.getDefault().buildQueryWithHost("www.googleapis.com",
							"storage/v1/b/chromium-browser-snapshots/o",
							"prefix", "Linux_x64/LAST_CHANGE", "fields", "items(metadata)");
					object = new HttpRequest(uri, holder).read().getJsonObject();
				} catch (HttpException e) {
					return null;
				}
				JSONArray array = object != null ? object.optJSONArray("items") : null;
				object = array != null && array.length() > 0 ? array.optJSONObject(0) : null;
				object = object != null ? object.optJSONObject("metadata") : null;
				String commit = object != null ? object.optString("cr-git-commit") : null;

				if (commit != null) {
					String chromeVersionResponse;
					try {
						Uri uri = ChanLocator.getDefault().buildQueryWithHost("chromium.googlesource.com",
								"chromium/src/+/" + commit + "/chrome/VERSION", "format", "TEXT");
						chromeVersionResponse = new HttpRequest(uri, holder).read().getString();
					} catch (HttpException e) {
						return null;
					}
					String webKitVersionResponse;
					try {
						Uri uri = ChanLocator.getDefault().buildQueryWithHost("chromium.googlesource.com",
								"chromium/src/+/" + commit + "/build/util/webkit_version.h.in", "format", "TEXT");
						webKitVersionResponse = new HttpRequest(uri, holder).read().getString();
					} catch (HttpException e) {
						return null;
					}
					if (chromeVersionResponse != null && webKitVersionResponse != null) {
						String chromiumVersionData;
						try {
							chromiumVersionData = new String(Base64.decode(chromeVersionResponse, Base64.DEFAULT));
						} catch (Exception e) {
							Log.persistent().stack(e);
							return null;
						}
						String webKitVersionData;
						try {
							webKitVersionData = new String(Base64.decode(webKitVersionResponse, Base64.DEFAULT));
						} catch (Exception e) {
							Log.persistent().stack(e);
							return null;
						}
						HashMap<String, Integer> chromeMap = new HashMap<>();
						for (String line : chromiumVersionData.split("\n")) {
							int index = line.indexOf('=');
							if (index >= 0) {
								String key = line.substring(0, index);
								String valueString = line.substring(index + 1);
								try {
									int value = Integer.parseInt(valueString);
									chromeMap.put(key.toLowerCase(Locale.US), value);
								} catch (NumberFormatException e) {
									// Ignore
								}
							}
						}
						HashMap<String, Integer> webKitMap = new HashMap<>();
						for (String line : webKitVersionData.split("\n")) {
							if (line.startsWith("#define")) {
								String[] nameValue = line.substring(7).trim().split(" ");
								if (nameValue.length == 2 && nameValue[0].startsWith("WEBKIT_VERSION_")) {
									try {
										webKitMap.put(nameValue[0].substring(15).toLowerCase(Locale.US),
												Integer.parseInt(nameValue[1]));
									} catch (NumberFormatException e) {
										// Ignore
									}
								}
							}
						}
						Integer chromeMajor = chromeMap.get("major");
						Integer chromeMinor = chromeMap.get("minor");
						Integer chromeBuild = chromeMap.get("build");
						Integer chromePatch = chromeMap.get("patch");
						Integer webKitMajor = webKitMap.get("major");
						Integer webKitMinor = webKitMap.get("minor");
						if (chromeMajor != null && chromeMinor != null) {
							if (chromeBuild == null) {
								chromeBuild = 0;
							}
							if (chromePatch == 0) {
								chromePatch = 0;
							}
							if (webKitMajor == null || webKitMinor == null) {
								webKitMajor = 537;
								webKitMinor = 36;
							}
							return chromeMajor + "." + chromeMinor + "." + chromeBuild + "." + chromePatch + "." +
									webKitMajor + "." + webKitMinor;
						}
					}
				}
				return null;
			}

			@Override
			protected void onPostExecute(String version) {
				if (version != null) {
					Preferences.setUserAgentReference(PREFIX + version);
					String userAgent = formatChromiumUserAgent(version);
					if (userAgent != null) {
						UserAgentProvider.this.userAgent = userAgent;
					}
				}
			}
		}.executeOnExecutor(ConcurrentUtils.SEPARATE_EXECUTOR);
	}

	public String getUserAgent() {
		return userAgent;
	}
}
{'fraction_non_alphanumeric': 0.13257175770375348, 'fraction_numerical': 0.010463665162714793, 'mean_word_length': 3.199113260781943, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5711133', 'n_tokens_mistral': 4187, 'n_tokens_neox': 3107, 'n_words': 670}