<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.metadata;

import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;

import org.apache.hadoop.metadata.json.Serialization$;
import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.DataTypes;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;

import java.util.ArrayList;
import java.util.Arrays;

public class DemoDataDriver {

	private static ArrayList<ITypedReferenceableInstance> tableArray;
	private static ArrayList<ITypedReferenceableInstance> lineageArray;

	private static final Logger LOG = LoggerFactory
			.getLogger(DemoDataDriver.class);

	private static final String DATABASE_TYPE = "hive_database";
	private static final String TABLE_TYPE = "hive_table";

	protected TypeSystem typeSystem;
	protected WebResource service;

	public void setUp() throws Exception {
		typeSystem = TypeSystem.getInstance();
		typeSystem.reset();

		String baseUrl = "http://localhost:21000/";

		DefaultClientConfig config = new DefaultClientConfig();
		Client client = Client.create(config);
		client.resource(UriBuilder.fromUri(baseUrl).build());

		service = client.resource(UriBuilder.fromUri(baseUrl).build());
	}

	protected AttributeDefinition createRequiredAttrDef(String name,
			IDataType dataType) {
		return new AttributeDefinition(name, dataType.getName(),
				Multiplicity.REQUIRED, false, null);
	}

	@SuppressWarnings("unchecked")
	protected HierarchicalTypeDefinition<TraitType> createTraitTypeDef(
			String name, ImmutableList<String> superTypes,
			AttributeDefinition... attrDefs) {
		return new HierarchicalTypeDefinition(TraitType.class, name,
				superTypes, attrDefs);
	}

	@SuppressWarnings("unchecked")
	protected HierarchicalTypeDefinition<ClassType> createClassTypeDef(
			String name, ImmutableList<String> superTypes,
			AttributeDefinition... attrDefs) {
		return new HierarchicalTypeDefinition(ClassType.class, name,
				superTypes, attrDefs);
	}

	public void submitEntity(ITypedReferenceableInstance tableInstance)
			throws Exception {
		String tableInstanceAsJSON = Serialization$.MODULE$
				.toJson(tableInstance);
		LOG.debug("tableInstance = " + tableInstanceAsJSON);

		WebResource resource = service.path("api/metadata/entities/submit")
				.path(TABLE_TYPE);

		ClientResponse clientResponse = resource
				.accept(MediaType.APPLICATION_JSON)
				.type(MediaType.APPLICATION_JSON)
				.method(HttpMethod.POST, ClientResponse.class,
						tableInstanceAsJSON);
		assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
	}

	public void getEntityList() throws Exception {
		ClientResponse clientResponse = service
				.path("api/metadata/entities/list/").path(TABLE_TYPE)
				.accept(MediaType.APPLICATION_JSON)
				.type(MediaType.APPLICATION_JSON)
				.method(HttpMethod.GET, ClientResponse.class);
		assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();

		String responseAsString = clientResponse.getEntity(String.class);
		JSONObject response = new JSONObject(responseAsString);
		final JSONArray list = response.getJSONArray("list");
		System.out.println("list = " + list);
		assert list != null;
		assert list.length() > 0;
	}

	private void createTypes() throws Exception {
		HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = createClassTypeDef(
				DATABASE_TYPE, ImmutableList.<String> of(),
				createRequiredAttrDef("name", DataTypes.STRING_TYPE),
				createRequiredAttrDef("description", DataTypes.STRING_TYPE));

		StructTypeDefinition structTypeDefinition = new StructTypeDefinition(
				"serdeType", new AttributeDefinition[] {
						createRequiredAttrDef("name", DataTypes.STRING_TYPE),
						createRequiredAttrDef("serde", DataTypes.STRING_TYPE) });

		HierarchicalTypeDefinition<ClassType> tableTypeDefinition = createClassTypeDef(
				TABLE_TYPE, ImmutableList.<String> of(),
				createRequiredAttrDef("name", DataTypes.STRING_TYPE),
				createRequiredAttrDef("description", DataTypes.STRING_TYPE),
				createRequiredAttrDef("type", DataTypes.STRING_TYPE),
				new AttributeDefinition("serde1", "serdeType",
						Multiplicity.REQUIRED, false, null),
				new AttributeDefinition("serde2", "serdeType",
						Multiplicity.REQUIRED, false, null),
				new AttributeDefinition("database", DATABASE_TYPE,
						Multiplicity.REQUIRED, true, null));

		HierarchicalTypeDefinition<TraitType> classificationTypeDefinition = createTraitTypeDef(
				"classification", ImmutableList.<String> of(),
				createRequiredAttrDef("tag", DataTypes.STRING_TYPE));

		HierarchicalTypeDefinition<ClassType> lineageTypeDefinition = createClassTypeDef(
				"HiveLineage",
				ImmutableList.<String> of(),
				createRequiredAttrDef("queryId", DataTypes.STRING_TYPE),
				createRequiredAttrDef("hiveId", DataTypes.STRING_TYPE),
				createRequiredAttrDef("user", DataTypes.STRING_TYPE),
				createRequiredAttrDef("queryStartTime", DataTypes.STRING_TYPE),
				createRequiredAttrDef("queryEndTime", DataTypes.STRING_TYPE),
				createRequiredAttrDef("query", DataTypes.STRING_TYPE),
				new AttributeDefinition("tableName", TABLE_TYPE,
						Multiplicity.REQUIRED, true, null),
				createRequiredAttrDef("success", DataTypes.STRING_TYPE),
				createRequiredAttrDef("executionEngine", DataTypes.STRING_TYPE),
				new AttributeDefinition("sourceTables", TABLE_TYPE,
						Multiplicity.COLLECTION, true, "forwardLineage"));

		typeSystem.defineTypes(ImmutableList.of(structTypeDefinition),
				ImmutableList.of(classificationTypeDefinition), ImmutableList
						.of(databaseTypeDefinition, tableTypeDefinition,
								lineageTypeDefinition));
	}

	private void submitTypes() throws Exception {
		String typesAsJSON = TypesSerialization.toJson(
				typeSystem,
				Arrays.asList(new String[] { DATABASE_TYPE, TABLE_TYPE,
						"serdeType", "classification" }));
		sumbitType(typesAsJSON, TABLE_TYPE);
	}

	private void sumbitType(String typesAsJSON, String type)
			throws JSONException {
		WebResource resource = service.path("api/metadata/types/submit").path(
				type);

		ClientResponse clientResponse = resource<commit_msg>Debugging Demo Driver.<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.metadata;

import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.DefaultClientConfig;

import org.apache.hadoop.metadata.json.Serialization$;
import org.apache.hadoop.metadata.json.TypesSerialization;
import org.apache.hadoop.metadata.types.AttributeDefinition;
import org.apache.hadoop.metadata.types.ClassType;
import org.apache.hadoop.metadata.types.DataTypes;
import org.apache.hadoop.metadata.types.HierarchicalTypeDefinition;
import org.apache.hadoop.metadata.types.IDataType;
import org.apache.hadoop.metadata.types.Multiplicity;
import org.apache.hadoop.metadata.types.StructTypeDefinition;
import org.apache.hadoop.metadata.types.TraitType;
import org.apache.hadoop.metadata.types.TypeSystem;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriBuilder;

import java.util.ArrayList;
import java.util.Arrays;

public class DemoDataDriver {

	private static ArrayList<ITypedReferenceableInstance> tableArray;
	private static ArrayList<ITypedReferenceableInstance> lineageArray;

	private static final Logger LOG = LoggerFactory
			.getLogger(DemoDataDriver.class);

	private static final String DATABASE_TYPE = "hive_database";
	private static final String TABLE_TYPE = "hive_table";

	protected TypeSystem typeSystem;
	protected WebResource service;

	public void setUp() throws Exception {
		typeSystem = TypeSystem.getInstance();
		typeSystem.reset();

		String baseUrl = "http://localhost:21000/";

		DefaultClientConfig config = new DefaultClientConfig();
		Client client = Client.create(config);
		client.resource(UriBuilder.fromUri(baseUrl).build());

		service = client.resource(UriBuilder.fromUri(baseUrl).build());
	}

	protected AttributeDefinition createRequiredAttrDef(String name,
			IDataType dataType) {
		return new AttributeDefinition(name, dataType.getName(),
				Multiplicity.REQUIRED, false, null);
	}

	@SuppressWarnings("unchecked")
	protected HierarchicalTypeDefinition<TraitType> createTraitTypeDef(
			String name, ImmutableList<String> superTypes,
			AttributeDefinition... attrDefs) {
		return new HierarchicalTypeDefinition(TraitType.class, name,
				superTypes, attrDefs);
	}

	@SuppressWarnings("unchecked")
	protected HierarchicalTypeDefinition<ClassType> createClassTypeDef(
			String name, ImmutableList<String> superTypes,
			AttributeDefinition... attrDefs) {
		return new HierarchicalTypeDefinition(ClassType.class, name,
				superTypes, attrDefs);
	}

	public void submitEntity(ITypedReferenceableInstance tableInstance)
			throws Exception {
		String tableInstanceAsJSON = Serialization$.MODULE$
				.toJson(tableInstance);
		LOG.debug("tableInstance = " + tableInstanceAsJSON);

		WebResource resource = service.path("api/metadata/entities/submit")
				.path(TABLE_TYPE);

		ClientResponse clientResponse = resource
				.accept(MediaType.APPLICATION_JSON)
				.type(MediaType.APPLICATION_JSON)
				.method(HttpMethod.POST, ClientResponse.class,
						tableInstanceAsJSON);
		assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();
	}

	public void getEntityList() throws Exception {
		ClientResponse clientResponse = service
				.path("api/metadata/entities/list/").path(TABLE_TYPE)
				.accept(MediaType.APPLICATION_JSON)
				.type(MediaType.APPLICATION_JSON)
				.method(HttpMethod.GET, ClientResponse.class);
		assert clientResponse.getStatus() == Response.Status.OK.getStatusCode();

		String responseAsString = clientResponse.getEntity(String.class);
		JSONObject response = new JSONObject(responseAsString);
		final JSONArray list = response.getJSONArray("list");
		System.out.println("list = " + list);
		assert list != null;
		assert list.length() > 0;
	}

	private void createTypes() throws Exception {
		HierarchicalTypeDefinition<ClassType> databaseTypeDefinition = createClassTypeDef(
				DATABASE_TYPE, ImmutableList.<String> of(),
				createRequiredAttrDef("name", DataTypes.STRING_TYPE),
				createRequiredAttrDef("description", DataTypes.STRING_TYPE));

		StructTypeDefinition structTypeDefinition = new StructTypeDefinition(
				"serdeType", new AttributeDefinition[] {
						createRequiredAttrDef("name", DataTypes.STRING_TYPE),
						createRequiredAttrDef("serde", DataTypes.STRING_TYPE) });

		HierarchicalTypeDefinition<ClassType> tableTypeDefinition = createClassTypeDef(
				TABLE_TYPE, ImmutableList.<String> of(),
				createRequiredAttrDef("name", DataTypes.STRING_TYPE),
				createRequiredAttrDef("description", DataTypes.STRING_TYPE),
				createRequiredAttrDef("type", DataTypes.STRING_TYPE),
				new AttributeDefinition("serde1", "serdeType",
						Multiplicity.REQUIRED, false, null),
				new AttributeDefinition("serde2", "serdeType",
						Multiplicity.REQUIRED, false, null),
				new AttributeDefinition("database", DATABASE_TYPE,
						Multiplicity.REQUIRED, true, null));

		HierarchicalTypeDefinition<TraitType> classificationTypeDefinition = createTraitTypeDef(
				"classification", ImmutableList.<String> of(),
				createRequiredAttrDef("tag", DataTypes.STRING_TYPE));

		HierarchicalTypeDefinition<ClassType> lineageTypeDefinition = createClassTypeDef(
				"HiveLineage",
				ImmutableList.<String> of(),
				createRequiredAttrDef("queryId", DataTypes.STRING_TYPE),
				createRequiredAttrDef("hiveId", DataTypes.STRING_TYPE),
				createRequiredAttrDef("user", DataTypes.STRING_TYPE),
				createRequiredAttrDef("queryStartTime", DataTypes.STRING_TYPE),
				createRequiredAttrDef("queryEndTime", DataTypes.STRING_TYPE),
				createRequiredAttrDef("query", DataTypes.STRING_TYPE),
				new AttributeDefinition("tableName", TABLE_TYPE,
						Multiplicity.REQUIRED, true, null),
				createRequiredAttrDef("success", DataTypes.STRING_TYPE),
				createRequiredAttrDef("executionEngine", DataTypes.STRING_TYPE),
				new AttributeDefinition("sourceTables", TABLE_TYPE,
						Multiplicity.COLLECTION, true, "forwardLineage"));

		typeSystem.defineTypes(ImmutableList.of(structTypeDefinition),
				ImmutableList.of(classificationTypeDefinition), ImmutableList
						.of(databaseTypeDefinition, tableTypeDefinition,
								lineageTypeDefinition));
	}

	private void submitTypes() throws Exception {
		String typesAsJSON = TypesSerialization.toJson(
				typeSystem,
				Arrays.asList(new String[] { DATABASE_TYPE, TABLE_TYPE,
						"serdeType", "classification", "HiveLineage" }));
		sumbitType(typesAsJSON, TABLE_TYPE);
		sumbitType(typesAsJSON, "HiveLineage");
	}

	private void sumbitType(String typesAsJSON, String type)
			throws JSONException {
		WebResource resource = service.path("api/metadata/types/submit").path(
				type);

		ClientResponse clientResponse = resource
{'fraction_non_alphanumeric': 0.09927360774818401, 'fraction_numerical': 0.001784121320249777, 'mean_word_length': 5.641980533220482, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3020738', 'n_tokens_mistral': 4828, 'n_tokens_neox': 4320, 'n_words': 942}