<commit_before>package com.jaselogic.adamsonelearn;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import android.os.Bundle;
import android.support.v4.app.ListFragment;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;

import com.jaselogic.adamsonelearn.DocumentManager.DocumentCookie;
import com.jaselogic.adamsonelearn.DocumentManager.ResponseReceiver;
import com.jaselogic.adamsonelearn.YearSelectAdapter.YearSelectListItem;

class CurriculumPageFragment {
	public static class YearSelectFragment extends ListFragment
			implements ResponseReceiver {
		public final static String SELECTOR_CURRICULUM_PAGE = "div.contentcontainer2 > table > tbody > tr > td:nth-of-type(2) > div:nth-of-type(3) div";
		public final static String SELECTOR_YEAR = "div[style*=background:#FF9]";
		
		private String cookie;
		private YearSelectAdapter adapter;
		private ArrayList<YearSelectListItem> yearArrayList;
		
		@Override
		public View onCreateView(LayoutInflater inflater, ViewGroup container,
				Bundle savedInstanceState) {
			View pageRootView = inflater.inflate(R.layout.fragment_updates, 
					container, false);
				
			yearArrayList = new ArrayList<YearSelectListItem>();	
			adapter = new YearSelectAdapter(getActivity(), yearArrayList);
			setListAdapter(adapter);
			
			//get original cookie
			cookie = ((Dashboard)getActivity()).cookie;
			
			//TODO: remove strings stdno pw
			new DocumentManager.DownloadDocumentTask(YearSelectFragment.this, 
				DocumentManager.PAGE_CURRICULUM, cookie).execute("stdno", "pw");
			
			return pageRootView;
		}
		
		@Override
		public void onResourceReceived(DocumentCookie result)
				throws IOException {
			if(result != null) {
				Elements curriculum = result.document.select(SELECTOR_CURRICULUM_PAGE);
				Iterator<Element> itr = curriculum.iterator();
				
				Pattern pattern = Pattern.compile("FF9");
				Matcher matcher;
				
				while(itr.hasNext()) {
					Element inner = itr.next();
					matcher = pattern.matcher(inner.attr("style"));
					if(matcher.find()) {
						//GET YEAR, SEMESTER
						Log.d("YEAR", inner.text().trim());
						//skip div with CCC
						itr.next();
						
						Element innest;
						while(itr.hasNext() && !(innest = itr.next()).attr("style").equals("height:20px;")) {
							if(innest.hasClass("curr")) {
								//GET PK, SUBJCODE, SUBJNAME, UNITS
								Log.d("SUBJ",
										innest.select(":root > table > tbody > tr:nth-of-type(1) td:nth-of-type(1)").text().trim() + " " +
										innest.select(":root > table > tbody > tr:nth-of-type(1) td:nth-of-type(2)").text().trim() + " " +
										innest.select(":root > table > tbody > tr:nth-of-type(1) td:nth-of-type(3)").text().trim() + " " +
										innest.select(":root > table > tbody > tr:nth-of-type(1) td:nth-of-type(4)").text().trim());
							}
						}
						//Log.d("SUBJ", msg)
					}
				}
				
				//Log.d("JUS!", curriculum.select(SELECTOR_YEAR).html());
			} else {
				Log.d("JUS!", "WALANG DOCUMENT");
			}
		}
		
	}
}<commit_msg>Add CSSPaths to extract more data from curriculum<commit_after>package com.jaselogic.adamsonelearn;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import android.os.Bundle;
import android.support.v4.app.ListFragment;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;

import com.jaselogic.adamsonelearn.DocumentManager.DocumentCookie;
import com.jaselogic.adamsonelearn.DocumentManager.ResponseReceiver;
import com.jaselogic.adamsonelearn.YearSelectAdapter.YearSelectListItem;

class CurriculumPageFragment {
	public static class YearSelectFragment extends ListFragment
			implements ResponseReceiver {
		public final static String SELECTOR_CURRICULUM_PAGE = "div.contentcontainer2 > table > tbody > tr > td:nth-of-type(2) > div:nth-of-type(3) div";
		public final static String SELECTOR_YEAR = "div[style*=background:#FF9]";
		
		private String cookie;
		private YearSelectAdapter adapter;
		private ArrayList<YearSelectListItem> yearArrayList;
		
		@Override
		public View onCreateView(LayoutInflater inflater, ViewGroup container,
				Bundle savedInstanceState) {
			View pageRootView = inflater.inflate(R.layout.fragment_updates, 
					container, false);
				
			yearArrayList = new ArrayList<YearSelectListItem>();	
			adapter = new YearSelectAdapter(getActivity(), yearArrayList);
			setListAdapter(adapter);
			
			//get original cookie
			cookie = ((Dashboard)getActivity()).cookie;
			
			//TODO: remove strings stdno pw
			new DocumentManager.DownloadDocumentTask(YearSelectFragment.this, 
				DocumentManager.PAGE_CURRICULUM, cookie).execute("stdno", "pw");
			
			return pageRootView;
		}
		
		@Override
		public void onResourceReceived(DocumentCookie result)
				throws IOException {
			if(result != null) {
				Elements curriculum = result.document.select(SELECTOR_CURRICULUM_PAGE);
				Iterator<Element> itr = curriculum.iterator();
				
				Pattern pattern = Pattern.compile("FF9");
				Matcher matcher;
				
				while(itr.hasNext()) {
					Element inner = itr.next();
					matcher = pattern.matcher(inner.attr("style"));
					if(matcher.find()) {
						//GET YEAR, SEMESTER
						Log.d("YEAR", inner.text().trim());
						//skip div with CCC
						itr.next();
						
						Element innest;
						while(itr.hasNext() && !(innest = itr.next()).attr("style").equals("height:20px;")) {
							if(innest.hasClass("curr")) {
								//GET PK, SUBJCODE, SUBJNAME, UNITS, PREREQ, COREQ
								Elements item = innest.select(":root > table > tbody");
								Log.d("SUBJ",
										item.select(":root > tr:nth-of-type(1) td:nth-of-type(1)").text().trim() + " " +
										item.select(":root > tr:nth-of-type(1) td:nth-of-type(2)").text().trim() + " " +
										item.select(":root > tr:nth-of-type(1) td:nth-of-type(3)").text().trim() + " " +
										item.select(":root > tr:nth-of-type(1) td:nth-of-type(4)").text().trim() + " " +
										"<" +
										item.select(":root > tr:nth-of-type(1) td:nth-of-type(5) span").text().trim()
										+">" + " " +
										"[" +
										item.select(":root > tr:nth-of-type(1) td:nth-of-type(6) span").text().trim()
										+ "]");
							}
						}
						//Log.d("SUBJ", msg)
					}
				}
				
				//Log.d("JUS!", curriculum.select(SELECTOR_YEAR).html());
			} else {
				Log.d("JUS!", "WALANG DOCUMENT");
			}
		}
		
	}
}
{'fraction_non_alphanumeric': 0.14417086917828537, 'fraction_numerical': 0.005339661821417977, 'mean_word_length': 3.5684281842818426, 'pattern_counts': {'":': 11, '<': 10, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1215913', 'n_tokens_mistral': 2725, 'n_tokens_neox': 2180, 'n_words': 388}