<commit_before>//
// methods for importing data
//
// Copyright (c) 2001 Virtual Terrain Project
// Free for all uses, see license.txt for details.
//

// For compilers that support precompilation, includes "wx/wx.h".
#include "wx/wxprec.h"

#ifndef WX_PRECOMP
#include "wx/wx.h"
#endif

#include "vtdata/DLG.h"
#include "vtdata/LULC.h"

#include "Frame.h"
#include "Helper.h"
// Layers
#include "StructLayer.h"
#include "WaterLayer.h"
#include "ElevLayer.h"
#include "ImageLayer.h"
#include "RawLayer.h"
#include "RoadLayer.h"
#include "VegLayer.h"
#include "UtilityLayer.h"
// Dialogs
#include "ImportVegDlg.h"
#include "VegFieldsDlg.h"
#include "Projection2Dlg.h"

#include "ogrsf_frmts.h"

//
// remember a set of directories, one for each layer type
//
wxString ImportDirectory[LAYER_TYPES];


//
// Ask the user for a filename, and import data from it.
//
void MainFrame::ImportData(LayerType ltype)
{
	// make a string which contains filters for the appropriate file types
	wxString filter = GetImportFilterString(ltype);

	// ask the user for a filename
	// default the same directory they used last time for a layer of this type
	wxFileDialog loadFile(NULL, "Import Data", ImportDirectory[ltype], "", filter, wxOPEN);
	bool bResult = (loadFile.ShowModal() == wxID_OK);
	if (!bResult)
		return;
	wxString strFileName = loadFile.GetPath();

	// remember the directory they used
	ImportDirectory[ltype] = loadFile.GetDirectory();

	ImportDataFromFile(ltype, strFileName, true);
}

//
// Import data of a given type from a file
//
void MainFrame::ImportDataFromFile(LayerType ltype, wxString strFileName, bool bRefresh)
{
	// check to see if the file is readable
	FILE *fp = fopen(strFileName, "rb");
	if (!fp)
		return;	// Cannot Open File
	fclose(fp);

	wxString msg = "Importing Data from ";
	msg += strFileName;
	OpenProgressDialog(msg);
<commit_msg>moved archive capability to ImportDataFromArchive from Frame.cpp<commit_after>//
// methods for importing data
//
// Copyright (c) 2001 Virtual Terrain Project
// Free for all uses, see license.txt for details.
//

// For compilers that support precompilation, includes "wx/wx.h".
#include "wx/wxprec.h"

#ifndef WX_PRECOMP
#include "wx/wx.h"
#endif

#include "vtdata/DLG.h"
#include "vtdata/LULC.h"
#include "vtdata/Unarchive.h"
#include "vtdata/boost/directory.h"
#include "vtdata/FilePath.h"

#include "Frame.h"
#include "Helper.h"
// Layers
#include "StructLayer.h"
#include "WaterLayer.h"
#include "ElevLayer.h"
#include "ImageLayer.h"
#include "RawLayer.h"
#include "RoadLayer.h"
#include "VegLayer.h"
#include "UtilityLayer.h"
// Dialogs
#include "ImportVegDlg.h"
#include "VegFieldsDlg.h"
#include "Projection2Dlg.h"

#include "ogrsf_frmts.h"

//
// remember a set of directories, one for each layer type
//
wxString ImportDirectory[LAYER_TYPES];


// Helper
void GetTempFolderName(char *path, const char *base)
{
	const char *tmp = base;
	const char *tmp1 = strrchr(base, '/');
	if (tmp1)
		tmp = tmp1+1;
	const char *tmp2 = strrchr(base, '\\');
	if (tmp2 && tmp2 > tmp)
		tmp = tmp2+1;
	const char *tmp3 = strrchr(base, ':');
	if (tmp3 && tmp3 > tmp)
		tmp = tmp3+1;

	const char *temp = getenv("TEMP");
	if (temp)
		strcpy(path, temp);
	else
#if WIN32
		strcpy(path, "C:/TEMP");
#else
		strcpy(path, "/tmp");
#endif
	strcat(path, "/");
	strcat(path, tmp);
	strcat(path, "_temp");
}


//
// Ask the user for a filename, and import data from it.
//
void MainFrame::ImportData(LayerType ltype)
{
	// make a string which contains filters for the appropriate file types
	wxString filter = GetImportFilterString(ltype);

	// ask the user for a filename
	// default the same directory they used last time for a layer of this type
	wxFileDialog loadFile(NULL, "Import Data", ImportDirectory[ltype], "", filter, wxOPEN);
	bool bResult = (loadFile.ShowModal() == wxID_OK);
	if (!bResult)
		return;
	wxString strFileName = loadFile.GetPath();

	// remember the directory they used
	ImportDirectory[ltype] = loadFile.GetDirectory();

	ImportDataFromArchive(ltype, strFileName, true);
}

/**
 * Import data of a given type from a file, which can potentially be an
 * archive file.  If it's an archive, it will be unarchived to a temporary
 * folder, and the contents will be imported.
 */
void MainFrame::ImportDataFromArchive(LayerType ltype, wxString fname_in,
									  bool bRefresh)
{
	// check file extension
	wxString fname = fname_in;
	wxString ext = fname.AfterLast('.');

	using namespace boost::filesystem;

	// check if it's an archive
	bool bExpandedArchive = false;
	char prepend_path[1024];
	GetTempFolderName(prepend_path, fname_in);

	if (ext.CmpNoCase("gz") == 0 || ext.CmpNoCase("tgz") == 0 ||
		ext.CmpNoCase("tar") == 0)
	{
		// try to uncompress
		int result;
		result = vtCreateDir(prepend_path);
		if (result == 0 && errno != EEXIST)
		{
			wxMessageBox("Couldn't create temporary directory to hold contents of archive.");
			return;
		}
		strcat(prepend_path, "/");

		result = ExpandTGZ((const char *)fname, prepend_path);
		if (result < 1)
		{
			wxMessageBox("Couldn't expand archive.");
			GetTempFolderName(prepend_path, fname_in);
			vtDestroyDir(prepend_path);
			return;
		}
		else if (result == 1)
		{
			// the archive contained a single file
			for (dir_it it(prepend_path); it != dir_it(); ++it)
			{
				if (get<is_directory>(it))
					continue;
				std::string name1 = *it;
				fname = prepend_path;
				fname += name1.c_str();
				break;
			}
		}
		else if (result > 1)
		{
			// probably SDTS
			// try to guess layer type from original file name
			if (fname.Contains(".hy") || fname.Contains(".HY"))
				ltype = LT_WATER;
			if (fname.Contains(".rd") || fname.Contains(".RD"))
				ltype = LT_ROAD;
			if (fname.Contains(".dem") || fname.Contains(".DEM"))
				ltype = LT_ELEVATION;
			if (fname.Contains(".ms") || fname.Contains(".MS"))
				ltype = LT_STRUCTURE;

			// look for the catalog file
			bool found = false;
			for (dir_it it(prepend_path); it != dir_it(); ++it)
			{
				std::string name1 = *it;
				wxString fname2 = name1.c_str();
				if (fname2.Right(8).CmpNoCase("catd.ddf") == 0)
				{
					fname = prepend_path;
					fname += fname2;
					found= true;
					break;
				}
			}
			if (!found)
			{
				wxMessageBox("Don't know what to do with contents of archive.");
				return;
			}
		}

		bExpandedArchive = true;
	}

	ImportDataFromFile(ltype, fname, bRefresh);

	if (bExpandedArchive)
	{
		// clean up after ourselves
		GetTempFolderName(prepend_path, fname_in);
		vtDestroyDir(prepend_path);
	}
}

void MainFrame::ImportDataFromFile(LayerType ltype, wxString strFileName,
								   bool bRefresh)
{
	// check to see if the file is readable
	FILE *fp = fopen(strFileName, "rb");
	if (!fp)
		return;	// Cannot Open File
	fclose(fp);

	wxString msg = "Importing Data from ";
	msg += strFileName;
	OpenProgressDialog(msg);

{'fraction_non_alphanumeric': 0.13596491228070176, 'fraction_numerical': 0.006725146198830409, 'mean_word_length': 3.821000704721635, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6252333', 'n_tokens_mistral': 2611, 'n_tokens_neox': 2387, 'n_words': 688}