<commit_before>* Add user to sudoers
	$ sudo usermod -a -G sudo <username>

* Login as another user
	$ sudo -u <username>

* Change password
	$ sudo passwd
	$ sudo passwd <username>

* Show index view of a list of images
	$ feh -i -f <image_file_list>

* Mirror a remote directory
	$ rsync -avz --delete -e ssh <EMAIL>:/leifeng/sopty_site sopty_site

* Mirror a local directory
	$ rsync -avz --delete <src dir> <dest dir>

* Extract information using XPath
	$ xidel <file|url> -e '//a/@href'url# extract all links of a html page
 
* Download a list of urls to disk for offline viewing
	$ wget -N -p -k --random-wait -i <url list> -o <log file>

* Download all files ending with a suffix from a web page
	$ wget --recursive --level=1 --no-directories --no-host-directories --accept <file type> <url>

* Get urls of a specify section of website
	$ wget --spider --wait 2 --random-wait -nv -e robots=off -r -X /wiki/lib/exe -l inf <url>

* Balance spacing of textual table
	$ column -t <file>
	$ column -t -s $'\t' <file> # balance columns separated by tab
	$ column -nt -s $'\t' <file> # balance columns separated by tab and also have empty cells, only works on debian release though

* Turn off text wrapping in less
	$ less -S <file>

* Print tab and newline characters
	$ cat -vet <file>

* Split a file by N lines
	$ split -d -l 10 <file> <output file prefix> # split file by 10 lines, the output file will be called prefix01, prefix02...

* Resize an image to not exceed specified size, shrink if necessary
	$ convert <image file> -resize 'widthxheight>' <resized image file>

* Resize an image to a percentage of original size, keep aspect ratio
	$ convert <image file> -resize <percent> <resized image file>

* Change image format
	$ convert <image.jpg> <image.png>

* Show basic image information
	$ identify <image file>

* List files in tar.gz/tar archives
	$ tar -tf <file.tar>
	$ tar -tzf <file.tar.gz>

* Copy the first n files to another place
	$ find <dir> -type f | head -n <N>|xargs cp -t <dest>

* Get file extension and name
	filename=$(basename "$fullfile")
	extension="${filename##*.}"
	filename="${filename%.*}"

* Update an alternative command
	$ sudo update-alternatives --config <command>

* Ubuntu/Debian application installation folder
	/usr/bin/<app>app 	# binary
	/usr/share/<app> 	# installation folder
	/etc/<app>app 		# configuration files

* Get the lines between two matching patterns
	$ sed -n ‘/start_pattern/, /end_pattern/ p’ <text file>

* Copy n random files from a directory
	$ ls <dir>|sort -R|head -n <n files>|while read f;cp $f <target dir>;done
<commit_msg>update rsync command example with more comments<commit_after>* Add user to sudoers
	$ sudo usermod -a -G sudo <username>

* Login as another user
	$ sudo -u <username>

* Change password
	$ sudo passwd
	$ sudo passwd <username>

* Show index view of a list of images
	$ feh -i -f <image_file_list>

* Mirror a remote directory
	$ rsync -avz --delete -e ssh <EMAIL>:/leifeng/sopty_site sopty_site

* Mirror a local directory
	$ rsync -avz --delete <src dir> <dest dir>
	# remove --delete option if you don't want to remove extra files in <dest dir>
	# consider --update option to skip updatting files that are newer in <dest dir>
	# if unsure, using --dry-run to test before the sync

* Extract information using XPath
	$ xidel <file|url> -e '//a/@href'url# extract all links of a html page
 
* Download a list of urls to disk for offline viewing
	$ wget -N -p -k --random-wait -i <url list> -o <log file>

* Download all files ending with a suffix from a web page
	$ wget --recursive --level=1 --no-directories --no-host-directories --accept <file type> <url>

* Get urls of a specify section of website
	$ wget --spider --wait 2 --random-wait -nv -e robots=off -r -X /wiki/lib/exe -l inf <url>

* Balance spacing of textual table
	$ column -t <file>
	$ column -t -s $'\t' <file> # balance columns separated by tab
	$ column -nt -s $'\t' <file> # balance columns separated by tab and also have empty cells, only works on debian release though

* Turn off text wrapping in less
	$ less -S <file>

* Print tab and newline characters
	$ cat -vet <file>

* Split a file by N lines
	$ split -d -l 10 <file> <output file prefix> # split file by 10 lines, the output file will be called prefix01, prefix02...

* Resize an image to not exceed specified size, shrink if necessary
	$ convert <image file> -resize 'widthxheight>' <resized image file>

* Resize an image to a percentage of original size, keep aspect ratio
	$ convert <image file> -resize <percent> <resized image file>

* Change image format
	$ convert <image.jpg> <image.png>

* Show basic image information
	$ identify <image file>

* List files in tar.gz/tar archives
	$ tar -tf <file.tar>
	$ tar -tzf <file.tar.gz>

* Copy the first n files to another place
	$ find <dir> -type f | head -n <N>|xargs cp -t <dest>

* Get file extension and name
	filename=$(basename "$fullfile")
	extension="${filename##*.}"
	filename="${filename%.*}"

* Update an alternative command
	$ sudo update-alternatives --config <command>

* Ubuntu/Debian application installation folder
	/usr/bin/<app>app 	# binary
	/usr/share/<app> 	# installation folder
	/etc/<app>app 		# configuration files

* Get the lines between two matching patterns
	$ sed -n ‘/start_pattern/, /end_pattern/ p’ <text file>

* Copy n random files from a directory
	$ ls <dir>|sort -R|head -n <n files>|while read f;cp $f <target dir>;done

{'fraction_non_alphanumeric': 0.11410750321042011, 'fraction_numerical': 0.00366905155017428, 'mean_word_length': 3.4469820554649266, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1704061', 'n_tokens_mistral': 1835, 'n_tokens_neox': 1757, 'n_words': 766}