|
import argparse |
|
import os |
|
import re |
|
|
|
from tqdm import tqdm |
|
from bs4 import BeautifulSoup |
|
from markdown import markdown |
|
from pathlib import Path |
|
|
|
|
|
def markdown_to_text(markdown_string): |
|
""" Converts a markdown string to plaintext """ |
|
|
|
|
|
html = markdown(markdown_string) |
|
|
|
html = re.sub(r'<!--((.|\n)*)-->', '', html) |
|
html = re.sub('<code>bash', '<code>', html) |
|
|
|
|
|
soup = BeautifulSoup(html, "html.parser") |
|
text = ''.join(soup.findAll(text=True)) |
|
|
|
text = re.sub('```(py|diff|python)', '', text) |
|
text = re.sub('```\n', '\n', text) |
|
text = re.sub('- .*', '', text) |
|
text = text.replace('...', '') |
|
text = re.sub('\n(\n)+', '\n\n', text) |
|
|
|
return text |
|
|
|
|
|
def main(): |
|
parser = argparse.ArgumentParser() |
|
parser.add_argument("--input-dir", help="input directory with markdown", type=str, |
|
default="transformers/docs/source/en/") |
|
parser.add_argument("--output-dir", help="output directory to store raw texts", type=str, |
|
default="docs") |
|
|
|
args = parser.parse_args() |
|
input_dir = Path(args.input_dir) |
|
output_dir = Path(args.output_dir) |
|
|
|
assert os.path.isdir(input_dir), "Input directory doesn't exist" |
|
|
|
files = input_dir.rglob("*") |
|
os.makedirs(output_dir, exist_ok=True) |
|
|
|
for file in tqdm(files): |
|
parent = file.parent.stem if file.parent.stem != input_dir.stem else "" |
|
if file.is_file(): |
|
with open(file, encoding="utf-8") as f: |
|
md = f.read() |
|
|
|
text = markdown_to_text(md) |
|
|
|
with open(output_dir / f"{parent}_{file.stem}.txt", "w", encoding="utf-8") as f: |
|
f.write(text) |
|
|
|
|
|
if __name__ == "__main__": |
|
main() |
|
|
